From 5175e7780a5d95c6189da10d2b970f687e614d82 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 22 Oct 2026 12:14:44 +0000 Subject: [PATCH 001/219] Backport #123999 to 26.8: Fix rows skipped when a sub-second `DateTime64` key is compared with an integer by `>` or `<` --- src/Storages/MergeTree/KeyCondition.cpp | 11 ++++++ ...etime64_key_integer_strict_bound.reference | 12 ++++++ ...25_datetime64_key_integer_strict_bound.sql | 39 +++++++++++++++++++ 3 files changed, 62 insertions(+) create mode 100644 tests/queries/0_stateless/05325_datetime64_key_integer_strict_bound.reference create mode 100644 tests/queries/0_stateless/05325_datetime64_key_integer_strict_bound.sql diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index 6bfad736d76c8..b45e96877aa3a 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -4035,6 +4035,17 @@ bool KeyCondition::extractAtomFromTree(const RPNBuilderTreeNode & node, const Bu (isNativeInteger(key_expr_type_not_null) || isDateTimeOrDateTime64(key_expr_type_not_null)) && (isNativeInteger(const_type) || isDateTimeOrDateTime64(const_type)); + /// `Range` closes the open integer bound of a strict comparison by stepping it by 1, a whole second here; + /// a scale-0 decimal compares the same and is not stepped. + if (cast_not_needed && (func_name == "less" || func_name == "greater") && isDateTime64(key_expr_type_not_null) + && assert_cast(*key_expr_type_not_null).getScale() > 0) + { + if (const_value.getType() == Field::Types::Int64) + const_value = DecimalField(Decimal256(const_value.safeGet()), 0); + else if (const_value.getType() == Field::Types::UInt64 || const_value.getType() == Field::Types::Bool) + const_value = DecimalField(Decimal256(const_value.safeGet()), 0); + } + if (!cast_not_needed && !key_expr_type_not_null->equals(*const_type)) { if (const_value.getType() == Field::Types::String) diff --git a/tests/queries/0_stateless/05325_datetime64_key_integer_strict_bound.reference b/tests/queries/0_stateless/05325_datetime64_key_integer_strict_bound.reference new file mode 100644 index 0000000000000..e8edf2d14d0f7 --- /dev/null +++ b/tests/queries/0_stateless/05325_datetime64_key_integer_strict_bound.reference @@ -0,0 +1,12 @@ +pk > 3 +pk < 3 +pk reversed 3 +pk < DateTime 3 +pk NOT <= 3 +pk monotonic chain 3 +pk pruning kept 0 +nullable pk > 3 +minmax > 4 +minmax < 3 +partition > 3 +partition < 3 diff --git a/tests/queries/0_stateless/05325_datetime64_key_integer_strict_bound.sql b/tests/queries/0_stateless/05325_datetime64_key_integer_strict_bound.sql new file mode 100644 index 0000000000000..e120d09168235 --- /dev/null +++ b/tests/queries/0_stateless/05325_datetime64_key_integer_strict_bound.sql @@ -0,0 +1,39 @@ +-- A strict comparison of a sub-second DateTime64 key with an integer or DateTime constant must not skip rows. + +DROP TABLE IF EXISTS t_pk; +DROP TABLE IF EXISTS t_pk_nullable; +DROP TABLE IF EXISTS t_minmax; +DROP TABLE IF EXISTS t_part; + +CREATE TABLE t_pk (dt DateTime64(3, 'UTC')) ENGINE = MergeTree ORDER BY dt SETTINGS index_granularity = 1; +INSERT INTO t_pk VALUES ('2020-01-01 00:00:00.000'), ('2020-01-01 00:00:00.300'), ('2020-01-01 00:00:00.500'), ('2020-01-01 00:00:01.000'); + +SELECT 'pk >', count() FROM t_pk WHERE dt > 1577836800; +SELECT 'pk <', count() FROM t_pk WHERE dt < 1577836801; +SELECT 'pk reversed', count() FROM t_pk WHERE 1577836800 < dt; +SELECT 'pk < DateTime', count() FROM t_pk WHERE dt < toDateTime('2020-01-01 00:00:01', 'UTC'); +SELECT 'pk NOT <=', count() FROM t_pk WHERE NOT (dt <= 1577836800); +SELECT 'pk monotonic chain', count() FROM t_pk WHERE toDateTime64(dt, 6) > 1577836800; +SELECT 'pk pruning kept', count() FROM t_pk WHERE dt < 1577836000 SETTINGS force_primary_key = 1, max_rows_to_read = 1; + +CREATE TABLE t_pk_nullable (dt Nullable(DateTime64(3, 'UTC'))) ENGINE = MergeTree ORDER BY dt SETTINGS allow_nullable_key = 1, index_granularity = 1; +INSERT INTO t_pk_nullable VALUES ('2020-01-01 00:00:00.000'), ('2020-01-01 00:00:00.300'), ('2020-01-01 00:00:00.500'), ('2020-01-01 00:00:01.000'); + +SELECT 'nullable pk >', count() FROM t_pk_nullable WHERE dt > 1577836800; + +CREATE TABLE t_minmax (time DateTime64(9, 'UTC'), INDEX idx_time time TYPE minmax GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple() SETTINGS index_granularity = 1; +INSERT INTO t_minmax VALUES (0::Decimal(9, 2)::DateTime64(9, 'UTC')), (0.01::Decimal(9, 2)::DateTime64(9, 'UTC')), (0.99::Decimal(9, 2)::DateTime64(9, 'UTC')), (1::Decimal(9, 2)::DateTime64(9, 'UTC')), (1.01::Decimal(9, 2)::DateTime64(9, 'UTC')); + +SELECT 'minmax >', count() FROM t_minmax WHERE time > 0 SETTINGS force_data_skipping_indices = 'idx_time'; +SELECT 'minmax <', count() FROM t_minmax WHERE time < 1 SETTINGS force_data_skipping_indices = 'idx_time'; + +CREATE TABLE t_part (dt DateTime64(3, 'UTC')) ENGINE = MergeTree PARTITION BY dt ORDER BY tuple(); +INSERT INTO t_part VALUES ('2020-01-01 00:00:00.000'), ('2020-01-01 00:00:00.300'), ('2020-01-01 00:00:00.500'), ('2020-01-01 00:00:01.000'); + +SELECT 'partition >', count() FROM t_part WHERE dt > 1577836800; +SELECT 'partition <', count() FROM t_part WHERE dt < 1577836801; + +DROP TABLE t_pk; +DROP TABLE t_pk_nullable; +DROP TABLE t_minmax; +DROP TABLE t_part; From 2b80972870fc66df5e54bdee17da7f61b8516f21 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 22 Oct 2026 16:32:14 +0000 Subject: [PATCH 002/219] Update autogenerated version to 26.8.34.13 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 1bc0134eb7c74..fd92cd2165712 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54546) +SET(VERSION_REVISION 54547) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 34) -SET(VERSION_GITHASH 4ca42662d1a0175631b2f340620d5a50dd049999) -SET(VERSION_DESCRIBE v26.8.34.1-lts) -SET(VERSION_STRING 26.8.34.1) +SET(VERSION_PATCH 35) +SET(VERSION_GITHASH e0cfed8d5e66d0e226afa2258a5956e3e22abff9) +SET(VERSION_DESCRIBE v26.8.35.1-lts) +SET(VERSION_STRING 26.8.35.1) # end of autochange From 80820076574b1b8d6f6b851ff8476ef858f7c463 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 22 Oct 2026 23:28:52 +0000 Subject: [PATCH 003/219] Backport #126906 to 26.8: Fix abort in `Protobuf` schema inference for a `map` with an unsupported value --- src/Formats/ProtobufSerializer.cpp | 3 +++ ...tobuf_map_with_unsupported_value.reference | 3 +++ ...429_protobuf_map_with_unsupported_value.sh | 13 ++++++++++++ ..._protobuf_map_with_unsupported_value.proto | 21 +++++++++++++++++++ 4 files changed, 40 insertions(+) create mode 100644 tests/queries/0_stateless/05429_protobuf_map_with_unsupported_value.reference create mode 100755 tests/queries/0_stateless/05429_protobuf_map_with_unsupported_value.sh create mode 100644 tests/queries/0_stateless/format_schemas/05429_protobuf_map_with_unsupported_value.proto diff --git a/src/Formats/ProtobufSerializer.cpp b/src/Formats/ProtobufSerializer.cpp index b9f13b1ef6f29..c49dac0b1d5ec 100644 --- a/src/Formats/ProtobufSerializer.cpp +++ b/src/Formats/ProtobufSerializer.cpp @@ -4225,6 +4225,9 @@ namespace if (!name_and_type) return std::nullopt; const auto * tuple_type = assert_cast(name_and_type->type.get()); + /// The key or the value of the map entry was skipped as unsupported, so the map is skipped as a whole. + if (tuple_type->getElements().size() != 2) + return std::nullopt; return NameAndTypePair{name_and_type->name, std::make_shared(tuple_type->getElements())}; } diff --git a/tests/queries/0_stateless/05429_protobuf_map_with_unsupported_value.reference b/tests/queries/0_stateless/05429_protobuf_map_with_unsupported_value.reference new file mode 100644 index 0000000000000..5453565c8dd8f --- /dev/null +++ b/tests/queries/0_stateless/05429_protobuf_map_with_unsupported_value.reference @@ -0,0 +1,3 @@ +BAD_ARGUMENTS +x UInt32 +supported_values Map(String, UInt32) diff --git a/tests/queries/0_stateless/05429_protobuf_map_with_unsupported_value.sh b/tests/queries/0_stateless/05429_protobuf_map_with_unsupported_value.sh new file mode 100755 index 0000000000000..e551cdbc68370 --- /dev/null +++ b/tests/queries/0_stateless/05429_protobuf_map_with_unsupported_value.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A `map` whose value type is unsupported is skipped in schema inference with +# `input_format_protobuf_skip_fields_with_unsupported_types_in_schema_inference = 1`. + +SCHEMADIR="$CUR_DIR/format_schemas" +$CLICKHOUSE_LOCAL -q "DESC file('nonexist', 'Protobuf') SETTINGS format_schema = '$SCHEMADIR/05429_protobuf_map_with_unsupported_value.proto:MessageWithMaps', input_format_protobuf_skip_fields_with_unsupported_types_in_schema_inference = 0" 2>&1 | grep -o -m1 -F 'BAD_ARGUMENTS' +$CLICKHOUSE_LOCAL -q "DESC file('nonexist', 'Protobuf') SETTINGS format_schema = '$SCHEMADIR/05429_protobuf_map_with_unsupported_value.proto:MessageWithMaps', input_format_protobuf_skip_fields_with_unsupported_types_in_schema_inference = 1" diff --git a/tests/queries/0_stateless/format_schemas/05429_protobuf_map_with_unsupported_value.proto b/tests/queries/0_stateless/format_schemas/05429_protobuf_map_with_unsupported_value.proto new file mode 100644 index 0000000000000..e616c064f520a --- /dev/null +++ b/tests/queries/0_stateless/format_schemas/05429_protobuf_map_with_unsupported_value.proto @@ -0,0 +1,21 @@ +syntax = "proto3"; + +message Empty { +} + +message Recursive { + Recursive next = 1; +} + +enum BigEnum { + SMALL = 0; + BIG = 100000; +} + +message MessageWithMaps { + uint32 x = 1; + map empty_values = 2; + map recursive_values = 3; + map big_enum_values = 4; + map supported_values = 5; +} From 5116f8f5debbaaf8467c4abf4f42c1901e69f5dd Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 22 Oct 2026 23:32:19 +0000 Subject: [PATCH 004/219] Backport #125497 to 26.8: Reject HTTP header values that RFC 2047-decode to CR, LF or NUL --- src/Access/HTTPAuthClient.h | 15 ++++++++-- src/Server/HTTP/ReadHeaders.cpp | 7 ++++- ...9_http_header_rfc2047_line_break.reference | 18 ++++++++++++ .../05339_http_header_rfc2047_line_break.sh | 29 +++++++++++++++++++ 4 files changed, 66 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/05339_http_header_rfc2047_line_break.reference create mode 100755 tests/queries/0_stateless/05339_http_header_rfc2047_line_break.sh diff --git a/src/Access/HTTPAuthClient.h b/src/Access/HTTPAuthClient.h index ca37bf851dbf0..961ee40e69b1f 100644 --- a/src/Access/HTTPAuthClient.h +++ b/src/Access/HTTPAuthClient.h @@ -3,6 +3,7 @@ #include #include +#include #include #include #include @@ -12,6 +13,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + struct HTTPAuthClientParams { Poco::URI uri; @@ -95,8 +101,13 @@ class HTTPBasicAuthClient : private HTTPAuthClient for (const auto & k : headers) { - if (this->getForwardHeaders().contains(k.first)) - request.add(k.first, k.second); + if (!this->getForwardHeaders().contains(k.first)) + continue; + /// Poco writes header values verbatim, so a line break would let the value inject extra headers + /// (e.g. an `Authorization` placed before ours) into the request to the authentication server. + if (k.second.find_first_of(std::string_view("\r\n\0", 3)) != std::string::npos) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Forwarded HTTP header \"{}\" contains CR, LF or NUL", k.first); + request.add(k.first, k.second); } Poco::Net::HTTPBasicCredentials basic_credentials{user_name, password}; diff --git a/src/Server/HTTP/ReadHeaders.cpp b/src/Server/HTTP/ReadHeaders.cpp index 142b47a8d6d2c..4a5ef642b68b9 100644 --- a/src/Server/HTTP/ReadHeaders.cpp +++ b/src/Server/HTTP/ReadHeaders.cpp @@ -87,7 +87,12 @@ void readHeaders( skipToNextLineOrEOF(in); Poco::trimRightInPlace(value); - headers.add(name, Poco::Net::MessageHeader::decodeWord(value)); + std::string decoded_value = Poco::Net::MessageHeader::decodeWord(value); + /// RFC 2047 decoding can produce bytes that are forbidden in a field value (RFC 9110, section 5.5), + /// e.g. `=0D=0A` would become a line break that splits the header when it is written out again. + if (decoded_value.find_first_of(std::string_view("\r\n\0", 3)) != std::string::npos) + throw Poco::Net::MessageException(fmt::format("Value of field \"{}\" contains CR, LF or NUL", name)); + headers.add(name, decoded_value); ++fields; } } diff --git a/tests/queries/0_stateless/05339_http_header_rfc2047_line_break.reference b/tests/queries/0_stateless/05339_http_header_rfc2047_line_break.reference new file mode 100644 index 0000000000000..21113e5c614b6 --- /dev/null +++ b/tests/queries/0_stateless/05339_http_header_rfc2047_line_break.reference @@ -0,0 +1,18 @@ +-- Q-encoded CRLF +contains CR, LF or NUL +400 +-- Q-encoded LF +contains CR, LF or NUL +400 +-- B-encoded CRLF +contains CR, LF or NUL +400 +-- Q-encoded NUL +contains CR, LF or NUL +400 +-- Encoded word in the middle of the value +contains CR, LF or NUL +400 +-- Other encoded words are still decoded +café au lait +café diff --git a/tests/queries/0_stateless/05339_http_header_rfc2047_line_break.sh b/tests/queries/0_stateless/05339_http_header_rfc2047_line_break.sh new file mode 100755 index 0000000000000..ef96f960e4b32 --- /dev/null +++ b/tests/queries/0_stateless/05339_http_header_rfc2047_line_break.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# An RFC 2047 encoded word must not decode into CR, LF or NUL inside a header value: +# such a value could split the header when it is forwarded (e.g. to an HTTP authentication server). + +function check_rejected() +{ + local header="$1" + ${CLICKHOUSE_CURL} -sS -H "$header" -w '%{http_code}\n' "${CLICKHOUSE_URL}&query=SELECT+1" | grep -o -F -e 'contains CR, LF or NUL' -e '400' +} + +echo "-- Q-encoded CRLF" +check_rejected 'X-Test: =?UTF-8?Q?x=0D=0AInjected:_1?=' +echo "-- Q-encoded LF" +check_rejected 'X-Test: =?UTF-8?Q?x=0AInjected:_1?=' +echo "-- B-encoded CRLF" +check_rejected 'X-Test: =?UTF-8?B?eA0KSW5qZWN0ZWQ6IDE=?=' +echo "-- Q-encoded NUL" +check_rejected 'X-Test: =?UTF-8?Q?x=00y?=' +echo "-- Encoded word in the middle of the value" +check_rejected 'X-Test: prefix =?UTF-8?Q?=0D=0A?= suffix' + +echo "-- Other encoded words are still decoded" +${CLICKHOUSE_CURL} -sS -H 'X-Test: =?UTF-8?Q?caf=C3=A9_au_lait?=' "${CLICKHOUSE_URL}&allow_get_client_http_header=1" -d "SELECT getClientHTTPHeader('X-Test')" +${CLICKHOUSE_CURL} -sS -H 'X-Test: =?UTF-8?B?Y2Fmw6k=?=' "${CLICKHOUSE_URL}&allow_get_client_http_header=1" -d "SELECT getClientHTTPHeader('X-Test')" From 29b9e70bc82b8ffbd6fe9700a383a7051ecbff21 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 22 Oct 2026 23:35:51 +0000 Subject: [PATCH 005/219] Backport #122164 to 26.8: Fix vertical TTL delete merge failing when the rows TTL reads a column no part stores --- src/Storages/MergeTree/IMergeTreeDataPart.h | 3 +- src/Storages/MergeTree/MergeTask.cpp | 21 +++++++++- ..._ttl_delete_unmaterialized_input.reference | 2 + ...rtical_ttl_delete_unmaterialized_input.sql | 40 +++++++++++++++++++ 4 files changed, 64 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/05258_vertical_ttl_delete_unmaterialized_input.reference create mode 100644 tests/queries/0_stateless/05258_vertical_ttl_delete_unmaterialized_input.sql diff --git a/src/Storages/MergeTree/IMergeTreeDataPart.h b/src/Storages/MergeTree/IMergeTreeDataPart.h index 86d129a5f38f3..682a244d769d3 100644 --- a/src/Storages/MergeTree/IMergeTreeDataPart.h +++ b/src/Storages/MergeTree/IMergeTreeDataPart.h @@ -464,7 +464,8 @@ class IMergeTreeDataPart : public std::enable_shared_from_thishasRowsTTL() || global_ctx.metadata_snapshot->hasAnyRowsWhereTTL(); + const auto & metadata = *global_ctx.metadata_snapshot; + if (!metadata.hasRowsTTL() && !metadata.hasAnyRowsWhereTTL()) + return false; + + /// `TTLDeleteFilterTransform` evaluates the rows TTLs before the merge, but only `TTLTransform`, + /// after the merge, fills the columns that no source part stores. + const auto & expired_columns = global_ctx.new_data_part->expired_columns; + auto is_missing = [&](const NameAndTypePair & column) + { + return Nested::tryGetColumnNameInStorage(column.name, expired_columns).has_value(); + }; + auto reads_missing_column = [&](const TTLDescription & ttl) + { + return std::ranges::any_of(ttl.expression_columns, is_missing) || std::ranges::any_of(ttl.where_expression_columns, is_missing); + }; + + if (metadata.hasRowsTTL() && reads_missing_column(metadata.getRowsTTL())) + return false; + + return std::ranges::none_of(metadata.getRowsWhereTTLs(), reads_missing_column); } bool MergeTask::isVerticalTTLDelete( diff --git a/tests/queries/0_stateless/05258_vertical_ttl_delete_unmaterialized_input.reference b/tests/queries/0_stateless/05258_vertical_ttl_delete_unmaterialized_input.reference new file mode 100644 index 0000000000000..04b2ce0808f88 --- /dev/null +++ b/tests/queries/0_stateless/05258_vertical_ttl_delete_unmaterialized_input.reference @@ -0,0 +1,2 @@ +modify_route 20 1 Horizontal +modify_route_control 20 1 Vertical diff --git a/tests/queries/0_stateless/05258_vertical_ttl_delete_unmaterialized_input.sql b/tests/queries/0_stateless/05258_vertical_ttl_delete_unmaterialized_input.sql new file mode 100644 index 0000000000000..7aee65903a627 --- /dev/null +++ b/tests/queries/0_stateless/05258_vertical_ttl_delete_unmaterialized_input.sql @@ -0,0 +1,40 @@ +-- A rows TTL whose input column no source part stores keeps the merge horizontal (#121960). +-- `*_control` tables store that column, so their merge stays a vertical TTL delete. + +SET optimize_throw_if_noop = 1; +SET materialize_ttl_after_modify = 0; +SYSTEM FLUSH LOGS part_log; + +-- One row per active part: a successful OPTIMIZE leaves one row, with the algorithm of the merge that wrote it. +CREATE VIEW merged_parts AS +SELECT p.table AS table, p.rows AS rows, p.delete_ttl_info_min > toDateTime(0) AS ttl_calculated, l.merge_algorithm AS merge_algorithm +FROM system.parts AS p +LEFT JOIN (SELECT table, part_name, merge_algorithm FROM system.part_log WHERE database = currentDatabase() AND event_type = 'MergeParts') AS l + ON p.table = l.table AND p.name = l.part_name +WHERE p.database = currentDatabase() AND p.active; + +-- The TTL is set with `materialize_ttl_after_modify = 0`, so the parts' TTL stays uncalculated. +-- `max_bytes_to_merge_at_max_space_in_pool = 1` keeps the background pool off regular merges, so the `OPTIMIZE` does the merge. +CREATE TABLE modify_route (id UInt64, v UInt64, w UInt64) +ENGINE = MergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 0, min_bytes_for_full_part_storage = 0, max_bytes_to_merge_at_max_space_in_pool = 1, + ratio_of_defaults_for_sparse_serialization = 1.0, enable_vertical_merge_algorithm = 1, + vertical_merge_algorithm_min_rows_to_activate = 1, vertical_merge_algorithm_min_columns_to_activate = 1, + vertical_merge_algorithm_min_bytes_to_activate = 0, vertical_merge_optimize_ttl_delete = 1; +CREATE TABLE modify_route_control (id UInt64, v UInt64, w UInt64, d DateTime) +ENGINE = MergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 0, min_bytes_for_full_part_storage = 0, max_bytes_to_merge_at_max_space_in_pool = 1, + ratio_of_defaults_for_sparse_serialization = 1.0, enable_vertical_merge_algorithm = 1, + vertical_merge_algorithm_min_rows_to_activate = 1, vertical_merge_algorithm_min_columns_to_activate = 1, + vertical_merge_algorithm_min_bytes_to_activate = 0, vertical_merge_optimize_ttl_delete = 1; +INSERT INTO modify_route SELECT number, number, number FROM numbers(10); +INSERT INTO modify_route SELECT number + 10, number, number FROM numbers(10); +INSERT INTO modify_route_control SELECT *, 0 FROM modify_route WHERE id < 10; +INSERT INTO modify_route_control SELECT *, 0 FROM modify_route WHERE id >= 10; +ALTER TABLE modify_route ADD COLUMN d DateTime; +ALTER TABLE modify_route MODIFY TTL d + INTERVAL 100 YEAR; +ALTER TABLE modify_route_control MODIFY TTL d + INTERVAL 100 YEAR; +OPTIMIZE TABLE modify_route FINAL; +OPTIMIZE TABLE modify_route_control FINAL; +SYSTEM FLUSH LOGS part_log; +SELECT * FROM merged_parts WHERE table LIKE 'modify_route%' ORDER BY table; From 9eeb58138e27de280860fd417220768ff630af43 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 23 Oct 2026 03:41:56 +0000 Subject: [PATCH 006/219] Backport #127281 to 26.8: Fix crash in MsgPack schema inference for a map with a nil or empty key --- .../Formats/Impl/MsgPackRowInputFormat.cpp | 4 ++-- ...ema_inference_untypeable_map_key.reference | 4 ++++ ...ck_schema_inference_untypeable_map_key.sql | 22 +++++++++++++++++++ 3 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/05441_msgpack_schema_inference_untypeable_map_key.reference create mode 100644 tests/queries/0_stateless/05441_msgpack_schema_inference_untypeable_map_key.sql diff --git a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp index b59fa9a055be6..8e630effe06df 100644 --- a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp @@ -745,10 +745,10 @@ DataTypePtr MsgPackSchemaReader::getDataType(const msgpack::object & object, siz msgpack::object_map object_map = object.via.map; if (object_map.size) { - auto key_type = removeNullable(getDataType(object_map.ptr[0].key, depth + 1)); + auto key_type = getDataType(object_map.ptr[0].key, depth + 1); auto value_type = getDataType(object_map.ptr[0].val, depth + 1); if (key_type && value_type) - return std::make_shared(key_type, value_type); + return std::make_shared(removeNullable(key_type), value_type); } return nullptr; } diff --git a/tests/queries/0_stateless/05441_msgpack_schema_inference_untypeable_map_key.reference b/tests/queries/0_stateless/05441_msgpack_schema_inference_untypeable_map_key.reference new file mode 100644 index 0000000000000..739d1f2b1342a --- /dev/null +++ b/tests/queries/0_stateless/05441_msgpack_schema_inference_untypeable_map_key.reference @@ -0,0 +1,4 @@ +Map(Int64, Nullable(Int64)) {1:1} +Map(Int64, Nullable(Int64)) {0:1} +Map(Int64, Nullable(Int64)) {2:3} +Map(Int64, Int64) {0:1} diff --git a/tests/queries/0_stateless/05441_msgpack_schema_inference_untypeable_map_key.sql b/tests/queries/0_stateless/05441_msgpack_schema_inference_untypeable_map_key.sql new file mode 100644 index 0000000000000..fb99be3512991 --- /dev/null +++ b/tests/queries/0_stateless/05441_msgpack_schema_inference_untypeable_map_key.sql @@ -0,0 +1,22 @@ +-- Tags: no-fasttest +-- no-fasttest: MsgPack format is not available in the fasttest environment + +-- A MsgPack map whose first key has no inferable type (nil, empty array, empty map, nested map with a nil key) +-- must be reported as "cannot determine type", not dereferenced. + +SET input_format_msgpack_number_of_columns = 1; + +-- {1: 1} +SELECT toTypeName(c1), c1 FROM format(MsgPack, '\x81\x01\x01'); + +-- {nil: 1}, {[]: 1}, {{}: 1}, {{nil: 1}: 1} +DESC format(MsgPack, '\x81\xc0\x01'); -- { serverError CANNOT_EXTRACT_TABLE_STRUCTURE } +DESC format(MsgPack, '\x81\x90\x01'); -- { serverError CANNOT_EXTRACT_TABLE_STRUCTURE } +DESC format(MsgPack, '\x81\x80\x01'); -- { serverError CANNOT_EXTRACT_TABLE_STRUCTURE } +DESC format(MsgPack, '\x81\x81\xc0\x01\x01'); -- { serverError CANNOT_EXTRACT_TABLE_STRUCTURE } + +-- {nil: 1}, {2: 3}: the type is taken from the second row. +SELECT toTypeName(c1), c1 FROM format(MsgPack, '\x81\xc0\x01\x81\x02\x03'); + +-- {nil: 1} with schema_inference_hints, which the error message suggests. +SELECT toTypeName(c1), c1 FROM format(MsgPack, '\x81\xc0\x01') SETTINGS schema_inference_hints = 'c1 Map(Int64, Int64)'; From 4090d028a80bf625450692dbf8f3ebec603a101d Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 23 Oct 2026 10:13:03 +0000 Subject: [PATCH 007/219] Backport #127136 to 26.8: Require drop privileges when restoring replacement access entities --- src/Access/AccessBackup.cpp | 73 ++++++---- src/Access/AccessBackup.h | 8 +- src/Backups/RestorerFromBackup.cpp | 11 +- .../test_backup_restore_new/test.py | 128 ++++++++++++++++++ 4 files changed, 182 insertions(+), 38 deletions(-) diff --git a/src/Access/AccessBackup.cpp b/src/Access/AccessBackup.cpp index 523ab1d167893..8f81ddd0bc205 100644 --- a/src/Access/AccessBackup.cpp +++ b/src/Access/AccessBackup.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -177,6 +178,23 @@ namespace } } }; + + AccessRights getRequiredAccessToRestoreGrants(const AccessRights & access) + { + AccessRights required_access; + for (const auto & element : access.getElements()) + { + /// Restoring a grant requires `GRANT OPTION` even if the backed-up entity does not have it. + if (element.is_partial_revoke && element.grant_option) + continue; + + if (element.is_partial_revoke) + required_access.revoke(element); + else + required_access.grantWithGrantOption(element); + } + return required_access; + } } @@ -339,12 +357,14 @@ void AccessRestorerFromBackup::loadFromBackup() } -AccessRightsElements AccessRestorerFromBackup::getRequiredAccess() const +AccessRights AccessRestorerFromBackup::getRequiredAccess() const { if (!loaded) throw Exception(ErrorCodes::LOGICAL_ERROR, "Access entities not loaded"); - AccessRightsElements res; + AccessRights res; + /// Like `CREATE ... OR REPLACE`, replacement requires `DROP` even if the entity does not exist. + const bool replace = (creation_mode == RestoreAccessCreationMode::kReplace); for (const auto & [id, entity_info] : entity_infos) { if (!entity_info.restore) @@ -356,57 +376,56 @@ AccessRightsElements AccessRestorerFromBackup::getRequiredAccess() const case User::TYPE: { const auto & user = typeid_cast(*entity); - res.emplace_back(AccessType::CREATE_USER, user.getName()); + AccessFlags flags = AccessType::CREATE_USER; + if (replace) + flags |= AccessType::DROP_USER; + res.grant(flags, user.getName()); if (!restore_with_current_grants) - { - auto elements = user.access.getElements(); - for (auto & element : elements) - { - if (!element.is_partial_revoke) - element.grant_option = true; - res.emplace_back(element); - } - } + res.makeUnion(getRequiredAccessToRestoreGrants(user.access)); if (!user.granted_roles.isEmpty()) - res.emplace_back(AccessType::ROLE_ADMIN); + res.grant(AccessType::ROLE_ADMIN); break; } case Role::TYPE: { const auto & role = typeid_cast(*entity); - res.emplace_back(AccessType::CREATE_ROLE, role.getName()); + AccessFlags flags = AccessType::CREATE_ROLE; + if (replace) + flags |= AccessType::DROP_ROLE; + res.grant(flags, role.getName()); if (!restore_with_current_grants) - { - auto elements = role.access.getElements(); - for (auto & element : elements) - { - if (!element.is_partial_revoke) - element.grant_option = true; - res.emplace_back(element); - } - } + res.makeUnion(getRequiredAccessToRestoreGrants(role.access)); if (!role.granted_roles.isEmpty()) - res.emplace_back(AccessType::ROLE_ADMIN); + res.grant(AccessType::ROLE_ADMIN); break; } case SettingsProfile::TYPE: { - res.emplace_back(AccessType::CREATE_SETTINGS_PROFILE); + AccessFlags flags = AccessType::CREATE_SETTINGS_PROFILE; + if (replace) + flags |= AccessType::DROP_SETTINGS_PROFILE; + res.grant(flags); break; } case RowPolicy::TYPE: { const auto & policy = typeid_cast(*entity); - res.emplace_back(AccessType::CREATE_ROW_POLICY, policy.getDatabase(), policy.getTableName()); + AccessFlags flags = AccessType::CREATE_ROW_POLICY; + if (replace) + flags |= AccessType::DROP_ROW_POLICY; + res.grant(flags, policy.getDatabase(), policy.getTableName()); break; } case Quota::TYPE: { - res.emplace_back(AccessType::CREATE_QUOTA); + AccessFlags flags = AccessType::CREATE_QUOTA; + if (replace) + flags |= AccessType::DROP_QUOTA; + res.grant(flags); break; } diff --git a/src/Access/AccessBackup.h b/src/Access/AccessBackup.h index 8219d1051035f..da0f3088a0330 100644 --- a/src/Access/AccessBackup.h +++ b/src/Access/AccessBackup.h @@ -11,7 +11,7 @@ class AccessControl; enum class AccessEntityType : uint8_t; struct IAccessEntity; using AccessEntityPtr = std::shared_ptr; -class AccessRightsElements; +class AccessRights; class IAccessStorage; class IBackup; using BackupPtr = std::shared_ptr; @@ -69,9 +69,9 @@ class AccessRestorerFromBackup /// Loads access entities from the backup. void loadFromBackup(); - /// Checks that the current user can do restoring. - /// Function loadFromBackup() must be called before that. - AccessRightsElements getRequiredAccess() const; + /// Returns the access required to restore the loaded entities. + /// `loadFromBackup` must be called first. + AccessRights getRequiredAccess() const; /// Generates random IDs for access entities we're restoring to insert them into an access storage; /// and finds IDs of existing access entities which are used as dependencies. diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index ad9315956e809..56e63dee650f6 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -26,7 +26,6 @@ #include #include #include -#include #include #include #include @@ -279,6 +278,7 @@ void RestorerFromBackup::loadSystemAccessTables() void RestorerFromBackup::checkAccessForObjectsFoundInBackup() const { AccessRightsElements required_access; + AccessRights required_access_rights; { std::lock_guard lock{mutex}; @@ -367,15 +367,12 @@ void RestorerFromBackup::checkAccessForObjectsFoundInBackup() const required_access.emplace_back(flags, table_name.database, table_name.table); } + required_access_rights = AccessRights{required_access}; + /// Partial revokes in restored grants must not remove the rights required to restore other objects. if (access_restorer) - insertAtEnd(required_access, access_restorer->getRequiredAccess()); + required_access_rights.makeUnion(access_restorer->getRequiredAccess()); } - /// We convert to AccessRights to check if we have the rights contained in our current access. - /// This handles the situation when restoring partial revoked grants: - /// GRANT SELECT ON *.* - /// REVOKE SELECT FROM systems.* - const auto & required_access_rights = AccessRights{required_access}; const auto & current_user_access_rights = context->getAccess()->getAccessRightsWithImplicit(); if (current_user_access_rights->contains(required_access_rights)) return; diff --git a/tests/integration/test_backup_restore_new/test.py b/tests/integration/test_backup_restore_new/test.py index a7e366e6a3c30..5948216ac36d5 100644 --- a/tests/integration/test_backup_restore_new/test.py +++ b/tests/integration/test_backup_restore_new/test.py @@ -1500,6 +1500,134 @@ def test_system_users_required_privileges(): instance.query("DROP USER u2") +# Access backups include every entity of a type, so these checks need this module's isolated server. +def test_restore_row_policy_replace_requires_scoped_drop(): + instance.query("CREATE DATABASE test") + instance.query( + "CREATE TABLE test.table (owner UInt64) ENGINE = MergeTree ORDER BY owner" + ) + instance.query("CREATE USER u2") + instance.query("GRANT READ ON DISK TO u2") + instance.query("GRANT CREATE ROW POLICY ON test.table TO u2") + instance.query( + "CREATE ROW POLICY rowpol1 ON test.table AS RESTRICTIVE USING 1 TO ALL" + ) + backup_name = new_backup_name() + instance.query(f"BACKUP TABLE system.row_policies TO {backup_name}") + instance.query("ALTER ROW POLICY rowpol1 ON test.table USING owner = 1") + + restore = f"RESTORE TABLE system.row_policies FROM {backup_name}" + select_filter = "SELECT select_filter FROM system.row_policies WHERE short_name = 'rowpol1' AND database = 'test' AND table = 'table'" + instance.query(f"{restore} SETTINGS create_access='if-not-exists'", user="u2") + assert instance.query(select_filter) == "owner = 1\n" + + error = instance.query_and_get_error( + f"{restore} SETTINGS create_access='replace'", user="u2" + ) + assert "ACCESS_DENIED" in error + assert "DROP ROW POLICY" in error + assert instance.query(select_filter) == "owner = 1\n" + + instance.query("GRANT DROP ROW POLICY ON test.another_table TO u2") + error = instance.query_and_get_error( + f"{restore} SETTINGS create_access='replace'", user="u2" + ) + assert "ACCESS_DENIED" in error + assert "DROP ROW POLICY" in error + assert instance.query(select_filter) == "owner = 1\n" + + instance.query("DROP ROW POLICY rowpol1 ON test.table") + error = instance.query_and_get_error( + f"{restore} SETTINGS create_access='replace'", user="u2" + ) + assert "ACCESS_DENIED" in error + assert "DROP ROW POLICY" in error + assert instance.query(select_filter) == "" + + instance.query(f"{restore} SETTINGS create_access='create'", user="u2") + assert instance.query(select_filter) == "1\n" + instance.query("ALTER ROW POLICY rowpol1 ON test.table USING owner = 1") + instance.query("GRANT DROP ROW POLICY ON test.table TO u2") + instance.query(f"{restore} SETTINGS create_access='replace'", user="u2") + assert instance.query(select_filter) == "1\n" + + +@pytest.mark.parametrize( + "kind,system_table,target", [("USER", "users", "u1"), ("ROLE", "roles", "r1")] +) +def test_restore_access_partial_revoke_cannot_cancel_drop(kind, system_table, target): + instance.query(f"CREATE {kind} {target} SETTINGS max_threads = 1") + instance.query(f"GRANT DROP {kind} ON * TO {target}") + instance.query(f"REVOKE DROP {kind} ON {target} FROM {target}") + backup_name = new_backup_name() + instance.query(f"BACKUP TABLE system.{system_table} TO {backup_name}") + instance.query(f"ALTER {kind} {target} SETTINGS max_threads = 2") + + # Create the actor after the backup so its grants cannot mask the target's partial revoke. + instance.query("CREATE USER u2") + instance.query("GRANT READ ON DISK TO u2") + instance.query(f"GRANT CREATE {kind} ON {target} TO u2") + instance.query(f"GRANT DROP {kind} ON * TO u2 WITH GRANT OPTION") + instance.query(f"REVOKE DROP {kind} ON {target} FROM u2") + restore = f"RESTORE TABLE system.{system_table} FROM {backup_name} SETTINGS create_access='replace'" + error = instance.query_and_get_error(restore, user="u2") + assert "ACCESS_DENIED" in error + assert f"DROP {kind}" in error + assert "max_threads = 2" in instance.query(f"SHOW CREATE {kind} {target}") + + # Ordinary `DROP` is sufficient; the target's grants exclude `DROP` on itself. + instance.query(f"GRANT DROP {kind} ON {target} TO u2") + instance.query(restore, user="u2") + assert "max_threads = 1" in instance.query(f"SHOW CREATE {kind} {target}") + + +def test_restore_access_partial_revokes_preserve_grant_and_table_requirements(): + instance.query("CREATE DATABASE test") + instance.query( + "CREATE TABLE test.table (owner UInt64) ENGINE = MergeTree ORDER BY owner" + ) + instance.query("INSERT INTO test.table VALUES (1)") + instance.query("CREATE ROLE r1 SETTINGS max_threads = 1") + instance.query("GRANT SELECT, INSERT ON test.* TO r1 WITH GRANT OPTION") + instance.query("REVOKE GRANT OPTION FOR SELECT ON test.table FROM r1") + instance.query("REVOKE INSERT ON test.table FROM r1") + backup_name = new_backup_name() + backup_tables = "TABLE system.roles, TABLE test.table" + instance.query(f"BACKUP {backup_tables} TO {backup_name}") + instance.query("ALTER ROLE r1 SETTINGS max_threads = 2") + + instance.query("CREATE USER u2") + instance.query("GRANT READ ON DISK TO u2") + instance.query("GRANT CREATE ROLE, DROP ROLE ON r1 TO u2") + instance.query("GRANT CREATE TABLE ON test.table TO u2") + instance.query("GRANT SELECT, INSERT ON test.* TO u2 WITH GRANT OPTION") + instance.query("REVOKE GRANT OPTION FOR SELECT ON test.table FROM u2") + restore = f"RESTORE TABLE system.roles FROM {backup_name} SETTINGS create_access='replace'" + error = instance.query_and_get_error(restore, user="u2") + assert "ACCESS_DENIED" in error + assert "SELECT" in error + assert "WITH GRANT OPTION" in error + assert "max_threads = 2" in instance.query("SHOW CREATE ROLE r1") + + instance.query("GRANT SELECT ON test.table TO u2 WITH GRANT OPTION") + instance.query(restore, user="u2") + assert "max_threads = 1" in instance.query("SHOW CREATE ROLE r1") + + instance.query("TRUNCATE TABLE test.table") + instance.query("REVOKE INSERT ON test.table FROM u2") + restore_with_table = ( + f"RESTORE {backup_tables} FROM {backup_name} SETTINGS create_access='replace'" + ) + error = instance.query_and_get_error(restore_with_table, user="u2") + assert "ACCESS_DENIED" in error + assert "INSERT" in error + assert instance.query("SELECT count() FROM test.table") == "0\n" + + instance.query("GRANT INSERT ON test.table TO u2") + instance.query(restore_with_table, user="u2") + assert instance.query("SELECT * FROM test.table") == "1\n" + + def test_system_users_async(): instance.query("CREATE USER u1 IDENTIFIED BY 'qwe123' SETTINGS custom_c = 3") From 94b20e7e2cf38a65257f571437683815e4528bef Mon Sep 17 00:00:00 2001 From: Shaohua Wang Date: Fri, 23 Oct 2026 00:01:18 +1100 Subject: [PATCH 008/219] fix --- src/Common/FieldBinaryEncoding.cpp | 9 ++ src/Core/Field.cpp | 61 ++++++------ src/Core/tests/gtest_field.cpp | 99 +++++++++++++++++++ .../tests/gtest_fields_binary_enciding.cpp | 28 ++++++ src/Core/tests/gtest_settings.cpp | 31 ++++++ 5 files changed, 200 insertions(+), 28 deletions(-) diff --git a/src/Common/FieldBinaryEncoding.cpp b/src/Common/FieldBinaryEncoding.cpp index 410f6abf2a9c2..f08c94fbe30dc 100644 --- a/src/Common/FieldBinaryEncoding.cpp +++ b/src/Common/FieldBinaryEncoding.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -11,6 +12,7 @@ namespace ErrorCodes { extern const int UNSUPPORTED_METHOD; extern const int INCORRECT_DATA; + extern const int ARGUMENT_OUT_OF_BOUND; } namespace @@ -262,6 +264,13 @@ DecimalField decodeDecimal(ReadBuffer & buf) { UInt32 scale = 0; readVarUInt(scale, buf); + /// The scale is attacker-controlled wire data (a `Dynamic` value or an `AggregateFunction` type + /// parameter carries its own binary-encoded Field); an out-of-range value later overruns a + /// fixed-size buffer in the decimal text writer, so reject it at the boundary, matching + /// `readDecimalFieldBinary` and `DataTypeDecimalBase`. + if (scale > DecimalUtils::max_precision) + throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Scale {} is out of bounds (max scale: {})", + scale, DecimalUtils::max_precision); T value{}; readBinaryLittleEndian(value, buf); return DecimalField(value, scale); diff --git a/src/Core/Field.cpp b/src/Core/Field.cpp index 5e9998838ce8a..6e3de2b8dbe57 100644 --- a/src/Core/Field.cpp +++ b/src/Core/Field.cpp @@ -3,8 +3,10 @@ #include #include #include +#include #include #include +#include #include #include #include @@ -33,6 +35,7 @@ extern const int INCORRECT_DATA; extern const int NOT_IMPLEMENTED; extern const int LOGICAL_ERROR; extern const int ILLEGAL_TYPE_OF_ARGUMENT; +extern const int ARGUMENT_OUT_OF_BOUND; } void Field::initEmptyContainer(Types::Which w) @@ -368,8 +371,26 @@ bool Field::operator== (const Field & rhs) const throw Exception(ErrorCodes::BAD_TYPE_OF_FIELD, "Bad type of Field"); } +template +static DecimalField readDecimalFieldBinary(ReadBuffer & buf) +{ + T value{}; + readBinary(value, buf); + UInt32 scale = 0; + readBinary(scale, buf); + /// The scale is attacker-controlled wire data; an out-of-range value later overruns a + /// fixed-size buffer in the decimal text writer, so reject it at the boundary. + if (scale > DecimalUtils::max_precision) + throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Scale {} is out of bounds (max scale: {})", + scale, DecimalUtils::max_precision); + return DecimalField(value, scale); +} + static Field getBinaryValue(UInt8 type, ReadBuffer & buf) { + /// Container fields recurse through readFieldBinary; the nesting depth is a property of the + /// wire data, so bound it here as the write side (FieldVisitorWriteBinary) already does. + checkStackSize(); switch (static_cast(type)) { case Field::Types::Null: @@ -480,37 +501,13 @@ static Field getBinaryValue(UInt8 type, ReadBuffer & buf) return bool(value); } case Field::Types::Decimal32: - { - Decimal value{}; - readBinary(value, buf); - UInt32 scale = 0 ; - readBinary(scale, buf); - return DecimalField(value, scale); - } + return readDecimalFieldBinary(buf); case Field::Types::Decimal64: - { - Decimal value{}; - readBinary(value, buf); - UInt32 scale = 0; - readBinary(scale, buf); - return DecimalField(value, scale); - } + return readDecimalFieldBinary(buf); case Field::Types::Decimal128: - { - Decimal value{}; - readBinary(value, buf); - UInt32 scale = 0; - readBinary(scale, buf); - return DecimalField(value, scale); - } + return readDecimalFieldBinary(buf); case Field::Types::Decimal256: - { - Decimal value{}; - readBinary(value, buf); - UInt32 scale = 0; - readBinary(scale, buf); - return DecimalField(value, scale); - } + return readDecimalFieldBinary(buf); case Field::Types::CustomType: return Field(); } @@ -649,6 +646,14 @@ void readQuoted(DecimalField & x, ReadBuffer & buf) } else scale = -exponent; + /// The scale comes from untrusted text: `restoreFromDump` is reached from the pre-auth native + /// settings reader for a `CUSTOM`-flagged setting (`SettingFieldCustom::parseFromString`), and an + /// exponent-form value such as `Decimal64_'1e-1000'` builds a scale that bypasses the binary + /// reader's check. An out-of-range scale later overruns a fixed-size buffer in the decimal text + /// writer, so reject it here, matching `readDecimalFieldBinary` and `DataTypeDecimalBase`. + if (scale > DecimalUtils::max_precision) + throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Scale {} is out of bounds (max scale: {})", + scale, DecimalUtils::max_precision); assertChar('\'', buf); x = DecimalField{value, scale}; } diff --git a/src/Core/tests/gtest_field.cpp b/src/Core/tests/gtest_field.cpp index b7014ff506563..68bfc3db544e8 100644 --- a/src/Core/tests/gtest_field.cpp +++ b/src/Core/tests/gtest_field.cpp @@ -1,8 +1,17 @@ #include #include +#include +#include +#include #include +namespace DB::ErrorCodes +{ + extern const int TOO_DEEP_RECURSION; + extern const int ARGUMENT_OUT_OF_BOUND; +} + using namespace DB; GTEST_TEST(Field, FromBool) @@ -222,3 +231,93 @@ GTEST_TEST(Field, CompareUUID) ASSERT_TRUE(one == one_again); ASSERT_FALSE(one == two); } + + +/// A malformed binary Field that declares an arbitrarily deep nesting must be rejected with +/// TOO_DEEP_RECURSION, not run the reader past the stack guard page. The bytes are hand-built +/// because the write side also bounds depth, so a legitimately serialized value cannot be this deep. +GTEST_TEST(Field, ReadBinaryDeeplyNestedThrowsNotOverflow) +{ + WriteBufferFromOwnString wb; + for (size_t i = 0; i < 500000; ++i) + { + writeBinary(UInt8(Field::Types::Array), wb); + writeBinary(size_t(1), wb); /// one child + } + writeBinary(UInt8(Field::Types::Null), wb); /// leaf + wb.finalize(); + + ReadBufferFromString rb(wb.str()); + try + { + readFieldBinary(rb); + FAIL() << "expected TOO_DEEP_RECURSION"; + } + catch (const Exception & e) + { + ASSERT_EQ(e.code(), ErrorCodes::TOO_DEEP_RECURSION); + } + + /// A shallow array still round-trips. + WriteBufferFromOwnString ok_wb; + writeFieldBinary(Field(Array{Field(UInt64(1)), Field(UInt64(2))}), ok_wb); + ok_wb.finalize(); + ReadBufferFromString ok_rb(ok_wb.str()); + Field roundtrip = readFieldBinary(ok_rb); + ASSERT_EQ(roundtrip.safeGet().size(), 2u); +} + + +/// A binary Decimal whose scale exceeds the type's maximum precision is rejected at the +/// deserialization boundary; otherwise the out-of-range scale overruns a fixed-size buffer in +/// the decimal text writer. +GTEST_TEST(Field, ReadBinaryDecimalOutOfRangeScaleThrows) +{ + WriteBufferFromOwnString wb; + writeBinary(UInt8(Field::Types::Decimal64), wb); + writeBinary(Int64(123), wb); /// value (Decimal is a POD 8-byte payload) + writeBinary(UInt32(1000), wb); /// scale far above max_precision = 18 + wb.finalize(); + + ReadBufferFromString rb(wb.str()); + try + { + readFieldBinary(rb); + FAIL() << "expected ARGUMENT_OUT_OF_BOUND"; + } + catch (const Exception & e) + { + ASSERT_EQ(e.code(), ErrorCodes::ARGUMENT_OUT_OF_BOUND); + } + + /// A valid scale round-trips. + WriteBufferFromOwnString ok_wb; + writeFieldBinary(Field(DecimalField(Decimal64(12345), 4)), ok_wb); + ok_wb.finalize(); + ReadBufferFromString ok_rb(ok_wb.str()); + Field roundtrip = readFieldBinary(ok_rb); + ASSERT_EQ(roundtrip.safeGet>().getScale(), 4u); +} + + +/// The text `Field` dump reader `restoreFromDump` shares the out-of-range-scale hazard with the +/// binary reader: it is reachable from the pre-auth native settings path for a `CUSTOM`-flagged +/// setting. The exponent form `Decimal64_'1e-1000'` builds a `DecimalField` with scale 1000 (the +/// `e`/`E` branch of `readDigits` bypasses the closing-quote check the plain-fraction form hits), +/// which an unfixed reader would hand to the decimal text writer. The scale must be rejected. +GTEST_TEST(Field, RestoreFromDumpDecimalOutOfRangeScaleThrows) +{ + try + { + Field::restoreFromDump("Decimal64_'1e-1000'"); + FAIL() << "expected ARGUMENT_OUT_OF_BOUND"; + } + catch (const Exception & e) + { + ASSERT_EQ(e.code(), ErrorCodes::ARGUMENT_OUT_OF_BOUND); + } + + /// An in-range exponent-form scale still round-trips. + Field restored = Field::restoreFromDump("Decimal64_'1e-5'"); + ASSERT_EQ(restored.safeGet>().getScale(), 5u); +} diff --git a/src/Core/tests/gtest_fields_binary_enciding.cpp b/src/Core/tests/gtest_fields_binary_enciding.cpp index b7e6a2f1027ad..6ec8f0e727ec9 100644 --- a/src/Core/tests/gtest_fields_binary_enciding.cpp +++ b/src/Core/tests/gtest_fields_binary_enciding.cpp @@ -1,13 +1,16 @@ #include +#include #include #include #include +#include using namespace DB; namespace DB::ErrorCodes { extern const int UNSUPPORTED_METHOD; + extern const int ARGUMENT_OUT_OF_BOUND; } @@ -63,3 +66,28 @@ GTEST_TEST(FieldBinaryEncoding, EncodeAndDecode) })); } + + +/// A binary-encoded `Decimal` whose scale exceeds the type's maximum precision must be rejected by +/// `decodeField`, not run on to the decimal text writer with an oversized scale. The scale is +/// attacker-controlled wire data (an `AggregateFunction` type parameter or a `Dynamic` value carries +/// its own binary-encoded `Field`). The bytes are hand-built because the encoder cannot produce an +/// out-of-range scale. +GTEST_TEST(FieldBinaryEncoding, DecodeDecimalOutOfRangeScaleThrows) +{ + WriteBufferFromOwnString ostr; + writeBinary(UInt8(0x09), ostr); /// FieldBinaryTypeIndex::Decimal64 + writeVarUInt(UInt32(1000), ostr); /// scale far above max_precision = 18 + writeBinaryLittleEndian(Int64(4242), ostr); + + ReadBufferFromString istr(ostr.str()); + try + { + decodeField(istr); + FAIL() << "expected ARGUMENT_OUT_OF_BOUND"; + } + catch (const Exception & e) + { + ASSERT_EQ(e.code(), ErrorCodes::ARGUMENT_OUT_OF_BOUND); + } +} diff --git a/src/Core/tests/gtest_settings.cpp b/src/Core/tests/gtest_settings.cpp index 70b38fb5dc1ad..963c74dbcbef7 100644 --- a/src/Core/tests/gtest_settings.cpp +++ b/src/Core/tests/gtest_settings.cpp @@ -23,6 +23,7 @@ namespace DB::ErrorCodes extern const int ATTEMPT_TO_READ_AFTER_EOF; extern const int TOO_LARGE_STRING_SIZE; extern const int INCORRECT_DATA; + extern const int ARGUMENT_OUT_OF_BOUND; } namespace @@ -445,3 +446,33 @@ GTEST_TEST(Settings, WellFormedSettingsStillReadUnderAClampedMemoryLimit) ASSERT_EQ(settings.get("totals_auto_threshold"), Field(0.25)); ASSERT_TRUE(in.eof()); } + + +GTEST_TEST(Settings, CustomFlaggedDecimalWithOutOfRangeScaleRejected) +{ + /// A per-query settings block arrives over the native TCP protocol and, in interserver mode, is + /// decoded by `Settings::read` before the interserver hash is checked. With `STRINGS_WITH_FLAGS`, + /// a built-in setting marked `CUSTOM` is parsed via `SettingFieldCustom::parseFromString` -> + /// `Field::restoreFromDump` -> `readQuoted`. An exponent-form decimal dump builds a `Decimal` + /// scale that, before the fix, bypassed the binary reader's check and reached the decimal text + /// writer (a fixed-size stack buffer) with an oversized scale. The scale must be rejected while + /// reading the settings, not formatted later. + WriteBufferFromOwnString out; + BaseSettingsHelpers::writeString("max_block_size", out); + BaseSettingsHelpers::writeFlags(BaseSettingsHelpers::Flags::CUSTOM, out); + BaseSettingsHelpers::writeString("Decimal64_'1e-1000'", out); + BaseSettingsHelpers::writeString("", out); /// end of settings + + const String wire = out.str(); + Settings settings; + ReadBufferFromString in(wire); + try + { + settings.read(in, SettingsWriteFormat::STRINGS_WITH_FLAGS); + FAIL() << "expected ARGUMENT_OUT_OF_BOUND"; + } + catch (const Exception & e) + { + ASSERT_EQ(e.code(), ErrorCodes::ARGUMENT_OUT_OF_BOUND); + } +} From 0f7eba4d00582d4a7b2927c238445d08cde41a74 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Fri, 23 Oct 2026 18:31:53 +0200 Subject: [PATCH 009/219] update From 9fe0099bcfff9acabbae3c4e6c3e743e3d0a6c59 Mon Sep 17 00:00:00 2001 From: Alexei Fedotov Date: Fri, 23 Oct 2026 18:44:19 +0200 Subject: [PATCH 010/219] Keep ClickHouse-internal release builds out of public reports and CIDB (backport of ClickHouse-internal#1) The YAML is regenerated with GITHUB_REPOSITORY=ClickHouse/ClickHouse-internal, which drops the Finish Workflow job. --- .github/workflows/release_branches.yml | 35 -------------------------- ci/workflows/release_branches.py | 11 +++++--- 2 files changed, 8 insertions(+), 38 deletions(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 7800092f13d84..1f6cc30dbf57a 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -2114,38 +2114,3 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Stress test (arm_msan)' --workflow "ReleaseBranchCI" --ci --timestamp - - finish_workflow: - runs-on: [self-hosted, style-checker-aarch64] - needs: [build_amd_asan_ubsan, build_amd_darwin, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_darwin, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_1_8, integration_tests_amd_asan_ubsan_db_disk_2_8, integration_tests_amd_asan_ubsan_db_disk_3_8, integration_tests_amd_asan_ubsan_db_disk_4_8, integration_tests_amd_asan_ubsan_db_disk_5_8, integration_tests_amd_asan_ubsan_db_disk_6_8, integration_tests_amd_asan_ubsan_db_disk_7_8, integration_tests_amd_asan_ubsan_db_disk_8_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, sign_macos_binary_amd_darwin, sign_macos_binary_arm_darwin, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_3_3, stateless_tests_amd_asan_ubsan_distributed_plan_parallel, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, unit_tests_asan_ubsan, unit_tests_msan, unit_tests_tsan] - if: ${{ always() }} - name: "Finish Workflow" - outputs: - data: ${{ steps.run.outputs.DATA }} - pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} - steps: - - name: Checkout code - uses: actions/checkout@v6 - with: - ref: ${{ env.CHECKOUT_REF }} - - - name: Prepare env script - run: | - rm -rf ./ci/tmp - mkdir -p ./ci/tmp - cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' - export PYTHONPATH=./ci:.: - - cat > ./ci/tmp/workflow_job.json << 'EOF' - ${{ toJson(job) }} - EOF - cat > ./ci/tmp/workflow_status.json << 'EOF' - ${{ toJson(needs) }} - EOF - ENV_SETUP_SCRIPT_EOF - - - name: Run - id: run - run: | - . ./ci/tmp/praktika_setup_env.sh - PYTHONUNBUFFERED=1 python3 -m praktika run 'Finish Workflow' --workflow "ReleaseBranchCI" --ci --timestamp diff --git a/ci/workflows/release_branches.py b/ci/workflows/release_branches.py index 5e738ce647786..bfff879cc774a 100644 --- a/ci/workflows/release_branches.py +++ b/ci/workflows/release_branches.py @@ -1,3 +1,5 @@ +import os + from praktika import Workflow from ci.defs.defs import ( @@ -23,6 +25,9 @@ artifact = artifact.add_tags({"retention": "long"}) clickhouse_binaries_with_tags.append(artifact) +# Reports and CIDB are public; a build in ClickHouse-internal, a full fork, must not reveal the fix there. +public_build_in_internal = os.getenv("GITHUB_REPOSITORY") == "ClickHouse/ClickHouse-internal" + workflow = Workflow.Config( name="ReleaseBranchCI", event=Workflow.Event.PUSH, @@ -68,10 +73,10 @@ ], dockers=DOCKERS, secrets=SECRETS + LOOM_SECRETS, - enable_job_filtering_by_changes=True, + enable_job_filtering_by_changes=not public_build_in_internal, enable_cache=True, - enable_report=True, - enable_cidb=True, + enable_report=not public_build_in_internal, + enable_cidb=not public_build_in_internal, enable_commit_status_on_failure=True, enable_slack_feed=True, pre_hooks=[ From 934b0808f473dde37fde894751bfbd30600af803 Mon Sep 17 00:00:00 2001 From: Alexander Tokmakov Date: Sun, 25 Oct 2026 17:01:50 +0100 Subject: [PATCH 011/219] update --- src/Common/FieldBinaryEncoding.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/Common/FieldBinaryEncoding.cpp b/src/Common/FieldBinaryEncoding.cpp index f08c94fbe30dc..aadabf7dcb74b 100644 --- a/src/Common/FieldBinaryEncoding.cpp +++ b/src/Common/FieldBinaryEncoding.cpp @@ -269,8 +269,8 @@ DecimalField decodeDecimal(ReadBuffer & buf) /// fixed-size buffer in the decimal text writer, so reject it at the boundary, matching /// `readDecimalFieldBinary` and `DataTypeDecimalBase`. if (scale > DecimalUtils::max_precision) - throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Scale {} is out of bounds (max scale: {})", - scale, DecimalUtils::max_precision); + throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Scale {} is out of bounds (max scale: {})", scale, DecimalUtils::max_precision); + T value{}; readBinaryLittleEndian(value, buf); return DecimalField(value, scale); From c9ff9b056f78648a2fbde45e552bf644bcc04edd Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 25 Oct 2026 23:06:56 +0000 Subject: [PATCH 012/219] Update autogenerated version to 26.8.35.10 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index fd92cd2165712..4404ff8b26e14 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54547) +SET(VERSION_REVISION 54548) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 35) -SET(VERSION_GITHASH e0cfed8d5e66d0e226afa2258a5956e3e22abff9) -SET(VERSION_DESCRIBE v26.8.35.1-lts) -SET(VERSION_STRING 26.8.35.1) +SET(VERSION_PATCH 36) +SET(VERSION_GITHASH 9fe0099bcfff9acabbae3c4e6c3e743e3d0a6c59) +SET(VERSION_DESCRIBE v26.8.36.1-lts) +SET(VERSION_STRING 26.8.36.1) # end of autochange From b47359294fe0f498ee19c5e31f18a9155b6dedae Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 26 Oct 2026 04:13:58 +0000 Subject: [PATCH 013/219] Backport #127419 to 26.8: Bump `brotli` from v1.2.0 to v1.2.1 (cherry picked from commit 60233e8acee62f28064f0e92e65493227330d835) --- contrib/brotli | 2 +- contrib/brotli-cmake/CMakeLists.txt | 51 ++++++++++++----------- src/CMakeLists.txt | 4 ++ src/IO/tests/gtest_brotli_decoder.cpp | 58 +++++++++++++++++++++++++++ 4 files changed, 91 insertions(+), 24 deletions(-) create mode 100644 src/IO/tests/gtest_brotli_decoder.cpp diff --git a/contrib/brotli b/contrib/brotli index d224526f8fa73..2d9c9b61f4e7d 160000 --- a/contrib/brotli +++ b/contrib/brotli @@ -1 +1 @@ -Subproject commit d224526f8fa736f24b2339e6f421a142de6586c8 +Subproject commit 2d9c9b61f4e7d3523c9dfb62e48fe91aaf86e812 diff --git a/contrib/brotli-cmake/CMakeLists.txt b/contrib/brotli-cmake/CMakeLists.txt index b89e81ecda19e..8c2b41721dc9a 100644 --- a/contrib/brotli-cmake/CMakeLists.txt +++ b/contrib/brotli-cmake/CMakeLists.txt @@ -9,35 +9,41 @@ set(BROTLI_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/brotli/c") set(BROTLI_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/brotli/c") set(SRCS - "${BROTLI_SOURCE_DIR}/enc/command.c" - "${BROTLI_SOURCE_DIR}/enc/fast_log.c" + "${BROTLI_SOURCE_DIR}/common/constants.c" + "${BROTLI_SOURCE_DIR}/common/context.c" + "${BROTLI_SOURCE_DIR}/common/dictionary.c" + "${BROTLI_SOURCE_DIR}/common/platform.c" + "${BROTLI_SOURCE_DIR}/common/shared_dictionary.c" + "${BROTLI_SOURCE_DIR}/common/transform.c" "${BROTLI_SOURCE_DIR}/dec/bit_reader.c" - "${BROTLI_SOURCE_DIR}/dec/state.c" - "${BROTLI_SOURCE_DIR}/dec/huffman.c" "${BROTLI_SOURCE_DIR}/dec/decode.c" - "${BROTLI_SOURCE_DIR}/enc/encode.c" - "${BROTLI_SOURCE_DIR}/enc/dictionary_hash.c" + "${BROTLI_SOURCE_DIR}/dec/huffman.c" + "${BROTLI_SOURCE_DIR}/dec/prefix.c" + "${BROTLI_SOURCE_DIR}/dec/state.c" + "${BROTLI_SOURCE_DIR}/dec/static_init.c" + "${BROTLI_SOURCE_DIR}/enc/backward_references.c" + "${BROTLI_SOURCE_DIR}/enc/backward_references_hq.c" + "${BROTLI_SOURCE_DIR}/enc/bit_cost.c" + "${BROTLI_SOURCE_DIR}/enc/block_splitter.c" + "${BROTLI_SOURCE_DIR}/enc/brotli_bit_stream.c" "${BROTLI_SOURCE_DIR}/enc/cluster.c" + "${BROTLI_SOURCE_DIR}/enc/command.c" + "${BROTLI_SOURCE_DIR}/enc/compound_dictionary.c" + "${BROTLI_SOURCE_DIR}/enc/compress_fragment.c" + "${BROTLI_SOURCE_DIR}/enc/compress_fragment_two_pass.c" + "${BROTLI_SOURCE_DIR}/enc/dictionary_hash.c" + "${BROTLI_SOURCE_DIR}/enc/encode.c" + "${BROTLI_SOURCE_DIR}/enc/encoder_dict.c" "${BROTLI_SOURCE_DIR}/enc/entropy_encode.c" + "${BROTLI_SOURCE_DIR}/enc/fast_log.c" + "${BROTLI_SOURCE_DIR}/enc/histogram.c" "${BROTLI_SOURCE_DIR}/enc/literal_cost.c" - "${BROTLI_SOURCE_DIR}/enc/compress_fragment_two_pass.c" + "${BROTLI_SOURCE_DIR}/enc/memory.c" + "${BROTLI_SOURCE_DIR}/enc/metablock.c" "${BROTLI_SOURCE_DIR}/enc/static_dict.c" - "${BROTLI_SOURCE_DIR}/enc/compress_fragment.c" - "${BROTLI_SOURCE_DIR}/enc/block_splitter.c" - "${BROTLI_SOURCE_DIR}/enc/backward_references_hq.c" - "${BROTLI_SOURCE_DIR}/enc/histogram.c" - "${BROTLI_SOURCE_DIR}/enc/brotli_bit_stream.c" + "${BROTLI_SOURCE_DIR}/enc/static_dict_lut.c" + "${BROTLI_SOURCE_DIR}/enc/static_init.c" "${BROTLI_SOURCE_DIR}/enc/utf8_util.c" - "${BROTLI_SOURCE_DIR}/enc/encoder_dict.c" - "${BROTLI_SOURCE_DIR}/enc/backward_references.c" - "${BROTLI_SOURCE_DIR}/enc/bit_cost.c" - "${BROTLI_SOURCE_DIR}/enc/metablock.c" - "${BROTLI_SOURCE_DIR}/enc/memory.c" - "${BROTLI_SOURCE_DIR}/common/dictionary.c" - "${BROTLI_SOURCE_DIR}/common/transform.c" - "${BROTLI_SOURCE_DIR}/common/platform.c" - "${BROTLI_SOURCE_DIR}/common/context.c" - "${BROTLI_SOURCE_DIR}/common/constants.c" ) add_library(_brotli ${SRCS}) @@ -45,4 +51,3 @@ add_library(ch_contrib::brotli ALIAS _brotli) target_include_directories(_brotli SYSTEM BEFORE PUBLIC "${BROTLI_SOURCE_DIR}/include") -target_compile_definitions(_brotli PRIVATE BROTLI_BUILD_PORTABLE=1) diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f47f4f2c2bce5..784fbf9f13cb9 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -978,6 +978,10 @@ if (ENABLE_TESTS) target_link_libraries(unit_tests_dbms PRIVATE ch_contrib::krb5) endif() + if (TARGET ch_contrib::brotli) + target_link_libraries(unit_tests_dbms PRIVATE ch_contrib::brotli) + endif() + if (TARGET ch_contrib::simdjson) target_link_libraries(unit_tests_dbms PRIVATE ch_contrib::simdjson) endif() diff --git a/src/IO/tests/gtest_brotli_decoder.cpp b/src/IO/tests/gtest_brotli_decoder.cpp new file mode 100644 index 0000000000000..d7b8ade6a3f3a --- /dev/null +++ b/src/IO/tests/gtest_brotli_decoder.cpp @@ -0,0 +1,58 @@ +#include "config.h" + +#if USE_BROTLI + +#include +#include + +#include +#include +#include +#include +#include + +TEST(BrotliDecoder, RejectInvalidBlockSwitch) +{ + // A block switch is invalid when the stream declares only one block type. + // From https://github.com/nodejs/node/blob/main/test/parallel/test-zlib-invalid-input.js. + constexpr std::array input = { + 0x20, 0x00, 0x10, 0x41, 0x41, 0x41, 0xfc, 0xff, 0xff, 0x07, 0x00, + 0x22, 0x28, 0xf0, 0x0b, 0x60, 0xf7, 0xf4, 0x1f, 0x00, 0x00, 0x00}; + + for (size_t chunk_size : {input.size(), size_t{1}}) + { + std::unique_ptr state( + BrotliDecoderCreateInstance(nullptr, nullptr, nullptr), &BrotliDecoderDestroyInstance); + ASSERT_NE(state.get(), nullptr); + + std::array output{}; + size_t total_output = 0; + BrotliDecoderResult result = BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT; + + for (size_t offset = 0; offset < input.size() && result == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT;) + { + size_t available_input = std::min(chunk_size, input.size() - offset); + const size_t provided_input = available_input; + const uint8_t * next_input = input.data() + offset; + do + { + uint8_t * next_output = output.data(); + size_t available_output = output.size(); + result = BrotliDecoderDecompressStream( + state.get(), &available_input, &next_input, &available_output, &next_output, nullptr); + total_output += output.size() - available_output; + ASSERT_LT(total_output, 64 * 1024 * 1024); + } + while (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT); + offset += provided_input - available_input; + if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) + ASSERT_GT(provided_input - available_input, 0); + } + + EXPECT_EQ(result, BROTLI_DECODER_RESULT_ERROR); + // -17 is the `BROTLI_DECODER_ERROR_FORMAT_BLOCK_SWITCH` code added by the fix. + EXPECT_EQ(BrotliDecoderGetErrorCode(state.get()), static_cast(-17)); + } +} + +#endif From e8d3ed4c72632c64020ef6a8bec58af7f72ef327 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 26 Oct 2026 18:39:51 +0000 Subject: [PATCH 014/219] Update autogenerated version to 26.8.36.3 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 4404ff8b26e14..2817be684f77a 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54548) +SET(VERSION_REVISION 54549) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 36) -SET(VERSION_GITHASH 9fe0099bcfff9acabbae3c4e6c3e743e3d0a6c59) -SET(VERSION_DESCRIBE v26.8.36.1-lts) -SET(VERSION_STRING 26.8.36.1) +SET(VERSION_PATCH 37) +SET(VERSION_GITHASH b47359294fe0f498ee19c5e31f18a9155b6dedae) +SET(VERSION_DESCRIBE v26.8.37.1-lts) +SET(VERSION_STRING 26.8.37.1) # end of autochange From 1eedcaa48bc66ffebe1efb1f31f5236657bddc5a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 23 Oct 2026 13:13:08 +0000 Subject: [PATCH 015/219] Backport #127318 to 26.8: Distrust the compromised eMudhra certificates --- .../include/Poco/Net/DistrustedCertificates.h | 44 ++++++ .../src/DistrustedCertificates.cpp | 95 ++++++++++++ base/poco/NetSSL_OpenSSL/src/SSLManager.cpp | 15 ++ contrib/grpc | 2 +- .../tests/gtest_distrusted_certificates.cpp | 137 ++++++++++++++++++ 5 files changed, 292 insertions(+), 1 deletion(-) create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/DistrustedCertificates.h create mode 100644 base/poco/NetSSL_OpenSSL/src/DistrustedCertificates.cpp create mode 100644 src/Common/tests/gtest_distrusted_certificates.cpp diff --git a/base/poco/NetSSL_OpenSSL/include/Poco/Net/DistrustedCertificates.h b/base/poco/NetSSL_OpenSSL/include/Poco/Net/DistrustedCertificates.h new file mode 100644 index 0000000000000..5aca47d6be881 --- /dev/null +++ b/base/poco/NetSSL_OpenSSL/include/Poco/Net/DistrustedCertificates.h @@ -0,0 +1,44 @@ +// +// DistrustedCertificates.h +// +// Library: NetSSL_OpenSSL +// Package: SSLCore +// Module: DistrustedCertificates +// +// Copyright (c) 2026, ClickHouse, Inc. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef NetSSL_DistrustedCertificates_INCLUDED +#define NetSSL_DistrustedCertificates_INCLUDED + + +#include +#include "Poco/Net/NetSSL.h" + + +namespace Poco +{ +namespace Net +{ + + + NetSSL_API int checkDistrustedCertificate(X509 * certificate); + /// Checks the certificate against the list of distrusted public keys built into + /// the binary, and returns `X509_V_OK` if it is not on the list, `X509_V_ERR_CERT_REVOKED` + /// if it is, or another `X509_V_ERR_*` code if the check itself failed. + /// + /// The list contains the keys of CA certificates that must not be trusted even if + /// the trust store of the operating system still contains them, e.g. because their + /// private keys are compromised. A key is identified by the SHA-256 of its DER-encoded + /// `SubjectPublicKeyInfo`, so that every certificate of the same key is rejected, + /// including cross-signed ones. + + +} +} // namespace Poco::Net + + +#endif // NetSSL_DistrustedCertificates_INCLUDED diff --git a/base/poco/NetSSL_OpenSSL/src/DistrustedCertificates.cpp b/base/poco/NetSSL_OpenSSL/src/DistrustedCertificates.cpp new file mode 100644 index 0000000000000..448d4f601d9d0 --- /dev/null +++ b/base/poco/NetSSL_OpenSSL/src/DistrustedCertificates.cpp @@ -0,0 +1,95 @@ +// +// DistrustedCertificates.cpp +// +// Library: NetSSL_OpenSSL +// Package: SSLCore +// Module: DistrustedCertificates +// +// Copyright (c) 2026, ClickHouse, Inc. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include "Poco/Net/DistrustedCertificates.h" +#include +#include +#include +#include + + +namespace +{ + using SPKIHash = std::array; + + consteval unsigned char hexDigit(char c) + { + if (c >= '0' && c <= '9') + return c - '0'; + if (c >= 'A' && c <= 'F') + return c - 'A' + 10; + throw "Invalid hex digit"; + } + + consteval SPKIHash fromHex(const char (&hex)[65]) + { + SPKIHash result{}; + for (size_t i = 0; i < result.size(); ++i) + result[i] = static_cast(hexDigit(hex[2 * i]) << 4 | hexDigit(hex[2 * i + 1])); + return result; + } + + /// SHA-256 of the DER-encoded `SubjectPublicKeyInfo` of every distrusted key. + constexpr SPKIHash distrusted_keys[] = + { + /// eMudhra: the private keys of the intermediate CAs below, issued in 2025, can be recovered + /// from their certificates. Each of them is issued by `emSign Root TLS CA - G3` or `emSign Root CS CA - G3` + /// and cross-signed by Sectigo `AAA Certificate Services`; the cross-signed certificates have the same keys. + /// The roots that issued or cross-signed them are distrusted as well. + /// https://emudhra.we-have-a-problem.com/ + fromHex("EBED6475C2C2D80FCB199F3FE8DD6E6B3CE60B77EB15D4E42C3DEC0C6F5B5105"), /// EM DV TLS CA - G3B + fromHex("43BFFA19D6EF651451F6841E019931E96F6F2EAEFF28AB4B6F1CE4576DE2717B"), /// EM OV TLS CA - G3B + fromHex("E0254C7FCDC265E2F0F8A428DABB75AB4AE039B4872D62F5EE57F0877D5A8A41"), /// EM EV TLS CA - G3B + fromHex("49DD353DC2B919A3770645117D0E60C16115E7D1BC259EEFE734B00EA434FE5F"), /// EM DV TLS CA - G3C + fromHex("63ECEFEE431D49523BC1136C4E72D0206A477086CAF606CD5CF025686C109CF8"), /// EM OV TLS CA - G3C + fromHex("5C0660A2B164910D8FF589C18E14552E9E0EC237DCEFE2CB736D41A4811F81A0"), /// EM EV TLS CA - G3C + fromHex("EC4FFA1AD0AC88081314B7AAFFEF6A91F97CD57291BB5BA686C5415F5EB4F6B6"), /// EM OV CS CA - G3B + fromHex("92C063D67E8F14DB71D288E11266FFDAF69CBF115F7BAB14F3BC2F68D25E0E30"), /// EM OV CS CA - G3C + fromHex("C619E4780ED8E0ABEDA24AD732567DE4383A7EA941338F6D1A7D1B49AC792FE9"), /// emSign Root TLS CA - G3 + fromHex("0A6C2B2CA5B364207954171987AC9D2178561569D8205457E77CA90C32EDE282"), /// emSign Root CS CA - G3 + fromHex("8D417DB2DD8BF5E3084D1E3F196D583849D81BDD4C00C70B9D39369E96B8C782"), /// emSign ECC Root CA - G3 + }; +} + + +namespace Poco { +namespace Net { + + +int checkDistrustedCertificate(X509 * certificate) +{ + X509_PUBKEY * key = X509_get_X509_PUBKEY(certificate); + if (key == nullptr) + return X509_V_ERR_UNSPECIFIED; + + /// Not `X509_pubkey_digest`: it hashes only the key bits, without the algorithm identifier. + unsigned char * der = nullptr; + int der_size = i2d_X509_PUBKEY(key, &der); + if (der_size <= 0) + return X509_V_ERR_OUT_OF_MEM; + + SPKIHash hash; + unsigned int hash_size = 0; + int ok = EVP_Digest(der, der_size, hash.data(), &hash_size, EVP_sha256(), nullptr); + OPENSSL_free(der); + if (ok != 1 || hash_size != hash.size()) + return X509_V_ERR_UNSPECIFIED; + + if (std::find(std::begin(distrusted_keys), std::end(distrusted_keys), hash) != std::end(distrusted_keys)) + return X509_V_ERR_CERT_REVOKED; + + return X509_V_OK; +} + + +} } // namespace Poco::Net diff --git a/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp b/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp index 94b66cdcf5f1e..fd4e31f53d258 100644 --- a/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp +++ b/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp @@ -14,6 +14,7 @@ #include "Poco/Net/SSLManager.h" #include "Poco/Net/Context.h" +#include "Poco/Net/DistrustedCertificates.h" #include "Poco/Net/Utility.h" #include "Poco/Net/PrivateKeyPassphraseHandler.h" #include "Poco/Net/RejectCertificateHandler.h" @@ -196,6 +197,20 @@ SSLManager::InvalidCertificateHandlerPtr SSLManager::clientCertificateHandler() int SSLManager::verifyCallback(bool server, int ok, X509_STORE_CTX* pStore) { + /// OpenSSL calls this for every certificate of the chain, with `ok` set to 1 if it has + /// no other errors, so this rejects a chain that contains a distrusted certificate at any + /// depth, whichever trust store (the system one, the configured one or the embedded one) + /// it was built from. The rejection is reported like any other verification error. + if (X509 * cert = X509_STORE_CTX_get_current_cert(pStore)) + { + int distrusted = checkDistrustedCertificate(cert); + if (distrusted != X509_V_OK) + { + X509_STORE_CTX_set_error(pStore, distrusted); + ok = 0; + } + } + if (!ok) { int depth = X509_STORE_CTX_get_error_depth(pStore); diff --git a/contrib/grpc b/contrib/grpc index a703d1f9448ff..68b2fe0b2b131 160000 --- a/contrib/grpc +++ b/contrib/grpc @@ -1 +1 @@ -Subproject commit a703d1f9448ff77deca3beb36dfc7d337a209d3e +Subproject commit 68b2fe0b2b1318f18aadee9027e86e8ceaa20af2 diff --git a/src/Common/tests/gtest_distrusted_certificates.cpp b/src/Common/tests/gtest_distrusted_certificates.cpp new file mode 100644 index 0000000000000..5931feea9f6b4 --- /dev/null +++ b/src/Common/tests/gtest_distrusted_certificates.cpp @@ -0,0 +1,137 @@ +#include "config.h" + +#if USE_SSL + +#include + +#include +#include + +#include +#include +#include + +#include +#include + + +namespace +{ + +/// `emSign ECC Root CA - G3`, distrusted. +constexpr std::string_view emsign_ecc_root_g3 = R"( +-----BEGIN CERTIFICATE----- +MIICTjCCAdOgAwIBAgIKPPYHqWhwDtqLhDAKBggqhkjOPQQDAzBrMQswCQYDVQQG +EwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNo +bm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0g +RzMwHhcNMTgwMjE4MTgzMDAwWhcNNDMwMjE4MTgzMDAwWjBrMQswCQYDVQQGEwJJ +TjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNobm9s +b2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0gRzMw +djAQBgcqhkjOPQIBBgUrgQQAIgNiAAQjpQy4LRL1KPOxst3iAhKAnjlfSU2fySU0 +WXTsuwYc58Byr+iuL+FBVIcUqEqy6HyC5ltqtdyzdc6LBtCGI79G1Y4PPwT01xyS +fvalY8L1X44uT6EYGQIrMgqCZH0Wk9GjQjBAMB0GA1UdDgQWBBR8XQKEE9TMipuB +zhccLikenEhjQjAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggq +hkjOPQQDAwNpADBmAjEAvvNhzwIQHWSVB7gYboiFBS+DCBeQyh+KTOgNG3qxrdWB +CUfvO6wIBHxcmbHtRwfSAjEAnbpV/KlK6O3t5nYBQnvI+GDZjVGLVTv7jHvrZQnD ++JbNR6iC8hZVdyR+EhCVBCyj +-----END CERTIFICATE----- + +)"; + +/// `EM DV TLS CA - G3B` cross-signed by `AAA Certificate Services`, distrusted by the key of the certificate issued by `emSign Root TLS CA - G3`. +constexpr std::string_view em_dv_tls_ca_g3b_cross_signed = R"( +-----BEGIN CERTIFICATE----- +MIIDzjCCAragAwIBAgIRAKAk/fS6Zlx4XOSgIEg4WLcwDQYJKoZIhvcNAQELBQAw +ezELMAkGA1UEBhMCR0IxGzAZBgNVBAgMEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G +A1UEBwwHU2FsZm9yZDEaMBgGA1UECgwRQ29tb2RvIENBIExpbWl0ZWQxITAfBgNV +BAMMGEFBQSBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczAeFw0yNTA1MTQwMDAwMDBaFw0y +ODEyMzEyMzU5NTlaMD8xCzAJBgNVBAYTAk5MMRMwEQYDVQQKDAplTXVkaHJhIEJW +MRswGQYDVQQDDBJFTSBEViBUTFMgQ0EgLSBHM0IwdjAQBgcqhkjOPQIBBgUrgQQA +IgNiAATnntnASpdzxHNeUREdqFrCvV5kqzdLgOEsvCA47B5PDjuBL8mFnh5nr7BO +L4vJf3SFumWqkR7opORbYtXMA9Vr5nizw4y03JN8okYOI4RTyJKTXSnUT5HhVga2 +PM7atGijggE1MIIBMTAfBgNVHSMEGDAWgBSgEQojPpbxB+zirynvgqV/0DCktDAd +BgNVHQ4EFgQU4h4v5xwZOQ5VS2LBICRXNSSKJRQwDgYDVR0PAQH/BAQDAgEGMBIG +A1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMC +MDEGA1UdIAQqMCgwCAYGZ4EMAQIBMA0GCysGAQQBg44hAQABMA0GCysGAQQBg44h +AQJkMEMGA1UdHwQ8MDowOKA2oDSGMmh0dHA6Ly9jcmwuY29tb2RvY2EuY29tL0FB +QUNlcnRpZmljYXRlU2VydmljZXMuY3JsMDQGCCsGAQUFBwEBBCgwJjAkBggrBgEF +BQcwAYYYaHR0cDovL29jc3AuY29tb2RvY2EuY29tMA0GCSqGSIb3DQEBCwUAA4IB +AQBzQ1IoqpdotM6gFrR4Q3YrKifOLeaj5XXunldpVE/vZn2TxC2y0s8oYiLR5ICE +wqVXSB7N6BM1Mp+Nif7Jbhb2IswAfSR4Iik8+B0iRmi7lYoNlycCgHjmsjOmZaxM +QkPmjXAJbG5f4V41OsvJZmS7lI3r0IuvEDVX3GAZ13on6IImSs9ybNpHQ7NCqkMQ +7wH4uAO1dtpFycgJSdHaCOhHjrox2BTYGLdpUrYwY3EOAjlZ8UGgsxs7mG+xquMl +bN1LFURwfnkVMP3T/DBIZ/ys2+lNFyxK3FlArrV4/000uTnVB1lO8oGORYdXoT5V +GUyRCEIDhF5TBUuKF4E3kf1h +-----END CERTIFICATE-----)"; + +/// `emSign Root CA - G1`, not affected. +constexpr std::string_view emsign_root_g1 = R"( +-----BEGIN CERTIFICATE----- +MIIDlDCCAnygAwIBAgIKMfXkYgxsWO3W2DANBgkqhkiG9w0BAQsFADBnMQswCQYD +VQQGEwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBU +ZWNobm9sb2dpZXMgTGltaXRlZDEcMBoGA1UEAxMTZW1TaWduIFJvb3QgQ0EgLSBH +MTAeFw0xODAyMTgxODMwMDBaFw00MzAyMTgxODMwMDBaMGcxCzAJBgNVBAYTAklO +MRMwEQYDVQQLEwplbVNpZ24gUEtJMSUwIwYDVQQKExxlTXVkaHJhIFRlY2hub2xv +Z2llcyBMaW1pdGVkMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEcxMIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAk0u76WaK7p1b1TST0Bsew+eeuGQz +f2N4aLTNLnF115sgxk0pvLZoYIr3IZpWNVrzdr3YzZr/k1ZLpVkGoZM0Kd0WNHVO +8oG0x5ZOrRkVUkr+PHB1cM2vK6sVmjM8qrOLqs1D/fXqcP/tzxE7lM5OMhbTI0Aq +d7OvPAEsbO2ZLIvZTmmYsvePQbAyeGHWDV/D+qJAkh1cF+ZwPjXnorfCYuKrpDhM +tTk1b+oDafo6VGiFbdbyL0NVHpENDtjVaqSW0RM8LHhQ6DqS0hdW5TUaQBw+jSzt +Od9C4INBdN+jzcKGYEho42kLVACL5HZpIQ15TjQIXhTCzLG3rdd8cIrHhQIDAQAB +o0IwQDAdBgNVHQ4EFgQU++8Nhp6w492pufEhF38+/PB3KxowDgYDVR0PAQH/BAQD +AgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAFn/8oz1h31x +PaOfG1vR2vjTnGs2vZupYeveFix0PZ7mddrXuqe8QhfnPZHr5X3dPpzxz5KsbEjM +wiI/aTvFthUvozXGaCocV685743QNcMYDHsAVhzNixl03r4PEuDQqqE/AjSxcM6d +GNYIAwlG7mDgfrbESQRRfXBgvKqy/3lyeqYdPV8q+Mri/Tm3R7nrft8EI6/6nAYH +6ftjk4BAtcZsCjEozgyfz7MjNYBBjWzEN3uBL4ChQEKF6dk4jeihU80Bv2noWgby +RQuQ+q7hv53yrlc8pa6yVvSLZUDp/TGBLPQ5Cdjua6e0ph0VpZj3AYHYhX3zUVxx +iN66zB+Afko= +-----END CERTIFICATE----- + +)"; + +using X509Ptr = std::unique_ptr; + +X509Ptr parseCertificate(std::string_view pem) +{ + std::unique_ptr bio(BIO_new_mem_buf(pem.data(), static_cast(pem.size())), BIO_free); + X509Ptr certificate(PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr), X509_free); + EXPECT_NE(certificate, nullptr); + return certificate; +} + +/// Verifies a trusted self-signed certificate the same way a TLS connection of a Poco context does it. +int verifyTrustedRoot(std::string_view pem) +{ + Poco::Net::Context context(Poco::Net::Context::CLIENT_USE, Poco::Net::Context::Params{}); + X509Ptr certificate = parseCertificate(pem); + X509_STORE * store = SSL_CTX_get_cert_store(context.sslContext()); + EXPECT_EQ(X509_STORE_add_cert(store, certificate.get()), 1); + + std::unique_ptr store_ctx(X509_STORE_CTX_new(), X509_STORE_CTX_free); + EXPECT_EQ(X509_STORE_CTX_init(store_ctx.get(), store, certificate.get(), nullptr), 1); + X509_STORE_CTX_set_verify_cb(store_ctx.get(), SSL_CTX_get_verify_callback(context.sslContext())); + + if (X509_verify_cert(store_ctx.get()) == 1) + return X509_V_OK; + return X509_STORE_CTX_get_error(store_ctx.get()); +} + +} + + +TEST(DistrustedCertificates, Check) +{ + EXPECT_EQ(Poco::Net::checkDistrustedCertificate(parseCertificate(emsign_ecc_root_g3).get()), X509_V_ERR_CERT_REVOKED); + EXPECT_EQ(Poco::Net::checkDistrustedCertificate(parseCertificate(em_dv_tls_ca_g3b_cross_signed).get()), X509_V_ERR_CERT_REVOKED); + EXPECT_EQ(Poco::Net::checkDistrustedCertificate(parseCertificate(emsign_root_g1).get()), X509_V_OK); +} + +TEST(DistrustedCertificates, RejectedEvenIfTrusted) +{ + EXPECT_EQ(verifyTrustedRoot(emsign_ecc_root_g3), X509_V_ERR_CERT_REVOKED); + EXPECT_EQ(verifyTrustedRoot(emsign_root_g1), X509_V_OK); +} + +#endif From 77e85afb1f28b5c8218f30e29043c0aa8a6bf50c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 23 Oct 2026 17:11:32 +0000 Subject: [PATCH 016/219] Backport #127568 to 26.8: Implement `writeFileUsingBlobWritingFunction` for Multi-Location Disks --- src/Common/setThreadName.h | 1 + .../DiskObjectStorageTransaction.cpp | 41 +++++++++++++++---- 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/src/Common/setThreadName.h b/src/Common/setThreadName.h index 3c062991b6ddf..6f9b59cd3f0c6 100644 --- a/src/Common/setThreadName.h +++ b/src/Common/setThreadName.h @@ -181,6 +181,7 @@ namespace DB M(BLOB_KILLER_TASK, "BlobKillerTask") \ M(BLOB_COPIER_TASK, "BlobCopierTask") \ M(DISK_OBJECT_STORAGE_COPY, "DiskObjStCopy") \ + M(DISK_OBJECT_STORAGE_WRITE_BLOB, "DiskObjStWrBlob") \ M(DISTRIBUTED_CACHE, "DistribCache") \ M(DISTRIBUTED_CACHE_DROP, "DropDistCache") \ diff --git a/src/Disks/DiskObjectStorage/DiskObjectStorageTransaction.cpp b/src/Disks/DiskObjectStorage/DiskObjectStorageTransaction.cpp index ff752d7c0272a..3ce962c4d0e7f 100644 --- a/src/Disks/DiskObjectStorage/DiskObjectStorageTransaction.cpp +++ b/src/Disks/DiskObjectStorage/DiskObjectStorageTransaction.cpp @@ -5,7 +5,9 @@ #include #include #include +#include #include +#include #if ENABLE_DISTRIBUTED_CACHE #include #endif @@ -50,7 +52,6 @@ namespace ErrorCodes { extern const int NOT_IMPLEMENTED; extern const int FAULT_INJECTED; - extern const int LOGICAL_ERROR; } void DiskObjectStorageTransaction::waitBlobRemoval(const StoredObjects & blobs) const @@ -437,38 +438,60 @@ void DiskObjectStorageTransaction::submitBlobForRemoval(const std::string & remo } /// This function is a simplified and adapted version of DiskObjectStorageTransaction::writeFile(). -/// TODO: This function is incorrect for multiple object storages, we need to remove it and use everywhere regular writeFile +/// TODO: Remove it and use regular writeFile everywhere. void DiskObjectStorageTransaction::writeFileUsingBlobWritingFunction( const String & path, WriteMode mode, WriteBlobFunction && write_blob_function) { - if (cluster->getConfiguration().size() > 1) - throw Exception(ErrorCodes::LOGICAL_ERROR, "writeFileUsingBlobWritingFunction not supported for replicated setup"); + const Location local_location = cluster->getLocalLocation(); + const auto local_object_storage = object_storages->takePointingTo(local_location); StoredObject object(metadata_transaction->generateObjectKeyForPath(path).serialize(), path); - written_blobs[cluster->getLocalLocation()].push_back(object); + written_blobs[local_location].push_back(object); /// See DiskObjectStorage::getBlobPath(). Strings blob_path; blob_path.reserve(2); blob_path.emplace_back(object.remote_path); - String objects_namespace = object_storages->takePointingTo(cluster->getLocalLocation())->getObjectsNamespace(); + String objects_namespace = local_object_storage->getObjectsNamespace(); if (!objects_namespace.empty()) blob_path.emplace_back(objects_namespace); /// We always use mode Rewrite because we simulate append using metadata and different files object.bytes_size = std::move(write_blob_function)(blob_path, WriteMode::Rewrite, /*object_attributes=*/std::nullopt); - addOperation([object, mode](MetadataTransactionPtr tx) + LocationSet written_locations{local_location}; + for (const auto & location : cluster->getEnabledLocations()) + if (written_locations.insert(location).second) + written_blobs[location].push_back(object); + + { + const ReadSettings read_settings = getReadSettings(); + const WriteSettings write_settings = getWriteSettings(); + ThreadPoolCallbackRunnerLocal runner(*copy_object_pool, ThreadName::DISK_OBJECT_STORAGE_WRITE_BLOB); + for (const auto & location : written_locations) + { + if (location == local_location) + continue; + + runner.enqueueAndKeepTrack([&, location] + { + local_object_storage->copyObjectToAnotherObjectStorage(object, object, read_settings, write_settings, *object_storages->takePointingTo(location)); + }); + } + runner.waitForAllToFinishAndRethrowFirstError(); + } + + addOperation([object, mode, missing_locations = cluster->findComplement(std::move(written_locations))](MetadataTransactionPtr tx) { if (mode == WriteMode::Rewrite) { LOG_TEST(getLogger("DiskObjectStorageTransaction"), "Writing blob for path {}, key {}, size {}", object.local_path, object.remote_path, object.bytes_size); - tx->recordBlobsReplication(object, /*missing_locations=*/{}); + tx->recordBlobsReplication(object, missing_locations); tx->createMetadataFile(object.local_path, {object}); } else { - tx->recordBlobsReplication(object, /*missing_locations=*/{}); + tx->recordBlobsReplication(object, missing_locations); tx->addBlobToMetadata(object.local_path, object); } }); From 65f06b455e8bbae1eff91f376861054100e3c4a9 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 23 Oct 2026 20:55:12 +0000 Subject: [PATCH 017/219] Backport #126905 to 26.8: Fix segfault in `MySQLDump` format on `CREATE TABLE` without column definitions --- .../Formats/Impl/MySQLDumpRowInputFormat.cpp | 9 +++- ...mysqldump_create_without_columns.reference | 51 +++++++++++++++++++ .../05428_mysqldump_create_without_columns.sh | 38 ++++++++++++++ 3 files changed, 96 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/05428_mysqldump_create_without_columns.reference create mode 100755 tests/queries/0_stateless/05428_mysqldump_create_without_columns.sh diff --git a/src/Processors/Formats/Impl/MySQLDumpRowInputFormat.cpp b/src/Processors/Formats/Impl/MySQLDumpRowInputFormat.cpp index 0c35e80bc0910..c60ec17480d7f 100644 --- a/src/Processors/Formats/Impl/MySQLDumpRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/MySQLDumpRowInputFormat.cpp @@ -377,8 +377,12 @@ static bool tryToExtractStructureFromCreateQuery(ReadBuffer & in, NamesAndTypesL if (!create_query) return false; + /// `CREATE TABLE ... LIKE` has no column list, and a column list with only indexes or constraints has no columns. + if (!create_query->columns_list) + return false; + const auto * create_defines = create_query->columns_list->as(); - if (!create_defines) + if (!create_defines || !create_defines->columns) return false; structure = getColumnsList(create_defines->columns); @@ -431,9 +435,10 @@ static void readFirstCreateAndInsertQueries(ReadBuffer & in, String & table_name if (type == MySQLQueryType::CREATE) { /// If we have CREATE query, we can extract columns names and types from it. + /// `tryToExtractStructureFromCreateQuery` reads the CREATE query up to and including its `;`, + /// so the next query must not be skipped: it can be the first INSERT query. if (tryToExtractStructureFromCreateQuery(in, structure_from_create)) column_names = structure_from_create.getNames(); - skipQuery(in); insert_query_present = skipToInsertQuery(table_name, in); } diff --git a/tests/queries/0_stateless/05428_mysqldump_create_without_columns.reference b/tests/queries/0_stateless/05428_mysqldump_create_without_columns.reference new file mode 100644 index 0000000000000..e838531bffe08 --- /dev/null +++ b/tests/queries/0_stateless/05428_mysqldump_create_without_columns.reference @@ -0,0 +1,51 @@ +LIKE `t0` +c1 Nullable(Int64) +c2 Nullable(String) +1 a +2 b +(KEY `k` (`x`)) ENGINE=InnoDB +c1 Nullable(Int64) +c2 Nullable(String) +1 a +2 b +(CONSTRAINT `c` CHECK (`x` > 0)) ENGINE=InnoDB +c1 Nullable(Int64) +c2 Nullable(String) +1 a +2 b +LIKE `t0` +1 a +2 b +1 a +2 b +3 c +1 a +2 b +3 c +(KEY `k` (`x`)) ENGINE=InnoDB +1 a +2 b +1 a +2 b +3 c +1 a +2 b +3 c +(CONSTRAINT `c` CHECK (`x` > 0)) ENGINE=InnoDB +1 a +2 b +1 a +2 b +3 c +1 a +2 b +3 c +(`x` int, `y` varchar(10)) ENGINE=InnoDB +1 a +2 b +1 a +2 b +3 c +1 a +2 b +3 c diff --git a/tests/queries/0_stateless/05428_mysqldump_create_without_columns.sh b/tests/queries/0_stateless/05428_mysqldump_create_without_columns.sh new file mode 100755 index 0000000000000..ab7e6079e8297 --- /dev/null +++ b/tests/queries/0_stateless/05428_mysqldump_create_without_columns.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A `CREATE TABLE` in a MySQL dump without column definitions (`CREATE TABLE ... LIKE`, or only indexes or constraints) +# gives no structure, so the structure is inferred from the data of the `INSERT`. + +function dump() +{ + printf 'CREATE TABLE `t1` %s;\n/*!40101 SET character_set_client = @saved_cs_client */;\nINSERT INTO `t1` VALUES (1,'"'"'a'"'"'),(2,'"'"'b'"'"');\n' "$1" +} + +for create in 'LIKE `t0`' '(KEY `k` (`x`)) ENGINE=InnoDB' '(CONSTRAINT `c` CHECK (`x` > 0)) ENGINE=InnoDB'; do + echo "$create" + dump "$create" | $CLICKHOUSE_LOCAL --input-format MySQLDump --query "DESCRIBE TABLE table" + dump "$create" | $CLICKHOUSE_LOCAL --input-format MySQLDump --query "SELECT * FROM table" +done + +# The `INSERT` immediately following the `CREATE TABLE` must not be skipped. + +function dump_single_insert() +{ + printf 'CREATE TABLE `t1` %s;\nINSERT INTO `t1` VALUES (1,'"'"'a'"'"'),(2,'"'"'b'"'"');\n' "$1" +} + +function dump_multiple_inserts() +{ + printf 'CREATE TABLE `t1` %s;\nINSERT INTO `t1` VALUES (1,'"'"'a'"'"');\nINSERT INTO `t1` VALUES (2,'"'"'b'"'"');\nINSERT INTO `t1` VALUES (3,'"'"'c'"'"');\n' "$1" +} + +for create in 'LIKE `t0`' '(KEY `k` (`x`)) ENGINE=InnoDB' '(CONSTRAINT `c` CHECK (`x` > 0)) ENGINE=InnoDB' '(`x` int, `y` varchar(10)) ENGINE=InnoDB'; do + echo "$create" + dump_single_insert "$create" | $CLICKHOUSE_LOCAL --input-format MySQLDump --query "SELECT * FROM table" + dump_multiple_inserts "$create" | $CLICKHOUSE_LOCAL --input-format MySQLDump --query "SELECT * FROM table" + dump_multiple_inserts "$create" | $CLICKHOUSE_LOCAL --input-format MySQLDump --structure 'x UInt32, y String' --query "SELECT * FROM table" +done From 4f293ae934e27b7edddfb5eaac947bb737b763ca Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 24 Oct 2026 03:26:40 +0000 Subject: [PATCH 018/219] Backport #127288 to 26.8: Write the serialized state of `AggregateFunction` in `RawBLOB` output --- .../Formats/Impl/RawBLOBRowOutputFormat.cpp | 20 +++++++++++++++---- .../Formats/Impl/RawBLOBRowOutputFormat.h | 8 ++++++-- ...05441_rawblob_aggregate_function.reference | 5 +++++ .../05441_rawblob_aggregate_function.sh | 18 +++++++++++++++++ 4 files changed, 45 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/05441_rawblob_aggregate_function.reference create mode 100755 tests/queries/0_stateless/05441_rawblob_aggregate_function.sh diff --git a/src/Processors/Formats/Impl/RawBLOBRowOutputFormat.cpp b/src/Processors/Formats/Impl/RawBLOBRowOutputFormat.cpp index 7d007ac8d451b..5bcbec3c45b6f 100644 --- a/src/Processors/Formats/Impl/RawBLOBRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/RawBLOBRowOutputFormat.cpp @@ -1,4 +1,6 @@ +#include #include +#include #include #include #include @@ -10,14 +12,24 @@ namespace DB RawBLOBRowOutputFormat::RawBLOBRowOutputFormat( WriteBuffer & out_, - SharedHeader header_) + SharedHeader header_, + const FormatSettings & format_settings_) : IRowOutputFormat(header_, out_) + , format_settings(format_settings_) { } -void RawBLOBRowOutputFormat::writeField(const IColumn & column, const ISerialization &, size_t row_num) +void RawBLOBRowOutputFormat::writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) { + /// `getDataAt` of `ColumnAggregateFunction` returns the bytes of the pointer to the state, not the state itself. + /// Write the serialized state, the same as its conversion to `String`. + if (typeid_cast(&column)) + { + serialization.serializeBinary(column, row_num, out, format_settings); + return; + } + if (!column.isNullAt(row_num)) { auto value = column.getDataAt(row_num); @@ -32,10 +44,10 @@ void registerOutputFormatRawBLOB(FormatFactory & factory) factory.registerOutputFormat("RawBLOB", []( WriteBuffer & buf, const Block & sample, - const FormatSettings &, + const FormatSettings & settings, FormatFilterInfoPtr /*format_filter_info*/) { - return std::make_shared(buf, std::make_shared(sample)); + return std::make_shared(buf, std::make_shared(sample), settings); }); /// The output is a verbatim copy of the column bytes, which are not guaranteed to be valid UTF-8 text. diff --git a/src/Processors/Formats/Impl/RawBLOBRowOutputFormat.h b/src/Processors/Formats/Impl/RawBLOBRowOutputFormat.h index e2afa209faaab..9418d3c33028c 100644 --- a/src/Processors/Formats/Impl/RawBLOBRowOutputFormat.h +++ b/src/Processors/Formats/Impl/RawBLOBRowOutputFormat.h @@ -1,5 +1,6 @@ #pragma once +#include #include @@ -28,12 +29,15 @@ class RawBLOBRowOutputFormat final : public IRowOutputFormat public: RawBLOBRowOutputFormat( WriteBuffer & out_, - SharedHeader header_); + SharedHeader header_, + const FormatSettings & format_settings_); String getName() const override { return "RawBLOBRowOutputFormat"; } private: - void writeField(const IColumn & column, const ISerialization &, size_t row_num) override; + void writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) override; + + const FormatSettings format_settings; }; } diff --git a/tests/queries/0_stateless/05441_rawblob_aggregate_function.reference b/tests/queries/0_stateless/05441_rawblob_aggregate_function.reference new file mode 100644 index 0000000000000..98c80be8e58ba --- /dev/null +++ b/tests/queries/0_stateless/05441_rawblob_aggregate_function.reference @@ -0,0 +1,5 @@ +0300000000000000 +c9b15b7fd5143f1d8f677be3dc4daab9 - +c9b15b7fd5143f1d8f677be3dc4daab9 - +9038957ba26f89fe09ee1964aad98576 - +9038957ba26f89fe09ee1964aad98576 - diff --git a/tests/queries/0_stateless/05441_rawblob_aggregate_function.sh b/tests/queries/0_stateless/05441_rawblob_aggregate_function.sh new file mode 100755 index 0000000000000..d6493c3c6808f --- /dev/null +++ b/tests/queries/0_stateless/05441_rawblob_aggregate_function.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + + +# `RawBLOB` must write the serialized state of an aggregate function, the same as its conversion to `String`, +# not the pointer to the state in memory. + +$CLICKHOUSE_CLIENT -q "SELECT sumState(number) FROM numbers(3) FORMAT RawBLOB" | od -An -tx1 | tr -d ' \n' +echo + +$CLICKHOUSE_CLIENT -q "SELECT quantileState(number) FROM numbers(100000) FORMAT RawBLOB" | md5sum +$CLICKHOUSE_CLIENT -q "SELECT CAST(quantileState(number) AS String) FROM numbers(100000) FORMAT RawBLOB" | md5sum + +$CLICKHOUSE_CLIENT -q "SELECT uniqExactState(number) FROM numbers(10) GROUP BY number % 2 ORDER BY number % 2 FORMAT RawBLOB" | md5sum +$CLICKHOUSE_CLIENT -q "SELECT CAST(uniqExactState(number) AS String) FROM numbers(10) GROUP BY number % 2 ORDER BY number % 2 FORMAT RawBLOB" | md5sum From b7c7ae4e5217494cff719c6ba1b84b17bc959595 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 23 Oct 2026 03:12:57 +0000 Subject: [PATCH 019/219] Backport #113574 to 26.8: Fix `RESTORE` of tables on read-only storage that provides their data (`backup_metadata_only`) --- src/Backups/BackupEntriesCollector.cpp | 26 ++- src/Backups/BackupEntriesCollector.h | 4 +- src/Backups/BackupUtils.cpp | 121 +++++++++++ src/Backups/BackupUtils.h | 17 ++ src/Backups/RestorerFromBackup.cpp | 77 ++++++- src/Backups/RestorerFromBackup.h | 4 + src/Disks/IStoragePolicy.h | 3 + src/Disks/StoragePolicy.cpp | 29 +++ src/Disks/StoragePolicy.h | 6 + src/Interpreters/Context.cpp | 9 + src/Interpreters/Context.h | 2 + .../MergeTree/registerStorageMergeTree.cpp | 1 + .../configs/disk_s3.xml | 8 + .../test_backup_restore_s3/test.py | 195 ++++++++++++++++++ .../test_backup_metadata_only.py | 134 ++++++++++++ ...le_function_table_without_source.reference | 3 + ...kup_table_function_table_without_source.sh | 19 ++ ...ing_table_unknown_storage_policy.reference | 4 + ...o_existing_table_unknown_storage_policy.sh | 53 +++++ 19 files changed, 707 insertions(+), 8 deletions(-) create mode 100644 tests/integration/test_backup_restore_s3/test_backup_metadata_only.py create mode 100644 tests/queries/0_stateless/05339_backup_table_function_table_without_source.reference create mode 100755 tests/queries/0_stateless/05339_backup_table_function_table_without_source.sh create mode 100644 tests/queries/0_stateless/05441_restore_into_existing_table_unknown_storage_policy.reference create mode 100755 tests/queries/0_stateless/05441_restore_into_existing_table_unknown_storage_policy.sh diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp index 763799a598bc0..8f8132dad5cad 100644 --- a/src/Backups/BackupEntriesCollector.cpp +++ b/src/Backups/BackupEntriesCollector.cpp @@ -587,7 +587,7 @@ void BackupEntriesCollector::gatherTablesMetadata() /// partition-related constraints. for (auto & [qualified_name, res_table_info] : table_infos) { - res_table_info.should_backup_data = shouldBackupTableData(qualified_name, res_table_info.storage, rmv_replace_target_ids); + res_table_info.should_backup_data = shouldBackupTableData(qualified_name, res_table_info.storage, res_table_info.create_table_query, rmv_replace_target_ids); if (!res_table_info.should_backup_data) continue; @@ -606,6 +606,19 @@ void BackupEntriesCollector::gatherTablesMetadata() res_table_info.storage->getName(), tableNameWithTypeToString(qualified_name.database, qualified_name.table, false)); } + + /// The data of a table on a storage policy marked `backup_metadata_only` is never backed up, the storage provides + /// it. A backup of some of its partitions would be restored as the whole table, so refuse it. + if (partitions + && BackupUtils::isDataOnBackupMetadataOnlyStoragePolicy(res_table_info.storage, res_table_info.create_table_query, context)) + { + throw Exception( + ErrorCodes::CANNOT_BACKUP_TABLE, + "Cannot backup specific partitions of {}: it is on a storage policy marked backup_metadata_only, so only its " + "definition is backed up, and it is restored with all the data of its storage", + tableNameWithTypeToString(qualified_name.database, qualified_name.table, false)); + } + res_table_info.partitions = partitions; } } @@ -887,8 +900,8 @@ void BackupEntriesCollector::makeBackupEntriesForTableData(const QualifiedTableN bool BackupEntriesCollector::shouldBackupTableData( const QualifiedTableName & table_name, - /// Used in the Cloud build. - [[maybe_unused]] const StoragePtr & storage, + const StoragePtr & storage, + const ASTPtr & create_table_query, const std::unordered_set & rmv_replace_target_ids) const { if (backup_settings.structure_only) @@ -900,6 +913,13 @@ bool BackupEntriesCollector::shouldBackupTableData( LOG_TRACE(log, "Skipping table data for {} (a target of a refreshable materialized view)", table_name.getFullName()); return false; } + + if (BackupUtils::isDataOnBackupMetadataOnlyStoragePolicy(storage, create_table_query, context)) + { + LOG_INFO(log, "Skipping table data for {} (on a storage policy marked backup_metadata_only)", table_name.getFullName()); + return false; + } + return true; } diff --git a/src/Backups/BackupEntriesCollector.h b/src/Backups/BackupEntriesCollector.h index 84c9819aa71e7..928f23abfbdbe 100644 --- a/src/Backups/BackupEntriesCollector.h +++ b/src/Backups/BackupEntriesCollector.h @@ -99,8 +99,8 @@ class BackupEntriesCollector : private boost::noncopyable void makeBackupEntriesForTableData(const QualifiedTableName & table_name); bool shouldBackupTableData( const QualifiedTableName & table_name, - /// Used in the Cloud build. - [[maybe_unused]] const StoragePtr & storage, + const StoragePtr & storage, + const ASTPtr & create_table_query, const std::unordered_set & rmv_replace_target_ids) const; void addBackupEntryUnlocked(const String & file_name, BackupEntryPtr backup_entry); diff --git a/src/Backups/BackupUtils.cpp b/src/Backups/BackupUtils.cpp index 879e8716a447e..183b42991d43c 100644 --- a/src/Backups/BackupUtils.cpp +++ b/src/Backups/BackupUtils.cpp @@ -3,9 +3,21 @@ #include #include #include +#include +#include #include #include +#include +#include +#include +#include +#include +#include +#include +#include +#include #include +#include #include @@ -149,4 +161,113 @@ bool isInnerTable(const String & /* database_name */, const String & table_name) return table_name.starts_with(".inner.") || table_name.starts_with(".inner_id.") || table_name.starts_with(".tmp.inner.") || table_name.starts_with(".tmp.inner_id."); } +namespace +{ + +/// The `MergeTreeData` behind `storage` if it is on a storage policy marked `backup_metadata_only`. Only a `MergeTreeData` +/// keeps its data on a storage policy: `Alias`, `Distributed` and `StorageProxy` report one but keep no data of their own. +/// Loads a lazily loaded table, whose `StorageTableProxy` reports no storage policy, but never runs the table function +/// behind a `StorageTableFunctionProxy`, which is checked for every table backed up or restored. +std::shared_ptr getMergeTreeOnBackupMetadataOnlyStoragePolicy(const StoragePtr & storage) +{ + auto merge_tree = castStorage(resolveStorageProxyLoading(storage), DeferredTable::Skip); + if (!merge_tree) + return nullptr; + auto policy = merge_tree->getStoragePolicy(); + if (!policy || !policy->isBackupMetadataOnly()) + return nullptr; + return merge_tree; +} + +void warnBackupMetadataOnlyIgnored(const StoragePtr & storage, const String & reason) +{ + LOG_WARNING( + getLogger("BackupUtils"), + "{} is on a storage policy marked backup_metadata_only, which BACKUP and RESTORE support only for tables of the MergeTree " + "family that are neither replicated nor shared, but {}: its data is backed up and restored as usual", + storage->getStorageID().getNameForLogs(), + reason); +} + +} + +bool isDataOnBackupMetadataOnlyStoragePolicy(const StoragePtr & storage, const ContextPtr & context) +{ + /// Only a table of the plain `MergeTree` family reopens the parts it finds on the storage. `RESTORE` creates a + /// `ReplicatedMergeTree` table in a mode that refuses them, a `SharedMergeTree` table sees only the parts in Keeper, + /// and a view looks for the data of its inner tables in the backup. For the others the flag is ignored. + if (const auto * materialized_view = typeid_cast(storage.get())) + { + if (materialized_view->hasInnerTable()) + { + auto target = materialized_view->tryGetTargetTable(); + if (target && getMergeTreeOnBackupMetadataOnlyStoragePolicy(target)) + warnBackupMetadataOnlyIgnored(storage, fmt::format("it is a materialized view with the inner table {}", target->getStorageID().getNameForLogs())); + } + return false; + } + + if (const auto * time_series = typeid_cast(storage.get())) + { + for (auto target_kind : StorageTimeSeries::getTargetKinds()) + { + if (!time_series->isInnerTable(target_kind)) + continue; + auto target = time_series->tryGetTargetTable(target_kind, context); + if (target && getMergeTreeOnBackupMetadataOnlyStoragePolicy(target)) + warnBackupMetadataOnlyIgnored(storage, fmt::format("it is a TimeSeries table with the inner table {}", target->getStorageID().getNameForLogs())); + } + return false; + } + + auto merge_tree = getMergeTreeOnBackupMetadataOnlyStoragePolicy(storage); + if (!merge_tree) + return false; + + if (!castStorage(merge_tree, DeferredTable::Skip)) + { + warnBackupMetadataOnlyIgnored(storage, fmt::format("it is a {} table", merge_tree->getName())); + return false; + } + return true; +} + +bool isDataOnBackupMetadataOnlyStoragePolicy(const StoragePtr & storage, const ASTPtr & create_table_query, const ContextPtr & context) +{ + if (storage) + return isDataOnBackupMetadataOnlyStoragePolicy(storage, context); + + const auto * create_query = create_table_query ? create_table_query->as() : nullptr; + return create_query && isCreatedOnBackupMetadataOnlyStoragePolicy(*create_query, context); +} + +bool isCreatedOnBackupMetadataOnlyStoragePolicy(const ASTCreateQuery & create_query, const ContextPtr & context) +{ + const auto * storage = create_query.storage; + if (!storage || !storage->engine || !storage->settings) + return false; + + /// Only an explicit `storage_policy` matters: a read-only policy can't be the default one. + const auto * policy_name = storage->settings->changes.tryGet("storage_policy"); + if (!policy_name) + return false; + + /// Only an engine of the `MergeTree` family keeps its data on a storage policy, see + /// `isDataOnBackupMetadataOnlyStoragePolicy`. `storage_policy` is a builtin setting of the `Log` family too, so what + /// identifies the family is the settings the engine declares, not that it declares this one. An unknown engine is + /// reported by `CREATE`. + const auto & storages = StorageFactory::instance().getAllStorages(); + auto it = storages.find(storage->engine->name); + if (it == storages.end() || it->second.features.has_builtin_setting_fn != MergeTreeSettings::hasBuiltin) + return false; + + /// Only the plain family reopens the parts it finds on the storage: the replicated and shared engines are + /// registered with `supports_replication`. + if (it->second.features.supports_replication) + return false; + + auto policy = context->tryGetStoragePolicy(policy_name->safeGet()); + return policy && policy->isBackupMetadataOnly(); +} + } diff --git a/src/Backups/BackupUtils.h b/src/Backups/BackupUtils.h index ba889eccc48e9..b17a14c365869 100644 --- a/src/Backups/BackupUtils.h +++ b/src/Backups/BackupUtils.h @@ -2,12 +2,14 @@ #include #include +#include namespace DB { class IBackup; class AccessRightsElements; +class ASTCreateQuery; class DDLRenamingMap; struct QualifiedTableName; @@ -28,6 +30,21 @@ bool compareRestoredDatabaseDef(const IAST & restored_database_create_query, con bool isInnerTable(const QualifiedTableName & table_name); bool isInnerTable(const String & database_name, const String & table_name); +/// A table on a storage policy marked `backup_metadata_only` gets its data from the storage itself, e.g. a shared +/// read-only dataset: `BACKUP` stores only the definition of the table, and `RESTORE` recreates the table with its UUID +/// from the backup, because the data is on the storage under a path derived from the UUID, and doesn't restore data. + +/// Whether `storage` keeps its data on a storage policy marked `backup_metadata_only`. Only for the plain `MergeTree` family: +/// for a replicated or shared `MergeTree` table, or a view with an inner table on such a policy, it warns and returns false. +bool isDataOnBackupMetadataOnlyStoragePolicy(const StoragePtr & storage, const ContextPtr & context); + +/// The same for a table that may not exist on this host: a `Replicated` database gives only the definition of a table +/// not created on this replica yet, and then the definition decides. +bool isDataOnBackupMetadataOnlyStoragePolicy(const StoragePtr & storage, const ASTPtr & create_table_query, const ContextPtr & context); + +/// Whether a table created by `create_query` is on a storage policy marked `backup_metadata_only`. +bool isCreatedOnBackupMetadataOnlyStoragePolicy(const ASTCreateQuery & create_query, const ContextPtr & context); + } } diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index 56e63dee650f6..49078930ef869 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -814,9 +814,32 @@ void RestorerFromBackup::createTable(const QualifiedTableName & table_name) "as a mount, or restore the selected partitions without mount_readonly_parts_from_snapshot."); #endif - /// Generate a new UUID for a table (the same table on different hosts must use the same UUID, `restore_coordination` will make it so). - /// The generated UUID will be ignored if the database does not support UUIDs. - restore_coordination->generateUUIDForTable(*create_table_query); + /// Without the data the table has nothing to find on the storage, so it is restored as an ordinary empty table, + /// with a new UUID: keeping the UUID would show all the data of the storage in a `structure_only` restore. + if (create_table_query->uuid != UUIDHelpers::Nil && restore_settings.shouldRestoreTableData() + && BackupUtils::isCreatedOnBackupMetadataOnlyStoragePolicy(*create_table_query, context)) + { + /// The table finds its data on the storage under a path derived from its UUID, so it keeps its UUID from the backup. + /// The other table can be in any database, so its name is not shown: the user may not be allowed to see it. + auto existing_table = DatabaseCatalog::instance().tryGetByUUID(create_table_query->uuid).second; + if (existing_table) + { + auto existing_table_id = existing_table->getStorageID(); + if (existing_table_id.database_name != table_name.database || existing_table_id.table_name != table_name.table) + throw Exception( + ErrorCodes::CANNOT_RESTORE_TABLE, + "Cannot restore {}: it is on a storage policy marked backup_metadata_only, so it keeps its UUID {} from " + "the backup to find its data on the storage, but another table already has this UUID", + tableNameWithTypeToString(table_name.database, table_name.table, false), + toString(create_table_query->uuid)); + } + } + else + { + /// Generate a new UUID for a table (the same table on different hosts must use the same UUID, `restore_coordination` will make it so). + /// The generated UUID will be ignored if the database does not support UUIDs. + restore_coordination->generateUUIDForTable(*create_table_query); + } /// Add the clause `IF NOT EXISTS` if that is specified in the restore settings. create_table_query->if_not_exists = (restore_settings.create_table == RestoreTableCreationMode::kCreateIfNotExists); @@ -991,12 +1014,20 @@ void RestorerFromBackup::insertDataToTable(const QualifiedTableName & table_name StoragePtr storage; String data_path_in_backup; std::optional partitions; + ASTPtr create_table_query_from_backup; { std::lock_guard lock{mutex}; auto & table_info = table_infos.at(table_name); storage = table_info.storage; data_path_in_backup = table_info.data_path_in_backup; partitions = table_info.partitions; + create_table_query_from_backup = table_info.create_table_query; + } + + if (BackupUtils::isDataOnBackupMetadataOnlyStoragePolicy(storage, context)) + { + skipTableDataOnBackupMetadataOnlyStoragePolicy(table_name, storage, partitions, create_table_query_from_backup); + return; } schedule( @@ -1004,6 +1035,46 @@ void RestorerFromBackup::insertDataToTable(const QualifiedTableName & table_name ThreadName::RESTORE_TABLE_DATA); } +void RestorerFromBackup::skipTableDataOnBackupMetadataOnlyStoragePolicy( + const QualifiedTableName & table_name, const StoragePtr & storage, const std::optional & partitions, const ASTPtr & create_table_query_from_backup) const +{ + if (partitions) + throw Exception( + ErrorCodes::CANNOT_RESTORE_TABLE, + "Cannot restore specific partitions of {}: it is on a storage policy marked backup_metadata_only, so it shows " + "all the data of its storage and the partition filter cannot be applied", + tableNameWithTypeToString(table_name.database, table_name.table, false)); + + /// The table must have its UUID from the backup, under which the storage keeps the data. This catches what `createTable` + /// doesn't, e.g. an existing table with `create_table = 'must-exist'`, or a database without UUIDs. + const auto * create_query = create_table_query_from_backup ? create_table_query_from_backup->as() : nullptr; + if (!create_query) + throw Exception(ErrorCodes::LOGICAL_ERROR, "The backup has no definition of {}", tableNameWithTypeToString(table_name.database, table_name.table, false)); + + const UUID uuid = storage->getStorageID().uuid; + const UUID uuid_in_backup = create_query->uuid; + if (uuid == UUIDHelpers::Nil || uuid_in_backup == UUIDHelpers::Nil) + throw Exception( + ErrorCodes::CANNOT_RESTORE_TABLE, + "Cannot restore {}: it is on a storage policy marked backup_metadata_only, so its data is not restored and it " + "must have its UUID from the backup to find the data on the storage, but {}", + tableNameWithTypeToString(table_name.database, table_name.table, false), + uuid == UUIDHelpers::Nil ? "its database doesn't give tables UUIDs" : "the backup records no UUID for it"); + + if (uuid != uuid_in_backup) + throw Exception( + ErrorCodes::CANNOT_RESTORE_TABLE, + "Cannot restore {}: it is on a storage policy marked backup_metadata_only, so its data is not restored and it " + "must have its UUID from the backup to find the data on the storage, but the table with UUID {} in the backup " + "has UUID {}", + tableNameWithTypeToString(table_name.database, table_name.table, false), + toString(uuid_in_backup), + toString(uuid)); + + LOG_INFO(log, "Not restoring the data of {}: it is on a storage policy marked backup_metadata_only", + tableNameWithTypeToString(table_name.database, table_name.table, false)); +} + void RestorerFromBackup::insertDataToTableImpl(const QualifiedTableName & table_name, StoragePtr storage, const String & data_path_in_backup, const std::optional & partitions) { try diff --git a/src/Backups/RestorerFromBackup.h b/src/Backups/RestorerFromBackup.h index fa754a049a1d7..aad3d2eb4232e 100644 --- a/src/Backups/RestorerFromBackup.h +++ b/src/Backups/RestorerFromBackup.h @@ -107,6 +107,10 @@ class RestorerFromBackup : public BackupMetadataFinder void insertDataToTable(const QualifiedTableName & table_name); void insertDataToTableImpl(const QualifiedTableName & table_name, StoragePtr storage, const String & data_path_in_backup, const std::optional & partitions); + /// The data of a table on a storage policy marked `backup_metadata_only` is not restored, see `BackupUtils`. + void skipTableDataOnBackupMetadataOnlyStoragePolicy( + const QualifiedTableName & table_name, const StoragePtr & storage, const std::optional & partitions, const ASTPtr & create_table_query_from_backup) const; + void runDataRestoreTasks(); void finalizeTables(); diff --git a/src/Disks/IStoragePolicy.h b/src/Disks/IStoragePolicy.h index d80d0b89a2ffb..66226f4cc425c 100644 --- a/src/Disks/IStoragePolicy.h +++ b/src/Disks/IStoragePolicy.h @@ -60,6 +60,9 @@ class IStoragePolicy DiskPtr getDiskByName(const String & disk_name) const; /// Get free space from most free disk virtual UInt64 getMaxUnreservedFreeSpace() const = 0; + /// The storage provides the data of tables on this policy (``), so `BACKUP` stores only their + /// metadata and `RESTORE` doesn't restore their data. + virtual bool isBackupMetadataOnly() const = 0; /// Reserves space on any volume with index > min_volume_index or returns error virtual ReservationPtrOrError reserve(UInt64 bytes, size_t min_volume_index) const = 0; /// Returns valid reservation or error diff --git a/src/Disks/StoragePolicy.cpp b/src/Disks/StoragePolicy.cpp index eab83e5abd0f3..7311e1e69aebc 100644 --- a/src/Disks/StoragePolicy.cpp +++ b/src/Disks/StoragePolicy.cpp @@ -117,11 +117,40 @@ StoragePolicy::StoragePolicy( "Disk move factor have to be in [0., 1.] interval, but set to {} in storage policy {}", move_factor, backQuote(name)); + backup_metadata_only = config.getBool(config_prefix + ".backup_metadata_only", false); + if (backup_metadata_only) + checkVolumesAreReadOnly(); + buildVolumeIndices(); LOG_TRACE(log, "Storage policy {} created, total volumes {}", name, volumes.size()); } +bool StoragePolicy::isBackupMetadataOnly() const +{ + /// Rechecked on every call, because a disk can become writable while the policy is live: `DiskLocal` clears its + /// read-only flag itself once its periodic check writes to the root directory again, and a reload of the + /// configuration that makes a disk writable keeps this policy, since the policy built from the new configuration + /// fails the check in the constructor and is discarded. So the check cannot be left to where a policy is built. + if (backup_metadata_only) + checkVolumesAreReadOnly(); + return backup_metadata_only; +} + +void StoragePolicy::checkVolumesAreReadOnly() const +{ + /// The data of a table on such a policy is not backed up, so it must come from the storage, not from writes to it. + for (const auto & volume : volumes) + { + if (!volume->isReadOnly()) + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, + "Storage policy {} is marked backup_metadata_only, but its volume {} is not read-only, " + "so BACKUP would leave out the data written to it", + backQuote(name), backQuote(volume->getName())); + } +} + + StoragePolicy::StoragePolicy(String name_, Volumes volumes_, double move_factor_) : volumes(std::move(volumes_)) , name(std::move(name_)) diff --git a/src/Disks/StoragePolicy.h b/src/Disks/StoragePolicy.h index eef7dc1aef764..fc65c63af5960 100644 --- a/src/Disks/StoragePolicy.h +++ b/src/Disks/StoragePolicy.h @@ -50,6 +50,9 @@ class StoragePolicy : public IStoragePolicy /// Get free space from most free disk UInt64 getMaxUnreservedFreeSpace() const override; + /// Throws if the policy is marked `backup_metadata_only` but one of its volumes is not read-only. + bool isBackupMetadataOnly() const override; + const String & getName() const override{ return name; } /// Returns valid reservation or nullptr @@ -101,7 +104,10 @@ class StoragePolicy : public IStoragePolicy /// filled more than total_size * move_factor double move_factor = 0.1; /// by default move factor is 10% + bool backup_metadata_only = false; + void buildVolumeIndices(); + void checkVolumesAreReadOnly() const; LoggerPtr log; }; diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 01696d5962828..cd97ff21caadb 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -7377,6 +7377,15 @@ StoragePolicyPtr Context::getStoragePolicy(const String & name) const return policy_selector->get(name); } +StoragePolicyPtr Context::tryGetStoragePolicy(const String & name) const +{ + std::lock_guard lock(shared->storage_policies_mutex); + + auto policy_selector = getStoragePolicySelector(lock); + + return policy_selector->tryGet(name); +} + StoragePolicyPtr Context::getStoragePolicyFromDisk(const String & disk_name) const { std::lock_guard lock(shared->storage_policies_mutex); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 916929569a4b1..f6a7dc9dfe8e8 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1835,6 +1835,8 @@ class Context: public ContextData, public std::enable_shared_from_this /// Provides storage politics schemes StoragePolicyPtr getStoragePolicy(const String & name) const; + /// Returns `nullptr` instead of throwing when the policy is not configured. + StoragePolicyPtr tryGetStoragePolicy(const String & name) const; StoragePolicyPtr getStoragePolicyFromDisk(const String & disk_name) const; diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index e07d37416dc3d..89507350ce8e1 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -2198,6 +2198,7 @@ Storage policies configuration markup: - `disk` — a disk within a volume. - `max_data_part_size_bytes` — the maximum size of a part that can be stored on any of the volume's disks. If the a size of a merged part estimated to be bigger than `max_data_part_size_bytes` then this part will be written to a next volume. Basically this feature allows to keep new/small parts on a hot (SSD) volume and move them to a cold (HDD) volume when they reach large size. Do not use this setting if your policy has only one volume. - `move_factor` — when the amount of available space gets lower than this factor, data automatically starts to move on the next volume if any (by default, 0.1). ClickHouse sorts existing parts by size from largest to smallest (in descending order) and selects parts with the total size that is sufficient to meet the `move_factor` condition. If the total size of all parts is insufficient, all parts will be moved. +- `backup_metadata_only` — the data of tables on this policy is provided by the storage itself, e.g. a shared read-only dataset (by default, `false`). All volumes of the policy must be read-only. `BACKUP` stores only the definitions of such tables, and `RESTORE` recreates them without restoring data and keeps their UUIDs from the backup, so that they find their data on the storage again. `BACKUP` and `RESTORE` of specific partitions of such a table fail. Only the plain `MergeTree` family is supported: a replicated or shared `MergeTree` table, or a materialized view or a `TimeSeries` table with an inner table on such a policy, is backed up and restored as usual, with a warning. - `perform_ttl_move_on_insert` — Disables TTL move on data part INSERT. By default (if enabled) if we insert a data part that already expired by the TTL move rule it immediately goes to a volume/disk declared in move rule. This can significantly slowdown insert in case if destination volume/disk is slow (e.g. S3). If disabled then already expired data part is written into a default volume and then right after moved to TTL volume. - `load_balancing` - Policy for disk balancing, `round_robin` or `least_used`. - `least_used_ttl_ms` - Configure timeout (in milliseconds) for the updating available space on all disks (`0` - update always, `-1` - never update, default is `60000`). Note, if the disk can be used by ClickHouse only and is not subject to a online filesystem resize/shrink you can use `-1`, in all other cases it is not recommended, since eventually it will lead to incorrect space distribution. diff --git a/tests/integration/test_backup_restore_s3/configs/disk_s3.xml b/tests/integration/test_backup_restore_s3/configs/disk_s3.xml index 6e82a13609704..53e114245a19a 100644 --- a/tests/integration/test_backup_restore_s3/configs/disk_s3.xml +++ b/tests/integration/test_backup_restore_s3/configs/disk_s3.xml @@ -52,6 +52,14 @@ + + +
+ disk_s3_plain +
+
+ true +
diff --git a/tests/integration/test_backup_restore_s3/test.py b/tests/integration/test_backup_restore_s3/test.py index 0a06846903071..fc832e898c5e3 100644 --- a/tests/integration/test_backup_restore_s3/test.py +++ b/tests/integration/test_backup_restore_s3/test.py @@ -1278,6 +1278,201 @@ def test_backup_restore_s3_plain(cluster): instance.query("DROP TABLE sample_restored SYNC") +def test_backup_restore_s3_plain_backup_metadata_only(cluster): + # A table on a storage policy marked `backup_metadata_only` keeps its data on the storage: `BACKUP` stores only its + # metadata, and `RESTORE` recreates it with its UUID from the backup, so it finds its data on the storage again. + # An `Alias` of such a table keeps no data, so it is backed up and restored like any other `Alias`. Materialized views + # and `TimeSeries` tables with inner tables on such a policy are not supported. + instance = cluster.instances["node"] + backup_name = new_backup_name() + backup_destination = f"S3('http://minio1:9001/root/data/backups/{backup_name}', 'minio', '{minio_secret_key}')" + policy = "policy_s3_plain_backup_metadata_only" + + def new_backup_destination(): + return f"S3('http://minio1:9001/root/data/backups/{new_backup_name()}', 'minio', '{minio_secret_key}')" + + instance.query( + """ + DROP TABLE IF EXISTS sample_alias SYNC; + DROP TABLE IF EXISTS sample_alias_copy SYNC; + DROP TABLE IF EXISTS sample_structure_only SYNC; + DROP TABLE IF EXISTS ts SYNC; + DROP TABLE IF EXISTS sample SYNC; + DROP TABLE IF EXISTS mv SYNC; + DROP TABLE IF EXISTS src SYNC; + CREATE TABLE sample (key Int, value String) ENGINE = MergeTree PARTITION BY key % 2 ORDER BY key + AS SELECT number, concat('name_', toString(number)) FROM numbers(100); + """ + ) + + def get_uuid(table): + return instance.query( + f"SELECT uuid FROM system.tables WHERE database = 'default' AND name = '{table}'" + ).strip() + + sample_uuid = get_uuid("sample") + + # Copy the parts to the read-only plain disk, where a table with the same UUID finds them. + minio = cluster.minio_client + table_data_path = ( + instance.query(f"SELECT data_paths[1] FROM system.tables WHERE uuid = '{sample_uuid}'") + .strip() + .replace("/var/lib/clickhouse/", "") + .strip("/") + ) + remote_blob_path = f"data/disks/disk_s3_plain/{table_data_path}" + remove_directory(minio, cluster.minio_bucket, remote_blob_path) + upload_directory( + minio, + cluster.minio_bucket, + os.path.join(instance.path, "database", table_data_path), + remote_blob_path, + use_relpath=True, + ) + + instance.query( + f""" + DROP TABLE sample SYNC; + ATTACH TABLE sample UUID '{sample_uuid}' (key Int, value String) ENGINE = MergeTree PARTITION BY key % 2 ORDER BY key + SETTINGS storage_policy = '{policy}'; + """ + ) + instance.query("CREATE TABLE sample_alias ENGINE = Alias(sample)") + assert instance.query("SELECT count() FROM sample") == "100\n" + assert instance.query("SELECT count() FROM sample_alias") == "100\n" + + instance.query(f"BACKUP TABLE sample, TABLE sample_alias TO {backup_destination}") + + # The backup has the definitions of the tables, but none of their data. + backup_manifest = ( + minio.get_object(cluster.minio_bucket, f"data/backups/{backup_name}/.backup").read().decode() + ) + assert "metadata/default/sample.sql" in backup_manifest, backup_manifest + assert "data/" not in backup_manifest, backup_manifest + + # Only the definition is backed up, so a backup of some partitions would restore as the whole table. + err = instance.query_and_get_error(f"BACKUP TABLE sample PARTITIONS 0 TO {new_backup_destination()}") + assert "CANNOT_BACKUP_TABLE" in err and "Cannot backup specific partitions" in err, err + + # The inner tables of a materialized view and a `TimeSeries` table on such a policy are not supported: the flag is + # ignored for them with a warning, and they are backed up as usual. + instance.query( + f""" + CREATE TABLE src (key Int) ENGINE = Null; + CREATE MATERIALIZED VIEW mv ENGINE = MergeTree ORDER BY key SETTINGS storage_policy = '{policy}' AS SELECT key FROM src; + """ + ) + instance.query( + f""" + CREATE TABLE ts ENGINE = TimeSeries SETTINGS recent_samples_ttl_seconds = 0 + DATA INNER ENGINE = MergeTree SETTINGS storage_policy = '{policy}' + TAGS INNER ENGINE = Memory + METRICS INNER ENGINE = ReplacingMergeTree SETTINGS storage_policy = '{policy}' + """, + settings={"allow_experimental_time_series_table": 1}, + ) + for table, kind in [("mv", "materialized view"), ("ts", "TimeSeries table")]: + instance.query(f"BACKUP TABLE {table} TO {new_backup_destination()}") + assert instance.contains_in_log( + f"default.{table} (" + get_uuid(table) + ), f"no warning for {table}" + assert instance.contains_in_log(f"but it is a {kind} with the inner table"), f"no warning for {table}" + + instance.query("DROP TABLE ts SYNC") + instance.query("DROP TABLE mv SYNC") + instance.query("DROP TABLE src SYNC") + + # The restored table keeps its UUID, so it can't be restored next to the source table. + err = instance.query_and_get_error( + f"RESTORE TABLE sample AS sample_copy FROM {backup_destination}" + ) + assert "CANNOT_RESTORE_TABLE" in err and "another table already has this UUID" in err, err + + # A restore that leaves out the data has nothing to find on the storage, so it keeps no UUID either: it is an + # ordinary empty table and fits next to the source table. + instance.query( + f"RESTORE TABLE sample AS sample_structure_only FROM {backup_destination} SETTINGS structure_only = 1" + ) + assert get_uuid("sample_structure_only") != sample_uuid + assert instance.query("SELECT count() FROM sample_structure_only") == "0\n" + instance.query("DROP TABLE sample_structure_only SYNC") + + # The other table can be in a database the restoring user may not see, so the error doesn't name it. + instance.query( + """ + CREATE DATABASE private_db; + RENAME TABLE sample TO private_db.secret_table; + CREATE DATABASE restore_db; + CREATE USER restore_user; + GRANT CREATE TABLE, INSERT ON restore_db.* TO restore_user; + GRANT READ ON S3 TO restore_user; + """ + ) + try: + err = instance.query_and_get_error( + f"RESTORE TABLE sample AS restore_db.sample FROM {backup_destination}", user="restore_user" + ) + assert "CANNOT_RESTORE_TABLE" in err and "another table already has this UUID" in err, err + assert "private_db" not in err and "secret_table" not in err, err + finally: + instance.query( + """ + RENAME TABLE private_db.secret_table TO sample; + DROP DATABASE private_db; + DROP DATABASE restore_db; + DROP USER restore_user; + """ + ) + + instance.query("DROP TABLE sample_alias SYNC") + instance.query("DROP TABLE sample SYNC") + + # The table shows all the data of its storage, so a partition filter can't be applied. + err = instance.query_and_get_error( + f"RESTORE TABLE sample PARTITIONS 0 FROM {backup_destination}" + ) + assert "CANNOT_RESTORE_TABLE" in err and "partition filter cannot be applied" in err, err + instance.query("DROP TABLE IF EXISTS sample SYNC") + + # A table in an `Ordinary` database has no UUID: its data would be under a path derived from its name. + instance.query( + "CREATE DATABASE ordinary_db ENGINE = Ordinary", + settings={"allow_deprecated_database_ordinary": 1}, + ) + err = instance.query_and_get_error( + f"RESTORE TABLE sample AS ordinary_db.sample FROM {backup_destination}" + ) + assert "CANNOT_RESTORE_TABLE" in err and "its database doesn't give tables UUIDs" in err, err + instance.query("DROP DATABASE ordinary_db SYNC") + + # An existing table with another UUID doesn't see the data of the table from the backup. + instance.query( + f""" + ATTACH TABLE sample UUID '{uuid.uuid4()}' (key Int, value String) ENGINE = MergeTree PARTITION BY key % 2 ORDER BY key + SETTINGS storage_policy = '{policy}' + """ + ) + err = instance.query_and_get_error( + f"RESTORE TABLE sample FROM {backup_destination} SETTINGS create_table = 'must-exist'" + ) + assert "CANNOT_RESTORE_TABLE" in err and f"the table with UUID {sample_uuid} in the backup has UUID" in err, err + instance.query("DROP TABLE sample SYNC") + + instance.query(f"RESTORE TABLE sample FROM {backup_destination}") + assert get_uuid("sample") == sample_uuid + assert instance.query("SELECT count(), sum(key) FROM sample") == "100\t4950\n" + + # The `Alias` gets a new UUID like any table without data on such a policy, also when it is restored under another name. + instance.query(f"RESTORE TABLE sample_alias FROM {backup_destination}") + instance.query(f"RESTORE TABLE sample_alias AS sample_alias_copy FROM {backup_destination}") + assert instance.query("SELECT count(), sum(key) FROM sample_alias") == "100\t4950\n" + assert instance.query("SELECT count(), sum(key) FROM sample_alias_copy") == "100\t4950\n" + + instance.query("DROP TABLE sample_alias SYNC") + instance.query("DROP TABLE sample_alias_copy SYNC") + instance.query("DROP TABLE sample SYNC") + + @pytest.mark.parametrize( "to_disk", [ diff --git a/tests/integration/test_backup_restore_s3/test_backup_metadata_only.py b/tests/integration/test_backup_restore_s3/test_backup_metadata_only.py new file mode 100644 index 0000000000000..ea15306c20940 --- /dev/null +++ b/tests/integration/test_backup_restore_s3/test_backup_metadata_only.py @@ -0,0 +1,134 @@ +import uuid + +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.config_cluster import minio_secret_key + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/disk_s3.xml"], + macros={"shard": "s1", "replica": "r1"}, + with_minio=True, + with_zookeeper=True, +) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/disk_s3.xml"], + macros={"shard": "s1", "replica": "r2"}, + with_minio=True, + with_zookeeper=True, +) + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + yield + finally: + cluster.shutdown() + + +def test_partitions_of_table_not_created_on_replica(start_cluster): + # A replica of a `Replicated` database backs up a table from its definition in Keeper before the table is created on + # the replica. A backup of some partitions of a table on a storage policy marked `backup_metadata_only` is refused from + # the definition then, as it would be restored as the whole table. + policy = "policy_s3_plain_backup_metadata_only" + zookeeper_path = f"/clickhouse/databases/rdb_{uuid.uuid4().hex}" + for node, replica in [(node1, "r1"), (node2, "r2")]: + node.query(f"CREATE DATABASE rdb ENGINE = Replicated('{zookeeper_path}', 's1', '{replica}')") + + node2.query("SYSTEM ENABLE FAILPOINT database_replicated_stop_entry_execution") + try: + node1.query( + f""" + CREATE TABLE rdb.t (key Int, value String) ENGINE = MergeTree PARTITION BY key % 2 ORDER BY key + SETTINGS storage_policy = '{policy}' + """, + settings={"distributed_ddl_task_timeout": 0}, + ) + assert node2.query("SELECT count() FROM system.tables WHERE database = 'rdb' AND name = 't'") == "0\n" + + err = node2.query_and_get_error( + f"BACKUP TABLE rdb.t PARTITIONS 0 TO S3('http://minio1:9001/root/data/backups/{uuid.uuid4().hex}', 'minio', '{minio_secret_key}')" + ) + assert "CANNOT_BACKUP_TABLE" in err and "Cannot backup specific partitions" in err, err + + # The flag is ignored for a `ReplicatedMergeTree` table known only by its definition too, which is backed up as usual. + node1.query( + f"CREATE TABLE rdb.t_replicated (key Int) ENGINE = ReplicatedMergeTree ORDER BY key SETTINGS storage_policy = '{policy}'", + settings={"distributed_ddl_task_timeout": 0}, + ) + assert node2.query("SELECT count() FROM system.tables WHERE database = 'rdb' AND name = 't_replicated'") == "0\n" + backup_destination = f"S3('http://minio1:9001/root/data/backups/{uuid.uuid4().hex}', 'minio', '{minio_secret_key}')" + node2.query(f"BACKUP TABLE rdb.t_replicated TO {backup_destination}") + finally: + node2.query("SYSTEM DISABLE FAILPOINT database_replicated_stop_entry_execution") + for node in [node1, node2]: + node.query("DROP DATABASE IF EXISTS rdb SYNC") + + +def test_disk_made_writable_by_config_reload(start_cluster): + # The policy is checked when it is created, and the reload keeps it when its new version fails the check, but the + # disk can still be made writable by the reload. Then the data written to it would be left out of the backup. + policy = "policy_s3_plain_backup_metadata_only" + config = "/etc/clickhouse-server/config.d/disk_s3.xml" + node1.query( + f"CREATE TABLE t_reload (key Int) ENGINE = MergeTree ORDER BY key SETTINGS storage_policy = '{policy}'" + ) + node1.replace_in_config(config, "true", "false") + try: + node1.query("SYSTEM RELOAD CONFIG") + err = node1.query_and_get_error( + f"BACKUP TABLE t_reload TO S3('http://minio1:9001/root/data/backups/{uuid.uuid4().hex}', 'minio', '{minio_secret_key}')" + ) + assert "INVALID_CONFIG_PARAMETER" in err and "is not read-only" in err, err + finally: + node1.replace_in_config(config, "false", "true") + node1.query("SYSTEM RELOAD CONFIG") + node1.query("DROP TABLE IF EXISTS t_reload SYNC") + + +def test_replicated_merge_tree_ignores_the_flag(start_cluster): + # `RESTORE` creates a `ReplicatedMergeTree` table in a mode that refuses the parts it finds on the storage, so such a + # table on a storage policy marked `backup_metadata_only` couldn't be restored from a backup of its definition: the flag + # is ignored for it with a warning, and it is backed up and restored as usual, with a new UUID. + policy = "policy_s3_plain_backup_metadata_only" + node1.query( + f""" + CREATE TABLE t_replicated (key Int) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{uuid.uuid4().hex}', 'r1') + ORDER BY key SETTINGS storage_policy = '{policy}' + """ + ) + try: + backup_destination = f"S3('http://minio1:9001/root/data/backups/{uuid.uuid4().hex}', 'minio', '{minio_secret_key}')" + node1.query(f"BACKUP TABLE t_replicated TO {backup_destination}") + assert node1.contains_in_log("but it is a ReplicatedMergeTree table: its data is backed up and restored as usual") + uuid_before = node1.query("SELECT uuid FROM system.tables WHERE name = 't_replicated'").strip() + node1.query("DROP TABLE t_replicated SYNC") + node1.query(f"RESTORE TABLE t_replicated FROM {backup_destination} SETTINGS structure_only = 1") + assert node1.query("SELECT uuid FROM system.tables WHERE name = 't_replicated'").strip() != uuid_before + finally: + node1.query("DROP TABLE IF EXISTS t_replicated SYNC") + + +def test_log_family_keeps_the_normal_uuid_path(start_cluster): + # `storage_policy` is a builtin setting of the `Log` family too, but those engines keep no data on a storage policy + # and never take the metadata-only data-skip path. They keep their ordinary backup and restore behavior, which + # includes giving a restored copy a new UUID: preserving the source's would collide with the table it was taken from. + policy = "policy_s3_plain_backup_metadata_only" + node1.query(f"CREATE TABLE t_log (key Int) ENGINE = Log SETTINGS storage_policy = '{policy}'") + try: + backup_destination = f"S3('http://minio1:9001/root/data/backups/{uuid.uuid4().hex}', 'minio', '{minio_secret_key}')" + node1.query(f"BACKUP TABLE t_log TO {backup_destination}") + # The source is left in place on purpose: a preserved UUID is refused as already taken. + node1.query(f"RESTORE TABLE t_log AS t_log_copy FROM {backup_destination} SETTINGS structure_only = 1") + distinct_uuids = node1.query( + "SELECT countDistinct(uuid) FROM system.tables WHERE database = 'default' AND name IN ('t_log', 't_log_copy')" + ).strip() + assert distinct_uuids == "2", distinct_uuids + finally: + node1.query("DROP TABLE IF EXISTS t_log SYNC") + node1.query("DROP TABLE IF EXISTS t_log_copy SYNC") diff --git a/tests/queries/0_stateless/05339_backup_table_function_table_without_source.reference b/tests/queries/0_stateless/05339_backup_table_function_table_without_source.reference new file mode 100644 index 0000000000000..8dd7f35b70168 --- /dev/null +++ b/tests/queries/0_stateless/05339_backup_table_function_table_without_source.reference @@ -0,0 +1,3 @@ +BACKUP_CREATED +RESTORED +t diff --git a/tests/queries/0_stateless/05339_backup_table_function_table_without_source.sh b/tests/queries/0_stateless/05339_backup_table_function_table_without_source.sh new file mode 100755 index 0000000000000..95c5a5cebd527 --- /dev/null +++ b/tests/queries/0_stateless/05339_backup_table_function_table_without_source.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# A table defined from a table function keeps no data of its own, so `BACKUP` and `RESTORE` store and recreate only its +# definition, without running the table function: they work while its source is missing. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +backup="Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}')" + +$CLICKHOUSE_CLIENT -q " +DROP TABLE IF EXISTS t; +CREATE TABLE t (x UInt64) AS file('${CLICKHOUSE_TEST_UNIQUE_NAME}_missing.csv', 'CSV'); +" +$CLICKHOUSE_CLIENT -q "BACKUP TABLE t TO $backup" | grep -o "BACKUP_CREATED" +$CLICKHOUSE_CLIENT -q "DROP TABLE t" +$CLICKHOUSE_CLIENT -q "RESTORE TABLE t FROM $backup" | grep -o "RESTORED" +$CLICKHOUSE_CLIENT -q "SELECT name FROM system.tables WHERE database = currentDatabase() AND name = 't'" +$CLICKHOUSE_CLIENT -q "DROP TABLE t" diff --git a/tests/queries/0_stateless/05441_restore_into_existing_table_unknown_storage_policy.reference b/tests/queries/0_stateless/05441_restore_into_existing_table_unknown_storage_policy.reference new file mode 100644 index 0000000000000..632d3fe0d3f87 --- /dev/null +++ b/tests/queries/0_stateless/05441_restore_into_existing_table_unknown_storage_policy.reference @@ -0,0 +1,4 @@ +restore into an existing table whose policy this host does not configure +3 6 +creating the table still reports the unknown policy +1 diff --git a/tests/queries/0_stateless/05441_restore_into_existing_table_unknown_storage_policy.sh b/tests/queries/0_stateless/05441_restore_into_existing_table_unknown_storage_policy.sh new file mode 100755 index 0000000000000..615176a9b4774 --- /dev/null +++ b/tests/queries/0_stateless/05441_restore_into_existing_table_unknown_storage_policy.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# `RESTORE` into an existing table must not resolve the storage policy named in the backup: nothing is +# created for it. Two configurations, because only the source defines `extra_policy`. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +WORK_DIR="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}" +trap 'rm -rf "${WORK_DIR}"' EXIT +rm -rf "${WORK_DIR}" +mkdir -p "${WORK_DIR}/backups" + +cat > "${WORK_DIR}/source.xml" < + + + local${WORK_DIR}/extra/ + + +
extra
+
+
+ ${WORK_DIR}/backups + +XML + +cat > "${WORK_DIR}/target.xml" < + ${WORK_DIR}/backups + +XML + +${CLICKHOUSE_LOCAL} --config-file "${WORK_DIR}/source.xml" --path "${WORK_DIR}/source" -m -q " + CREATE DATABASE db ENGINE = Atomic; + CREATE TABLE db.t (x UInt64) ENGINE = MergeTree ORDER BY x SETTINGS storage_policy = 'extra_policy'; + INSERT INTO db.t VALUES (1), (2), (3); + BACKUP TABLE db.t TO File('${WORK_DIR}/backups/b') FORMAT Null; +" + +echo "restore into an existing table whose policy this host does not configure" +${CLICKHOUSE_LOCAL} --config-file "${WORK_DIR}/target.xml" --path "${WORK_DIR}/target" -m -q " + CREATE DATABASE db ENGINE = Atomic; + CREATE TABLE db.t (x UInt64) ENGINE = MergeTree ORDER BY x; + RESTORE TABLE db.t FROM File('${WORK_DIR}/backups/b') SETTINGS allow_different_table_def = 1 FORMAT Null; + SELECT count(), sum(x) FROM db.t; +" + +echo "creating the table still reports the unknown policy" +${CLICKHOUSE_LOCAL} --config-file "${WORK_DIR}/target.xml" --path "${WORK_DIR}/target_create" -m -q " + CREATE DATABASE db ENGINE = Atomic; + RESTORE TABLE db.t FROM File('${WORK_DIR}/backups/b') FORMAT Null; +" 2>&1 | grep -c -m1 "UNKNOWN_POLICY" From 394937fc576f1122eb48a2dca5c72272bf892f69 Mon Sep 17 00:00:00 2001 From: Julia Kartseva Date: Fri, 23 Oct 2026 14:57:30 +0000 Subject: [PATCH 020/219] Call `getTargetKinds` on the table, it is not static on 26.8 On `master` `StorageTimeSeries::getTargetKinds` is a `static constexpr` returning every possible target kind, so the backported code called it on the class. On 26.8 it is a non-static const member returning the kinds this table actually has, which does not compile as a qualified call: error: call to non-static member function without an object argument The loop already holds `time_series` and filters every kind through `isInnerTable`, so calling the member on that table keeps the behavior: it iterates this table's own kinds instead of all of them, and the filter makes the two sets agree. Co-Authored-By: Claude Opus 5 --- src/Backups/BackupUtils.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Backups/BackupUtils.cpp b/src/Backups/BackupUtils.cpp index 183b42991d43c..1b067d940beee 100644 --- a/src/Backups/BackupUtils.cpp +++ b/src/Backups/BackupUtils.cpp @@ -209,7 +209,7 @@ bool isDataOnBackupMetadataOnlyStoragePolicy(const StoragePtr & storage, const C if (const auto * time_series = typeid_cast(storage.get())) { - for (auto target_kind : StorageTimeSeries::getTargetKinds()) + for (auto target_kind : time_series->getTargetKinds()) { if (!time_series->isInnerTable(target_kind)) continue; From 9e382d074e7aaff3dff5036dc3119e6582d4c884 Mon Sep 17 00:00:00 2001 From: Julia Kartseva Date: Fri, 23 Oct 2026 20:24:33 +0000 Subject: [PATCH 021/219] Validate the partitions of a table before its data is left out The branch decides `should_backup_data` and leaves the loop at once when it is false, while on master the partition checks run first, so that naming a partition of a table that cannot back one up is still refused however the data came to be left out. The cherry-pick put the new `backup_metadata_only` check after that early exit, where it cannot be reached: leaving the data out is exactly what the flag does, so `BACKUP ... PARTITIONS` of such a table was accepted and restored as the whole table. The two checks move above the exit, as on master. `test_backup_restore_s3_plain_backup_metadata_only` and `test_partitions_of_table_not_created_on_replica` assert the refusal; the first of them also left its table behind when it failed there, which is what then made `test_backup_restore_with_s3_throttle` restore that table into a UUID that was still taken. Co-Authored-By: Claude Opus 5 --- src/Backups/BackupEntriesCollector.cpp | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp index 8f8132dad5cad..24b622386ffb2 100644 --- a/src/Backups/BackupEntriesCollector.cpp +++ b/src/Backups/BackupEntriesCollector.cpp @@ -589,14 +589,15 @@ void BackupEntriesCollector::gatherTablesMetadata() { res_table_info.should_backup_data = shouldBackupTableData(qualified_name, res_table_info.storage, res_table_info.create_table_query, rmv_replace_target_ids); - if (!res_table_info.should_backup_data) - continue; - const auto & database_info = database_infos.at(qualified_name.database); auto it = database_info.tables.find(qualified_name.table); if (it == database_info.tables.end()) continue; + /// Validated before `should_backup_data` is acted on, so it holds however the data came to be left out - + /// `structure_only`, a refreshable materialized view target, or a `backup_metadata_only` storage policy. + /// Naming a partition is a user error in each of those cases, and answering it by silently ignoring the + /// clause is worse than refusing the query. const auto & partitions = it->second.partitions; if (partitions && res_table_info.storage && !res_table_info.storage->supportsBackupPartition()) { @@ -619,6 +620,10 @@ void BackupEntriesCollector::gatherTablesMetadata() tableNameWithTypeToString(qualified_name.database, qualified_name.table, false)); } + /// An excluded table contributes no partitions, so the scope below is left unset for it. + if (!res_table_info.should_backup_data) + continue; + res_table_info.partitions = partitions; } } From b1de37fecd8735f1f79f5de3454210f19a206269 Mon Sep 17 00:00:00 2001 From: Julia Kartseva Date: Fri, 23 Oct 2026 20:34:28 +0000 Subject: [PATCH 022/219] Leave out the `TimeSeries` targets instead of calling `getTargetKinds` on the table `StorageTimeSeries::getTargetKinds` is a `static constexpr` on master and a non-static const member on this branch, so the cherry-pick did not compile. Calling it on the table would work, but it is the only thing the branch needs it for, and it only produces a warning: `isDataOnBackupMetadataOnlyStoragePolicy` answers `false` for a `TimeSeries` table either way, as the table keeps no data of its own. So the lookup of its inner tables is left out, with the assertion of that warning dropped from the test, and `context` becomes unused here. Co-Authored-By: Claude Opus 5 --- src/Backups/BackupUtils.cpp | 17 +++++------------ .../integration/test_backup_restore_s3/test.py | 14 +++----------- 2 files changed, 8 insertions(+), 23 deletions(-) diff --git a/src/Backups/BackupUtils.cpp b/src/Backups/BackupUtils.cpp index 1b067d940beee..5362628001edd 100644 --- a/src/Backups/BackupUtils.cpp +++ b/src/Backups/BackupUtils.cpp @@ -191,7 +191,8 @@ void warnBackupMetadataOnlyIgnored(const StoragePtr & storage, const String & re } -bool isDataOnBackupMetadataOnlyStoragePolicy(const StoragePtr & storage, const ContextPtr & context) +/// `context` is unused here: only the `TimeSeries` targets left out on this branch needed it. +bool isDataOnBackupMetadataOnlyStoragePolicy(const StoragePtr & storage, [[maybe_unused]] const ContextPtr & context) { /// Only a table of the plain `MergeTree` family reopens the parts it finds on the storage. `RESTORE` creates a /// `ReplicatedMergeTree` table in a mode that refuses them, a `SharedMergeTree` table sees only the parts in Keeper, @@ -207,18 +208,10 @@ bool isDataOnBackupMetadataOnlyStoragePolicy(const StoragePtr & storage, const C return false; } - if (const auto * time_series = typeid_cast(storage.get())) - { - for (auto target_kind : time_series->getTargetKinds()) - { - if (!time_series->isInnerTable(target_kind)) - continue; - auto target = time_series->tryGetTargetTable(target_kind, context); - if (target && getMergeTreeOnBackupMetadataOnlyStoragePolicy(target)) - warnBackupMetadataOnlyIgnored(storage, fmt::format("it is a TimeSeries table with the inner table {}", target->getStorageID().getNameForLogs())); - } + /// The warning for the inner tables of a `TimeSeries` table is left out: it needs `getTargetKinds`, which is + /// not static on this branch. The flag is ignored for such a table either way, as it keeps no data itself. + if (typeid_cast(storage.get())) return false; - } auto merge_tree = getMergeTreeOnBackupMetadataOnlyStoragePolicy(storage); if (!merge_tree) diff --git a/tests/integration/test_backup_restore_s3/test.py b/tests/integration/test_backup_restore_s3/test.py index fc832e898c5e3..6710b5c52615c 100644 --- a/tests/integration/test_backup_restore_s3/test.py +++ b/tests/integration/test_backup_restore_s3/test.py @@ -1362,23 +1362,15 @@ def get_uuid(table): CREATE MATERIALIZED VIEW mv ENGINE = MergeTree ORDER BY key SETTINGS storage_policy = '{policy}' AS SELECT key FROM src; """ ) - instance.query( - f""" - CREATE TABLE ts ENGINE = TimeSeries SETTINGS recent_samples_ttl_seconds = 0 - DATA INNER ENGINE = MergeTree SETTINGS storage_policy = '{policy}' - TAGS INNER ENGINE = Memory - METRICS INNER ENGINE = ReplacingMergeTree SETTINGS storage_policy = '{policy}' - """, - settings={"allow_experimental_time_series_table": 1}, - ) - for table, kind in [("mv", "materialized view"), ("ts", "TimeSeries table")]: + # A `TimeSeries` table is left out: the warning for its inner tables needs `getTargetKinds`, which is not + # static on this branch, so `isDataOnBackupMetadataOnlyStoragePolicy` does not look at them here. + for table, kind in [("mv", "materialized view")]: instance.query(f"BACKUP TABLE {table} TO {new_backup_destination()}") assert instance.contains_in_log( f"default.{table} (" + get_uuid(table) ), f"no warning for {table}" assert instance.contains_in_log(f"but it is a {kind} with the inner table"), f"no warning for {table}" - instance.query("DROP TABLE ts SYNC") instance.query("DROP TABLE mv SYNC") instance.query("DROP TABLE src SYNC") From 0db8dfd9c3b554e50ffcff4a476bc425f40770a1 Mon Sep 17 00:00:00 2001 From: Julia Kartseva Date: Fri, 23 Oct 2026 20:41:48 +0000 Subject: [PATCH 023/219] Leave out the diagnostics and the proxy resolution the branch has no helpers for Trimmed to what the original change is about - a table on a `backup_metadata_only` storage policy has its data left out of a backup and a restore - with everything the branch cannot express removed: - `castStorage`, `DeferredTable` and `resolveStorageProxyLoading` come from #121332, which is not on this branch, so the casts are plain `std::dynamic_pointer_cast`. A table behind a `StorageProxy` is no longer resolved, so its data is backed up as usual rather than left out. - `warnBackupMetadataOnlyIgnored` and the lookups of the inner tables of a materialized view and a `TimeSeries` table are gone. They only logged, and the answer does not depend on them: the flag is ignored for both kinds either way. The test asserts the backups still succeed instead of asserting the warnings. `context` is unused in the overload that is left, and the includes the removed code needed are dropped. Co-Authored-By: Claude Opus 5 --- src/Backups/BackupUtils.cpp | 44 ++++--------------- .../test_backup_restore_s3/test.py | 14 ++---- 2 files changed, 13 insertions(+), 45 deletions(-) diff --git a/src/Backups/BackupUtils.cpp b/src/Backups/BackupUtils.cpp index 5362628001edd..71be481e1bca6 100644 --- a/src/Backups/BackupUtils.cpp +++ b/src/Backups/BackupUtils.cpp @@ -14,10 +14,8 @@ #include #include #include -#include #include #include -#include #include @@ -166,11 +164,11 @@ namespace /// The `MergeTreeData` behind `storage` if it is on a storage policy marked `backup_metadata_only`. Only a `MergeTreeData` /// keeps its data on a storage policy: `Alias`, `Distributed` and `StorageProxy` report one but keep no data of their own. -/// Loads a lazily loaded table, whose `StorageTableProxy` reports no storage policy, but never runs the table function -/// behind a `StorageTableFunctionProxy`, which is checked for every table backed up or restored. +/// A proxy is not resolved here, unlike on master, where `resolveStorageProxyLoading` loads a lazily loaded table: the +/// helpers it needs are not on this branch, so a table behind a proxy keeps its data backed up as usual. std::shared_ptr getMergeTreeOnBackupMetadataOnlyStoragePolicy(const StoragePtr & storage) { - auto merge_tree = castStorage(resolveStorageProxyLoading(storage), DeferredTable::Skip); + auto merge_tree = std::dynamic_pointer_cast(storage); if (!merge_tree) return nullptr; auto policy = merge_tree->getStoragePolicy(); @@ -179,16 +177,6 @@ std::shared_ptr getMergeTreeOnBackupMetadataOnlyStoragePolicy(con return merge_tree; } -void warnBackupMetadataOnlyIgnored(const StoragePtr & storage, const String & reason) -{ - LOG_WARNING( - getLogger("BackupUtils"), - "{} is on a storage policy marked backup_metadata_only, which BACKUP and RESTORE support only for tables of the MergeTree " - "family that are neither replicated nor shared, but {}: its data is backed up and restored as usual", - storage->getStorageID().getNameForLogs(), - reason); -} - } /// `context` is unused here: only the `TimeSeries` targets left out on this branch needed it. @@ -197,32 +185,18 @@ bool isDataOnBackupMetadataOnlyStoragePolicy(const StoragePtr & storage, [[maybe /// Only a table of the plain `MergeTree` family reopens the parts it finds on the storage. `RESTORE` creates a /// `ReplicatedMergeTree` table in a mode that refuses them, a `SharedMergeTree` table sees only the parts in Keeper, /// and a view looks for the data of its inner tables in the backup. For the others the flag is ignored. - if (const auto * materialized_view = typeid_cast(storage.get())) - { - if (materialized_view->hasInnerTable()) - { - auto target = materialized_view->tryGetTargetTable(); - if (target && getMergeTreeOnBackupMetadataOnlyStoragePolicy(target)) - warnBackupMetadataOnlyIgnored(storage, fmt::format("it is a materialized view with the inner table {}", target->getStorageID().getNameForLogs())); - } - return false; - } - - /// The warning for the inner tables of a `TimeSeries` table is left out: it needs `getTargetKinds`, which is - /// not static on this branch. The flag is ignored for such a table either way, as it keeps no data itself. - if (typeid_cast(storage.get())) + /// + /// Master warns when it ignores the flag, naming the inner table that carries it. The warning is left out here: + /// it needs the inner tables of a `TimeSeries` table through `getTargetKinds`, which is not static on this + /// branch, and the answer below does not depend on it. + if (typeid_cast(storage.get()) || typeid_cast(storage.get())) return false; auto merge_tree = getMergeTreeOnBackupMetadataOnlyStoragePolicy(storage); if (!merge_tree) return false; - if (!castStorage(merge_tree, DeferredTable::Skip)) - { - warnBackupMetadataOnlyIgnored(storage, fmt::format("it is a {} table", merge_tree->getName())); - return false; - } - return true; + return std::dynamic_pointer_cast(merge_tree) != nullptr; } bool isDataOnBackupMetadataOnlyStoragePolicy(const StoragePtr & storage, const ASTPtr & create_table_query, const ContextPtr & context) diff --git a/tests/integration/test_backup_restore_s3/test.py b/tests/integration/test_backup_restore_s3/test.py index 6710b5c52615c..2ea4a9f956d60 100644 --- a/tests/integration/test_backup_restore_s3/test.py +++ b/tests/integration/test_backup_restore_s3/test.py @@ -1354,22 +1354,16 @@ def get_uuid(table): err = instance.query_and_get_error(f"BACKUP TABLE sample PARTITIONS 0 TO {new_backup_destination()}") assert "CANNOT_BACKUP_TABLE" in err and "Cannot backup specific partitions" in err, err - # The inner tables of a materialized view and a `TimeSeries` table on such a policy are not supported: the flag is - # ignored for them with a warning, and they are backed up as usual. + # The inner tables of a materialized view and a `TimeSeries` table on such a policy are not supported: the flag + # is ignored for them and they are backed up as usual. Master also warns, naming the inner table that carries + # the flag; that warning is left out on this branch, so only the backup itself is asserted here. instance.query( f""" CREATE TABLE src (key Int) ENGINE = Null; CREATE MATERIALIZED VIEW mv ENGINE = MergeTree ORDER BY key SETTINGS storage_policy = '{policy}' AS SELECT key FROM src; """ ) - # A `TimeSeries` table is left out: the warning for its inner tables needs `getTargetKinds`, which is not - # static on this branch, so `isDataOnBackupMetadataOnlyStoragePolicy` does not look at them here. - for table, kind in [("mv", "materialized view")]: - instance.query(f"BACKUP TABLE {table} TO {new_backup_destination()}") - assert instance.contains_in_log( - f"default.{table} (" + get_uuid(table) - ), f"no warning for {table}" - assert instance.contains_in_log(f"but it is a {kind} with the inner table"), f"no warning for {table}" + instance.query(f"BACKUP TABLE mv TO {new_backup_destination()}") instance.query("DROP TABLE mv SYNC") instance.query("DROP TABLE src SYNC") From 10cf7bed3537ed2d505b0546a2eab18df5b575a7 Mon Sep 17 00:00:00 2001 From: Julia Kartseva Date: Sat, 24 Oct 2026 03:25:50 +0000 Subject: [PATCH 024/219] Drop the assertion of the warning this branch no longer writes `test_replicated_merge_tree_ignores_the_flag` asserted the line `warnBackupMetadataOnlyIgnored` writes for an engine of the `MergeTree` family that is not the plain one. That warning was left out of this branch along with the rest of the diagnostics, so the assertion fails while the behavior it guards - the flag ignored, the data backed up as usual, a new UUID after a `structure_only` restore - is unchanged and still asserted. Co-Authored-By: Claude Opus 5 --- .../test_backup_restore_s3/test_backup_metadata_only.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration/test_backup_restore_s3/test_backup_metadata_only.py b/tests/integration/test_backup_restore_s3/test_backup_metadata_only.py index ea15306c20940..6018762638d4b 100644 --- a/tests/integration/test_backup_restore_s3/test_backup_metadata_only.py +++ b/tests/integration/test_backup_restore_s3/test_backup_metadata_only.py @@ -94,7 +94,8 @@ def test_disk_made_writable_by_config_reload(start_cluster): def test_replicated_merge_tree_ignores_the_flag(start_cluster): # `RESTORE` creates a `ReplicatedMergeTree` table in a mode that refuses the parts it finds on the storage, so such a # table on a storage policy marked `backup_metadata_only` couldn't be restored from a backup of its definition: the flag - # is ignored for it with a warning, and it is backed up and restored as usual, with a new UUID. + # is ignored for it, and it is backed up and restored as usual, with a new UUID. Master also warns, naming the engine; + # that warning is left out on this branch, so only the behavior is asserted. policy = "policy_s3_plain_backup_metadata_only" node1.query( f""" @@ -105,7 +106,6 @@ def test_replicated_merge_tree_ignores_the_flag(start_cluster): try: backup_destination = f"S3('http://minio1:9001/root/data/backups/{uuid.uuid4().hex}', 'minio', '{minio_secret_key}')" node1.query(f"BACKUP TABLE t_replicated TO {backup_destination}") - assert node1.contains_in_log("but it is a ReplicatedMergeTree table: its data is backed up and restored as usual") uuid_before = node1.query("SELECT uuid FROM system.tables WHERE name = 't_replicated'").strip() node1.query("DROP TABLE t_replicated SYNC") node1.query(f"RESTORE TABLE t_replicated FROM {backup_destination} SETTINGS structure_only = 1") From b353565f7b196b2c89bde4e01f3220e9f0d67224 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 24 Oct 2026 08:54:48 +0000 Subject: [PATCH 025/219] Backport #127439 to 26.8: Fix `LOGICAL_ERROR` in ReplacingMergeTree variants on empty block after insert deduplication --- .../MergeTree/MergeTreeDataWriter.cpp | 5 +++ ...t_mv_dedup_empty_partition_block.reference | 3 ++ ..._insert_mv_dedup_empty_partition_block.sql | 39 +++++++++++++++++++ 3 files changed, 47 insertions(+) create mode 100644 tests/queries/0_stateless/05443_async_insert_mv_dedup_empty_partition_block.reference create mode 100644 tests/queries/0_stateless/05443_async_insert_mv_dedup_empty_partition_block.sql diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 0df5d547c2a03..f008dd050d143 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -592,6 +592,11 @@ Block MergeTreeDataWriter::mergeBlock( span.addAttribute("clickhouse.rows", block_size); span.addAttribute("clickhouse.columns", header->columns()); + /// The block can lose all its rows before this point, e.g. when insert deduplication removes + /// duplicate rows of a materialized view block. Merging algorithms do not expect an empty input. + if (block_size == 0) + return *header; + auto get_merging_algorithm = [&]() -> std::shared_ptr { switch (merging_params.mode) diff --git a/tests/queries/0_stateless/05443_async_insert_mv_dedup_empty_partition_block.reference b/tests/queries/0_stateless/05443_async_insert_mv_dedup_empty_partition_block.reference new file mode 100644 index 0000000000000..1ee31d8b55207 --- /dev/null +++ b/tests/queries/0_stateless/05443_async_insert_mv_dedup_empty_partition_block.reference @@ -0,0 +1,3 @@ +src 0 1 +src 1 2 +tgt 0 1 diff --git a/tests/queries/0_stateless/05443_async_insert_mv_dedup_empty_partition_block.sql b/tests/queries/0_stateless/05443_async_insert_mv_dedup_empty_partition_block.sql new file mode 100644 index 0000000000000..71c125e9ef63e --- /dev/null +++ b/tests/queries/0_stateless/05443_async_insert_mv_dedup_empty_partition_block.sql @@ -0,0 +1,39 @@ +-- Two async inserts with the same `insert_deduplication_token` but different data are coalesced +-- into one batch. In the materialized view target, the rows of the two tokens land in different +-- partitions. For the partition of the second (duplicate) token, self-deduplication removes all +-- rows, and the recalculated block for that partition is empty. The empty block used to reach +-- `MergeTreeDataWriter::mergeBlock` and trigger the "Required source after the first merge is +-- not 0" logical error with `optimize_on_insert` enabled and a `ReplacingMergeTree` target. + +DROP TABLE IF EXISTS mv_dedup_empty; +DROP TABLE IF EXISTS tgt_dedup_empty; +DROP TABLE IF EXISTS src_dedup_empty; + +CREATE TABLE src_dedup_empty (p UInt64, v UInt64) ENGINE = MergeTree ORDER BY v; +CREATE TABLE tgt_dedup_empty (p UInt64, v UInt64) ENGINE = ReplacingMergeTree PARTITION BY p ORDER BY v + SETTINGS non_replicated_deduplication_window = 100; +CREATE MATERIALIZED VIEW mv_dedup_empty TO tgt_dedup_empty AS SELECT p, v FROM src_dedup_empty; + +SET async_insert = 1; +SET wait_for_async_insert = 0; +SET async_insert_deduplicate = 1; +SET deduplicate_blocks_in_dependent_materialized_views = 1; +SET optimize_on_insert = 1; +-- Prevent an automatic flush between the two inserts; the flush below is explicit. +SET async_insert_busy_timeout_min_ms = 600000; +SET async_insert_busy_timeout_max_ms = 600000; +SET async_insert_use_adaptive_busy_timeout = 0; + +INSERT INTO src_dedup_empty SETTINGS insert_deduplication_token = 'tok' VALUES (0, 1); +INSERT INTO src_dedup_empty SETTINGS insert_deduplication_token = 'tok' VALUES (1, 2); + +SYSTEM FLUSH ASYNC INSERT QUEUE src_dedup_empty; + +-- The source table has no deduplication window, so it keeps both rows. In the target table the +-- second insert is a duplicate of the first by token, so only the first row survives. +SELECT 'src', p, v FROM src_dedup_empty ORDER BY v; +SELECT 'tgt', p, v FROM tgt_dedup_empty ORDER BY v; + +DROP TABLE mv_dedup_empty; +DROP TABLE tgt_dedup_empty; +DROP TABLE src_dedup_empty; From 8ad157d60bd34960dd13a9583325bceb00c962bc Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 23 Oct 2026 20:04:36 +0000 Subject: [PATCH 026/219] Backport #127286 to 26.8: Reject too deep cursors instead of exhausting the stack --- src/Core/Streaming/CursorTree.cpp | 9 +++ .../05441_deep_refresh_cursor.reference | 5 ++ .../0_stateless/05441_deep_refresh_cursor.sh | 68 +++++++++++++++++++ 3 files changed, 82 insertions(+) create mode 100644 tests/queries/0_stateless/05441_deep_refresh_cursor.reference create mode 100755 tests/queries/0_stateless/05441_deep_refresh_cursor.sh diff --git a/src/Core/Streaming/CursorTree.cpp b/src/Core/Streaming/CursorTree.cpp index 7f5933bd00bf3..eccbe6c4fdb2b 100644 --- a/src/Core/Streaming/CursorTree.cpp +++ b/src/Core/Streaming/CursorTree.cpp @@ -17,8 +17,14 @@ namespace DB namespace ErrorCodes { extern const int INVALID_CURSOR_LOOKUP; + extern const int BAD_ARGUMENTS; } +/// The tree is traversed and destroyed recursively, so its depth must be bounded. +/// Real cursors are shallow (e.g. `partition_id.block_number`), but the dotted paths may come +/// from a query (`STREAM CURSOR`) or from external storage (the refresh cursor in data lake metadata). +static constexpr size_t MAX_CURSOR_TREE_DEPTH = 256; + namespace { @@ -182,6 +188,9 @@ CursorTreeNodePtr buildCursorTree(const Map & collapsed_tree) VectorWithMemoryTracking path; boost::split(path, dotted_path, boost::is_any_of(".")); + if (path.size() > MAX_CURSOR_TREE_DEPTH) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cursor path is too deep: {} components, the maximum is {}", path.size(), MAX_CURSOR_TREE_DEPTH); + CursorTreeNode * node = root.get(); for (size_t i = 0; i + 1 < path.size(); ++i) node = node->getSubtreeOrCreate(path[i]).get(); diff --git a/tests/queries/0_stateless/05441_deep_refresh_cursor.reference b/tests/queries/0_stateless/05441_deep_refresh_cursor.reference new file mode 100644 index 0000000000000..61014011d5b1b --- /dev/null +++ b/tests/queries/0_stateless/05441_deep_refresh_cursor.reference @@ -0,0 +1,5 @@ +BAD_ARGUMENTS +2 +TOO_DEEP_RECURSION +BAD_ARGUMENTS +still alive diff --git a/tests/queries/0_stateless/05441_deep_refresh_cursor.sh b/tests/queries/0_stateless/05441_deep_refresh_cursor.sh new file mode 100755 index 0000000000000..99c3112200bdb --- /dev/null +++ b/tests/queries/0_stateless/05441_deep_refresh_cursor.sh @@ -0,0 +1,68 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# A cursor with an absurdly deep structure must be rejected with an exception instead of exhausting the stack: +# both in `STREAM CURSOR` of a query and in the refresh cursor that is stored in Iceberg metadata. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +DEEP_PATH="a$(printf '.a%.0s' {1..100000})" +echo "SELECT 1 FROM system.one STREAM CURSOR {'${DEEP_PATH}': 1}" | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&max_query_size=10000000" --data-binary @- | grep -o -m1 'BAD_ARGUMENTS' + +TABLE_PATH="${USER_FILES_PATH}/${CLICKHOUSE_DATABASE}_deep_refresh_cursor/" +rm -rf "${TABLE_PATH}" + +${CLICKHOUSE_CLIENT} --query " + CREATE TABLE src (id Int64) ENGINE = MergeTree ORDER BY id + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1; + CREATE TABLE tgt (id Int64) ENGINE = IcebergLocal('${TABLE_PATH}'); + CREATE MATERIALIZED VIEW mv REFRESH EVERY 1 YEAR APPEND INCREMENTAL TO tgt EMPTY AS SELECT id FROM src; + INSERT INTO src VALUES (1), (2); + SYSTEM REFRESH VIEW mv; + SYSTEM WAIT VIEW mv; + SELECT count() FROM tgt; +" + +# Replace the refresh cursor of the current snapshot with a forged one in a new metadata version. +forge_cursor() +{ + python3 -I - "${TABLE_PATH}metadata" "$1" <<'EOF' +import base64, json, os, re, struct, sys + +metadata_dir, kind = sys.argv[1], sys.argv[2] +versions = {int(re.match(r'v?(\d+)', name).group(1)): name for name in os.listdir(metadata_dir) if name.endswith('.metadata.json')} +last = max(versions) +with open(os.path.join(metadata_dir, versions[last])) as f: + metadata = json.load(f) + +if kind == 'nested': + # Binary `Field`: 100000 nested single-element arrays around `UInt64` 1. + payload = (bytes([17]) + struct.pack('= 0x80: + varuint.append((n & 0x7F) | 0x80) + n >>= 7 + varuint.append(n) + payload = bytes([26]) + struct.pack('&1 | grep -o -m1 -E 'TOO_DEEP_RECURSION|BAD_ARGUMENTS' +done + +${CLICKHOUSE_CLIENT} --query "SELECT 'still alive'; DROP TABLE mv; DROP TABLE tgt; DROP TABLE src;" +rm -rf "${TABLE_PATH}" From f5dc3f0161535e870aaaf65d926b17021ecabd7a Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sat, 24 Oct 2026 23:15:12 +0000 Subject: [PATCH 027/219] Remove the refresh cursor part of `05441_deep_refresh_cursor` in the backport to 26.8 The 26.8 branch has neither `REFRESH ... APPEND INCREMENTAL` nor the refresh cursor stored in Iceberg metadata, so this part of the test failed with a syntax error. Keep only the `STREAM CURSOR` check, which exists on 26.8. https://s3.amazonaws.com/clickhouse-test-reports/json.html?PR=127635&sha=ff0689119079ed0e62d07208fc18361a84423a02&name_0=BackportPR&name_1=Stateless%20tests%20%28amd_asan_ubsan%2C%20distributed%20plan%2C%20parallel%29 https://github.com/ClickHouse/ClickHouse/pull/127635 Co-Authored-By: Claude Opus 5.5 --- .../05441_deep_refresh_cursor.reference | 4 -- .../0_stateless/05441_deep_refresh_cursor.sh | 60 +------------------ 2 files changed, 1 insertion(+), 63 deletions(-) diff --git a/tests/queries/0_stateless/05441_deep_refresh_cursor.reference b/tests/queries/0_stateless/05441_deep_refresh_cursor.reference index 61014011d5b1b..2481fe970f6a6 100644 --- a/tests/queries/0_stateless/05441_deep_refresh_cursor.reference +++ b/tests/queries/0_stateless/05441_deep_refresh_cursor.reference @@ -1,5 +1 @@ BAD_ARGUMENTS -2 -TOO_DEEP_RECURSION -BAD_ARGUMENTS -still alive diff --git a/tests/queries/0_stateless/05441_deep_refresh_cursor.sh b/tests/queries/0_stateless/05441_deep_refresh_cursor.sh index 99c3112200bdb..7d3019b268338 100755 --- a/tests/queries/0_stateless/05441_deep_refresh_cursor.sh +++ b/tests/queries/0_stateless/05441_deep_refresh_cursor.sh @@ -1,7 +1,6 @@ #!/usr/bin/env bash # Tags: no-fasttest -# A cursor with an absurdly deep structure must be rejected with an exception instead of exhausting the stack: -# both in `STREAM CURSOR` of a query and in the refresh cursor that is stored in Iceberg metadata. +# A cursor with an absurdly deep structure in `STREAM CURSOR` of a query must be rejected with an exception instead of exhausting the stack. CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -9,60 +8,3 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) DEEP_PATH="a$(printf '.a%.0s' {1..100000})" echo "SELECT 1 FROM system.one STREAM CURSOR {'${DEEP_PATH}': 1}" | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&max_query_size=10000000" --data-binary @- | grep -o -m1 'BAD_ARGUMENTS' - -TABLE_PATH="${USER_FILES_PATH}/${CLICKHOUSE_DATABASE}_deep_refresh_cursor/" -rm -rf "${TABLE_PATH}" - -${CLICKHOUSE_CLIENT} --query " - CREATE TABLE src (id Int64) ENGINE = MergeTree ORDER BY id - SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1; - CREATE TABLE tgt (id Int64) ENGINE = IcebergLocal('${TABLE_PATH}'); - CREATE MATERIALIZED VIEW mv REFRESH EVERY 1 YEAR APPEND INCREMENTAL TO tgt EMPTY AS SELECT id FROM src; - INSERT INTO src VALUES (1), (2); - SYSTEM REFRESH VIEW mv; - SYSTEM WAIT VIEW mv; - SELECT count() FROM tgt; -" - -# Replace the refresh cursor of the current snapshot with a forged one in a new metadata version. -forge_cursor() -{ - python3 -I - "${TABLE_PATH}metadata" "$1" <<'EOF' -import base64, json, os, re, struct, sys - -metadata_dir, kind = sys.argv[1], sys.argv[2] -versions = {int(re.match(r'v?(\d+)', name).group(1)): name for name in os.listdir(metadata_dir) if name.endswith('.metadata.json')} -last = max(versions) -with open(os.path.join(metadata_dir, versions[last])) as f: - metadata = json.load(f) - -if kind == 'nested': - # Binary `Field`: 100000 nested single-element arrays around `UInt64` 1. - payload = (bytes([17]) + struct.pack('= 0x80: - varuint.append((n & 0x7F) | 0x80) - n >>= 7 - varuint.append(n) - payload = bytes([26]) + struct.pack('&1 | grep -o -m1 -E 'TOO_DEEP_RECURSION|BAD_ARGUMENTS' -done - -${CLICKHOUSE_CLIENT} --query "SELECT 'still alive'; DROP TABLE mv; DROP TABLE tgt; DROP TABLE src;" -rm -rf "${TABLE_PATH}" From 7e7aee2c11fdf8ee8bf386d987fcdbcbd4ab07f6 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 25 Oct 2026 07:45:39 +0000 Subject: [PATCH 028/219] Backport #127453 to 26.8: Respect input_format_null_as_default for nulls in non-nullable Arrow fields --- .../Impl/ArrowIPC/RecordBatchDecoder.cpp | 85 ++++++++++++- .../Impl/ArrowIPC/RecordBatchDecoder.h | 10 +- ...ll_as_default_non_nullable_field.reference | 33 +++++ ...rrow_null_as_default_non_nullable_field.sh | 118 ++++++++++++++++++ 4 files changed, 240 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/05443_arrow_null_as_default_non_nullable_field.reference create mode 100755 tests/queries/0_stateless/05443_arrow_null_as_default_non_nullable_field.sh diff --git a/src/Processors/Formats/Impl/ArrowIPC/RecordBatchDecoder.cpp b/src/Processors/Formats/Impl/ArrowIPC/RecordBatchDecoder.cpp index cac181c39f5b7..1f05e7aeb77c4 100644 --- a/src/Processors/Formats/Impl/ArrowIPC/RecordBatchDecoder.cpp +++ b/src/Processors/Formats/Impl/ArrowIPC/RecordBatchDecoder.cpp @@ -96,6 +96,25 @@ DataTypePtr matchColumnNullability(const DataTypePtr & type, const ColumnPtr & c return t; } +/// Marks as nullable the fields of `copy` (a copy of `original`) whose originals are in `fields`. +void declareNullable(ArrowField & copy, const ArrowField & original, const UnorderedSetWithMemoryTracking & fields) +{ + if (fields.contains(&original)) + copy.nullable = true; + /// Promotion depends on each batch's nulls, and a `Variant` type can't be converted to another one. + if (original.type.kind == TypeKind::Union) + return; + for (size_t i = 0; i < original.type.children.size(); ++i) + declareNullable(copy.type.children[i], original.type.children[i], fields); + /// The map entries and keys stay non-nullable, as `parseSchema` requires, so their nulls are rejected. + if (original.type.kind == TypeKind::Map) + { + ArrowField & entries = copy.type.children.at(0); + entries.nullable = false; + entries.type.children.at(0).nullable = false; + } +} + /// Expand an Arrow LSB-first bitmap into one byte per row (0 or 1). With `invert` each output byte is /// flipped, turning the validity bitmap (1 = valid) into a ClickHouse null map (1 = null). Unpacks 8 bits /// at a time via SWAR (the same trick as the Parquet decoder), scalar for the trailing < 8 bits. @@ -883,6 +902,10 @@ ColumnPtr RecordBatchDecoder::decodeDictionary( { const Slice indices_slice = nextBuffer(); ColumnPtr values = registry.get(field.dictionary->id); + /// The values of a field declared non-nullable are decoded non-nullable. When its indices contain nulls, + /// `decodeColumns` decodes a nullable copy of the field, so the values need a NULL key as well. + if (field.nullable && !values->isNullable() && values->canBeInsideNullable()) + values = makeNullable(values); const size_t dict_size = values->size(); const int bits = field.dictionary->index_bit_width; @@ -1221,6 +1244,38 @@ ColumnPtr RecordBatchDecoder::decodeField( return inner; } +bool RecordBatchDecoder::collectUndeclaredNullFields( + const ArrowField & field, size_t & node_offset, UnorderedSetWithMemoryTracking & fields) const +{ + const auto * nodes = current_batch->nodes(); + if (!nodes || node_index + node_offset >= nodes->size()) + return false; + const flatbuf::FieldNode & node = *nodes->Get(static_cast(node_index + node_offset)); + ++node_offset; + + /// Unions, `null`-typed and unsupported fields have no validity bitmap: `decodeField` rejects their nulls. + if (!field.nullable && node.null_count() != 0 + && (field.dictionary + || (field.type.kind != TypeKind::Union && field.type.kind != TypeKind::Null && field.type.kind != TypeKind::Unsupported))) + fields.insert(&field); + + /// The values of a dictionary-encoded field are in a separate DictionaryBatch. + if (field.dictionary) + return true; + + /// Mirrors the layouts `skipField` walks. The buffer layout of other unsupported types is unknown. + if (field.type.kind == TypeKind::Unsupported && field.type.skip_layout != ArrowType::SkipLayout::RunEndEncoded + && field.type.skip_layout != ArrowType::SkipLayout::ListView) + return false; + + for (const ArrowField & child : field.type.children) + { + if (!collectUndeclaredNullFields(child, node_offset, fields)) + return false; + } + return true; +} + void RecordBatchDecoder::skipField(const ArrowField & field) { /// Consume this field's FieldNode, mirroring decodeField. @@ -1353,7 +1408,8 @@ RecordBatchDecoder::DecodedColumns RecordBatchDecoder::decodeColumns( const flatbuf::RecordBatch & batch, const PODArray & body, const ArrowFields & fields, const UnorderedSetWithMemoryTracking * keep_top_level_fields, const UnorderedMapWithMemoryTracking * target_types_, - const VectorWithMemoryTracking * reachable_buffers) + const VectorWithMemoryTracking * reachable_buffers, + bool read_undeclared_nulls) { target_types = target_types_; current_batch = &batch; @@ -1401,15 +1457,33 @@ RecordBatchDecoder::DecodedColumns RecordBatchDecoder::decodeColumns( continue; } + /// Nulls in fields declared non-nullable are read as NULL, as the Apache Arrow library did. Decoding a + /// nullable copy of the field keeps the types derived from the schema consistent with the decoded columns. + /// Values of delta dictionary batches are merged into one column, so their types must not depend on each + /// batch's nulls: the dictionary path does not set `read_undeclared_nulls` and keeps rejecting them. + std::optional nullable_copy; + if (read_undeclared_nulls) + { + UnorderedSetWithMemoryTracking undeclared_null_fields; + size_t node_offset = 0; + collectUndeclaredNullFields(field, node_offset, undeclared_null_fields); + if (!undeclared_null_fields.empty()) + { + nullable_copy = field; + declareNullable(*nullable_copy, field, undeclared_null_fields); + } + } + const ArrowField & effective_field = nullable_copy ? *nullable_copy : field; + DecodedColumn decoded; decoded.name = field.name; - decoded.type = fieldToCHType(field, settings, field.nullable, /*allow_null_type=*/true); + decoded.type = fieldToCHType(effective_field, settings, effective_field.nullable, /*allow_null_type=*/true); /// A top-level dictionary-encoded field decodes into a LowCardinality column of its value type. - if (field.dictionary && decoded.type->canBeInsideLowCardinality()) + if (effective_field.dictionary && decoded.type->canBeInsideLowCardinality()) decoded.type = std::make_shared(decoded.type); /// `normalized_name` seeds the recursive `date32` numeric type hint (looked up in `target_types`); /// nested fields derive their hints from it as the decoder recurses. See decodeField/decodeInner. - decoded.column = decodeField(field, /*allow_low_cardinality=*/true, /*target_hint=*/nullptr, normalized_name); + decoded.column = decodeField(effective_field, /*allow_low_cardinality=*/true, /*target_hint=*/nullptr, normalized_name); /// `decodeField` may wrap a struct in Nullable based on the requested type hint even when /// `fieldToCHType` did not (setting off); reconcile the reported type with the decoded column so the /// column/type pair stays consistent for the subsequent cast. @@ -1607,7 +1681,8 @@ RecordBatchDecoder::decodeBatch( const UnorderedMapWithMemoryTracking * target_types_, const VectorWithMemoryTracking * reachable_buffers) { - return decodeColumns(batch, body, schema.fields, keep_top_level_fields, target_types_, reachable_buffers); + return decodeColumns( + batch, body, schema.fields, keep_top_level_fields, target_types_, reachable_buffers, /*read_undeclared_nulls=*/true); } VectorWithMemoryTracking RecordBatchDecoder::reachableTopLevelBuffers( diff --git a/src/Processors/Formats/Impl/ArrowIPC/RecordBatchDecoder.h b/src/Processors/Formats/Impl/ArrowIPC/RecordBatchDecoder.h index 5166ed393843f..700be74f67638 100644 --- a/src/Processors/Formats/Impl/ArrowIPC/RecordBatchDecoder.h +++ b/src/Processors/Formats/Impl/ArrowIPC/RecordBatchDecoder.h @@ -81,11 +81,14 @@ class RecordBatchDecoder const VectorWithMemoryTracking * reachable_buffers = nullptr); /// Decodes an explicit list of fields (used for dictionary batches, which carry one value column). + /// With `read_undeclared_nulls`, nulls in fields declared non-nullable are read as NULL (by decoding a + /// copy of the field that declares them nullable) instead of being rejected. DecodedColumns decodeColumns( const flatbuf::RecordBatch & batch, const PODArray & body, const ArrowFields & fields, const UnorderedSetWithMemoryTracking * keep_top_level_fields = nullptr, const UnorderedMapWithMemoryTracking * target_types = nullptr, - const VectorWithMemoryTracking * reachable_buffers = nullptr); + const VectorWithMemoryTracking * reachable_buffers = nullptr, + bool read_undeclared_nulls = false); /// The buffers (indices into `batch.buffers()`) referenced by the requested top-level fields, as a /// 0/1 mask of length `batch.buffers()->size()`. Computed by the same cursor walk decoding uses @@ -122,6 +125,11 @@ class RecordBatchDecoder /// reading or materializing its data. Used to skip an unrequested top-level column while keeping the /// flat node/buffer cursors aligned for the columns that follow. void skipField(const ArrowField & field); + /// Walks the FieldNodes of `field`'s subtree, starting `node_offset` nodes after the next one, without + /// consuming them, and collects the fields declared non-nullable whose nodes report nulls. Returns false + /// when the walk cannot continue (missing nodes or an unknown layout); decoding then reports the error. + bool collectUndeclaredNullFields( + const ArrowField & field, size_t & node_offset, UnorderedSetWithMemoryTracking & fields) const; ColumnPtr decodeInner(const ArrowField & field, size_t rows, const DataTypePtr & target_hint, const String & path); ColumnPtr decodeUnion(const ArrowField & field, size_t rows); ColumnPtr decodeDictionary( diff --git a/tests/queries/0_stateless/05443_arrow_null_as_default_non_nullable_field.reference b/tests/queries/0_stateless/05443_arrow_null_as_default_non_nullable_field.reference new file mode 100644 index 0000000000000..8b590edc9e2bb --- /dev/null +++ b/tests/queries/0_stateless/05443_arrow_null_as_default_non_nullable_field.reference @@ -0,0 +1,33 @@ +Arrow +1 a x [1,0] (1) +3 c x [] (0) +42 default none [] (0) +CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN +1 a x Nullable(String) +3 c x Nullable(String) +\N \N \N Nullable(String) +[('x'),(NULL),('x')] +CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN +ArrowStream +1 a x [1,0] (1) +3 c x [] (0) +42 default none [] (0) +CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN +1 a x Nullable(String) +3 c x Nullable(String) +\N \N \N Nullable(String) +[('x'),(NULL),('x')] +CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN +Batches +[0] +[1,2] +[7] +[1,2] +[7] +[NULL] +CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN +Rejected +declared non-nullable but its FieldNode reports 1 nulls +declared non-nullable but its FieldNode reports 1 nulls +declared non-nullable but its FieldNode reports 1 nulls +declared non-nullable but its FieldNode reports 1 nulls diff --git a/tests/queries/0_stateless/05443_arrow_null_as_default_non_nullable_field.sh b/tests/queries/0_stateless/05443_arrow_null_as_default_non_nullable_field.sh new file mode 100755 index 0000000000000..cf8c4ebf9b88b --- /dev/null +++ b/tests/queries/0_stateless/05443_arrow_null_as_default_non_nullable_field.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: needs pyarrow + +# Arrow fields declared non-nullable that still contain nulls are read as nulls: with +# `input_format_null_as_default = 1` they become defaults (including column `DEFAULT` expressions) in non-nullable +# columns; with 0 they are rejected for non-nullable columns and kept as NULL for `Nullable` ones. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +TMP_DIR="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}" +mkdir -p "$TMP_DIR" +trap 'rm -rf "$TMP_DIR"' EXIT + +python3 - "$TMP_DIR" <<'PYEOF' +import struct +import sys +import pyarrow as pa +import pyarrow.ipc as ipc + +out = sys.argv[1] +columns = { + "i": pa.array([1, None, 3], pa.int64()), + "s": pa.array(["a", None, "c"]), + "d": pa.array(["x", None, "x"]).dictionary_encode(), + "arr": pa.array([[1, None], [], None], pa.list_(pa.field("item", pa.int64(), nullable=False))), + "tup": pa.array([{"x": 1}, {"x": None}, None], pa.struct([pa.field("x", pa.int64(), nullable=False)])), +} +schema = pa.schema([pa.field(name, arr.type, nullable=False) for name, arr in columns.items()]) +table = pa.Table.from_arrays(list(columns.values()), schema=schema) +with ipc.new_file(f"{out}/data.arrow", schema) as w: + w.write_table(table) +with ipc.new_stream(f"{out}/data.arrows", schema) as w: + w.write_table(table) + +item = pa.field("item", pa.int64(), nullable=False) +list_type = pa.list_(item) +lists = pa.array([[None], [7]], list_type) + +# The nulls first appear in the second batch. +schema = pa.schema([pa.field("l", list_type, nullable=False)]) +with ipc.new_stream(f"{out}/batches.arrows", schema) as w: + w.write_batch(pa.record_batch([pa.array([[1, 2]], list_type)], schema=schema)) + w.write_batch(pa.record_batch([lists], schema=schema)) + +union_type = pa.sparse_union([pa.field("l", list_type, nullable=False)]) +union = pa.Array.from_buffers(union_type, 2, [None, pa.array([0, 0], pa.int8()).buffers()[1]], children=[lists]) +dict_of_lists = pa.DictionaryArray.from_arrays(pa.array([1, 0], pa.int32()), lists) +for name, arr in (("union", union), ("dictionary", dict_of_lists)): + table = pa.Table.from_arrays([arr], names=["c"]) + with ipc.new_stream(f"{out}/{name}.arrows", table.schema) as w: + w.write_table(table) + +# pyarrow refuses nulls in map entries and keys, so write a list of structs and retag the field as a map. +def table_pos(b, table, slot): + vtable = table - struct.unpack_from("&1 \ + | grep -o -m1 "CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN" + $CLICKHOUSE_LOCAL -q "SELECT i, s, d, toTypeName(d) FROM file('$file', $format, 'i Nullable(Int64), s Nullable(String), d Nullable(String)') ORDER BY i SETTINGS input_format_null_as_default = 0" + $CLICKHOUSE_LOCAL -q "SELECT groupArray(tuple(d)) FROM file('$file', $format, 'd LowCardinality(Nullable(String))') SETTINGS input_format_null_as_default = 0" + $CLICKHOUSE_LOCAL -q "SELECT d FROM file('$file', $format, 'd String') SETTINGS input_format_null_as_default = 0" 2>&1 \ + | grep -o -m1 "CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN" +done + +echo "Batches" +$CLICKHOUSE_LOCAL -q "SELECT l FROM file('$TMP_DIR/batches.arrows', ArrowStream) ORDER BY l" +$CLICKHOUSE_LOCAL -q "SELECT l FROM file('$TMP_DIR/batches.arrows', ArrowStream, 'l Array(Nullable(Int64))') ORDER BY l SETTINGS input_format_null_as_default = 0" +$CLICKHOUSE_LOCAL -q "SELECT l FROM file('$TMP_DIR/batches.arrows', ArrowStream, 'l Array(Int64)') SETTINGS input_format_null_as_default = 0" 2>&1 \ + | grep -o -m1 "CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN" + +# Nulls in non-nullable children of unions and of dictionary values, and in map entries and keys, are rejected. +echo "Rejected" +for name in union dictionary map_entries map_keys; do + $CLICKHOUSE_LOCAL -q "SELECT c FROM file('$TMP_DIR/$name.arrows', ArrowStream)" 2>&1 | grep -o -m1 "declared non-nullable but its FieldNode reports [0-9]* nulls" +done + +$CLICKHOUSE_CLIENT -q "DROP TABLE t_arrow_null_as_default" From a8d7ab4032c368845ecccd197ce2506f45cbb3b4 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 26 Oct 2026 03:39:54 +0000 Subject: [PATCH 029/219] Backport #127657 to 26.8: Fix distributed `INSERT ... SELECT FROM` cluster functions ignoring column `DEFAULT` expressions --- src/Interpreters/InterpreterInsertQuery.cpp | 8 +- src/Storages/IStorageCluster.cpp | 2 +- src/Storages/IStorageCluster.h | 11 +- .../StorageObjectStorageCluster.cpp | 9 +- .../StorageObjectStorageCluster.h | 3 +- src/Storages/StorageFileCluster.cpp | 16 +-- src/Storages/StorageFileCluster.h | 3 +- src/Storages/StorageURLCluster.cpp | 16 +-- src/Storages/StorageURLCluster.h | 3 +- ...cluster_function_column_defaults.reference | 18 +++ ...select_cluster_function_column_defaults.sh | 113 ++++++++++++++++++ 11 files changed, 176 insertions(+), 26 deletions(-) create mode 100644 tests/queries/0_stateless/05446_insert_select_cluster_function_column_defaults.reference create mode 100755 tests/queries/0_stateless/05446_insert_select_cluster_function_column_defaults.sh diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index a158eaf3ba07a..9402931bb6987 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -1126,11 +1126,13 @@ std::optional InterpreterInsertQuery::distributedWriteIntoReplica /// the forwarded query as a secondary query does not convert it again: it creates a plain storage that /// expands the globs and reads every file on its own instead of taking its share of the read tasks from /// the initiator, so N nodes insert the data N times. Rewrite the source the same way `IStorageCluster::read` - /// does for a `SELECT`: the function becomes its `*Cluster` variant with the cluster name argument, and the - /// structure and format arguments are added so that the nodes do not infer the schema again. + /// does for a `SELECT`: the function becomes its `*Cluster` variant with the cluster name argument. The other + /// arguments stay as written, so that the nodes take the structure from the insertion table with its `DEFAULT` + /// expressions unless the query gives one. { auto & select_to_send = query_to_send->as().select->as(); - src_storage_cluster->updateQueryToSendIfNeeded(select_to_send.list_of_selects->children.at(0), src_snapshot, local_context); + src_storage_cluster->updateQueryToSendIfNeeded( + select_to_send.list_of_selects->children.at(0), src_snapshot, local_context, /*add_structure_and_format=*/false); } String query_str; diff --git a/src/Storages/IStorageCluster.cpp b/src/Storages/IStorageCluster.cpp index 87c61dcae9cce..c6c040a629582 100644 --- a/src/Storages/IStorageCluster.cpp +++ b/src/Storages/IStorageCluster.cpp @@ -120,7 +120,7 @@ void IStorageCluster::read( query_to_send = interpreter.getQueryInfo().query->clone(); } - updateQueryToSendIfNeeded(query_to_send, storage_snapshot, context); + updateQueryToSendIfNeeded(query_to_send, storage_snapshot, context, /*add_structure_and_format=*/true); RestoreQualifiedNamesVisitor::Data data; data.distributed_table = DatabaseAndTableWithAlias(*getTableExpression(query_info.query->as(), 0)); diff --git a/src/Storages/IStorageCluster.h b/src/Storages/IStorageCluster.h index 6ecfdd22dc193..144fda69e3195 100644 --- a/src/Storages/IStorageCluster.h +++ b/src/Storages/IStorageCluster.h @@ -59,8 +59,15 @@ class IStorageCluster : public IStorage /// function (`url`, `s3`, ...) that `parallel_replicas_for_cluster_engines` converted into this cluster /// storage into its `*Cluster` variant with the cluster name argument, so that the nodes take their read /// tasks from the initiator instead of reading every file on their own. Called by `read` and by the - /// distributed `INSERT ... SELECT` in `InterpreterInsertQuery`, which forwards the query the same way. - virtual void updateQueryToSendIfNeeded(ASTPtr & /*query*/, const StorageSnapshotPtr & /*storage_snapshot*/, const ContextPtr & /*context*/) {} + /// distributed `INSERT ... SELECT` in `InterpreterInsertQuery`. + /// A `SELECT` needs the structure and format so all nodes produce the initiator's header. For a forwarded + /// `INSERT ... SELECT`, leave the arguments as written: an omitted or `'auto'` structure then takes the + /// destination table's columns and `DEFAULT` expressions on each node. + virtual void updateQueryToSendIfNeeded( + ASTPtr & /*query*/, + const StorageSnapshotPtr & /*storage_snapshot*/, + const ContextPtr & /*context*/, + bool /*add_structure_and_format*/) {} protected: virtual void updateBeforeRead(const ContextPtr &) {} diff --git a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp index 8e04a3f576f20..a42ceb4e9c310 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp @@ -275,7 +275,8 @@ std::optional StorageObjectStorageCluster::totalBytes(ContextPtr query_c void StorageObjectStorageCluster::updateQueryToSendIfNeeded( ASTPtr & query, const DB::StorageSnapshotPtr & storage_snapshot, - const ContextPtr & context) + const ContextPtr & context, + bool add_structure_and_format) { auto * table_function = extractTableFunctionFromSelectQuery(query); if (!table_function) @@ -313,7 +314,8 @@ void StorageObjectStorageCluster::updateQueryToSendIfNeeded( if (!endsWith(table_function->name, "Cluster")) { - configuration->addStructureAndFormatToArgsIfNeeded(args, structure, configuration->format, context, /*with_structure=*/true); + if (add_structure_and_format) + configuration->addStructureAndFormatToArgsIfNeeded(args, structure, configuration->format, context, /*with_structure=*/true); /// When a non-cluster table function (e.g. `s3`) was auto-converted to cluster mode /// by the `parallel_replicas_for_cluster_engines` setting, rename it to the Cluster variant @@ -334,7 +336,8 @@ void StorageObjectStorageCluster::updateQueryToSendIfNeeded( { ASTPtr cluster_name_arg = args.front(); args.erase(args.begin()); - configuration->addStructureAndFormatToArgsIfNeeded(args, structure, configuration->format, context, /*with_structure=*/true); + if (add_structure_and_format) + configuration->addStructureAndFormatToArgsIfNeeded(args, structure, configuration->format, context, /*with_structure=*/true); args.insert(args.begin(), cluster_name_arg); } if (settings_temporary_storage) diff --git a/src/Storages/ObjectStorage/StorageObjectStorageCluster.h b/src/Storages/ObjectStorage/StorageObjectStorageCluster.h index 9815b14f6a173..2d7f64ea87467 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageCluster.h +++ b/src/Storages/ObjectStorage/StorageObjectStorageCluster.h @@ -77,7 +77,8 @@ class StorageObjectStorageCluster : public IStorageCluster void updateQueryToSendIfNeeded( ASTPtr & query, const StorageSnapshotPtr & storage_snapshot, - const ContextPtr & context) override; + const ContextPtr & context, + bool add_structure_and_format) override; const String engine_name; const StorageObjectStorageConfigurationPtr configuration; diff --git a/src/Storages/StorageFileCluster.cpp b/src/Storages/StorageFileCluster.cpp index d8b6a06f28413..8b41dc6c6fdd1 100644 --- a/src/Storages/StorageFileCluster.cpp +++ b/src/Storages/StorageFileCluster.cpp @@ -105,18 +105,20 @@ void StorageFileCluster::updateBeforeRead(const ContextPtr & context) checkWriteAccessIfFilesAreRenamed(context); } -void StorageFileCluster::updateQueryToSendIfNeeded(DB::ASTPtr & query, const StorageSnapshotPtr & storage_snapshot, const DB::ContextPtr & context) +void StorageFileCluster::updateQueryToSendIfNeeded( + DB::ASTPtr & query, const StorageSnapshotPtr & storage_snapshot, const DB::ContextPtr & context, bool add_structure_and_format) { auto * table_function = extractTableFunctionFromSelectQuery(query); if (!table_function) throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected SELECT query from table function fileCluster, got '{}'", query->formatForErrorMessage()); - TableFunctionFileCluster::updateStructureAndFormatArgumentsIfNeeded( - table_function, - storage_snapshot->metadata->getColumns().getAll().toNamesAndTypesDescription(), - format_name, - context - ); + if (add_structure_and_format) + TableFunctionFileCluster::updateStructureAndFormatArgumentsIfNeeded( + table_function, + storage_snapshot->metadata->getColumns().getAll().toNamesAndTypesDescription(), + format_name, + context + ); } RemoteQueryExecutor::Extension StorageFileCluster::getTaskIteratorExtension( diff --git a/src/Storages/StorageFileCluster.h b/src/Storages/StorageFileCluster.h index fab05dd2f6459..b656f7fbbabbe 100644 --- a/src/Storages/StorageFileCluster.h +++ b/src/Storages/StorageFileCluster.h @@ -36,7 +36,8 @@ class StorageFileCluster : public IStorageCluster StorageMetadataPtr) const override; private: - void updateQueryToSendIfNeeded(ASTPtr & query, const StorageSnapshotPtr & storage_snapshot, const ContextPtr & context) override; + void updateQueryToSendIfNeeded( + ASTPtr & query, const StorageSnapshotPtr & storage_snapshot, const ContextPtr & context, bool add_structure_and_format) override; void updateBeforeRead(const ContextPtr & context) override; Strings paths; diff --git a/src/Storages/StorageURLCluster.cpp b/src/Storages/StorageURLCluster.cpp index e6378b222182a..3d89d3d8f7cfc 100644 --- a/src/Storages/StorageURLCluster.cpp +++ b/src/Storages/StorageURLCluster.cpp @@ -109,7 +109,8 @@ StorageURLCluster::StorageURLCluster( setInMemoryMetadata(storage_metadata); } -void StorageURLCluster::updateQueryToSendIfNeeded(ASTPtr & query, const StorageSnapshotPtr & storage_snapshot, const ContextPtr & context) +void StorageURLCluster::updateQueryToSendIfNeeded( + ASTPtr & query, const StorageSnapshotPtr & storage_snapshot, const ContextPtr & context, bool add_structure_and_format) { auto * table_function = extractTableFunctionFromSelectQuery(query); if (!table_function) @@ -119,12 +120,13 @@ void StorageURLCluster::updateQueryToSendIfNeeded(ASTPtr & query, const StorageS if (!expression_list) throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected SELECT query from table function urlCluster, got '{}'", query->formatForErrorMessage()); - TableFunctionURLCluster::updateStructureAndFormatArgumentsIfNeeded( - table_function, - storage_snapshot->metadata->getColumns().getAll().toNamesAndTypesDescription(), - format_name, - context - ); + if (add_structure_and_format) + TableFunctionURLCluster::updateStructureAndFormatArgumentsIfNeeded( + table_function, + storage_snapshot->metadata->getColumns().getAll().toNamesAndTypesDescription(), + format_name, + context + ); /// When a non-cluster table function (`url`) was auto-converted to cluster mode /// by the `parallel_replicas_for_cluster_engines` setting, rename it to the Cluster variant diff --git a/src/Storages/StorageURLCluster.h b/src/Storages/StorageURLCluster.h index bee90c4ba7b0d..9c267b5e1bc6a 100644 --- a/src/Storages/StorageURLCluster.h +++ b/src/Storages/StorageURLCluster.h @@ -38,7 +38,8 @@ class StorageURLCluster : public IStorageCluster StorageMetadataPtr) const override; private: - void updateQueryToSendIfNeeded(ASTPtr & query, const StorageSnapshotPtr & storage_snapshot, const ContextPtr & context) override; + void updateQueryToSendIfNeeded( + ASTPtr & query, const StorageSnapshotPtr & storage_snapshot, const ContextPtr & context, bool add_structure_and_format) override; String uri; String format_name; diff --git a/tests/queries/0_stateless/05446_insert_select_cluster_function_column_defaults.reference b/tests/queries/0_stateless/05446_insert_select_cluster_function_column_defaults.reference new file mode 100644 index 0000000000000..3f826b9aceefd --- /dev/null +++ b/tests/queries/0_stateless/05446_insert_select_cluster_function_column_defaults.reference @@ -0,0 +1,18 @@ +--- s3Cluster, column list --- +1 2000-01-01 7 unknown 10 +2 2001-02-03 5 x 6 +--- s3Cluster, asterisk --- +1 2000-01-01 7 unknown 10 +2 2001-02-03 5 x 6 +--- urlCluster --- +1 2000-01-01 7 unknown 10 +2 2001-02-03 5 x 6 +--- s3 with parallel replicas --- +1 2000-01-01 7 unknown 10 +2 2001-02-03 5 x 6 +--- s3Cluster, explicit structure --- +1 1.50 +--- s3Cluster, explicit structure as key-value argument --- +1 1.50 +--- ClickPipes shape, DEFAULT now64(3) with TTL --- +3 1 diff --git a/tests/queries/0_stateless/05446_insert_select_cluster_function_column_defaults.sh b/tests/queries/0_stateless/05446_insert_select_cluster_function_column_defaults.sh new file mode 100755 index 0000000000000..676f062b7e386 --- /dev/null +++ b/tests/queries/0_stateless/05446_insert_select_cluster_function_column_defaults.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, zookeeper +# Tag no-fasttest: needs MinIO and a Replicated table + +# A distributed `INSERT ... SELECT FROM ` must fill the fields missing in the files with the +# `DEFAULT` expressions of the destination table, the same as a plain `INSERT ... SELECT FROM s3(...)`. +# +# The forwarded query used to carry the structure of the source storage, which has names and types only, in +# place of `'auto'`. The nodes then did not take the structure from the insertion table and filled the missing +# fields with the default values of their types (0, '', 1970-01-01). +# +# All nodes of the clusters below are this server. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +set -eu + +S3_DIR="http://localhost:11111/test/${CLICKHOUSE_DATABASE}/05446" + +# The first file has none of the defaulted fields, the second one has all of them. +$CLICKHOUSE_CLIENT -q " + INSERT INTO FUNCTION s3('${S3_DIR}/part_1.jsonl', 'JSONEachRow', 'id UInt32') VALUES (1)" +$CLICKHOUSE_CLIENT -q " + INSERT INTO FUNCTION s3('${S3_DIR}/part_2.jsonl', 'JSONEachRow', 'id UInt32, d Date, q UInt32, c String, e UInt32') + VALUES (2, '2001-02-03', 5, 'x', 6)" + +COLUMNS="id UInt32, d Date DEFAULT '2000-01-01', q UInt32 DEFAULT 7, c String DEFAULT 'unknown', e UInt32 DEFAULT id * 10" + +$CLICKHOUSE_CLIENT -q " + DROP TABLE IF EXISTS dst_05446 SYNC; + CREATE TABLE dst_05446 (${COLUMNS}) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/dst_05446', 'r1') ORDER BY id; +" + +S3_GLOB="${S3_DIR}/part_{1,2}.jsonl" +CLUSTER="test_cluster_one_shard_three_replicas_localhost" +PARALLEL_REPLICAS="enable_parallel_replicas = 1, automatic_parallel_replicas_mode = 0, max_parallel_replicas = 3, + cluster_for_parallel_replicas = '${CLUSTER}', parallel_replicas_for_cluster_engines = 1" + +function check() +{ + local table=$1 + echo "--- $2 ---" + $CLICKHOUSE_CLIENT -q "SELECT * FROM ${table} ORDER BY id" + $CLICKHOUSE_CLIENT -q "TRUNCATE TABLE ${table}" +} + +# The shape of the queries of object storage ClickPipes. +$CLICKHOUSE_CLIENT -q " + INSERT INTO dst_05446 (id, d, q, c, e) SELECT id, d, q, c, e FROM s3Cluster('${CLUSTER}', '${S3_GLOB}', 'JSONEachRow') + SETTINGS parallel_distributed_insert_select = 2" +check dst_05446 "s3Cluster, column list" + +$CLICKHOUSE_CLIENT -q " + INSERT INTO dst_05446 SELECT * FROM s3Cluster('${CLUSTER}', '${S3_GLOB}', 'JSONEachRow') + SETTINGS parallel_distributed_insert_select = 2" +check dst_05446 "s3Cluster, asterisk" + +$CLICKHOUSE_CLIENT -q " + INSERT INTO dst_05446 SELECT * FROM urlCluster('${CLUSTER}', '${S3_GLOB}', 'JSONEachRow') + SETTINGS parallel_distributed_insert_select = 2" +check dst_05446 "urlCluster" + +$CLICKHOUSE_CLIENT -q " + INSERT INTO dst_05446 SELECT * FROM s3('${S3_GLOB}', 'JSONEachRow') + SETTINGS parallel_distributed_insert_select = 2, ${PARALLEL_REPLICAS}" +check dst_05446 "s3 with parallel replicas" + +# An explicit structure must reach the nodes as written, positional or `key = value`. The `SELECT` list has an +# expression, so the nodes cannot take the structure from the insertion table: with `'auto'` they would infer +# `v` as `Float64` and insert '1.5' instead of '1.50'. +$CLICKHOUSE_CLIENT -q "INSERT INTO FUNCTION s3('${S3_DIR}/typed.jsonl', 'LineAsString', 'l String') VALUES ('{\"id\":1,\"v\":1.50}')" +$CLICKHOUSE_CLIENT -q " + DROP TABLE IF EXISTS typed_05446 SYNC; + CREATE TABLE typed_05446 (id UInt32, v String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/typed_05446', 'r1') ORDER BY id; +" + +$CLICKHOUSE_CLIENT -q " + INSERT INTO typed_05446 SELECT id, toString(v) FROM s3Cluster('${CLUSTER}', '${S3_DIR}/typed.jsonl', 'JSONEachRow', 'id UInt32, v String') + SETTINGS parallel_distributed_insert_select = 2" +check typed_05446 "s3Cluster, explicit structure" + +$CLICKHOUSE_CLIENT -q " + INSERT INTO typed_05446 SELECT id, toString(v) FROM s3Cluster('${CLUSTER}', '${S3_DIR}/typed.jsonl', format = 'JSONEachRow', structure = 'id UInt32, v String') + SETTINGS parallel_distributed_insert_select = 2" +check typed_05446 "s3Cluster, explicit structure as key-value argument" + +# The shape of the queries of object storage ClickPipes, with a `DEFAULT now64(3)` column missing in the files and +# a TTL on it: with the default values of the type the rows got `1970-01-01` and the TTL deleted them. +$CLICKHOUSE_CLIENT -q "INSERT INTO FUNCTION s3('${S3_DIR}/clickpipes_a.json', 'LineAsString', 'l String') VALUES ('{\"document\":\"d1\"}'), ('{\"document\":\"d2\"}')" +$CLICKHOUSE_CLIENT -q "INSERT INTO FUNCTION s3('${S3_DIR}/clickpipes_b.json', 'LineAsString', 'l String') VALUES ('{\"document\":\"d3\"}')" +$CLICKHOUSE_CLIENT -q " + DROP TABLE IF EXISTS clickpipes_05446 SYNC; + CREATE TABLE clickpipes_05446 (document String, _synced_at DateTime64(3, 'UTC') DEFAULT now64(3)) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/clickpipes_05446', 'r1') ORDER BY document + TTL toDateTime(_synced_at) + INTERVAL 30 DAY; +" + +$CLICKHOUSE_CLIENT --param_url="${S3_DIR}/clickpipes_{a,b}.json" --param_format=JSONEachRow -q " + INSERT INTO \`${CLICKHOUSE_DATABASE}\`.\`clickpipes_05446\` (\`_synced_at\`, \`document\`) + SELECT \`_synced_at\` AS \`_synced_at\`, \`document\` AS \`document\` + FROM s3Cluster('${CLUSTER}', {url:String}, NOSIGN, {format:String}) + SETTINGS parallel_distributed_insert_select = 2, insert_deduplicate = 0, optimize_on_insert = 0" +$CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE clickpipes_05446 FINAL" +echo "--- ClickPipes shape, DEFAULT now64(3) with TTL ---" +$CLICKHOUSE_CLIENT -q "SELECT count(), min(_synced_at) > now64(3) - INTERVAL 1 HOUR FROM clickpipes_05446" + +$CLICKHOUSE_CLIENT -q " + DROP TABLE dst_05446 SYNC; + DROP TABLE typed_05446 SYNC; + DROP TABLE clickpipes_05446 SYNC; +" From 30f76be4e44a74cf8d7150508434c36e338c6d42 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 26 Oct 2026 17:56:27 +0000 Subject: [PATCH 030/219] Backport #127717 to 26.8: Take an explicitly configured text_log.level into account when updating logger levels --- .../_server_settings_outside_source.mdx | 2 +- programs/server/config.xml | 4 +- src/Loggers/Loggers.cpp | 9 +++ .../__init__.py | 0 .../configs/logger.xml | 9 +++ .../test_text_log_level_after_reload/test.py | 65 +++++++++++++++++++ 6 files changed, 87 insertions(+), 2 deletions(-) create mode 100644 tests/integration/test_text_log_level_after_reload/__init__.py create mode 100644 tests/integration/test_text_log_level_after_reload/configs/logger.xml create mode 100644 tests/integration/test_text_log_level_after_reload/test.py diff --git a/docs/reference/settings/server-settings/_server_settings_outside_source.mdx b/docs/reference/settings/server-settings/_server_settings_outside_source.mdx index 23af0e0fa6040..f72dee5492310 100644 --- a/docs/reference/settings/server-settings/_server_settings_outside_source.mdx +++ b/docs/reference/settings/server-settings/_server_settings_outside_source.mdx @@ -1444,7 +1444,7 @@ Additionally: | Setting | Description | Default Value | |---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------| -| `level` | Maximum Message Level (by default `Trace`) which will be stored in a table. | `Trace` | +| `level` | Maximum message level stored in the table. When set explicitly, the root logger level is raised to at least this value, so `text_log` can be more verbose than the server log file (`logger.level`). When not set, messages up to the most verbose level of the configured log outputs (`logger.level`, `console_log_level`, `errorlog_level`, `syslog_level`) are stored. | `Trace` (limited by the other log outputs when not set) | **Example** diff --git a/programs/server/config.xml b/programs/server/config.xml index 5fe5bd70ca412..36eb35e48440b 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -1354,7 +1354,9 @@ system diff --git a/src/Loggers/Loggers.cpp b/src/Loggers/Loggers.cpp index a44ea3d596a5b..708517eb61e9a 100644 --- a/src/Loggers/Loggers.cpp +++ b/src/Loggers/Loggers.cpp @@ -260,6 +260,9 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log String text_log_level_str = config.getString("text_log.level", "trace"); int text_log_level = Poco::Logger::parseLevel(text_log_level_str); + if (config.has("text_log.level")) + max_log_level = std::max(text_log_level, max_log_level); + DB::SystemLogQueueSettings log_settings; log_settings.flush_interval_milliseconds = config.getUInt64("text_log.flush_interval_milliseconds", DB::TextLog::getDefaultFlushIntervalMilliseconds()); @@ -382,6 +385,12 @@ void Loggers::updateLevels(Poco::Util::AbstractConfiguration & config, Poco::Log } split->setLevel("Syslog", syslog_level); + if (allowTextLog() && config.has("text_log.level")) + { + int text_log_level = Poco::Logger::parseLevel(config.getString("text_log.level")); + max_log_level = std::max(text_log_level, max_log_level); + } + // Global logging level (it can be overridden for specific loggers). logger.setLevel(max_log_level); diff --git a/tests/integration/test_text_log_level_after_reload/__init__.py b/tests/integration/test_text_log_level_after_reload/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_text_log_level_after_reload/configs/logger.xml b/tests/integration/test_text_log_level_after_reload/configs/logger.xml new file mode 100644 index 0000000000000..811ec0c3320d3 --- /dev/null +++ b/tests/integration/test_text_log_level_after_reload/configs/logger.xml @@ -0,0 +1,9 @@ + + + information + /var/log/clickhouse-server/clickhouse-server.log + + + trace + + diff --git a/tests/integration/test_text_log_level_after_reload/test.py b/tests/integration/test_text_log_level_after_reload/test.py new file mode 100644 index 0000000000000..6bb8f1264f391 --- /dev/null +++ b/tests/integration/test_text_log_level_after_reload/test.py @@ -0,0 +1,65 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +# The file log is capped at `information`, text_log wants `trace`. +# The root logger level must stay at `trace` (the most verbose sink), both at +# startup and after the levels are re-applied on a configuration reload. +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance("node", main_configs=["configs/logger.xml"]) + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def trace_rows_since(node, since): + node.query("SYSTEM FLUSH LOGS") + return int( + node.query( + f"SELECT count() FROM system.text_log WHERE level = 'Trace' AND event_time_microseconds > '{since}'" + ) + ) + + +def now(node): + return node.query("SELECT now64(6)").strip() + + +def test_text_log_keeps_trace_after_reload(start_cluster): + # Startup: the root level must already include text_log.level when the loggers are + # built, before the first configuration reload, so early Trace messages are kept. + node.query("SYSTEM FLUSH LOGS") + assert ( + int( + node.query( + "SELECT count() FROM system.text_log WHERE level = 'Trace' AND logger_name = 'Application' AND message LIKE 'Initialized DateLUT%'" + ) + ) + > 0 + ) + + since = now(node) + for _ in range(5): + node.query("SELECT 1") + assert trace_rows_since(node, since) > 0 + + # Re-applies the logger levels from the (unchanged) configuration. + node.query("SYSTEM RELOAD CONFIG") + + since = now(node) + for _ in range(5): + node.query("SELECT 1") + assert trace_rows_since(node, since) > 0 + + # The file log must still be capped at `information`. + log = node.exec_in_container( + ["bash", "-c", "cat /var/log/clickhouse-server/clickhouse-server.log"] + ) + assert "" not in log + assert "" not in log From ec7c53ce8b560433991b4509646cd0e1ec345fe7 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 27 Oct 2026 01:30:26 +0000 Subject: [PATCH 031/219] Backport #127411 to 26.8: Bump `c-ares` from 1.34.8 to 1.34.9 --- contrib/c-ares | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/c-ares b/contrib/c-ares index c7a3138dcfe3b..bc91829cce983 160000 --- a/contrib/c-ares +++ b/contrib/c-ares @@ -1 +1 @@ -Subproject commit c7a3138dcfe3bb0eaaf10c0c24c36dc66dc790ab +Subproject commit bc91829cce983a35ec01f5dbc05465858a3df469 From 528e2e4bf60396ddebda8863f77fae403bbff466 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 27 Oct 2026 09:51:02 +0000 Subject: [PATCH 032/219] Backport #123622 to 26.8: Send the query id with OpenAI AI function requests --- src/Functions/AI/AnthropicProvider.cpp | 2 +- src/Functions/AI/AnthropicProvider.h | 2 +- src/Functions/AI/IAIProvider.cpp | 6 +- src/Functions/AI/IAIProvider.h | 35 ++++++----- src/Functions/AI/OpenAIProvider.cpp | 23 +++++-- src/Functions/AI/OpenAIProvider.h | 2 +- src/Functions/FunctionBaseAI.cpp | 13 +++- src/Functions/FunctionBaseAI.h | 2 + src/Functions/aiEmbed.cpp | 5 +- src/Functions/aiSimilarity.cpp | 5 +- tests/integration/test_ai_functions/test.py | 69 +++++++++++++++++++++ 11 files changed, 134 insertions(+), 30 deletions(-) diff --git a/src/Functions/AI/AnthropicProvider.cpp b/src/Functions/AI/AnthropicProvider.cpp index fb8ae2afd4e87..720f7af8fc8d4 100644 --- a/src/Functions/AI/AnthropicProvider.cpp +++ b/src/Functions/AI/AnthropicProvider.cpp @@ -40,7 +40,7 @@ HTTPHeaderEntries AnthropicProvider::getHeaders() const return headers; } -AIResponse AnthropicProvider::call(const AIRequest & ai_request, const ConnectionTimeouts & timeouts) +AIResponse AnthropicProvider::call(const AIChatRequest & ai_request, const ConnectionTimeouts & timeouts) { Poco::JSON::Object::Ptr root = new Poco::JSON::Object; root->set("model", ai_request.model); diff --git a/src/Functions/AI/AnthropicProvider.h b/src/Functions/AI/AnthropicProvider.h index cd91752b31cc9..122e1c1b383ae 100644 --- a/src/Functions/AI/AnthropicProvider.h +++ b/src/Functions/AI/AnthropicProvider.h @@ -22,7 +22,7 @@ class AnthropicProvider : public IAIProvider public: AnthropicProvider(const String & endpoint_, const String & api_key_, const String & api_version_); - AIResponse call(const AIRequest & request, const ConnectionTimeouts & timeouts) override; + AIResponse call(const AIChatRequest & request, const ConnectionTimeouts & timeouts) override; HTTPHeaderEntries getHeaders() const override; private: diff --git a/src/Functions/AI/IAIProvider.cpp b/src/Functions/AI/IAIProvider.cpp index d88ec81183c2d..072231adbf8cd 100644 --- a/src/Functions/AI/IAIProvider.cpp +++ b/src/Functions/AI/IAIProvider.cpp @@ -18,7 +18,7 @@ namespace ErrorCodes extern const int RECEIVED_ERROR_FROM_REMOTE_IO_SERVER; } -String sanitizeForLog(std::string_view input) +String sanitize(std::string_view input) { String output; output.reserve(input.size()); @@ -45,7 +45,7 @@ String formatProviderError(int status_code, const String & response_body) String msg = err->optValue("message", ""); String type = err->optValue("type", ""); if (!msg.empty()) - return fmt::format("HTTP {} [{}]: {}", status_code, sanitizeForLog(type), sanitizeForLog(msg)); + return fmt::format("HTTP {} [{}]: {}", status_code, sanitize(type), sanitize(msg)); } } } @@ -55,7 +55,7 @@ String formatProviderError(int status_code, const String & response_body) constexpr size_t max_len = 256; const std::string_view snippet = std::string_view(response_body).substr(0, max_len); - return fmt::format("HTTP {}: {}", status_code, sanitizeForLog(snippet)); + return fmt::format("HTTP {}: {}", status_code, sanitize(snippet)); } AIProviderHTTPException::AIProviderHTTPException(Poco::Net::HTTPResponse::HTTPStatus http_status_, PreformattedMessage msg) diff --git a/src/Functions/AI/IAIProvider.h b/src/Functions/AI/IAIProvider.h index 850218e093347..c5ce5937b8c13 100644 --- a/src/Functions/AI/IAIProvider.h +++ b/src/Functions/AI/IAIProvider.h @@ -31,12 +31,25 @@ class AIProviderHTTPException : public Exception const char * className() const noexcept override { return "DB::AIProviderHTTPException"; } }; +/// Extra info sent with each request. Useful for debugging. +struct AIRequest +{ + /// Name of the AI SQL function that produced this request, e.g. "aiGenerate". + String function_name; + + /// The current query id. Different on different shards of a distributed query. + String query_id; + + /// The initial query id. Same on all shards of a distributed query, empty for a background mutation. + String initial_query_id; +}; + /** Parameters for a single AI chat completion request. * - * Each row processed by an AI function produces one AIRequest. + * Each row processed by an AI chat function produces one AIChatRequest. * The provider serializes it into the HTTP body format expected by the API. */ -struct AIRequest +struct AIChatRequest : AIRequest { /// Constant instruction that guides the model's behavior (persona, format, constraints). /// Sent as a system message (OpenAI) or top-level field (Anthropic). @@ -63,10 +76,6 @@ struct AIRequest /// Maximum number of tokens the model may generate in its response. This is a per-request limit, not a per-query limit. UInt64 max_tokens = 0; - - /// SQL name of the AI function that produced this request (e.g. "aiGenerate"). - /// Emitted by OpenAIProvider as the `X-ClickHouse-AI-Function` header; ignored by other providers. - String function_name; }; /// Canonical, provider-independent reason the model stopped generating. Each provider maps its @@ -104,7 +113,7 @@ struct AIResponse * Embedding APIs typically accept multiple inputs per call, so inputs is a vector. * The provider serializes it into the HTTP body format expected by the API. */ -struct AIEmbeddingRequest +struct AIEmbeddingRequest : AIRequest { /// Texts to embed. Providers send these in a single batched HTTP request. VectorWithMemoryTracking inputs; @@ -115,10 +124,6 @@ struct AIEmbeddingRequest /// Optional target dimensionality for the output vectors. 0 means use the model's native size. /// Supported by OpenAI's `text-embedding-3-*` models; providers that ignore it return the native size. UInt64 dimensions = 0; - - /// SQL name of the AI function that produced this request (e.g. "aiEmbed"). - /// Emitted by OpenAIProvider as the `X-ClickHouse-AI-Function` header; ignored by other providers. - String function_name; }; /// Response from a single embedding request. `embeddings` is aligned 1:1 with `AIEmbeddingRequest::inputs`. @@ -143,7 +148,7 @@ class IAIProvider virtual ~IAIProvider() = default; /// Send a chat completion request and return the parsed response. - virtual AIResponse call(const AIRequest & ai_request, const ConnectionTimeouts & timeouts) = 0; + virtual AIResponse call(const AIChatRequest & ai_request, const ConnectionTimeouts & timeouts) = 0; /// Provider-specific headers (credentials, API version) sent with every request. virtual HTTPHeaderEntries getHeaders() const = 0; @@ -165,8 +170,8 @@ AIProviderPtr createAIProvider( /// Build an error message from a provider's non-200 HTTP response, for use in an exception that is logged. String formatProviderError(int status_code, const String & response_body); -/// Replace control characters (including `\t \n \r`) with spaces so provider-controlled text cannot -/// forge log lines or corrupt a terminal when embedded in a logged exception. -String sanitizeForLog(std::string_view input); +/// Replace control characters (including `\t \n \r`) with spaces. +/// Needed to block provider-controlled text from forging log lines or corrupt a terminal when embedded in a logged exception. +String sanitize(std::string_view input); } diff --git a/src/Functions/AI/OpenAIProvider.cpp b/src/Functions/AI/OpenAIProvider.cpp index c307b2cfa6a16..44ab7006c1483 100644 --- a/src/Functions/AI/OpenAIProvider.cpp +++ b/src/Functions/AI/OpenAIProvider.cpp @@ -19,6 +19,21 @@ namespace ErrorCodes extern const int MALFORMED_AI_PROVIDER_RESPONSE; } +namespace +{ + +void setCommonHeaders(Poco::Net::HTTPRequest & http_request, const AIRequest & ai_request) +{ + chassert(!ai_request.function_name.empty()); + http_request.set("X-ClickHouse-AI-Function", ai_request.function_name); + if (!ai_request.query_id.empty()) + http_request.set("X-ClickHouse-Query-Id", sanitize(ai_request.query_id)); + if (!ai_request.initial_query_id.empty()) + http_request.set("X-ClickHouse-Initial-Query-Id", sanitize(ai_request.initial_query_id)); +} + +} + OpenAIProvider::OpenAIProvider(const String & endpoint_, const String & api_key_) : endpoint(endpoint_) , api_key(api_key_) @@ -33,7 +48,7 @@ HTTPHeaderEntries OpenAIProvider::getHeaders() const return {{"Authorization", "Bearer " + api_key}}; } -AIResponse OpenAIProvider::call(const AIRequest & ai_request, const ConnectionTimeouts & timeouts) +AIResponse OpenAIProvider::call(const AIChatRequest & ai_request, const ConnectionTimeouts & timeouts) { Poco::JSON::Object::Ptr root = new Poco::JSON::Object; root->set("model", ai_request.model); @@ -70,8 +85,7 @@ AIResponse OpenAIProvider::call(const AIRequest & ai_request, const ConnectionTi http_request.setContentType("application/json"); for (const auto & header : getHeaders()) http_request.set(header.name, header.value); - chassert(!ai_request.function_name.empty()); - http_request.set("X-ClickHouse-AI-Function", ai_request.function_name); + setCommonHeaders(http_request, ai_request); http_request.setContentLength(body.size()); auto & out_stream = session->sendRequest(http_request); @@ -181,8 +195,7 @@ AIEmbeddingResponse OpenAIProvider::embed(const AIEmbeddingRequest & ai_embeddin http_request.setContentType("application/json"); for (const auto & header : getHeaders()) http_request.set(header.name, header.value); - chassert(!ai_embedding_request.function_name.empty()); - http_request.set("X-ClickHouse-AI-Function", ai_embedding_request.function_name); + setCommonHeaders(http_request, ai_embedding_request); http_request.setContentLength(body.size()); auto & out_stream = session->sendRequest(http_request); diff --git a/src/Functions/AI/OpenAIProvider.h b/src/Functions/AI/OpenAIProvider.h index 38d65fc6b8138..bcfe89c6d411b 100644 --- a/src/Functions/AI/OpenAIProvider.h +++ b/src/Functions/AI/OpenAIProvider.h @@ -33,7 +33,7 @@ class OpenAIProvider : public IAIProvider public: OpenAIProvider(const String & endpoint_, const String & api_key_); - AIResponse call(const AIRequest & ai_request, const ConnectionTimeouts & timeouts) override; + AIResponse call(const AIChatRequest & ai_request, const ConnectionTimeouts & timeouts) override; HTTPHeaderEntries getHeaders() const override; bool supportsEmbeddings() const override { return true; } diff --git a/src/Functions/FunctionBaseAI.cpp b/src/Functions/FunctionBaseAI.cpp index 7aea1b1826401..31e354cf35c78 100644 --- a/src/Functions/FunctionBaseAI.cpp +++ b/src/Functions/FunctionBaseAI.cpp @@ -371,6 +371,8 @@ FunctionBaseAI::EmbeddingResult FunctionBaseAI::embedTexts( const String & model, UInt64 dimensions, const String & function_name, + const String & query_id, + const String & initial_query_id, const VectorWithMemoryTracking & inputs, size_t max_batch_size, UInt64 max_retries, @@ -396,6 +398,8 @@ FunctionBaseAI::EmbeddingResult FunctionBaseAI::embedTexts( ai_embedding_request.model = model; ai_embedding_request.dimensions = dimensions; ai_embedding_request.function_name = function_name; + ai_embedding_request.query_id = query_id; + ai_embedding_request.initial_query_id = initial_query_id; ai_embedding_request.inputs.reserve(batch_end - batch_start); for (size_t k = batch_start; k < batch_end; ++k) ai_embedding_request.inputs.emplace_back(inputs[k]); @@ -494,6 +498,9 @@ ColumnPtr FunctionBaseAI::executeImpl(const ColumnsWithTypeAndName & arguments, auto timeouts = ConnectionTimeouts::getHTTPTimeouts(settings, getContext()->getServerSettings()); timeouts.receive_timeout = Poco::Timespan(static_cast(timeout_sec) /*s*/, 0 /*us*/); + const String query_id = getContext()->getCurrentQueryId(); + const String & initial_query_id = getContext()->getClientInfo().initial_query_id; + auto result_col = removeNullable(result_type)->createColumn(); auto null_map_col = prompt_nullable ? ColumnUInt8::create(input_rows_count, static_cast(0)) : nullptr; @@ -541,7 +548,7 @@ ColumnPtr FunctionBaseAI::executeImpl(const ColumnsWithTypeAndName & arguments, try { - AIRequest ai_request; + AIChatRequest ai_request; ai_request.system_prompt = system_prompt; ai_request.user_message = user_message; ai_request.response_format = response_format; @@ -549,6 +556,8 @@ ColumnPtr FunctionBaseAI::executeImpl(const ColumnsWithTypeAndName & arguments, ai_request.temperature = temperature; ai_request.max_tokens = max_tokens; ai_request.function_name = getName(); + ai_request.query_id = query_id; + ai_request.initial_query_id = initial_query_id; ++total_api_calls; @@ -560,7 +569,7 @@ ColumnPtr FunctionBaseAI::executeImpl(const ColumnsWithTypeAndName & arguments, /// `raw_finish_reason` is provider-controlled text; sanitize control characters before /// interpolating it into an exception message that reaches the logs and `system.query_log`. - const String safe_finish_reason = sanitizeForLog(ai_response.raw_finish_reason); + const String safe_finish_reason = sanitize(ai_response.raw_finish_reason); /// Reject incomplete responses, throw plain DB::Exception so it is classified as non-retriable switch (ai_response.finish_reason) diff --git a/src/Functions/FunctionBaseAI.h b/src/Functions/FunctionBaseAI.h index 41751061fa203..d337464ddbc3a 100644 --- a/src/Functions/FunctionBaseAI.h +++ b/src/Functions/FunctionBaseAI.h @@ -162,6 +162,8 @@ class FunctionBaseAI : public IFunction const String & model, UInt64 dimensions, const String & function_name, + const String & query_id, + const String & initial_query_id, const VectorWithMemoryTracking & inputs, size_t max_batch_size, UInt64 max_retries, diff --git a/src/Functions/aiEmbed.cpp b/src/Functions/aiEmbed.cpp index 26cb68c767f6c..4aa2f606e8110 100644 --- a/src/Functions/aiEmbed.cpp +++ b/src/Functions/aiEmbed.cpp @@ -161,7 +161,10 @@ class FunctionAiEmbed final : public IFunction } auto embedding_result = FunctionBaseAI::embedTexts( - *provider, model, dimensions, getName(), inputs, max_batch_size, max_retries, retry_delay_ms, throw_on_error, *quota_tracker, timeouts); + *provider, model, dimensions, + getName(), getContext()->getCurrentQueryId(), getContext()->getClientInfo().initial_query_id, + inputs, max_batch_size, max_retries, + retry_delay_ms, throw_on_error, *quota_tracker, timeouts); auto data_col = ColumnVector::create(); /// float32 is standard embedding API output auto offsets_col = ColumnArray::ColumnOffsets::create(); diff --git a/src/Functions/aiSimilarity.cpp b/src/Functions/aiSimilarity.cpp index 7c35c9bf94bd6..9696d9f3ca9d8 100644 --- a/src/Functions/aiSimilarity.cpp +++ b/src/Functions/aiSimilarity.cpp @@ -199,7 +199,10 @@ class FunctionAiSimilarity final : public IFunction } auto embedding_result = FunctionBaseAI::embedTexts( - *provider, model, dimensions, getName(), inputs, max_batch_size, max_retries, retry_delay_ms, throw_on_error, *quota_tracker, timeouts); + *provider, model, dimensions, getName(), + getContext()->getCurrentQueryId(), getContext()->getClientInfo().initial_query_id, + inputs, max_batch_size, max_retries, + retry_delay_ms, throw_on_error, *quota_tracker, timeouts); ProfileEvents::increment(ProfileEvents::AIAPICalls, embedding_result.api_calls); ProfileEvents::increment(ProfileEvents::AIInputTokens, embedding_result.input_tokens); diff --git a/tests/integration/test_ai_functions/test.py b/tests/integration/test_ai_functions/test.py index 1c7e063b7f5d1..e72e058621a6d 100644 --- a/tests/integration/test_ai_functions/test.py +++ b/tests/integration/test_ai_functions/test.py @@ -1561,6 +1561,75 @@ def test_function_name_header(started_cluster): assert last_request()["headers"].get("x-clickhouse-ai-function") == name +def test_query_id_header(started_cluster): + """The OpenAI provider sends the `query_id` of the query that makes the request as the + `X-ClickHouse-Query-Id` header and its `initial_query_id` as the `X-ClickHouse-Initial-Query-Id` header: + on a remote shard the first is the shard's own query id, the second is the initiator's one. + Covers the chat and the embedding paths.""" + local_cases = [ + "SELECT aiGenerate('hi', map('credentials', 'ai_mock'))", + "SELECT aiEmbed('hi', 'test-embed-model', map('credentials', 'ai_embed'))", + ] + for i, query in enumerate(local_cases): + qid = unique_query_id(f"query_id_header_{i}") + instance.query(query, query_id=qid) + headers = last_request()["headers"] + assert headers.get("x-clickhouse-query-id") == qid + assert headers.get("x-clickhouse-initial-query-id") == qid + + remote_cases = [ + "SELECT aiGenerate(toString(dummy), map('credentials', 'ai_mock')) FROM remote('127.0.0.2', system.one)", + "SELECT aiEmbed(toString(dummy), 'test-embed-model', map('credentials', 'ai_embed')) FROM remote('127.0.0.2', system.one)", + ] + for i, query in enumerate(remote_cases): + qid = unique_query_id(f"query_id_header_remote_{i}") + instance.query(query, query_id=qid) + headers = last_request()["headers"] + sent_query_id = headers.get("x-clickhouse-query-id") + assert sent_query_id and sent_query_id != qid + assert headers.get("x-clickhouse-initial-query-id") == qid + instance.query("SYSTEM FLUSH LOGS query_log") + assert ( + instance.query( + f"SELECT initial_query_id FROM system.query_log WHERE query_id = '{sent_query_id}' AND NOT is_initial_query AND type = 'QueryFinish'" + ).strip() + == qid + ) + + # A background mutation sends the mutation task's own `current_query_id` and has no `initial_query_id`. + instance.query("DROP TABLE IF EXISTS query_id_header SYNC") + instance.query("CREATE TABLE query_id_header (k UInt64, s String) ENGINE = MergeTree ORDER BY k") + instance.query("INSERT INTO query_id_header VALUES (1, 'a')") + instance.query( + "ALTER TABLE query_id_header UPDATE s = aiGenerate('hi', map('credentials', 'ai_mock')) WHERE 1", + settings={"mutations_sync": 2, "allow_nondeterministic_mutations": 1}, + ) + table_uuid = instance.query("SELECT uuid FROM system.tables WHERE database = 'default' AND name = 'query_id_header'").strip() + headers = last_request()["headers"] + assert headers.get("x-clickhouse-query-id") == f"{table_uuid}::all_1_1_0_2" + assert "x-clickhouse-initial-query-id" not in headers + instance.query("DROP TABLE query_id_header SYNC") + + +def test_query_id_header_control_characters(started_cluster): + """A query id from the native protocol can contain control characters. They are replaced with spaces + in the `X-ClickHouse-Query-Id` and `X-ClickHouse-Initial-Query-Id` headers, so no extra header is injected.""" + suffix = uuid.uuid4().hex[:8] + qid = f"query_id_crlf_{suffix}\r\nX-Injected: 1" + instance.query("SELECT aiGenerate('hi', map('credentials', 'ai_mock'))", query_id=qid) + instance.query("SYSTEM FLUSH LOGS query_log") + assert ( + instance.query( + f"SELECT count() FROM system.query_log WHERE query_id = 'query_id_crlf_{suffix}\\r\\nX-Injected: 1' AND type = 'QueryFinish'" + ).strip() + == "1" + ) + headers = last_request()["headers"] + assert headers.get("x-clickhouse-query-id") == f"query_id_crlf_{suffix} X-Injected: 1" + assert headers.get("x-clickhouse-initial-query-id") == f"query_id_crlf_{suffix} X-Injected: 1" + assert "x-injected" not in headers + + def test_embed_retry_respects_api_call_quota(started_cluster): """The embedding path enforces the same per-attempt API-call quota: a retriable HTTP 500 is not retried past `ai_function_max_api_calls_per_query`.""" From ec21f68590a34dec1f3c945758c8818b23f4eb66 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 27 Oct 2026 15:15:33 +0000 Subject: [PATCH 033/219] Backport #125510 to 26.8: Do not share line editor state between concurrent embedded client sessions --- src/Client/ClientBase.cpp | 6 +- src/Client/ClientBaseHelpers.cpp | 10 +- src/Client/ClientBaseHelpers.h | 4 +- src/Client/ReplxxLineReader.cpp | 21 ++-- src/Client/ReplxxLineReader.h | 19 ++- ...rminal_concurrent_sessions_input.reference | 2 + ...5_webterminal_concurrent_sessions_input.sh | 110 ++++++++++++++++++ 7 files changed, 147 insertions(+), 25 deletions(-) create mode 100644 tests/queries/0_stateless/05445_webterminal_concurrent_sessions_input.reference create mode 100755 tests/queries/0_stateless/05445_webterminal_concurrent_sessions_input.sh diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 5ab920a0665ff..925b73cb464a0 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -4763,13 +4763,13 @@ void ClientBase::runInteractive() #if USE_REPLXX - replxx::Replxx::highlighter_callback_with_pos_t highlight_callback{}; + ReplxxLineReader::Highlighter highlight_callback{}; if (ConfigHelper::getBool(getClientConfiguration(), "highlight", true)) { - highlight_callback = [this](const String & query, std::vector & colors, int pos) + highlight_callback = [this](const String & query, std::vector & colors, int pos, bool & last_is_delimiter) { - highlight(query, colors, *client_context, pos, rainbow_parentheses); + highlight(query, colors, *client_context, pos, rainbow_parentheses, &last_is_delimiter); }; } diff --git a/src/Client/ClientBaseHelpers.cpp b/src/Client/ClientBaseHelpers.cpp index 6b485bab3c582..1d9fd59c18080 100644 --- a/src/Client/ClientBaseHelpers.cpp +++ b/src/Client/ClientBaseHelpers.cpp @@ -247,7 +247,7 @@ static void highlightWithLexer(const String & query, std::vector & colors, const Context & context, int cursor_position, bool rainbow_parentheses, bool lexer_fallback) +void highlight(const String & query, std::vector & colors, const Context & context, int cursor_position, bool rainbow_parentheses, bool * last_is_delimiter, bool lexer_fallback) { using namespace replxx; @@ -602,14 +602,16 @@ void highlight(const String & query, std::vector & colors } /// This is a callback for the client/local app to better find query end. Note: this is a kludge, remove it. + if (!last_is_delimiter) + return; if (last_token.type == TokenType::Semicolon || last_token.type == TokenType::VerticalDelimiter || query.ends_with(';') || query.ends_with("\\G")) /// This is for raw data in INSERT queries, which is not necessarily tokenized. { - ReplxxLineReader::setLastIsDelimiter(true); + *last_is_delimiter = true; } else if (last_token.type != TokenType::Whitespace) { - ReplxxLineReader::setLastIsDelimiter(false); + *last_is_delimiter = false; } } @@ -618,7 +620,7 @@ String highlighted(const String & query, const Context & context, bool rainbow_p const size_t num_code_points = countCodePointsWithSeqLength(query, query.data() + query.size()); std::vector colors(num_code_points, replxx::Replxx::Color::DEFAULT); - highlight(query, colors, context, 0, rainbow_parentheses, lexer_fallback); + highlight(query, colors, context, 0, rainbow_parentheses, nullptr, lexer_fallback); String res; size_t query_size = query.size(); diff --git a/src/Client/ClientBaseHelpers.h b/src/Client/ClientBaseHelpers.h index d106413dee0de..97191753a4cfe 100644 --- a/src/Client/ClientBaseHelpers.h +++ b/src/Client/ClientBaseHelpers.h @@ -27,7 +27,9 @@ bool isCloudEndpoint(const std::string & host); /// When `lexer_fallback` is set, input that cannot be parsed as complete queries (e.g. the query /// fragments found in the documentation) is highlighted token-by-token using the lexer instead of being /// left unhighlighted. The line editor keeps the default (parser-only) behaviour. -void highlight(const String & query, std::vector & colors, const Context & context, int cursor_position, bool rainbow_parentheses, bool lexer_fallback = false); +/// If `last_is_delimiter` is not null, it is set to whether the query ends with a delimiter (and left +/// intact if the query ends with whitespace). +void highlight(const String & query, std::vector & colors, const Context & context, int cursor_position, bool rainbow_parentheses, bool * last_is_delimiter, bool lexer_fallback = false); String highlighted(const String & query, const Context & context, bool rainbow_parentheses, bool lexer_fallback = false); #endif diff --git a/src/Client/ReplxxLineReader.cpp b/src/Client/ReplxxLineReader.cpp index 9ef83c3974eac..6dd54968f79da 100644 --- a/src/Client/ReplxxLineReader.cpp +++ b/src/Client/ReplxxLineReader.cpp @@ -328,12 +328,6 @@ void convertHistoryFile(const std::string & path, replxx::Replxx & rx) namespace DB { -static bool replxx_last_is_delimiter = false; -void ReplxxLineReader::setLastIsDelimiter(bool flag) -{ - replxx_last_is_delimiter = flag; -} - ReplxxLineReader::ReplxxLineReader(ReplxxLineReader::Options && options) : LineReader ( @@ -385,8 +379,8 @@ ReplxxLineReader::ReplxxLineReader(ReplxxLineReader::Options && options) } } - /// The handler of `SIGWINCH` is process-wide. In the embedded mode the process is the server, - /// and the pseudoterminal is not its controlling terminal, so the signal is never delivered. + /// Signals are process-wide: an embedded client must not take over `SIGWINCH` of the server. + /// Its terminal size is updated by the server, and replxx queries it on every refresh. if (!options.embedded_mode) rx.install_window_change_handler(); @@ -417,7 +411,12 @@ ReplxxLineReader::ReplxxLineReader(ReplxxLineReader::Options && options) rx.set_indent_multiline(false); if (highlighter) - rx.set_highlighter_callback(highlighter); + { + rx.set_highlighter_callback([this](const String & query, std::vector & colors, int cursor_position) + { + highlighter(query, colors, cursor_position, last_is_delimiter); + }); + } /// As-you-type autocompletion: show the matching suggestions as inline "ghost" hints, with /// the same priority ordering as Tab completion. replxx renders a single hint inline after @@ -564,9 +563,9 @@ ReplxxLineReader::ReplxxLineReader(ReplxxLineReader::Options && options) /// a newline into the edit buffer and does not use the lexer, so the paste stays under a /// single prompt even with `--highlight 0`. The explicit `--multiline` continuation still /// requires the highlighter, preserving the previous behavior. - if (!replxx_last_is_delimiter && ((highlighter && multiline) || hasInputData())) + if (!last_is_delimiter && ((highlighter && multiline) || hasInputData())) return rx.invoke(Replxx::ACTION::NEW_LINE, code); - replxx_last_is_delimiter = false; + last_is_delimiter = false; return rx.invoke(Replxx::ACTION::COMMIT_LINE, code); }; /// bind C-j to ENTER action. diff --git a/src/Client/ReplxxLineReader.h b/src/Client/ReplxxLineReader.h index f7813514374da..1603acf773e6d 100644 --- a/src/Client/ReplxxLineReader.h +++ b/src/Client/ReplxxLineReader.h @@ -12,6 +12,11 @@ namespace DB class ReplxxLineReader : public LineReader { public: + /// The highlighter callback of replxx, which also reports whether the last token of the + /// query is a delimiter: it sets `last_is_delimiter` to true or false, or leaves it intact + /// when this cannot be determined (e.g. the query ends with whitespace). + using Highlighter = std::function & colors, int cursor_position, bool & last_is_delimiter)>; + struct Options { Suggest & suggest; @@ -26,7 +31,7 @@ class ReplxxLineReader : public LineReader Patterns extenders; Patterns delimiters; std::span word_break_characters; - replxx::Replxx::highlighter_callback_with_pos_t highlighter; + Highlighter highlighter; std::istream & input_stream = std::cin; std::ostream & output_stream = std::cout; int in_fd = STDIN_FILENO; @@ -40,10 +45,6 @@ class ReplxxLineReader : public LineReader void enableBracketedPaste() override; void disableBracketedPaste() override; - /// If highlight is on, we will set a flag to denote whether the last token is a delimiter. - /// This is useful to determine the behavior of key when multiline is enabled. - static void setLastIsDelimiter(bool flag); - /// Set text to be prepopulated in the next readLine call void setInitialText(const String & text) override; private: @@ -76,7 +77,13 @@ class ReplxxLineReader : public LineReader bool hintChosen(); replxx::Replxx rx; - replxx::Replxx::highlighter_callback_with_pos_t highlighter; + Highlighter highlighter; + + /// If highlight is on, the highlighter reports whether the last token is a delimiter. + /// This is used to determine the behavior of key when multiline is enabled. + /// It is a member rather than a global, because several readers can work concurrently + /// in one process (client sessions embedded into the server). + bool last_is_delimiter = false; /// Suggestion dictionary; used both for Tab completion and as-you-type hints, and updated /// with identifiers from committed queries to prioritize recently-used words. diff --git a/tests/queries/0_stateless/05445_webterminal_concurrent_sessions_input.reference b/tests/queries/0_stateless/05445_webterminal_concurrent_sessions_input.reference new file mode 100644 index 0000000000000..e6c10cc871478 --- /dev/null +++ b/tests/queries/0_stateless/05445_webterminal_concurrent_sessions_input.reference @@ -0,0 +1,2 @@ +session 0: ok +session 1: ok diff --git a/tests/queries/0_stateless/05445_webterminal_concurrent_sessions_input.sh b/tests/queries/0_stateless/05445_webterminal_concurrent_sessions_input.sh new file mode 100755 index 0000000000000..71c889eb14248 --- /dev/null +++ b/tests/queries/0_stateless/05445_webterminal_concurrent_sessions_input.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Concurrent web terminal sessions run in the same server process and must not share +# the state of the line editor: the input of each session is sent byte by byte, +# interleaved with the other session, with multi-byte UTF-8 characters split +# between the sessions, and each session must decode its own query correctly. + +python3 - "${CLICKHOUSE_HOST}" "${CLICKHOUSE_PORT_HTTP}" <<'PYEOF' +import base64 +import json +import os +import socket +import struct +import sys +import time + +host, port = sys.argv[1], int(sys.argv[2]) + + +def recv_exact(sock, n): + buf = b"" + while len(buf) < n: + chunk = sock.recv(n - len(buf)) + if not chunk: + raise EOFError("connection closed") + buf += chunk + return buf + + +def send_frame(sock, opcode, payload): + mask = os.urandom(4) + masked = bytes(b ^ mask[i % 4] for i, b in enumerate(payload)) + if len(payload) < 126: + header = bytes([0x80 | opcode, 0x80 | len(payload)]) + else: + header = bytes([0x80 | opcode, 0x80 | 126]) + struct.pack(">H", len(payload)) + sock.sendall(header + mask + masked) + + +def read_frame(sock): + header = recv_exact(sock, 2) + opcode = header[0] & 0x0F + length = header[1] & 0x7F + if length == 126: + length = struct.unpack(">H", recv_exact(sock, 2))[0] + elif length == 127: + length = struct.unpack(">Q", recv_exact(sock, 8))[0] + return opcode, recv_exact(sock, length) + + +def read_until(sock, needle, timeout): + """Accumulate the terminal output until it contains `needle`.""" + deadline = time.monotonic() + timeout + output = b"" + while needle not in output: + if time.monotonic() > deadline: + raise TimeoutError(f"{needle!r} not found in {output[-500:]!r}") + opcode, payload = read_frame(sock) + if opcode == 0x08: + raise EOFError(f"closed by the server: {payload!r}") + if opcode == 0x09: + send_frame(sock, 0x0A, payload) + elif opcode == 0x02: + output += payload + return output + + +def connect(): + sock = socket.create_connection((host, port), timeout=60) + key = base64.b64encode(os.urandom(16)).decode() + request = ( + f"GET /webterminal HTTP/1.1\r\n" + f"Host: {host}:{port}\r\n" + f"Origin: http://{host}:{port}\r\n" + f"Upgrade: websocket\r\n" + f"Connection: Upgrade\r\n" + f"Sec-WebSocket-Key: {key}\r\n" + f"Sec-WebSocket-Version: 13\r\n\r\n" + ) + sock.sendall(request.encode()) + response = b"" + while b"\r\n\r\n" not in response: + response += recv_exact(sock, 1) + assert response.startswith(b"HTTP/1.1 101"), response + send_frame(sock, 0x01, json.dumps({"type": "auth", "user": "default", "password": ""}).encode()) + read_until(sock, b":) ", 60) + return sock + + +# Different multi-byte characters in each session, so that a mixed-up decoder state +# produces a different string. +texts = ["привет", "€😀日本"] +sessions = [connect() for _ in texts] +queries = [f"SELECT hex('{text}') AS h_{i}\r".encode() for i, text in enumerate(texts)] + +for pos in range(max(len(q) for q in queries)): + for sock, query in zip(sessions, queries): + if pos < len(query): + send_frame(sock, 0x02, query[pos:pos + 1]) + +for i, (sock, text) in enumerate(zip(sessions, texts)): + expected = text.encode().hex().upper().encode() + read_until(sock, expected, 60) + print(f"session {i}: ok") + sock.close() +PYEOF From e3e1648e15939759e72235f6b5f7257789b93630 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 27 Oct 2026 16:05:46 +0000 Subject: [PATCH 034/219] Backport #125698 to 26.8: Stop AI functions mid-block on `KILL QUERY` and `max_execution_time` --- src/Functions/AI/AIQuotaTracker.cpp | 19 +++++ src/Functions/AI/AIQuotaTracker.h | 14 +++- src/Interpreters/Context.cpp | 3 +- .../test_ai_functions/mock_ai_server.py | 53 ++++++++++++ tests/integration/test_ai_functions/test.py | 81 +++++++++++++++++++ 5 files changed, 168 insertions(+), 2 deletions(-) diff --git a/src/Functions/AI/AIQuotaTracker.cpp b/src/Functions/AI/AIQuotaTracker.cpp index 374e22d370371..247021e2636ae 100644 --- a/src/Functions/AI/AIQuotaTracker.cpp +++ b/src/Functions/AI/AIQuotaTracker.cpp @@ -1,4 +1,5 @@ #include +#include #include namespace DB @@ -7,6 +8,7 @@ namespace DB namespace ErrorCodes { extern const int LIMIT_EXCEEDED; + extern const int TIMEOUT_EXCEEDED; } bool AIQuotaTracker::quotasExceededLocked() @@ -39,14 +41,31 @@ bool AIQuotaTracker::quotasExceededLocked() return false; } +void AIQuotaTracker::throwIfCancelled() const +{ + auto status = query_status.lock(); + if (!status) + return; + + /// `checkTimeLimit` throws on `KILL QUERY` and when `max_execution_time` is exceeded. + /// Under `timeout_overflow_mode = 'break'`, it returns false instead of throwing. + /// A function cannot return fewer rows than it received, so throw in that case too. + if (!status->checkTimeLimit()) + throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Timeout exceeded: elapsed time limit reached in an AI function"); +} + bool AIQuotaTracker::checkQuotas() { + throwIfCancelled(); + std::lock_guard lock(mutex); return quotasExceededLocked(); } bool AIQuotaTracker::recordApiCall() { + throwIfCancelled(); + std::lock_guard lock(mutex); /// Don't start a new request once any quota is known-exhausted (e.g. another thread's response diff --git a/src/Functions/AI/AIQuotaTracker.h b/src/Functions/AI/AIQuotaTracker.h index 515821b0f9f64..c9836c29a6491 100644 --- a/src/Functions/AI/AIQuotaTracker.h +++ b/src/Functions/AI/AIQuotaTracker.h @@ -3,11 +3,14 @@ #include #include +#include #include namespace DB { +class QueryStatus; + /// Tracks AI-function quota usage for one query. A single instance is shared by every AI function /// call in the query context (owned by the query `Context`) and updated concurrently from the /// pipeline threads, so the counters are guarded by `mutex`. It is per query-execution context: a @@ -16,16 +19,21 @@ namespace DB /// /// The API-call limit is a hard cap within a context, while token limits are best effort (we only /// know the usage after the call returns). +/// +/// `checkQuotas` and `recordApiCall` also throw once the query is killed or exceeds +/// `max_execution_time`, so no new request is dispatched for it. class AIQuotaTracker { public: AIQuotaTracker( UInt64 max_input_tokens_, UInt64 max_output_tokens_, - UInt64 max_api_calls_, bool throw_on_quota_exceeded_) + UInt64 max_api_calls_, bool throw_on_quota_exceeded_, + std::weak_ptr query_status_) : max_input_tokens(max_input_tokens_) , max_output_tokens(max_output_tokens_) , max_api_calls(max_api_calls_) , throw_on_quota_exceeded(throw_on_quota_exceeded_) + , query_status(std::move(query_status_)) {} /// Check the token quotas (and the sticky exceeded flag). Returns true if a limit is met or @@ -51,6 +59,7 @@ class AIQuotaTracker const UInt64 max_output_tokens; const UInt64 max_api_calls; const bool throw_on_quota_exceeded; + const std::weak_ptr query_status; std::mutex mutex; bool quota_exceeded TSA_GUARDED_BY(mutex) = false; @@ -62,6 +71,9 @@ class AIQuotaTracker /// per `throw_on_quota_exceeded`) when a token quota is met. Shared by `checkQuotas` and /// `recordApiCall` so a call is never started once a quota is known-exhausted. bool quotasExceededLocked() TSA_REQUIRES(mutex); + + /// Throws if the query was killed or exceeded its time limit. + void throwIfCancelled() const; }; } diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index cd97ff21caadb..f8c548748a064 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -9013,7 +9013,8 @@ AIQuotaTrackerPtr Context::getAIQuotaTracker() const settings_ref[Setting::ai_function_max_input_tokens_per_query], settings_ref[Setting::ai_function_max_output_tokens_per_query], settings_ref[Setting::ai_function_max_api_calls_per_query], - settings_ref[Setting::ai_function_throw_on_quota_exceeded]); + settings_ref[Setting::ai_function_throw_on_quota_exceeded], + query_context->process_list_elem); } return query_context->ai_quota_tracker; } diff --git a/tests/integration/test_ai_functions/mock_ai_server.py b/tests/integration/test_ai_functions/mock_ai_server.py index 41368618f5b7e..d14ba6bbeb8d4 100644 --- a/tests/integration/test_ai_functions/mock_ai_server.py +++ b/tests/integration/test_ai_functions/mock_ai_server.py @@ -8,6 +8,12 @@ `aiTranslate`'s `instructions` argument is forwarded in the prompt, or that the `Authorization` header is omitted when the named collection has no `api_key`). Header names are lower-cased for case-insensitive lookup. + GET /concurrency — returns JSON `{"requests": N, "max_concurrency": M}` describing + the requests the slow endpoint below has served since the last `/reset-concurrency`, where + `max_concurrency` is the highest number that were ever being served at the same moment. + GET /reset-concurrency — zeroes the counters above + POST /v1/chat/slow — like `/v1/chat/completions`, but sleeps SLOW_RESPONSE_SECONDS + before answering. GET /set-flaky?count=N — arm the flaky endpoints below to fail their next N requests with a simulated transient network error (used to exercise retries). `count=0` disarms. POST /v1/chat/flaky — like `/v1/chat/completions`, but drops the connection without @@ -59,6 +65,7 @@ import http.server import json import threading +import time from urllib.parse import urlparse, parse_qs MOCK_PORT = 18123 @@ -74,6 +81,13 @@ # Set via `GET /set-flaky?count=N`. Used to exercise the network-error retry path. FLAKY = {"fails_remaining": 0} +# How long the slow endpoint takes to answer. +SLOW_RESPONSE_SECONDS = 0.5 + +# Requests the slow endpoint has served, and the high-water mark of how many it was serving +# simultaneously, since the last `/reset-concurrency`. Guarded by `_LOCK`. +CONCURRENCY = {"requests": 0, "in_flight": 0, "max_concurrency": 0} + def extract_user_message(body): data = json.loads(body) @@ -237,6 +251,24 @@ def do_GET(self): self._send_json(200, snapshot) return + if parsed.path == "/concurrency": + with _LOCK: + snapshot = { + "requests": CONCURRENCY["requests"], + "max_concurrency": CONCURRENCY["max_concurrency"], + } + self._send_json(200, snapshot) + return + + if parsed.path == "/reset-concurrency": + with _LOCK: + CONCURRENCY.update(requests=0, in_flight=0, max_concurrency=0) + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.end_headers() + self.wfile.write(b"OK") + return + if parsed.path == "/set-flaky": qs = parse_qs(parsed.query) with _LOCK: @@ -260,6 +292,10 @@ def do_POST(self): LAST_REQUEST["body"] = body LAST_REQUEST["headers"] = {k.lower(): v for k, v in self.headers.items()} + if parsed.path == "/v1/chat/slow": + self._serve_slowly(lambda: make_success_response(extract_user_message(body))) + return + if parsed.path in ("/v1/chat/flaky", "/v1/embeddings_flaky"): with _LOCK: should_fail = FLAKY["fails_remaining"] > 0 @@ -414,6 +450,23 @@ def do_POST(self): self.send_response(404) self.end_headers() + def _serve_slowly(self, make_response): + """Sleep, then answer, counting the request as in flight for the duration of the sleep.""" + with _LOCK: + CONCURRENCY["requests"] += 1 + CONCURRENCY["in_flight"] += 1 + CONCURRENCY["max_concurrency"] = max( + CONCURRENCY["max_concurrency"], CONCURRENCY["in_flight"] + ) + try: + time.sleep(SLOW_RESPONSE_SECONDS) + response = make_response() + finally: + with _LOCK: + CONCURRENCY["in_flight"] -= 1 + + self._send_json(200, response) + def _send_json(self, status, obj): body = json.dumps(obj).encode("utf-8") self.send_response(status) diff --git a/tests/integration/test_ai_functions/test.py b/tests/integration/test_ai_functions/test.py index e72e058621a6d..8fefed37a4a0b 100644 --- a/tests/integration/test_ai_functions/test.py +++ b/tests/integration/test_ai_functions/test.py @@ -98,6 +98,14 @@ def started_cluster() -> typing.Generator[ClickHouseCluster, None, None]: f"model = 'test-model', " f"api_key = 'test-key'" ) + # Slow endpoint, used to observe how many requests an AI function sends before it is stopped. + instance.query( + f"CREATE NAMED COLLECTION ai_slow AS " + f"provider = 'openai', " + f"endpoint = 'http://localhost:{MOCK_PORT}/v1/chat/slow', " + f"model = 'test-model', " + f"api_key = 'test-key'" + ) instance.query( f"CREATE NAMED COLLECTION ai_error AS " f"provider = 'openai', " @@ -2246,3 +2254,76 @@ def test_api_call_quota_ignores_subquery_settings(started_cluster): assert subquery_only == 64, ( f"expected all 64 rows to run (a quota set only in the subquery is ignored), got {subquery_only}" ) + + +SLOW_CHAT_CALL = "aiClassify(toString(number), ['positive','negative'], map('credentials', 'ai_slow'))" + + +def _concurrency_stats(): + return json.loads( + instance.exec_in_container( + ["curl", "-s", f"http://localhost:{MOCK_PORT}/concurrency"] + ) + ) + + +def _reset_concurrency(): + instance.exec_in_container( + ["curl", "-s", f"http://localhost:{MOCK_PORT}/reset-concurrency"] + ) + + +def test_kill_query_stops_issuing_requests(started_cluster): + """`KILL QUERY` stops an AI function mid-block: no new request is sent after the kill. The 400 rows + form one block, which at 0.5s per request would otherwise take 200s and send all 400 requests.""" + _reset_concurrency() + qid = unique_query_id("ai_kill") + request = instance.get_query_request( + f"SELECT {SLOW_CHAT_CALL} FROM numbers(400) FORMAT Null", + query_id=qid, + ) + + wait_condition( + _concurrency_stats, lambda stats: stats["requests"] >= 2, max_attempts=100, delay=0.1 + ) + instance.query(f"KILL QUERY WHERE query_id = '{qid}' SYNC") + + assert "Query was cancelled" in request.get_error() + requests = _concurrency_stats()["requests"] + assert requests < 40, f"the killed query kept sending requests: {requests} of 400 were sent" + + instance.query("SYSTEM FLUSH LOGS") + exception_code = instance.query( + f"SELECT exception_code FROM system.query_log " + f"WHERE query_id = '{qid}' AND type = 'ExceptionWhileProcessing'" + ).strip() + assert exception_code == "394" + assert int(get_profile_events(qid, "ExceptionWhileProcessing")["api_calls"]) == requests + + +def test_max_execution_time_stops_issuing_requests(started_cluster): + """`max_execution_time` stops an AI function mid-block like `KILL QUERY`: the 400 rows form one block + that would take 200s at 0.5s per request, and the query fails after 2s having sent a fraction of them.""" + _reset_concurrency() + error = instance.query_and_get_error( + f"SELECT countIf(r != '') FROM (SELECT {SLOW_CHAT_CALL} AS r FROM numbers(400))", + settings={"max_execution_time": 2}, + ) + assert "TIMEOUT_EXCEEDED" in error + requests = _concurrency_stats()["requests"] + assert requests < 40, f"the timed-out query kept sending requests: {requests} of 400 were sent" + + +def test_max_execution_time_break_stops_issuing_requests(started_cluster): + """With `timeout_overflow_mode = 'break'` the query returns without an error once the time limit is + reached, and no further requests are sent.""" + _reset_concurrency() + instance.query( + f"SELECT countIf(r != '') FROM (SELECT {SLOW_CHAT_CALL} AS r FROM numbers(400))", + settings={ + "max_execution_time": 2, + "timeout_overflow_mode": "break", + }, + ) + requests = _concurrency_stats()["requests"] + assert requests < 40, f"the query kept sending requests past the time limit: {requests} of 400 were sent" From ca54b1011f5900f956f47cc45be32aea82228890 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 27 Oct 2026 22:41:40 +0000 Subject: [PATCH 035/219] Backport #119353 to 26.8: Respect `remote_url_allow_hosts` in the Hive metastore catalog --- src/Databases/DataLake/HiveCatalog.cpp | 14 ++++- .../configs/config_with_hosts.xml | 1 + .../test_allowed_url_from_config/test.py | 51 +++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) diff --git a/src/Databases/DataLake/HiveCatalog.cpp b/src/Databases/DataLake/HiveCatalog.cpp index 086f389e81a1f..421fe2b73d9d8 100644 --- a/src/Databases/DataLake/HiveCatalog.cpp +++ b/src/Databases/DataLake/HiveCatalog.cpp @@ -10,8 +10,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -74,8 +76,9 @@ std::pair parseHostPort(const String & url) } -HiveCatalog::HiveCatalog(const std::string & warehouse_, const std::string & base_url_, DB::ContextPtr) +HiveCatalog::HiveCatalog(const std::string & warehouse_, const std::string & base_url_, DB::ContextPtr context_) : ICatalog(warehouse_) + , DB::WithContext(context_) , base_url(base_url_) { std::lock_guard lock(client_mutex); @@ -98,6 +101,10 @@ void HiveCatalog::reconnectUnlocked() const TSA_REQUIRES(client_mutex) auto [host, port] = parseHostPort(base_url); + /// Also rejects control characters such as NUL, which `getaddrinfo` would truncate the host at. + getContext()->getRemoteHostFilter().checkAndGetCanonicalHostAndPort( + host + ":" + std::to_string(port), static_cast(port), "Hive metastore address"); + socket = std::make_shared(host, port); transport = std::make_shared(socket); protocol = std::make_shared(transport); @@ -129,6 +136,11 @@ void HiveCatalog::executeWithRetry(Func && func) const { reconnectUnlocked(); } + catch (const DB::Exception &) + { + /// Non-transient (e.g. `UNACCEPTABLE_URL` from `remote_url_allow_hosts`), retrying cannot help. + throw; + } catch (const std::exception & ex) { LOG_TRACE( diff --git a/tests/integration/test_allowed_url_from_config/configs/config_with_hosts.xml b/tests/integration/test_allowed_url_from_config/configs/config_with_hosts.xml index d4ac4986c22dc..ad199feefc1ed 100644 --- a/tests/integration/test_allowed_url_from_config/configs/config_with_hosts.xml +++ b/tests/integration/test_allowed_url_from_config/configs/config_with_hosts.xml @@ -2,6 +2,7 @@ host:80 ^[a-z]*\.ru$ + .*\.allowed\.example diff --git a/tests/integration/test_allowed_url_from_config/test.py b/tests/integration/test_allowed_url_from_config/test.py index 23c28067f9de4..25c07fc5744fa 100644 --- a/tests/integration/test_allowed_url_from_config/test.py +++ b/tests/integration/test_allowed_url_from_config/test.py @@ -412,3 +412,54 @@ def test_schema_inference(start_cluster): error = node7.query_and_get_error("desc url('http://test.com', 'TSVRaw')") assert "not allowed" in error assert error.find("ReadWriteBufferFromHTTPBase") == -1 + + +def test_hive_catalog_host_filter(start_cluster): + query = ( + "CREATE DATABASE test_hive_host_filter ENGINE = DataLakeCatalog('thrift://localhost:1') " + "SETTINGS catalog_type = 'hive', warehouse = 'test_warehouse'" + ) + settings = {"allow_experimental_database_hms_catalog": 1} + try: + # Control first: `node4` declares no `remote_url_allow_hosts` at all, so its filter stays + # uninitialized and allows everything - the same statement gets past the check and fails at + # the network instead. Without this arm, a filter that rejected everything would also pass. + error = node4.query_and_get_error(query, settings=settings) + if "USE_HIVE" in error: + pytest.skip("ClickHouse was built without Hive support") + assert "UNACCEPTABLE_URL" not in error, error + + # `node5` declares an empty `remote_url_allow_hosts`, which allows nothing. + error = node5.query_and_get_error(query, settings=settings) + assert "UNACCEPTABLE_URL" in error, error + # The check must run before the connect, so the refusal carries no connection error. + assert "Connection refused" not in error, error + finally: + node4.query("DROP DATABASE IF EXISTS test_hive_host_filter") + node5.query("DROP DATABASE IF EXISTS test_hive_host_filter") + + +def test_hive_catalog_host_filter_rejects_nul(start_cluster): + def create(url): + return node1.query_and_get_error( + f"CREATE DATABASE test_hive_host_filter_nul ENGINE = DataLakeCatalog('{url}') " + "SETTINGS catalog_type = 'hive', warehouse = 'test_warehouse'", + settings={"allow_experimental_database_hms_catalog": 1}, + ) + + try: + # Control: the suffix rule in `config_with_hosts.xml` admits this host, so it gets past the + # filter and fails at name resolution instead. + error = create("thrift://hms.allowed.example:1") + if "USE_HIVE" in error: + pytest.skip("ClickHouse was built without Hive support") + assert "UNACCEPTABLE_URL" not in error, error + + # The same rule matches the whole string including the NUL, but `getaddrinfo` would stop at the + # NUL and connect to 127.0.0.1. The host must be refused before any connection attempt. + error = create("thrift://127.0.0.1\\0.allowed.example:1") + assert "BAD_ARGUMENTS" in error, error + assert "Unexpected character" in error, error + assert "Connection refused" not in error, error + finally: + node1.query("DROP DATABASE IF EXISTS test_hive_host_filter_nul") From 8f4135ec404dfee663fed7ef8c0cd478e9f2d150 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 29 Oct 2026 15:02:19 +0000 Subject: [PATCH 036/219] Backport #124000 to 26.8: Make test 01035_avg four times cheaper --- tests/queries/0_stateless/01035_avg.reference | 8 ++++---- tests/queries/0_stateless/01035_avg.sql | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/01035_avg.reference b/tests/queries/0_stateless/01035_avg.reference index 1f47b3a8b7ea3..192bceec0560d 100644 --- a/tests/queries/0_stateless/01035_avg.reference +++ b/tests/queries/0_stateless/01035_avg.reference @@ -1,6 +1,6 @@ nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan --0.5 -0.5 -0.5 -0.5 -0.5 -0.5 127.493856 32355.57552 499999.5 499999.5 499999.5 499999.5 499999.5 499999.5 -0.000500002 0.49999949943727 -0.000005 -0.000004999999999999992 --2767.546272 -999999 --0.50000449943727 +-0.5 -0.5 -0.5 -0.5 -0.5 -0.5 127.467744 31470.715104 124999.5 124999.5 124999.5 124999.5 124999.5 124999.5 -0.0005000056 0.12499949943764 -0.000005000000000000001 -0.0000050000000000000334 +-3182.944672 +249999 +-0.12500449943764 1 1 \N \N 1 1 diff --git a/tests/queries/0_stateless/01035_avg.sql b/tests/queries/0_stateless/01035_avg.sql index 0f7baddaec577..a8aae4480e66b 100644 --- a/tests/queries/0_stateless/01035_avg.sql +++ b/tests/queries/0_stateless/01035_avg.sql @@ -27,7 +27,7 @@ SELECT avg(i8), avg(i16), avg(i32), avg(i64), avg(i128), avg(i256), avg(f32), avg(f64), avg(d32), avg(d64), avg(d128), avg(d256) FROM test_01035_avg; -INSERT INTO test_01035_avg (u64) SELECT number FROM system.numbers LIMIT 1000000; +INSERT INTO test_01035_avg (u64) SELECT number FROM system.numbers LIMIT 250000; SELECT avg(i8), avg(i16), avg(i32), avg(i64), avg(i128), avg(i256), avg(u8), avg(u16), avg(u32), avg(u64), avg(u128), avg(u256), From 5164fa190203adfce589ebf99b6ca704c8e50d4a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 29 Oct 2026 09:20:06 +0000 Subject: [PATCH 037/219] Backport #123341 to 26.8: Fix possible crash due to use-after-free of memory reservation after DROP WORKLOAD --- src/Common/Scheduler/IWorkloadNode.h | 4 + .../Nodes/SpaceShared/AllocationQueue.cpp | 3 + .../Scheduler/Nodes/TimeShared/FifoQueue.h | 3 + src/Common/Scheduler/Nodes/WorkloadNode.h | 29 ++++ .../tests/gtest_workload_resource_manager.cpp | 133 ++++++++++++++++++ .../Scheduler/WorkloadResourceManager.cpp | 4 + ...d_drop_memory_reservation_in_use.reference | 1 + ...workload_drop_memory_reservation_in_use.sh | 35 +++++ 8 files changed, 212 insertions(+) create mode 100644 tests/queries/0_stateless/05239_workload_drop_memory_reservation_in_use.reference create mode 100755 tests/queries/0_stateless/05239_workload_drop_memory_reservation_in_use.sh diff --git a/src/Common/Scheduler/IWorkloadNode.h b/src/Common/Scheduler/IWorkloadNode.h index 26662046efb40..44e2cdd86980a 100644 --- a/src/Common/Scheduler/IWorkloadNode.h +++ b/src/Common/Scheduler/IWorkloadNode.h @@ -102,6 +102,10 @@ class IWorkloadNode /// Returns true if this workload node has workload children virtual bool hasWorkloadChildren() const = 0; + /// Fails the requests and allocations still in the queue of this leaf workload. Called on the scheduler thread when + /// the workload is dropped, before the node is destroyed: query threads may still reach the queue through older versions. + virtual void purgeQueue() = 0; + /// Introspection. Calls a visitor for self and every internal node. Do not recurse into workload children. virtual void forEachSchedulerNode(std::function visitor) = 0; }; diff --git a/src/Common/Scheduler/Nodes/SpaceShared/AllocationQueue.cpp b/src/Common/Scheduler/Nodes/SpaceShared/AllocationQueue.cpp index 8e680060124ce..715f1b80411b7 100644 --- a/src/Common/Scheduler/Nodes/SpaceShared/AllocationQueue.cpp +++ b/src/Common/Scheduler/Nodes/SpaceShared/AllocationQueue.cpp @@ -181,6 +181,9 @@ void AllocationQueue::purgeQueue() allocated = 0; allocations = 0; is_not_usable = true; + + // Older hierarchy versions can retain this retired queue after its workload is destroyed. + workload = nullptr; } void AllocationQueue::propagateUpdate(ISpaceSharedNode &, Update &&) diff --git a/src/Common/Scheduler/Nodes/TimeShared/FifoQueue.h b/src/Common/Scheduler/Nodes/TimeShared/FifoQueue.h index 52909645aca91..9079a5b60436e 100644 --- a/src/Common/Scheduler/Nodes/TimeShared/FifoQueue.h +++ b/src/Common/Scheduler/Nodes/TimeShared/FifoQueue.h @@ -127,6 +127,9 @@ class FifoQueue : public ISchedulerQueue requests_to_fail.push_back(request); } cancelActivation(); + + // Older hierarchy versions can retain this retired queue after its workload is destroyed. + workload = nullptr; } // Now notify all collected requests about the failure without holding the mutex auto exception = std::make_exception_ptr( diff --git a/src/Common/Scheduler/Nodes/WorkloadNode.h b/src/Common/Scheduler/Nodes/WorkloadNode.h index ea055c8842743..6c644005ca317 100644 --- a/src/Common/Scheduler/Nodes/WorkloadNode.h +++ b/src/Common/Scheduler/Nodes/WorkloadNode.h @@ -442,6 +442,7 @@ class WorkloadNodeCommon : public BaseNode, public IWorkloadNode struct QueueOrChildrenBranch { NodePtr queue; /// FifoQueue or AllocationQueue node is used if there are no children + bool queue_dropped = false; /// `queue` has been purged by `purgeDroppedQueue` ChildrenBranch branch; /// Used if there is at least one child WorkloadSettings settings; CostUnit unit = CostUnit::IOByte; @@ -495,6 +496,17 @@ class WorkloadNodeCommon : public BaseNode, public IWorkloadNode Traits::updateQueue(queue, settings, unit); } + /// Fails the requests and allocations of a dropped workload, the same way `removeQueue` does + /// when the workload gets its first child. The queue itself stays alive while older hierarchy + /// versions reference it. + void purgeDroppedQueue() + { + if (!queue || std::exchange(queue_dropped, true)) + return; + detach(queue); + Traits::purgeQueue(queue); + } + private: void removeQueue() { @@ -639,6 +651,18 @@ class WorkloadNodeCommon : public BaseNode, public IWorkloadNode : BaseNode(event_queue_, SchedulerNodeInfo(settings)) {} + ~WorkloadNodeCommon() override + { + /// Queues and constraints outlive this node while older versions are still referenced by classifiers. + /// The queue clears its own pointer in `purgeQueue` under its mutex, because query threads read it there. + purgeQueue(); // No-op if the workload was dropped through `purgeQueue` already + forEachSchedulerNode([this](ISchedulerNode * node) + { + if (node != impl.branch.queue.get()) + node->workload = nullptr; + }); + } + void attachWorkloadChild(const WorkloadNodePtr & child) final { SCHED_DBG("{} -- attachWorkloadChild(child={})", this->getPath(), child->getWorkload()); @@ -716,6 +740,11 @@ class WorkloadNodeCommon : public BaseNode, public IWorkloadNode } } + void purgeQueue() final + { + impl.branch.purgeDroppedQueue(); + } + bool hasWorkloadChildren() const final { return impl.branch.queue == nullptr; diff --git a/src/Common/Scheduler/Nodes/tests/gtest_workload_resource_manager.cpp b/src/Common/Scheduler/Nodes/tests/gtest_workload_resource_manager.cpp index bd1d35a18af27..42778c126794f 100644 --- a/src/Common/Scheduler/Nodes/tests/gtest_workload_resource_manager.cpp +++ b/src/Common/Scheduler/Nodes/tests/gtest_workload_resource_manager.cpp @@ -475,6 +475,45 @@ TEST(SchedulerWorkloadResourceManager, DropNotEmptyQueue) // failed the request), the thread-local `Request` must be reset back to `Finished`. Otherwise // the next `ResourceGuard` on the same thread chasserts in `enqueue` because the request is // still in `Dequeued` state with `exception` set. +TEST(SchedulerWorkloadResourceManager, DropWorkloadFailsPendingRequests) +{ + ResourceTest t; + + t.query("CREATE RESOURCE res1 (WRITE DISK disk, READ DISK disk)"); + t.query("CREATE WORKLOAD all SETTINGS max_io_requests = 1"); + t.query("CREATE WORKLOAD intermediate IN all"); + + std::barrier sync_before_enqueue(2); + std::barrier sync_before_drop(3); + std::barrier sync_after_drop(2); + t.async("intermediate", "res1", [&] (ResourceLink link) + { + TestGuard g(t, link, 1); + sync_before_enqueue.arrive_and_wait(); + sync_before_drop.arrive_and_wait(); // 1st resource request is consuming + sync_after_drop.arrive_and_wait(); // 1st resource request is still consuming + }); + + sync_before_enqueue.arrive_and_wait(); // to maintain correct order of resource requests + + t.async("intermediate", "res1", [&] (ResourceLink link) + { + { + TestGuard g(t, link, 1, EnqueueOnly); + sync_before_drop.arrive_and_wait(); // 2nd resource request is enqueued + g.waitFailed("is about to be destructed"); + } + // The dropped workload's queue is kept alive by this classifier, but it must not accept new requests. + EXPECT_THROW(ResourceGuard(ResourceGuard::Metrics::getIOWrite(), link, 1), DB::Exception); + }); + + sync_before_drop.arrive_and_wait(); + t.query("DROP WORKLOAD intermediate"); + sync_after_drop.arrive_and_wait(); + + t.wait(); // Wait for threads to finish before destructing locals +} + TEST(SchedulerWorkloadResourceManager, ResourceGuardDefaultLockResetsOnFailure) { ResourceTest t; @@ -2697,6 +2736,100 @@ TEST(SchedulerWorkloadResourceManager, MemoryReservationDropQueueWhilePending) EXPECT_EQ(threw.load(), waiter_count); } +TEST(SchedulerWorkloadResourceManager, RetiredAllocationQueueOutlivesWorkload) +{ + ResourceTest t; + + t.query("CREATE RESOURCE memory (MEMORY RESERVATION)"); + t.query("CREATE WORKLOAD A SETTINGS max_memory = 100"); + t.query("CREATE WORKLOAD B SETTINGS max_memory = 100"); + + // Only `B`'s classifier retains the old hierarchy version after the temporary classifier for `A` is released. + ClassifierPtr classifier_b = t.manager->acquire("B"); + auto * retired_queue = t.manager->acquire("A")->get("memory").allocation_queue; + ASSERT_NE(retired_queue, nullptr); + t.executeFromScheduler("memory", [&] + { + EXPECT_EQ(retired_queue->getWorkloadName(), "A"); + }); + + t.query("CREATE WORKLOAD child IN A"); + t.executeFromScheduler("memory", [&] + { + // Check retirement while `A` is still alive: this fails deterministically without relying on + // ASan or on the contents of freed memory when the retired queue is eventually destroyed. + EXPECT_TRUE(retired_queue->getWorkloadName().empty()); + }); + + t.query("DROP WORKLOAD child"); + EXPECT_NE(retired_queue, t.manager->acquire("A")->get("memory").allocation_queue); + t.query("DROP WORKLOAD A"); + + // Destroy the original queue after its workload, through the classifier for a different workload. + classifier_b.reset(); +} + +TEST(SchedulerWorkloadResourceManager, MemoryReservationDropWorkloadFailsRunningAllocation) +{ + ResourceTest t; + + t.query("CREATE RESOURCE memory (MEMORY RESERVATION)"); + t.query("CREATE WORKLOAD all SETTINGS max_memory = 100"); + t.query("CREATE WORKLOAD A IN all"); + t.query("CREATE WORKLOAD B IN all"); + + ClassifierPtr c_a = t.manager->acquire("A"); + TestAllocation a(c_a->get("memory"), "a", 60); + a.waitSync(); + + // The classifier keeps `A`'s queue alive, but the allocation must not keep running outside the limits of `all`. + t.query("DROP WORKLOAD A"); + EXPECT_THROW(a.waitSync(), Exception); + a.setSize(80); + EXPECT_THROW(a.waitSync(), Exception); + + // The memory of the failed allocation is available to the rest of the hierarchy. + ClassifierPtr c_b = t.manager->acquire("B"); + TestAllocation b(c_b->get("memory"), "b", 100); + b.waitSync(); +} + +TEST(SchedulerWorkloadResourceManager, MemoryReservationDropWorkloadWhileAllocating) +{ + ResourceTest t; + + t.query("CREATE RESOURCE memory (MEMORY RESERVATION)"); + t.query("CREATE WORKLOAD A SETTINGS max_memory = 100, max_waiting_queries = 0"); + + ClassifierPtr c_a = t.manager->acquire("A"); + ResourceLink link = c_a->get("memory"); + + // Keep inserting allocations into `A`'s queue while `A` is dropped. Every one is rejected and the rejection reads + // the workload's name: by `max_waiting_queries` before the purge, as an unusable queue after it. This lets TSan + // check those reads against the drop and the destruction of the workload node. + std::atomic dropped{false}; + std::atomic rejected{0}; + std::thread allocator([&] + { + while (!dropped.load() || rejected.load() < 100) + { + try + { + TestAllocation allocation(link, "racer", 10); + } + catch (const Exception &) + { + rejected.fetch_add(1); + } + } + }); + + t.query("DROP WORKLOAD A"); + dropped.store(true); + allocator.join(); + EXPECT_GE(rejected.load(), 100); +} + // Regression: simulate the race where the owner's destructor reaches `queue.removeAllocation` // AFTER `updateMinMaxAllocated`/`updateQueueLimit` has already rejected the allocation. The // guard in `removeAllocation` must keep an already-failed allocation out of diff --git a/src/Common/Scheduler/WorkloadResourceManager.cpp b/src/Common/Scheduler/WorkloadResourceManager.cpp index 0348710be62b8..9255595a94e8e 100644 --- a/src/Common/Scheduler/WorkloadResourceManager.cpp +++ b/src/Common/Scheduler/WorkloadResourceManager.cpp @@ -148,6 +148,10 @@ void WorkloadResourceManager::Resource::deleteNode(const NodeInfo & info) root_node.reset(); } + // Fail the queue's requests while the node is fully alive: query threads may still reach the queue through + // older versions, and the queue's error messages read the workload's name. + n->purgeQueue(); + node_for_workload.erase(info.name); updateCurrentVersion(); diff --git a/tests/queries/0_stateless/05239_workload_drop_memory_reservation_in_use.reference b/tests/queries/0_stateless/05239_workload_drop_memory_reservation_in_use.reference new file mode 100644 index 0000000000000..d00491fd7e5bb --- /dev/null +++ b/tests/queries/0_stateless/05239_workload_drop_memory_reservation_in_use.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/05239_workload_drop_memory_reservation_in_use.sh b/tests/queries/0_stateless/05239_workload_drop_memory_reservation_in_use.sh new file mode 100755 index 0000000000000..f49e7937d18fe --- /dev/null +++ b/tests/queries/0_stateless/05239_workload_drop_memory_reservation_in_use.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Tags: no-parallel + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +workload=w_$CLICKHOUSE_TEST_UNIQUE_NAME +query_id=q_$CLICKHOUSE_TEST_UNIQUE_NAME + +function cleanup() +{ + $CLICKHOUSE_CLIENT -nm -q "DROP WORKLOAD IF EXISTS $workload" >& /dev/null || : + $CLICKHOUSE_CLIENT -q "DROP RESOURCE IF EXISTS memory" >& /dev/null || : +} +trap cleanup EXIT + +$CLICKHOUSE_CLIENT -nm -q " +CREATE OR REPLACE RESOURCE memory (MEMORY RESERVATION); +CREATE WORKLOAD $workload SETTINGS max_memory = '1Gi'; +" + +# The query keeps the dropped workload's queue alive. DROP WORKLOAD fails the query's memory reservation, +# so the query may end on its own before KILL. +$CLICKHOUSE_CLIENT --query_id "$query_id" --workload "$workload" --max_block_size 1 -q "SELECT sleepEachRow(1) FROM numbers(300) FORMAT Null" 2>/dev/null & + +while [[ $($CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes WHERE query_id = '$query_id'") != 1 ]]; do + : +done + +$CLICKHOUSE_CLIENT -q "DROP WORKLOAD $workload" +$CLICKHOUSE_CLIENT -q "KILL QUERY WHERE query_id = '$query_id' SYNC FORMAT Null SETTINGS kill_throw_if_noop = false" +wait + +$CLICKHOUSE_CLIENT -q "SELECT 1" From ae5b8c56d7e97c37c86526eceb1170c29e3d7429 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Thu, 29 Oct 2026 10:07:18 +0000 Subject: [PATCH 038/219] Give the workloads of `RetiredAllocationQueueOutlivesWorkload` a common root This branch allows a single root workload, so the test failed with "The second root is not allowed" before checking anything. `A` and `B` now share the parent `all`, as in the other tests of the backport of https://github.com/ClickHouse/ClickHouse/pull/123341. Failure: https://s3.amazonaws.com/clickhouse-test-reports/json.html?PR=128123&sha=10c57992af3077a27b7aaab8eadd798da82cfaad&name_0=BackportPR&name_1=Unit%20tests%20%28asan_ubsan%29 Co-Authored-By: Claude Opus 5.5 --- .../Nodes/tests/gtest_workload_resource_manager.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/Common/Scheduler/Nodes/tests/gtest_workload_resource_manager.cpp b/src/Common/Scheduler/Nodes/tests/gtest_workload_resource_manager.cpp index 42778c126794f..3ea85d499877f 100644 --- a/src/Common/Scheduler/Nodes/tests/gtest_workload_resource_manager.cpp +++ b/src/Common/Scheduler/Nodes/tests/gtest_workload_resource_manager.cpp @@ -2741,8 +2741,10 @@ TEST(SchedulerWorkloadResourceManager, RetiredAllocationQueueOutlivesWorkload) ResourceTest t; t.query("CREATE RESOURCE memory (MEMORY RESERVATION)"); - t.query("CREATE WORKLOAD A SETTINGS max_memory = 100"); - t.query("CREATE WORKLOAD B SETTINGS max_memory = 100"); + // This branch allows a single root workload, so `A` and `B` share a parent. + t.query("CREATE WORKLOAD all"); + t.query("CREATE WORKLOAD A IN all SETTINGS max_memory = 100"); + t.query("CREATE WORKLOAD B IN all SETTINGS max_memory = 100"); // Only `B`'s classifier retains the old hierarchy version after the temporary classifier for `A` is released. ClassifierPtr classifier_b = t.manager->acquire("B"); From 7135eddc9ba4d4d681a9e98a5e8524d78cb31667 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Thu, 29 Oct 2026 15:06:47 +0000 Subject: [PATCH 039/219] Do not use `kill_throw_if_noop` in `05239_workload_drop_memory_reservation_in_use` The setting does not exist on this branch, so `KILL QUERY` failed with `UNKNOWN_SETTING`. An empty `KILL QUERY` match does not throw here anyway. https://s3.amazonaws.com/clickhouse-test-reports/praktika.html?PR=128124&sha=3fefe47eec25477b4a1e8620279a6f1d8f94ef61&name_0=BackportPR&name_1=Stateless%20tests%20%28amd_asan_ubsan%2C%20db%20disk%2C%20distributed%20plan%2C%20sequential%2C%203%2F3%29 https://github.com/ClickHouse/ClickHouse/pull/128124 Co-Authored-By: Claude Opus 5.5 --- .../05239_workload_drop_memory_reservation_in_use.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/05239_workload_drop_memory_reservation_in_use.sh b/tests/queries/0_stateless/05239_workload_drop_memory_reservation_in_use.sh index f49e7937d18fe..ce5710669e693 100755 --- a/tests/queries/0_stateless/05239_workload_drop_memory_reservation_in_use.sh +++ b/tests/queries/0_stateless/05239_workload_drop_memory_reservation_in_use.sh @@ -29,7 +29,7 @@ while [[ $($CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes WHERE que done $CLICKHOUSE_CLIENT -q "DROP WORKLOAD $workload" -$CLICKHOUSE_CLIENT -q "KILL QUERY WHERE query_id = '$query_id' SYNC FORMAT Null SETTINGS kill_throw_if_noop = false" +$CLICKHOUSE_CLIENT -q "KILL QUERY WHERE query_id = '$query_id' SYNC FORMAT Null" wait $CLICKHOUSE_CLIENT -q "SELECT 1" From 28144f599d7c2a8097a4a379f38eac6325921557 Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Thu, 29 Oct 2026 13:33:59 +0000 Subject: [PATCH 040/219] Make the progress indicator frame per-instance in `ProgressIndication` `ProgressIndication::writeProgress` kept the frame of the spinning indicator in a function-local `static`, which is shared by all clients in the process. Concurrent sessions of the web terminal (embedded clients) increment it from several threads, which TSan reports as a data race. It fails every `Stress test (amd_tsan)` run since `05445_webterminal_concurrent_sessions_input` came with the backport of #125510. Backport of 3329e92f2ab from #125514 (only this commit; the rest of the PR is a feature). https://s3.amazonaws.com/clickhouse-test-reports/praktika.html?REF=26.10&sha=da23b6505a74b35f9ef57cb2a3a26665db3fd763&name_0=ReleaseBranchCI https://github.com/ClickHouse/ClickHouse/pull/125514 (cherry picked from commit 3329e92f2abb7375afe57306e0637aa4bb0f34a3) Co-Authored-By: Claude Opus 5.5 --- src/Common/ProgressIndication.cpp | 5 ++--- src/Common/ProgressIndication.h | 4 ++++ 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/src/Common/ProgressIndication.cpp b/src/Common/ProgressIndication.cpp index 172f2b735e52a..def1f44b11f8d 100644 --- a/src/Common/ProgressIndication.cpp +++ b/src/Common/ProgressIndication.cpp @@ -129,7 +129,6 @@ void ProgressIndication::writeProgress(WriteBufferFromFileDescriptor & message, { std::lock_guard lock(progress_mutex); - static size_t increment = 0; static const char * indicators[8] = { "\033[1;30m→\033[0m", "\033[1;31m↘\033[0m", @@ -141,7 +140,7 @@ void ProgressIndication::writeProgress(WriteBufferFromFileDescriptor & message, "\033[1m↗\033[0m", }; - const char * indicator = indicators[increment % 8]; + const char * indicator = indicators[indicator_frame % 8]; auto [terminal_width, terminal_height] = getTerminalSize(in_fd, err_fd); @@ -287,7 +286,7 @@ void ProgressIndication::writeProgress(WriteBufferFromFileDescriptor & message, } message << CLEAR_TO_END_OF_LINE; - ++increment; + ++indicator_frame; message.next(); } diff --git a/src/Common/ProgressIndication.h b/src/Common/ProgressIndication.h index 517a23f0c4a87..68df926b9b81b 100644 --- a/src/Common/ProgressIndication.h +++ b/src/Common/ProgressIndication.h @@ -106,6 +106,10 @@ class ProgressIndication /// to check whether progress output needs to be cleared. size_t written_progress_chars = 0; + /// Selects the frame of the spinning indicator. It is per instance, because several clients + /// (e.g. the sessions of the web terminal) can show progress concurrently in one process. + size_t indicator_frame = 0; + /// The server periodically sends information about how much data was read since last time. /// This information is stored here. Progress progress; From 6b6d6a7cb7f431c9ea70b50a2ba35d9471c523b9 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 28 Oct 2026 14:34:04 +0000 Subject: [PATCH 041/219] Backport #124999 to 26.8: Add server setting `ai_function_allow_sql_named_collections` --- .../regular-functions/ai-functions.mdx | 18 +++++ src/Core/ServerSettings.cpp | 5 ++ src/Functions/FunctionBaseAI.cpp | 11 +++ ...ions_allow_sql_named_collections.reference | 34 +++++++++ ...i_functions_allow_sql_named_collections.sh | 74 +++++++++++++++++++ 5 files changed, 142 insertions(+) create mode 100644 tests/queries/0_stateless/05353_ai_functions_allow_sql_named_collections.reference create mode 100755 tests/queries/0_stateless/05353_ai_functions_allow_sql_named_collections.sh diff --git a/docs/reference/functions/regular-functions/ai-functions.mdx b/docs/reference/functions/regular-functions/ai-functions.mdx index c842b0943e6f5..159c6f3ec20fa 100644 --- a/docs/reference/functions/regular-functions/ai-functions.mdx +++ b/docs/reference/functions/regular-functions/ai-functions.mdx @@ -114,6 +114,24 @@ SELECT aiGenerate(body, map('temperature', '0.2', 'system_prompt', 'You are ters All AI-related settings are listed in [Settings](/reference/settings/session-settings) under the `ai_function_` prefix. +### Server configuration {#server-configuration} + +The server setting `ai_function_allow_sql_named_collections` (default `true`) controls which named collections AI functions accept. When set to `false`, AI functions accept only named collections defined in the server configuration file, and using a collection created with `CREATE NAMED COLLECTION` throws an exception. This applies both to the `credentials` parameter and to the default-credentials settings. Changing the setting requires a server restart. + +```xml + + false + + + openai + https://api.openai.com/v1/chat/completions + gpt-4o-mini + sk-... + + + +``` + ### Restricting endpoint hosts {#restricting-endpoint-hosts} The `endpoint` URL in an AI named collection is an outbound destination the server connects to under its own identity, potentially carrying (if specified) the named collection's `api_key` in the request headers. By default, ClickHouse permits any host. To restrict functions to a specific set of providers, configure [`remote_url_allow_hosts`](/reference/settings/server-settings/settings/remote#remote_url_allow_hosts) in the server config, e.g.: diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index 5d47a1b810c50..04408d509ca3e 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -885,6 +885,11 @@ A value of `0` means no limitation. ```xml 400 ``` +)", 0) \ + DECLARE(Bool, ai_function_allow_sql_named_collections, true, R"( +Allows AI functions (such as `aiGenerate`, `aiEmbed`, `aiSimilarity`) to use named collections created with `CREATE NAMED COLLECTION`. +When disabled, AI functions accept only named collections defined in the server configuration file, and using any other collection throws an exception. +This applies both to the `credentials` parameter and to the `ai_function_text_default_credentials` and `ai_function_embedding_default_credentials` settings. )", 0) \ DECLARE(UInt64, max_table_num_to_throw, 0lu, R"( If number of tables is greater than this value, server will throw an exception. diff --git a/src/Functions/FunctionBaseAI.cpp b/src/Functions/FunctionBaseAI.cpp index 31e354cf35c78..1751184e8e2ec 100644 --- a/src/Functions/FunctionBaseAI.cpp +++ b/src/Functions/FunctionBaseAI.cpp @@ -49,8 +49,14 @@ namespace Setting extern const SettingsBool ai_function_allow_insecure_endpoint; } +namespace ServerSetting +{ + extern const ServerSettingsBool ai_function_allow_sql_named_collections; +} + namespace ErrorCodes { + extern const int ACCESS_DENIED; extern const int BAD_ARGUMENTS; extern const int AI_PROVIDER_RESPONSE_TRUNCATED; extern const int AI_PROVIDER_RESPONSE_INCOMPLETE; @@ -250,6 +256,11 @@ FunctionBaseAI::AIParams FunctionBaseAI::resolveAIParams( context->checkAccess(AccessType::NAMED_COLLECTION, credentials); const auto & collection = NamedCollectionFactory::instance().get(credentials); + if (!context->getServerSettings()[ServerSetting::ai_function_allow_sql_named_collections] + && collection->getSourceId() != NamedCollection::SourceId::CONFIG) + throw Exception(ErrorCodes::ACCESS_DENIED, + "AI named collection '{}' is not defined in the server configuration file, and the server setting " + "'ai_function_allow_sql_named_collections' is disabled", credentials); AIParams params; params.collection.collection_name = credentials; diff --git a/tests/queries/0_stateless/05353_ai_functions_allow_sql_named_collections.reference b/tests/queries/0_stateless/05353_ai_functions_allow_sql_named_collections.reference new file mode 100644 index 0000000000000..af474ec4afd6e --- /dev/null +++ b/tests/queries/0_stateless/05353_ai_functions_allow_sql_named_collections.reference @@ -0,0 +1,34 @@ +== default +-- config collection, credentials parameter +0 +0 +0 +-- SQL collection, credentials parameter +0 +0 +0 +-- config collection, default-credentials settings +0 +0 +0 +-- SQL collection, default-credentials settings +0 +0 +0 +== ai_function_allow_sql_named_collections = 0 +-- config collection, credentials parameter +0 +0 +0 +-- SQL collection, credentials parameter +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +-- config collection, default-credentials settings +0 +0 +0 +-- SQL collection, default-credentials settings +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED diff --git a/tests/queries/0_stateless/05353_ai_functions_allow_sql_named_collections.sh b/tests/queries/0_stateless/05353_ai_functions_allow_sql_named_collections.sh new file mode 100755 index 0000000000000..97e374cac1974 --- /dev/null +++ b/tests/queries/0_stateless/05353_ai_functions_allow_sql_named_collections.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash + +# The server setting `ai_function_allow_sql_named_collections = 0` restricts AI functions to named +# collections defined in the server configuration file. Collections created with `CREATE NAMED COLLECTION` +# are rejected, whether passed in the `credentials` parameter or via the default-credentials settings. +# The check runs in `resolveAIParams`, before the zero-row early return, so empty inputs exercise it +# without any HTTP call. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +function config() +{ + echo " + $1 + + + openai + https://ai.example.com/v1/chat/completions + chat-model + fake-key + + + openai + https://ai.example.com/v1/embeddings + fake-key + + + " +} + +QUERIES=" +CREATE NAMED COLLECTION ai_sql_chat AS + provider = 'openai', endpoint = 'https://ai.example.com/v1/chat/completions', model = 'chat-model', api_key = 'fake-key'; +CREATE NAMED COLLECTION ai_sql_embed AS + provider = 'openai', endpoint = 'https://ai.example.com/v1/embeddings', api_key = 'fake-key'; +CREATE TABLE tab (x String) ENGINE = Memory; + +SELECT '-- config collection, credentials parameter'; +SELECT count() FROM (SELECT aiGenerate(x, map('credentials', 'ai_config_chat')) AS r FROM tab); +SELECT count() FROM (SELECT aiEmbed(x, 'embed-model', map('credentials', 'ai_config_embed')) AS r FROM tab); +SELECT count() FROM (SELECT aiSimilarity(x, x, 'embed-model', map('credentials', 'ai_config_embed')) AS r FROM tab); + +SELECT '-- SQL collection, credentials parameter'; +SELECT count() FROM (SELECT aiGenerate(x, map('credentials', 'ai_sql_chat')) AS r FROM tab); +SELECT count() FROM (SELECT aiEmbed(x, 'embed-model', map('credentials', 'ai_sql_embed')) AS r FROM tab); +SELECT count() FROM (SELECT aiSimilarity(x, x, 'embed-model', map('credentials', 'ai_sql_embed')) AS r FROM tab); + +SELECT '-- config collection, default-credentials settings'; +SET ai_function_text_default_credentials = 'ai_config_chat', ai_function_embedding_default_credentials = 'ai_config_embed'; +SELECT count() FROM (SELECT aiGenerate(x) AS r FROM tab); +SELECT count() FROM (SELECT aiEmbed(x, 'embed-model') AS r FROM tab); +SELECT count() FROM (SELECT aiSimilarity(x, x, 'embed-model') AS r FROM tab); + +SELECT '-- SQL collection, default-credentials settings'; +SET ai_function_text_default_credentials = 'ai_sql_chat', ai_function_embedding_default_credentials = 'ai_sql_embed'; +SELECT count() FROM (SELECT aiGenerate(x) AS r FROM tab); +SELECT count() FROM (SELECT aiEmbed(x, 'embed-model') AS r FROM tab); +SELECT count() FROM (SELECT aiSimilarity(x, x, 'embed-model') AS r FROM tab); +" + +# Prints result rows as they are and replaces each error with its code name. +function run() +{ + $CLICKHOUSE_LOCAL --config-file <(config "$1") --ignore-error -q "$QUERIES" 2>&1 \ + | grep -v -e '^Received exception:$' -e '^(query: ' \ + | sed -E 's/.*Code: [0-9]+\. DB::Exception: .*\(([A-Z_]+)\)$/\1/' +} + +echo "== default" +run "" +echo "== ai_function_allow_sql_named_collections = 0" +run "0" From 5f3825927b6206a497fc2eb82bff715f4f8b0c59 Mon Sep 17 00:00:00 2001 From: George Larionov Date: Thu, 29 Oct 2026 14:29:39 +0000 Subject: [PATCH 042/219] Do not rely on `--ignore-error` printing errors in `clickhouse-local` 26.8 `clickhouse-local` does not print errors under `--ignore-error` (changed in #119572), so run each query separately. https://s3.amazonaws.com/clickhouse-test-reports/json.html?PR=128063&sha=445a346409e70a7a13c660be6b8cb22ec8f0e3e9&name_0=BackportPR&name_1=Stateless%20tests%20%28amd_asan_ubsan%2C%20distributed%20plan%2C%20parallel%29 https://github.com/ClickHouse/ClickHouse/pull/128063 Co-Authored-By: Claude Opus 5.5 --- ...i_functions_allow_sql_named_collections.sh | 56 ++++++++++--------- 1 file changed, 31 insertions(+), 25 deletions(-) diff --git a/tests/queries/0_stateless/05353_ai_functions_allow_sql_named_collections.sh b/tests/queries/0_stateless/05353_ai_functions_allow_sql_named_collections.sh index 97e374cac1974..e093d6f0d3918 100755 --- a/tests/queries/0_stateless/05353_ai_functions_allow_sql_named_collections.sh +++ b/tests/queries/0_stateless/05353_ai_functions_allow_sql_named_collections.sh @@ -30,42 +30,48 @@ function config() " } -QUERIES=" +SETUP=" CREATE NAMED COLLECTION ai_sql_chat AS provider = 'openai', endpoint = 'https://ai.example.com/v1/chat/completions', model = 'chat-model', api_key = 'fake-key'; CREATE NAMED COLLECTION ai_sql_embed AS provider = 'openai', endpoint = 'https://ai.example.com/v1/embeddings', api_key = 'fake-key'; CREATE TABLE tab (x String) ENGINE = Memory; +" -SELECT '-- config collection, credentials parameter'; -SELECT count() FROM (SELECT aiGenerate(x, map('credentials', 'ai_config_chat')) AS r FROM tab); -SELECT count() FROM (SELECT aiEmbed(x, 'embed-model', map('credentials', 'ai_config_embed')) AS r FROM tab); -SELECT count() FROM (SELECT aiSimilarity(x, x, 'embed-model', map('credentials', 'ai_config_embed')) AS r FROM tab); - -SELECT '-- SQL collection, credentials parameter'; -SELECT count() FROM (SELECT aiGenerate(x, map('credentials', 'ai_sql_chat')) AS r FROM tab); -SELECT count() FROM (SELECT aiEmbed(x, 'embed-model', map('credentials', 'ai_sql_embed')) AS r FROM tab); -SELECT count() FROM (SELECT aiSimilarity(x, x, 'embed-model', map('credentials', 'ai_sql_embed')) AS r FROM tab); +# Runs one query in its own process and prints its result, or the code name of its error. +# `--ignore-error` is not used: older `clickhouse-local` versions do not print errors with it. +function query() +{ + $CLICKHOUSE_LOCAL --config-file <(config "$1") -q "$SETUP $2" 2>&1 \ + | sed -E 's/.*Code: [0-9]+\. DB::Exception: .*\(([A-Z_]+)\)$/\1/' +} -SELECT '-- config collection, default-credentials settings'; -SET ai_function_text_default_credentials = 'ai_config_chat', ai_function_embedding_default_credentials = 'ai_config_embed'; -SELECT count() FROM (SELECT aiGenerate(x) AS r FROM tab); -SELECT count() FROM (SELECT aiEmbed(x, 'embed-model') AS r FROM tab); -SELECT count() FROM (SELECT aiSimilarity(x, x, 'embed-model') AS r FROM tab); +# $1 is the server config fragment, $2 and $3 are the chat and embedding collections. +function queries_with_credentials() +{ + query "$1" "SELECT count() FROM (SELECT aiGenerate(x, map('credentials', '$2')) AS r FROM tab)" + query "$1" "SELECT count() FROM (SELECT aiEmbed(x, 'embed-model', map('credentials', '$3')) AS r FROM tab)" + query "$1" "SELECT count() FROM (SELECT aiSimilarity(x, x, 'embed-model', map('credentials', '$3')) AS r FROM tab)" +} -SELECT '-- SQL collection, default-credentials settings'; -SET ai_function_text_default_credentials = 'ai_sql_chat', ai_function_embedding_default_credentials = 'ai_sql_embed'; -SELECT count() FROM (SELECT aiGenerate(x) AS r FROM tab); -SELECT count() FROM (SELECT aiEmbed(x, 'embed-model') AS r FROM tab); -SELECT count() FROM (SELECT aiSimilarity(x, x, 'embed-model') AS r FROM tab); -" +function queries_with_default_credentials() +{ + local set="SET ai_function_text_default_credentials = '$2', ai_function_embedding_default_credentials = '$3';" + query "$1" "$set SELECT count() FROM (SELECT aiGenerate(x) AS r FROM tab)" + query "$1" "$set SELECT count() FROM (SELECT aiEmbed(x, 'embed-model') AS r FROM tab)" + query "$1" "$set SELECT count() FROM (SELECT aiSimilarity(x, x, 'embed-model') AS r FROM tab)" +} -# Prints result rows as they are and replaces each error with its code name. function run() { - $CLICKHOUSE_LOCAL --config-file <(config "$1") --ignore-error -q "$QUERIES" 2>&1 \ - | grep -v -e '^Received exception:$' -e '^(query: ' \ - | sed -E 's/.*Code: [0-9]+\. DB::Exception: .*\(([A-Z_]+)\)$/\1/' + echo "-- config collection, credentials parameter" + queries_with_credentials "$1" ai_config_chat ai_config_embed + echo "-- SQL collection, credentials parameter" + queries_with_credentials "$1" ai_sql_chat ai_sql_embed + echo "-- config collection, default-credentials settings" + queries_with_default_credentials "$1" ai_config_chat ai_config_embed + echo "-- SQL collection, default-credentials settings" + queries_with_default_credentials "$1" ai_sql_chat ai_sql_embed } echo "== default" From df67f2ba8b4473c300822fb73309cd5726d9631a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 29 Oct 2026 17:43:18 +0000 Subject: [PATCH 043/219] Backport #116502 to 26.8: Fix concurrent stores of the same format schema failing with CANNOT_OPEN_FILE --- src/Formats/FormatSchemaInfo.cpp | 5 ++- ..._schema_cache_leftover_temp_file.reference | 2 + ..._format_schema_cache_leftover_temp_file.sh | 45 +++++++++++++++++++ 3 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05045_format_schema_cache_leftover_temp_file.reference create mode 100755 tests/queries/0_stateless/05045_format_schema_cache_leftover_temp_file.sh diff --git a/src/Formats/FormatSchemaInfo.cpp b/src/Formats/FormatSchemaInfo.cpp index 9dc88c55f0e2a..a59ea6fe5d122 100644 --- a/src/Formats/FormatSchemaInfo.cpp +++ b/src/Formats/FormatSchemaInfo.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include namespace DB { @@ -288,7 +289,9 @@ void FormatSchemaInfo::storeSchemaOnDisk(const fs::path & file_path, const Strin fs::create_directory(dir_path); } - auto temp_path = fs::path(file_path.string() + ".tmp"); + /// The final file name is a hash of the schema source, so every writer of one schema targets + /// the same path; a shared temporary name would let them overwrite and delete each other's files. + auto temp_path = fs::path(file_path.string() + "." + getRandomASCIIString(8) + ".tmp"); try { diff --git a/tests/queries/0_stateless/05045_format_schema_cache_leftover_temp_file.reference b/tests/queries/0_stateless/05045_format_schema_cache_leftover_temp_file.reference new file mode 100644 index 0000000000000..f619cfc86a8a5 --- /dev/null +++ b/tests/queries/0_stateless/05045_format_schema_cache_leftover_temp_file.reference @@ -0,0 +1,2 @@ +data Array(Array(Int32)) +leftover file kept diff --git a/tests/queries/0_stateless/05045_format_schema_cache_leftover_temp_file.sh b/tests/queries/0_stateless/05045_format_schema_cache_leftover_temp_file.sh new file mode 100755 index 0000000000000..24f329eb412f0 --- /dev/null +++ b/tests/queries/0_stateless/05045_format_schema_cache_leftover_temp_file.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A schema given as a string is published into the schema cache through a temporary file. A +# temporary file already sitting at that path belongs to another query storing the same schema, so +# storing must neither fail because of it nor delete it. + +WORKDIR="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}" +rm -rf "$WORKDIR" +mkdir -p "$WORKDIR" + +SCHEMA='@0x844f048b15c12dab;\nstruct M { data @0 :List(List(Int32)); }' +QUERY="DESC format(CapnProto, '') +SETTINGS + format_schema_source = 'string', + format_schema = '${SCHEMA}', + format_schema_message_name = 'M'" + +# The cache directory of clickhouse-local is relative to the working directory. +(cd "$WORKDIR" && ${CLICKHOUSE_LOCAL} --logger.console=0 --query "$QUERY" > /dev/null) + +CACHED=$(basename "$(ls "$WORKDIR"/__cache__/*.capnp)") + +if [ -z "$CACHED" ]; then + echo "FAIL: the first query stored no schema file to learn the name from" +fi + +# Drop the published schema and leave a temporary file of it behind, so the next query has to store +# the schema again while that temporary path is taken. +rm -f "$WORKDIR"/__cache__/*.capnp +echo 'leftover' > "$WORKDIR/__cache__/$CACHED.tmp" + +(cd "$WORKDIR" && ${CLICKHOUSE_LOCAL} --logger.console=0 --query "$QUERY") + +if [ -f "$WORKDIR/__cache__/$CACHED.tmp" ]; then + echo 'leftover file kept' +else + echo 'leftover file removed' +fi + +rm -rf "$WORKDIR" From bb8270bc87856d6976287b40135b8af7362bee0f Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 30 Oct 2026 08:43:19 +0000 Subject: [PATCH 044/219] Backport #117802 to 26.8: Move AI functions to the end of PREWHERE --- src/Functions/FunctionBaseAI.h | 1 + src/Functions/FunctionDynamicAdaptor.h | 2 + src/Functions/FunctionVariantAdaptor.h | 2 + src/Functions/FunctionsMiscellaneous.h | 14 +++ src/Functions/IFunction.h | 11 +++ src/Functions/IFunctionAdaptors.h | 3 + src/Functions/aiEmbed.cpp | 1 + src/Functions/aiSimilarity.cpp | 1 + .../MergeTree/MergeTreeWhereOptimizer.cpp | 46 +++++++++- .../MergeTree/MergeTreeWhereOptimizer.h | 12 ++- tests/integration/test_ai_functions/test.py | 48 ++++++++++ ...16_ai_functions_last_in_prewhere.reference | 15 +++ .../05316_ai_functions_last_in_prewhere.sh | 92 +++++++++++++++++++ 13 files changed, 241 insertions(+), 7 deletions(-) create mode 100644 tests/queries/0_stateless/05316_ai_functions_last_in_prewhere.reference create mode 100755 tests/queries/0_stateless/05316_ai_functions_last_in_prewhere.sh diff --git a/src/Functions/FunctionBaseAI.h b/src/Functions/FunctionBaseAI.h index d337464ddbc3a..c04f1889ba798 100644 --- a/src/Functions/FunctionBaseAI.h +++ b/src/Functions/FunctionBaseAI.h @@ -59,6 +59,7 @@ class FunctionBaseAI : public IFunction bool isDeterministicInScopeOfQuery() const override { return false; } bool isSuitableForConstantFolding() const override { return false; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } + bool isExpensive() const override { return true; } /// Handle Nullable cols explicitly, since setting this to true may call functions with arbitrary input values bool useDefaultImplementationForNulls() const override { return false; } bool useDefaultImplementationForConstants() const override { return false; } diff --git a/src/Functions/FunctionDynamicAdaptor.h b/src/Functions/FunctionDynamicAdaptor.h index 47a60c762973a..788b6d825b127 100644 --- a/src/Functions/FunctionDynamicAdaptor.h +++ b/src/Functions/FunctionDynamicAdaptor.h @@ -72,6 +72,8 @@ class FunctionBaseDynamicAdaptor final : public IFunctionBase bool isSpatialPredicate() const override { return function_overload_resolver->isSpatialPredicate(); } + bool isExpensive() const override { return function_overload_resolver->isExpensive(); } + private: /// We remember the original IFunctionOverloadResolver to be able to build function for types inside Dynamic column. std::shared_ptr function_overload_resolver; diff --git a/src/Functions/FunctionVariantAdaptor.h b/src/Functions/FunctionVariantAdaptor.h index ccdd426a1c93c..83dbf3cd587d1 100644 --- a/src/Functions/FunctionVariantAdaptor.h +++ b/src/Functions/FunctionVariantAdaptor.h @@ -78,6 +78,8 @@ class FunctionBaseVariantAdaptor final : public IFunctionBase bool isSpatialPredicate() const override { return function_overload_resolver->isSpatialPredicate(); } + bool isExpensive() const override { return function_overload_resolver->isExpensive(); } + private: /// We remember the original IFunctionOverloadResolver to be able to build function for types inside Variant column. std::shared_ptr function_overload_resolver; diff --git a/src/Functions/FunctionsMiscellaneous.h b/src/Functions/FunctionsMiscellaneous.h index f1183243713ce..99849c951bd1a 100644 --- a/src/Functions/FunctionsMiscellaneous.h +++ b/src/Functions/FunctionsMiscellaneous.h @@ -100,6 +100,16 @@ class ExecutableFunctionExpression final : public IExecutableFunction SignaturePtr signature; }; +/// Whether a function in the body of a lambda reports `isExpensive`. Nested lambdas are covered through +/// their own `FunctionCapture`, which is a `FUNCTION` node of this body. +inline bool isLambdaBodyExpensive(const ExpressionActions & expression_actions) +{ + for (const auto & node : expression_actions.getActionsDAG().getNodes()) + if (node.type == ActionsDAG::ActionType::FUNCTION && node.function_base->isExpensive()) + return true; + return false; +} + /// Executes expression. Uses for lambda functions implementation. Can't be created from factory. class FunctionExpression final : public IFunctionBase { @@ -134,6 +144,8 @@ class FunctionExpression final : public IFunctionBase bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + bool isExpensive() const override { return isLambdaBodyExpensive(*expression_actions); } + const DataTypes & getArgumentTypes() const override { return argument_types; } const DataTypePtr & getResultType() const override { return capture->return_type; } @@ -292,6 +304,8 @@ class FunctionCapture final : public IFunctionBase return true; } + bool isExpensive() const override { return isLambdaBodyExpensive(*expression_actions); } + const DataTypes & getArgumentTypes() const override { return capture->captured_types; } const DataTypePtr & getResultType() const override { return return_type; } diff --git a/src/Functions/IFunction.h b/src/Functions/IFunction.h index 4c564f19dfecc..d3c4e66ffb6ff 100644 --- a/src/Functions/IFunction.h +++ b/src/Functions/IFunction.h @@ -220,6 +220,12 @@ class IFunctionBase : public IResolvedFunction */ virtual bool isVolumeReducing() const { return false; } + /** Returns true if the function costs far more than reading its arguments, e.g. one network + * request per row. Can be used to postpone the execution of such functions as much as possible + * (e.g. PREWHERE vs. WHERE) + */ + virtual bool isExpensive() const { return false; } + /** Returns true if this is a spatial predicate for which bbox-disjoint pruning is safe. * Specifically: if the bounding boxes of the geometry arguments are disjoint, * the function is guaranteed to return 0/false for all such rows. @@ -444,6 +450,9 @@ class IFunctionOverloadResolver : public std::enable_shared_from_thisisVolumeReducing(); } + bool isExpensive() const override { return function->isExpensive(); } + bool isInjective(const ColumnsWithTypeAndName & sample_columns) const override { return function->isInjective(sample_columns); } ComparisonOrderDomain getComparisonOrderDomain() const override @@ -138,6 +140,7 @@ class FunctionToOverloadResolverAdaptor final : public IFunctionOverloadResolver bool isDeterministicInScopeOfQuery() const override { return function->isDeterministicInScopeOfQuery(); } bool isInjective(const ColumnsWithTypeAndName & columns) const override { return function->isInjective(columns); } bool isSpatialPredicate() const override { return function->isSpatialPredicate(); } + bool isExpensive() const override { return function->isExpensive(); } String getName() const override { return function->getName(); } bool isStateful() const override { return function->isStateful(); } diff --git a/src/Functions/aiEmbed.cpp b/src/Functions/aiEmbed.cpp index 4aa2f606e8110..b227ed597e27f 100644 --- a/src/Functions/aiEmbed.cpp +++ b/src/Functions/aiEmbed.cpp @@ -78,6 +78,7 @@ class FunctionAiEmbed final : public IFunction bool isSuitableForConstantFolding() const override { return false; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } + bool isExpensive() const override { return true; } /// Handle Nullable cols explicitly, since setting this to true may call func with arbitrary input values bool useDefaultImplementationForNulls() const override { return false; } diff --git a/src/Functions/aiSimilarity.cpp b/src/Functions/aiSimilarity.cpp index 9696d9f3ca9d8..d6907497dc60d 100644 --- a/src/Functions/aiSimilarity.cpp +++ b/src/Functions/aiSimilarity.cpp @@ -104,6 +104,7 @@ class FunctionAiSimilarity final : public IFunction bool isSuitableForConstantFolding() const override { return false; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } + bool isExpensive() const override { return true; } /// Handle Nullable cols explicitly, since setting this to true may call func with arbitrary input values bool useDefaultImplementationForNulls() const override { return false; } diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index 7f5460817a718..09baddde3b971 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -393,6 +393,7 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTree bool has_invalid_column = false; bool may_use_primary_index = true; bool viable = false; + bool expensive = false; }; std::vector infos; @@ -400,7 +401,7 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTree for (const auto & conjunct : conjuncts) { - ConjunctInfo info{conjunct, {}, {}, false, true, false}; + ConjunctInfo info{conjunct, {}, {}, false, true, false, false}; collectColumns(conjunct, nullptr, table_columns, info.columns, info.has_invalid_column, info.may_use_primary_index); /// Resolve each column to its physical storage name so that subcolumns @@ -428,6 +429,8 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTree /// Do not move conditions involving all queried columns. && info.columns.size() < queried_columns.size(); + info.expensive = isExpensiveExpression(conjunct); + infos.push_back(std::move(info)); } @@ -437,6 +440,7 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTree /// same column (e.g. `map.key_k0` and `map.key_k1`) in one group, so they are moved to /// PREWHERE together and arrive adjacent for the prewhere-splitting step. /// Non-viable conjuncts stay as individual Conditions. + /// 'expensive' conjuncts each get their own group. /// /// We use a simple linear search to find groups (WHERE clauses are short). struct Group @@ -449,20 +453,29 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTree auto find_group_idx = [&](const NameSet & cols) -> std::optional { for (size_t g = 0; g < groups.size(); ++g) - if (groups[g].storage_columns == cols) + if (groups[g].storage_columns == cols && !infos[groups[g].indices.front()].expensive) return g; return std::nullopt; }; + /// Index of the group of each viable conjunct. + std::vector group_of(infos.size()); + for (size_t i = 0; i < infos.size(); ++i) { if (!infos[i].viable) continue; - auto g = find_group_idx(infos[i].storage_columns); + auto g = infos[i].expensive ? std::nullopt : find_group_idx(infos[i].storage_columns); if (!g.has_value()) + { + group_of[i] = groups.size(); groups.push_back({infos[i].storage_columns, {i}}); + } else + { + group_of[i] = *g; groups[*g].indices.push_back(i); + } } /// Emit Conditions in the original AND-chain order: @@ -491,8 +504,7 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTree else { /// Emit the whole column-set group at the position of the first conjunct. - auto g_idx = find_group_idx(info.storage_columns); - const auto & group = groups[*g_idx]; + const auto & group = groups[group_of[i]]; for (size_t idx : group.indices) emitted[idx] = true; @@ -516,6 +528,7 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTree cond.columns_size = getColumnsSize(group_columns); cond.viable = true; cond.good = group_good; + cond.expensive = info.expensive; if (where_optimizer_context.use_statistics) { @@ -687,6 +700,10 @@ std::optional MergeTreeWhereOptimizer:: if (!it->viable) break; + /// If any non-expensive conditions stay in WHERE then don't move expensive conditions to PREWHERE + if (it->expensive && std::ranges::any_of(where_conditions, [](const Condition & cond) { return !cond.expensive; })) + break; + if (!where_optimizer_context.move_all_conditions_to_prewhere) { bool moved_enough = false; @@ -878,4 +895,23 @@ NameSet MergeTreeWhereOptimizer::determineArrayJoinedNames(const ASTSelectQuery return array_joined_names; } +bool MergeTreeWhereOptimizer::isExpensiveExpression(const RPNBuilderTreeNode & node) +{ + if (!node.isFunction()) + return false; + + auto function_node = node.toFunctionNode(); + if (auto function_base = function_node.getFunctionBase(); function_base && function_base->isExpensive()) + return true; + + size_t arguments_size = function_node.getArgumentsSize(); + for (size_t i = 0; i < arguments_size; ++i) + { + if (isExpensiveExpression(function_node.getArgumentAt(i))) + return true; + } + + return false; +} + } diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.h b/src/Storages/MergeTree/MergeTreeWhereOptimizer.h index a5c115872b2a1..8089e072eb111 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.h +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.h @@ -76,6 +76,10 @@ class MergeTreeWhereOptimizer : private boost::noncopyable /// Can condition be moved to prewhere? bool viable = false; + /// Does the condition contain a function that reports `isExpensive`? + /// Such a condition is moved to PREWHERE after all other conditions. + bool expensive = false; + /// Does the condition presumably have good selectivity? bool good = false; @@ -103,10 +107,11 @@ class MergeTreeWhereOptimizer : private boost::noncopyable names += n.getColumnName(); } return fmt::format( - "Condition(exp:{} viable: {}, good: {}, min_position_in_primary_key: {}, estimated_row_count: {}, " + "Condition(exp:{} viable: {}, expensive: {}, good: {}, min_position_in_primary_key: {}, estimated_row_count: {}, " "columns_size: {}, bytes_per_rejected_row: {}, table_columns.size: {})", names, viable, + expensive, good, min_position_in_primary_key, estimated_row_count, @@ -117,7 +122,7 @@ class MergeTreeWhereOptimizer : private boost::noncopyable auto tuple() const { - return std::make_tuple(!viable, !good, -min_position_in_primary_key, bytes_per_rejected_row, table_columns.size()); + return std::make_tuple(!viable, expensive, !good, -min_position_in_primary_key, bytes_per_rejected_row, table_columns.size()); } /// Is condition a better candidate for moving to PREWHERE? @@ -183,6 +188,9 @@ class MergeTreeWhereOptimizer : private boost::noncopyable static NameSet determineArrayJoinedNames(const ASTSelectQuery & select); + /// Whether the expression contains a function that reports `isExpensive`. + static bool isExpensiveExpression(const RPNBuilderTreeNode & node); + ConditionSelectivityEstimatorPtr estimator; const NameSet table_columns; diff --git a/tests/integration/test_ai_functions/test.py b/tests/integration/test_ai_functions/test.py index 8fefed37a4a0b..a84b2af3cd87d 100644 --- a/tests/integration/test_ai_functions/test.py +++ b/tests/integration/test_ai_functions/test.py @@ -777,6 +777,54 @@ def test_filter_where(started_cluster): assert lines == ["also good", "great product"] +PREWHERE_SETTINGS = { + "enable_analyzer": 1, + "optimize_move_to_prewhere": 1, + "move_all_conditions_to_prewhere": 1, + "allow_reorder_prewhere_conditions": 1, +} + + +def count_filter_calls(prefix, where): + """Runs `SELECT count()` over 16 rows, half of them with `flag = 1`, and returns the number of + rows and of AI requests.""" + instance.query( + "CREATE TABLE IF NOT EXISTS prewhere_ai (flag UInt8, text String) ENGINE = MergeTree ORDER BY tuple()" + ) + instance.query("TRUNCATE TABLE prewhere_ai") + instance.query( + "INSERT INTO prewhere_ai SELECT number % 2, 'row ' || toString(number) FROM numbers(16)" + ) + qid = unique_query_id(prefix) + rows = instance.query( + f"SELECT count() FROM prewhere_ai WHERE {where}", + query_id=qid, + settings=PREWHERE_SETTINGS, + ) + return int(rows), int(get_profile_events(qid)["api_calls"]) + + +def test_filter_prewhere_after_cheap_condition(started_cluster): + """The AI condition is moved to PREWHERE after `flag = 1`, so only the rows passing it are sent.""" + rows, api_calls = count_filter_calls( + "filter_prewhere_after_cheap", + "aiFilter(text, 'matches', map('credentials', 'ai_mock')) AND flag = 1", + ) + assert rows == 8 + assert api_calls == 8 + + +def test_filter_prewhere_behind_condition_left_in_where(started_cluster): + """`length(text) = flag` uses every queried column, so it stays in WHERE. The AI condition + stays there too and is evaluated only for the rows passing it, which are none.""" + rows, api_calls = count_filter_calls( + "filter_prewhere_behind_where", + "length(text) = flag AND aiFilter(text, 'matches', map('credentials', 'ai_mock'))", + ) + assert rows == 0 + assert api_calls == 0 + + def test_filter_truncated_response_throw(started_cluster): """aiFilter shares the FunctionBaseAI rejection path. A provider-signalled incomplete reply (here `finish_reason="length"`) is an error under the default `ai_function_throw_on_error=1`, diff --git a/tests/queries/0_stateless/05316_ai_functions_last_in_prewhere.reference b/tests/queries/0_stateless/05316_ai_functions_last_in_prewhere.reference new file mode 100644 index 0000000000000..ef26043724e37 --- /dev/null +++ b/tests/queries/0_stateless/05316_ai_functions_last_in_prewhere.reference @@ -0,0 +1,15 @@ +aiGenerate: in PREWHERE, last +aiClassify: in PREWHERE, last +aiExtract: in PREWHERE, last +aiTranslate: in PREWHERE, last +aiFilter: in PREWHERE, last +aiRedact: in PREWHERE, last +aiEmbed: in PREWHERE, last +aiSimilarity: in PREWHERE, last +AI inside a lambda: in PREWHERE, last +AI and a cheap condition on the same column: in PREWHERE, last +AI inside OR: in PREWHERE, last +AI condition alone: in PREWHERE, last +AI behind a condition left in WHERE: not in PREWHERE +Reordering disabled, AI written first: in PREWHERE, not last +Reordering disabled, AI written last: in PREWHERE, last diff --git a/tests/queries/0_stateless/05316_ai_functions_last_in_prewhere.sh b/tests/queries/0_stateless/05316_ai_functions_last_in_prewhere.sh new file mode 100755 index 0000000000000..3d2767230c09a --- /dev/null +++ b/tests/queries/0_stateless/05316_ai_functions_last_in_prewhere.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# An AI function issues one request per row. PREWHERE evaluates its conditions as read +# steps in order, each on the rows that passed the previous ones, so a condition with an +# AI function is moved to PREWHERE after all other conditions. +# +# The order is taken from the PREWHERE filter column of `EXPLAIN json = 1`, which lists the +# conditions as `and(...)` arguments in their final order. The AI condition is last if the +# AI function appears after every cheap condition (`flag = 1`, `text LIKE`). +# +# Only `EXPLAIN` is used, so no HTTP call is made. + +CHAT_CREDS="ai_creds_${CLICKHOUSE_DATABASE}" +VEC_CREDS="ai_vec_creds_${CLICKHOUSE_DATABASE}" + +$CLICKHOUSE_CLIENT -q " + CREATE TABLE tab (flag UInt8, text String, other String) ENGINE = MergeTree ORDER BY tuple(); + INSERT INTO tab SELECT number % 2, 'row ' || toString(number), 'x' FROM numbers(16); + CREATE NAMED COLLECTION ${CHAT_CREDS} AS provider = 'openai', endpoint = 'http://localhost:1/v1/chat/completions', model = 'test-model', api_key = 'test-key'; + CREATE NAMED COLLECTION ${VEC_CREDS} AS provider = 'openai', endpoint = 'http://localhost:1/v1/embeddings', api_key = 'test-key'; +" + +CHAT="map('credentials', '${CHAT_CREDS}')" +VEC="map('credentials', '${VEC_CREDS}')" + +# Offset of the last match of a pattern in a string, empty if there is none. +function last_offset() +{ + grep -boE "$2" <<< "$1" | tail -1 | cut -d: -f1 +} + +# Prints whether the AI condition is in PREWHERE and whether it comes after the cheap ones. +# The settings that decide whether and how conditions are moved are pinned rather than left +# to test randomization. +function ai_position() +{ + local label=$1 where=$2 allow_reorder=${3:-1} + local column + column=$($CLICKHOUSE_CLIENT -q " + EXPLAIN json = 1, actions = 1 SELECT other FROM tab WHERE ${where} + SETTINGS enable_analyzer = 1, optimize_move_to_prewhere = 1, query_plan_enable_optimizations = 1, + move_all_conditions_to_prewhere = 1, allow_reorder_prewhere_conditions = ${allow_reorder}" \ + | grep -oE '"Prewhere filter column": "[^"]*"') + + local ai cheap + ai=$(grep -boE 'ai[A-Z]' <<< "$column" | head -1 | cut -d: -f1) + if [[ -z $ai ]]; then echo "${label}: not in PREWHERE"; return; fi + for pattern in 'equals\(__table1\.flag' 'like\(__table1\.text'; do + cheap=$(last_offset "$column" "$pattern") + if [[ -n $cheap ]] && (( cheap > ai )); then echo "${label}: in PREWHERE, not last"; return; fi + done + echo "${label}: in PREWHERE, last" +} + +ai_position 'aiGenerate' "aiGenerate(text, ${CHAT}) != '' AND flag = 1" +ai_position 'aiClassify' "aiClassify(text, ['a', 'b'], ${CHAT}) = 'a' AND flag = 1" +ai_position 'aiExtract' "aiExtract(text, 'the topic', ${CHAT}) != '' AND flag = 1" +ai_position 'aiTranslate' "aiTranslate(text, 'French', ${CHAT}) != '' AND flag = 1" +ai_position 'aiFilter' "aiFilter(text, 'matches', ${CHAT}) AND flag = 1" +ai_position 'aiRedact' "aiRedact(text, ['email'], ${CHAT}) != '' AND flag = 1" +ai_position 'aiEmbed' "length(aiEmbed(text, 'test-model', ${VEC})) != 0 AND flag = 1" +ai_position 'aiSimilarity' "aiSimilarity(text, 'reference', 'test-model', ${VEC}) != 0 AND flag = 1" + +# A lambda body is not among the condition's children, so the capture reports the trait. +ai_position 'AI inside a lambda' "arrayExists(x -> aiFilter(x, 'matches', ${CHAT}), [text]) AND flag = 1" + +# Conditions on the same columns are grouped and moved together. The AI condition is kept +# out of the group, otherwise it would move ahead of `flag = 1` with `text LIKE`. +ai_position 'AI and a cheap condition on the same column' "aiFilter(text, 'matches', ${CHAT}) AND text LIKE '%1%' AND flag = 1" + +# An `OR` is not split, so the whole disjunction is expensive and moved last. +ai_position 'AI inside OR' "(flag = 1 OR aiFilter(text, 'matches', ${CHAT})) AND text LIKE '%1%'" + +# With no other condition the AI condition is still moved. +ai_position 'AI condition alone' "aiFilter(text, 'matches', ${CHAT})" + +# A condition using every queried column is not moved, so the AI condition stays in WHERE behind it. +ai_position 'AI behind a condition left in WHERE' "length(text) = flag + length(other) AND aiFilter(text, 'matches', ${CHAT})" + +# Without reordering the written order is kept, the AI condition included. +ai_position 'Reordering disabled, AI written first' "aiFilter(text, 'matches', ${CHAT}) AND flag = 1" 0 +ai_position 'Reordering disabled, AI written last' "flag = 1 AND aiFilter(text, 'matches', ${CHAT})" 0 + +$CLICKHOUSE_CLIENT -q " + DROP NAMED COLLECTION ${CHAT_CREDS}; + DROP NAMED COLLECTION ${VEC_CREDS}; + DROP TABLE tab; +" From 9020c4d46fb2003f2dc3f57e8ddfd759cbaed9cc Mon Sep 17 00:00:00 2001 From: George Larionov Date: Fri, 30 Oct 2026 13:40:07 +0000 Subject: [PATCH 045/219] Pin `query_plan_optimize_prewhere` in `05316_ai_functions_last_in_prewhere` `clickhouse-test` randomizes it on this branch; with `0` nothing is moved to PREWHERE and the test fails. Co-Authored-By: Claude Opus 5.5 --- .../queries/0_stateless/05316_ai_functions_last_in_prewhere.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/05316_ai_functions_last_in_prewhere.sh b/tests/queries/0_stateless/05316_ai_functions_last_in_prewhere.sh index 3d2767230c09a..e9c830efb2d0d 100755 --- a/tests/queries/0_stateless/05316_ai_functions_last_in_prewhere.sh +++ b/tests/queries/0_stateless/05316_ai_functions_last_in_prewhere.sh @@ -42,7 +42,7 @@ function ai_position() local column column=$($CLICKHOUSE_CLIENT -q " EXPLAIN json = 1, actions = 1 SELECT other FROM tab WHERE ${where} - SETTINGS enable_analyzer = 1, optimize_move_to_prewhere = 1, query_plan_enable_optimizations = 1, + SETTINGS enable_analyzer = 1, optimize_move_to_prewhere = 1, query_plan_enable_optimizations = 1, query_plan_optimize_prewhere = 1, move_all_conditions_to_prewhere = 1, allow_reorder_prewhere_conditions = ${allow_reorder}" \ | grep -oE '"Prewhere filter column": "[^"]*"') From 5b92f1d281ee2abe4c867b87849c468db161947c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 30 Oct 2026 14:00:38 +0000 Subject: [PATCH 046/219] Backport #127797 to 26.8: Fix flaky test `05055_format_schema_source_query_background_access` when the session date is ahead of the server date --- .../05055_format_schema_source_query_background_access.sh | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/05055_format_schema_source_query_background_access.sh b/tests/queries/0_stateless/05055_format_schema_source_query_background_access.sh index ff237fd5db5b5..20fc7c819b314 100755 --- a/tests/queries/0_stateless/05055_format_schema_source_query_background_access.sh +++ b/tests/queries/0_stateless/05055_format_schema_source_query_background_access.sh @@ -14,7 +14,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) BG_MESSAGE="M_bg_${CLICKHOUSE_DATABASE}" FG_MESSAGE="M_fg_${CLICKHOUSE_DATABASE}" -START=$(${CLICKHOUSE_CLIENT} --query "SELECT toString(now64(6))") +START=$(${CLICKHOUSE_CLIENT} --query "SELECT toUnixTimestamp64Micro(now64(6))") ${CLICKHOUSE_CLIENT} --query " CREATE TABLE dest_bg (s String) ENGINE = File(ProtobufSingle) @@ -33,10 +33,11 @@ do ${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS text_log" # The logger is checked too because this query is itself written to the text log, so otherwise # an iteration would match the previous one instead of the background exception. + # `event_date` is in the server time zone, and the session one is randomized. refused=$(${CLICKHOUSE_CLIENT} --query " SELECT count() > 0 FROM system.text_log - WHERE event_date >= toDate('${START}') - AND event_time_microseconds >= toDateTime64('${START}', 6) + WHERE event_date >= yesterday() + AND event_time_microseconds >= fromUnixTimestamp64Micro(${START}) AND logger_name LIKE '%StorageBuffer%' AND message LIKE '%can only be executed on behalf of a user%'") [ "$refused" = "1" ] && break From 78c8782e65c0c05f0063ca1215d48d9f1aee083b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 30 Oct 2026 16:07:01 +0000 Subject: [PATCH 047/219] Backport #126345 to 26.8: Fix flaky test `03821_drop_workload_during_query_race` --- .../0_stateless/03821_drop_workload_during_query_race.sh | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/03821_drop_workload_during_query_race.sh b/tests/queries/0_stateless/03821_drop_workload_during_query_race.sh index e94aca50620da..95280b1f9347e 100755 --- a/tests/queries/0_stateless/03821_drop_workload_during_query_race.sh +++ b/tests/queries/0_stateless/03821_drop_workload_during_query_race.sh @@ -32,13 +32,16 @@ CALL_TIMEOUT=30 function thread_query() { local TIMELIMIT=$((SECONDS+TIMEOUT)) + local out while [ $SECONDS -lt "$TIMELIMIT" ] do # Run a short query that uses CPU scheduling # Use smaller numbers so queries finish quickly - # Ignore expected errors when workload is dropped during query - timeout "$CALL_TIMEOUT" $CLICKHOUSE_CLIENT --format Null -q "SELECT sum(number) FROM numbers(100000) SETTINGS workload = '$WORKLOAD_NAME'" 2>&1 \ - | { grep -v -e "RESOURCE_ACCESS_DENIED" -e "INVALID_SCHEDULER_NODE" -e "There is no resource" -e "^$" || true; } + # A dropped workload fails the query with an expected error; its report spans several lines, so drop it whole + out=$(timeout "$CALL_TIMEOUT" $CLICKHOUSE_CLIENT --format Null -q "SELECT sum(number) FROM numbers(100000) SETTINGS workload = '$WORKLOAD_NAME'" 2>&1) + if [ -n "$out" ] && ! grep -q -e "RESOURCE_ACCESS_DENIED" -e "INVALID_SCHEDULER_NODE" -e "There is no resource" <<< "$out"; then + echo "$out" + fi done } From 8021f41368fe1713041d8ae79d1ed54edb94b0f8 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 30 Oct 2026 23:43:39 +0000 Subject: [PATCH 048/219] Backport #122484 to 26.8: Issue AI function requests concurrently from a server-wide thread pool --- .../regular-functions/ai-functions.mdx | 30 ++ programs/local/LocalServer.cpp | 8 + programs/server/Server.cpp | 13 + src/Common/CurrentMetrics.cpp | 3 + src/Common/ProfileEvents.cpp | 3 + src/Common/setThreadName.h | 1 + src/Core/ServerSettings.cpp | 15 + src/Core/Settings.cpp | 18 +- src/Core/SettingsChangesHistory.cpp | 3 +- src/Functions/AI/AIQuotaTracker.cpp | 20 +- src/Functions/AI/AIQuotaTracker.h | 16 +- src/Functions/AI/AIRequestExecutor.cpp | 269 +++++++++++ src/Functions/AI/AIRequestExecutor.h | 41 ++ src/Functions/AI/IAIProvider.h | 3 + src/Functions/FunctionBaseAI.cpp | 441 ++++++++---------- src/Functions/FunctionBaseAI.h | 36 +- src/Functions/aiEmbed.cpp | 36 +- src/Functions/aiSimilarity.cpp | 38 +- src/IO/SharedThreadPools.h | 3 +- src/Interpreters/Context.cpp | 5 + .../test_ai_functions/mock_ai_server.py | 51 +- tests/integration/test_ai_functions/test.py | 325 ++++++++++++- .../0_stateless/03300_ai_functions.reference | 2 +- .../0_stateless/03300_ai_functions.sql | 3 + ...42_ai_functions_named_collection_access.sh | 4 +- ...functions_compatibility_defaults.reference | 4 +- ...28_ai_functions_compatibility_defaults.sql | 13 +- ...i_functions_retry_setting_limits.reference | 2 + ...5394_ai_functions_retry_setting_limits.sql | 31 ++ 29 files changed, 1083 insertions(+), 354 deletions(-) create mode 100644 src/Functions/AI/AIRequestExecutor.cpp create mode 100644 src/Functions/AI/AIRequestExecutor.h create mode 100644 tests/queries/0_stateless/05394_ai_functions_retry_setting_limits.reference create mode 100644 tests/queries/0_stateless/05394_ai_functions_retry_setting_limits.sql diff --git a/docs/reference/functions/regular-functions/ai-functions.mdx b/docs/reference/functions/regular-functions/ai-functions.mdx index 159c6f3ec20fa..edf769b482e0f 100644 --- a/docs/reference/functions/regular-functions/ai-functions.mdx +++ b/docs/reference/functions/regular-functions/ai-functions.mdx @@ -28,6 +28,7 @@ All functions are sharing a common infrastructure that provides: - **Quota enforcement**: Per-query limits on tokens ([`ai_function_max_input_tokens_per_query`](/reference/settings/session-settings/ai-function#ai_function_max_input_tokens_per_query), [`ai_function_max_output_tokens_per_query`](/reference/settings/session-settings/ai-function#ai_function_max_output_tokens_per_query)) and API calls ([`ai_function_max_api_calls_per_query`](/reference/settings/session-settings/ai-function#ai_function_max_api_calls_per_query)). - **Retry with backoff**: Transient failures are retried ([`ai_function_max_retries`](/reference/settings/session-settings/ai-function#ai_function_max_retries)) with exponential backoff ([`ai_function_retry_initial_delay_ms`](/reference/settings/session-settings/ai-function#ai_function_retry_initial_delay_ms)). +- **Concurrent requests**: Requests are issued from a server-wide thread pool, several at a time (controlled by [`ai_function_max_concurrent_requests_per_thread`](/reference/settings/session-settings/ai-function#ai_function_max_concurrent_requests_per_thread)). ## Configuration {#configuration} @@ -156,6 +157,35 @@ By default, AI functions reject an `endpoint` that would send data in cleartext Note that in either case the provider receives the input data in cleartext after TLS termination; TLS protects the data only on the network path between the server and the provider. +### Throughput {#throughput} + +The AI functions differ from most traditional ClickHouse functions in that their throughput is almost entirely bound by the remote endpoint they send requests to rather than local CPU or I/O. Because of this they benefit significantly from increased concurrency above the standard `max_threads` used throughout ClickHouse. For this reason the requests are issued from a dedicated server-wide thread pool. Its size (`ai_function_max_thread_pool_size`, a server setting) caps how many AI provider requests the whole server has in flight at once, across every query. + +[`ai_function_max_concurrent_requests_per_thread`](/reference/settings/session-settings/ai-function#ai_function_max_concurrent_requests_per_thread) bounds each query thread's share of that pool, so a query running on several threads (at most `max_threads`) can have up to that many requests in flight per thread. To stop queries from raising it, put a ceiling on the setting with a [settings profile constraint](/concepts/features/configuration/settings/constraints-on-settings). The ceiling is still per thread, so a query can have up to the ceiling times its number of threads in flight. For example: + +```xml + + + + + 16 + + + + +``` + +The default is `8`. A value above `1` loosens two guarantees: + +- A request's token usage is only known once its response arrives, so [`ai_function_max_input_tokens_per_query`](/reference/settings/session-settings/ai-function#ai_function_max_input_tokens_per_query) and [`ai_function_max_output_tokens_per_query`](/reference/settings/session-settings/ai-function#ai_function_max_output_tokens_per_query) may overshoot by one request's worth per request in flight, rather than by one request's worth in total. +- A wave of requests is dispatched before any of their errors is seen, so a query that fails on its first row may still have issued, and been billed for, as many requests as the concurrency allows. + +[`ai_function_max_api_calls_per_query`](/reference/settings/session-settings/ai-function#ai_function_max_api_calls_per_query) is unaffected and stays an exact cap, because a slot is reserved before each request is dispatched. + + +For a materialized view, the settings must go on the `INSERT` that feeds the view's source table, not on the `CREATE MATERIALIZED VIEW` statement: the quotas are read from the top-level query, and a `SETTINGS` clause in a view definition is ignored. + + ## Supported providers {#supported-providers} | Provider | `provider` value | Chat functions | Notes | diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 99ee12067d987..42e7a57dd6a86 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -215,6 +215,9 @@ namespace ServerSetting extern const ServerSettingsUInt64 max_iceberg_manifest_decode_thread_pool_size; extern const ServerSettingsUInt64 max_iceberg_manifest_decode_thread_pool_free_size; extern const ServerSettingsUInt64 iceberg_manifest_decode_thread_pool_queue_size; + extern const ServerSettingsNonZeroUInt64 ai_function_max_thread_pool_size; + extern const ServerSettingsUInt64 ai_function_max_thread_pool_free_size; + extern const ServerSettingsUInt64 ai_function_thread_pool_queue_size; extern const ServerSettingsUInt64 page_cache_history_window_ms; extern const ServerSettingsString page_cache_policy; extern const ServerSettingsDouble page_cache_size_ratio; @@ -470,6 +473,11 @@ void LocalServer::initialize(Poco::Util::Application & self) server_settings[ServerSetting::max_iceberg_manifest_decode_thread_pool_size], server_settings[ServerSetting::max_iceberg_manifest_decode_thread_pool_free_size], server_settings[ServerSetting::iceberg_manifest_decode_thread_pool_queue_size]); + + getAIRequestThreadPool().initialize( + server_settings[ServerSetting::ai_function_max_thread_pool_size], + server_settings[ServerSetting::ai_function_max_thread_pool_free_size], + server_settings[ServerSetting::ai_function_thread_pool_queue_size]); } diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 9884fc325c09d..c57083a0bb06e 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -428,6 +428,9 @@ namespace ServerSetting extern const ServerSettingsUInt64 max_iceberg_manifest_decode_thread_pool_size; extern const ServerSettingsUInt64 max_iceberg_manifest_decode_thread_pool_free_size; extern const ServerSettingsUInt64 iceberg_manifest_decode_thread_pool_queue_size; + extern const ServerSettingsNonZeroUInt64 ai_function_max_thread_pool_size; + extern const ServerSettingsUInt64 ai_function_max_thread_pool_free_size; + extern const ServerSettingsUInt64 ai_function_thread_pool_queue_size; extern const ServerSettingsUInt64 page_cache_history_window_ms; extern const ServerSettingsString page_cache_policy; extern const ServerSettingsDouble page_cache_size_ratio; @@ -2001,6 +2004,11 @@ try server_settings[ServerSetting::max_iceberg_manifest_decode_thread_pool_free_size], server_settings[ServerSetting::iceberg_manifest_decode_thread_pool_queue_size]); + getAIRequestThreadPool().initialize( + server_settings[ServerSetting::ai_function_max_thread_pool_size], + server_settings[ServerSetting::ai_function_max_thread_pool_free_size], + server_settings[ServerSetting::ai_function_thread_pool_queue_size]); + std::string path_str = getCanonicalPath(String(server_settings[ServerSetting::path]), original_working_directory); fs::path path = path_str; @@ -3053,6 +3061,11 @@ try new_server_settings[ServerSetting::max_iceberg_manifest_decode_thread_pool_free_size], new_server_settings[ServerSetting::iceberg_manifest_decode_thread_pool_queue_size]); + getAIRequestThreadPool().reloadConfiguration( + new_server_settings[ServerSetting::ai_function_max_thread_pool_size], + new_server_settings[ServerSetting::ai_function_max_thread_pool_free_size], + new_server_settings[ServerSetting::ai_function_thread_pool_queue_size]); + global_context->setMergeWorkload(new_server_settings[ServerSetting::merge_workload]); global_context->setMutationWorkload(new_server_settings[ServerSetting::mutation_workload]); global_context->setThrowOnUnknownWorkload(new_server_settings[ServerSetting::throw_on_unknown_workload]); diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index 8be37f19d9536..b3e428d1171fb 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -218,6 +218,9 @@ M(DistributedIndexAnalysisThreads, "Number of threads in the thread pool for distributed index analysis.") \ M(DistributedIndexAnalysisThreadsActive, "Number of threads in the thread pool for distributed index analysis running a task.") \ M(DistributedIndexAnalysisThreadsScheduled, "Number of queued or active jobs in the distributed index analysis thread pool.") \ + M(AIRequestThreads, "Number of threads in the AI request thread pool.") \ + M(AIRequestThreadsActive, "Number of threads in the AI request thread pool running a task.") \ + M(AIRequestThreadsScheduled, "Number of queued or active jobs in the AI request thread pool.") \ M(ObjectStorageQueueRegisteredServers, "Number of registered servers in StorageS3(Azure)Queue")\ M(IcebergCatalogThreads, "Number of threads in the IcebergCatalog thread pool.") \ M(IcebergCatalogThreadsActive, "Number of threads in the IcebergCatalog thread pool running a task.") \ diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 3a17b07f8f6a7..76ff1b08cd0a7 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -1663,6 +1663,9 @@ The server successfully detected this situation and will download merged part fr M(AIInputTokens, "Total prompt tokens consumed across all AI function calls in the query.", ValueType::Number) \ M(AIOutputTokens, "Total completion tokens consumed across all AI function calls in the query.", ValueType::Number) \ M(AIAPICalls, "Number of HTTP requests dispatched to AI providers.", ValueType::Number) \ + M(AIAPICallsRetried, "Number of AI provider requests retried after a transient failure.", ValueType::Number) \ + M(AIAPICallsFailed, "Number of AI provider requests that failed after all retries.", ValueType::Number) \ + M(AIAPICallsThrottled, "Number of AI provider requests rejected with HTTP 429 (Too Many Requests).", ValueType::Number) \ M(AIRowsProcessed, "Number of rows that received an AI result.", ValueType::Number) \ M(AIRowsSkipped, "Number of rows that received a default value due to quota or error.", ValueType::Number) \ \ diff --git a/src/Common/setThreadName.h b/src/Common/setThreadName.h index 6f9b59cd3f0c6..cf5f56738fcd9 100644 --- a/src/Common/setThreadName.h +++ b/src/Common/setThreadName.h @@ -9,6 +9,7 @@ namespace DB M(AGGREGATED_ZOOKEEPER_LOG, "AggrZooLog") \ M(AGGREGATOR_DESTRUCTION, "AggregDestruct") \ M(AGGREGATOR_POOL, "AggregatorPool") \ + M(AI_REQUEST, "AIRequest") \ M(ARROW_FILE, "ArrowFile") \ M(ARROW_FLIGHT, "ArrowFlight") \ M(ARROW_FLIGHT_EXPR, "ArrowFlightExpr") \ diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index 04408d509ca3e..8aeef728f2cb1 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -193,6 +193,15 @@ Maximum number of idle standby threads to keep in the thread pool for decoding I )", 0) \ DECLARE(UInt64, iceberg_manifest_decode_thread_pool_queue_size, 10000, R"( The maximum number of jobs that can be scheduled on the thread pool for decoding Iceberg data manifest files. +)", 0) \ + DECLARE(NonZeroUInt64, ai_function_max_thread_pool_size, 1000, R"( +Maximum total number of threads to use for requests to AI providers. One thread holds one request, so this caps how many AI provider requests the whole server has in flight at once, across all queries. [`ai_function_max_concurrent_requests_per_thread`](/reference/settings/session-settings/ai-function#ai_function_max_concurrent_requests_per_thread) bounds each query thread's share of it. +)", 0) \ + DECLARE(UInt64, ai_function_max_thread_pool_free_size, 0, R"( +Maximum number of idle standby threads to keep in the thread pool for requests to AI providers. +)", 0) \ + DECLARE(UInt64, ai_function_thread_pool_queue_size, 10'000, R"( +The maximum number of jobs that can be scheduled on the thread pool for requests to AI providers. )", 0) \ DECLARE(UInt64, max_format_parsing_thread_pool_free_size, 0, R"( Maximum number of idle standby threads to keep in the thread pool for parsing input. @@ -3670,6 +3679,12 @@ ChangeableSettingsMap collectChangeableServerSettings(ContextPtr context) {getIcebergManifestDecodeThreadPool().isInitialized() ? std::to_string(getIcebergManifestDecodeThreadPool().get().getMaxFreeThreads()) : "0", ChangeableWithoutRestart::Yes}}, {"iceberg_manifest_decode_thread_pool_queue_size", {getIcebergManifestDecodeThreadPool().isInitialized() ? std::to_string(getIcebergManifestDecodeThreadPool().get().getQueueSize()) : "0", ChangeableWithoutRestart::Yes}}, + {"ai_function_max_thread_pool_size", + {getAIRequestThreadPool().isInitialized() ? std::to_string(getAIRequestThreadPool().get().getMaxThreads()) : "0", ChangeableWithoutRestart::Yes}}, + {"ai_function_max_thread_pool_free_size", + {getAIRequestThreadPool().isInitialized() ? std::to_string(getAIRequestThreadPool().get().getMaxFreeThreads()) : "0", ChangeableWithoutRestart::Yes}}, + {"ai_function_thread_pool_queue_size", + {getAIRequestThreadPool().isInitialized() ? std::to_string(getAIRequestThreadPool().get().getQueueSize()) : "0", ChangeableWithoutRestart::Yes}}, {"abort_on_logical_error", {std::to_string(DB::abort_on_logical_error), ChangeableWithoutRestart::Yes}}, diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 3beb8564864b8..332099c64c708 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -8782,11 +8782,11 @@ Control the minimum number of payload columns from the left side required for en DECLARE(UInt64, ai_function_request_timeout_sec, 60, R"( Timeout in seconds for individual HTTP requests made by AI functions (AI chat completions and embedding API calls). If a request does not complete within this time, it is considered failed and may be retried according to `ai_function_max_retries`. )", BETA) \ - DECLARE(UInt64, ai_function_max_retries, 1, R"( -Maximum number of retry attempts for transient errors per individual API request. Each retry uses exponential backoff starting from `ai_function_retry_initial_delay_ms`. + DECLARE(UInt64, ai_function_max_retries, 3, R"( +Maximum number of retry attempts for transient errors per individual API request. Each retry uses exponential backoff starting from `ai_function_retry_initial_delay_ms`. At most 5. )", BETA) \ DECLARE(UInt64, ai_function_retry_initial_delay_ms, 1000, R"( -Initial delay in milliseconds before the first retry of a failed AI function API request. The delay doubles on each subsequent attempt (exponential backoff). For example, with default settings: 1000ms, 2000ms, 4000ms. +Initial delay in milliseconds before the first retry of a failed AI function API request. The delay doubles on each subsequent attempt (exponential backoff). For example, with default settings: 1000ms, 2000ms, 4000ms. At most 10000. )", BETA) \ DECLARE(Bool, ai_function_throw_on_error, true, R"( If true (default), an AI function call that fails permanently after exhausting all retries aborts the query with an exception. If false, the failed row receives the default value for the column type (empty string for String) and processing continues. @@ -8809,6 +8809,18 @@ If true (default), exceeding an AI function quota limit (`ai_function_max_input_ )", BETA) \ DECLARE(NonZeroUInt64, ai_function_embedding_max_batch_size, 100, R"( Maximum number of texts to include in a single HTTP request made by the embedding functions (`aiEmbed`, `aiSimilarity`). Texts are grouped into batches of this size to reduce API call overhead. For example, 500 unique texts with a batch size of 100 result in 5 HTTP requests. +)", BETA) \ + DECLARE(NonZeroUInt64, ai_function_max_concurrent_requests_per_thread, 8, R"( +Maximum number of provider requests one query thread has in flight at the same time. `1` issues requests one at a time. + +A query running on several threads (at most `max_threads`) can have up to this many requests in flight per thread. Use a [settings profile constraint](/concepts/features/configuration/settings/constraints-on-settings) (`...`) to put a ceiling on it that a query cannot raise. The ceiling is still per thread. + +A value above `1` loosens two guarantees: + +- A request's token usage is only known once its response arrives, so the token quotas (`ai_function_max_input_tokens_per_query`, `ai_function_max_output_tokens_per_query`) may overshoot by one request's worth per request in flight - that is, by up to this many requests' worth rather than by one. +- A whole wave of requests is dispatched before any of their errors is seen, so a query that fails on its first row may still have issued, and been billed for, up to this many requests. + +`ai_function_max_api_calls_per_query` is unaffected and stays an exact cap, because a slot is reserved before each request is dispatched. )", BETA) \ DECLARE(String, ai_function_text_default_credentials, "", R"( Name of the named collection used by the text AI functions (`aiGenerate`, `aiClassify`, `aiFilter`, `aiExtract`, `aiTranslate`, `aiRedact`) when the call does not pass `credentials` in its parameter map. Empty means no default: such calls must pass `credentials` explicitly. A chat-completions endpoint differs from an embeddings one, so this is separate from `ai_function_embedding_default_credentials`. diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index faefbc68af3a8..b6ee1b4351242 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -55,7 +55,8 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"validate_group_by_all_key_types", true, true, "The validation of the key types that `GROUP BY ALL` expands the `SELECT` expressions into is kept under `compatibility` with 26.7: the previous value is deliberately equal to the new one, because 26.7 already rejected such a key and only a version before 26.7 restores the earlier acceptance."}, {"allow_executable_tables", true, true, "New setting to disable reading through the `executable` table function and from `Executable` and `ExecutablePool` tables."}, {"allow_experimental_ai_functions", false, false, "The setting is obsolete, AI functions are beta now and enabled by default."}, - {"ai_function_max_retries", 0, 1, "Retry a transient API error once by default, so a single 429 or 5xx from the provider does not fail the query."}, + {"ai_function_max_retries", 0, 3, "Retry a transient API error up to three times by default, so a 429 or 5xx from the provider does not fail the query and an overloaded rate-limited provider is retried under concurrent requests."}, + {"ai_function_max_concurrent_requests_per_thread", 1, 8, "New setting"}, {"adaptive_aggregator_freeze_threshold_bytes", 4194304, 4194304, "New setting bounding the adaptive aggregator's frozen local tables in bytes, whichever of it and the key-count threshold is reached first; 0 disables the byte bound."}, {"query_plan_aggregation_bucket_top_k", false, true, "New setting to toggle the plan optimization that materializes only each two-level bucket's best n groups when a final aggregation feeds ORDER BY over its outputs with LIMIT n and the per-bucket selection is provably exact."}, {"enable_group_by_top_k_optimization", false, true, "New setting to control the TopK filtering optimization during aggregation in `GROUP BY key ORDER BY key LIMIT N` queries."}, diff --git a/src/Functions/AI/AIQuotaTracker.cpp b/src/Functions/AI/AIQuotaTracker.cpp index 247021e2636ae..0d10a70c0db42 100644 --- a/src/Functions/AI/AIQuotaTracker.cpp +++ b/src/Functions/AI/AIQuotaTracker.cpp @@ -8,6 +8,7 @@ namespace DB namespace ErrorCodes { extern const int LIMIT_EXCEEDED; + extern const int QUERY_WAS_CANCELLED; extern const int TIMEOUT_EXCEEDED; } @@ -43,15 +44,28 @@ bool AIQuotaTracker::quotasExceededLocked() void AIQuotaTracker::throwIfCancelled() const { - auto status = query_status.lock(); - if (!status) + if (!isCancelled()) return; /// `checkTimeLimit` throws on `KILL QUERY` and when `max_execution_time` is exceeded. /// Under `timeout_overflow_mode = 'break'`, it returns false instead of throwing. /// A function cannot return fewer rows than it received, so throw in that case too. - if (!status->checkTimeLimit()) + if (auto status = query_status.lock(); status && !status->checkTimeLimit()) throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Timeout exceeded: elapsed time limit reached in an AI function"); + + /// The query has finished, and a request it submitted is still queued. + throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Query was cancelled"); +} + +bool AIQuotaTracker::isCancelled() const +{ + if (cancelled) + return true; + + auto status = query_status.lock(); + if (status && !status->checkTimeLimitSoft()) + cancelled = true; + return cancelled; } bool AIQuotaTracker::checkQuotas() diff --git a/src/Functions/AI/AIQuotaTracker.h b/src/Functions/AI/AIQuotaTracker.h index c9836c29a6491..b2d018863b857 100644 --- a/src/Functions/AI/AIQuotaTracker.h +++ b/src/Functions/AI/AIQuotaTracker.h @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -37,8 +38,8 @@ class AIQuotaTracker {} /// Check the token quotas (and the sticky exceeded flag). Returns true if a limit is met or - /// exceeded, false otherwise. Should be called before issuing an API call. The API-call limit is - /// enforced separately by `recordApiCall`. + /// exceeded, false otherwise. Lets a caller skip building and submitting a request that + /// `recordApiCall` would refuse anyway. The API-call limit is enforced separately by `recordApiCall`. bool checkQuotas(); /// Count one outbound API call against the request quota, only while under the limit. Should be @@ -53,6 +54,11 @@ class AIQuotaTracker /// so this is kept separate and called only after the response is parsed. void recordTokens(UInt64 in_tokens, UInt64 out_tokens); + /// Throws if the query was killed or exceeded its time limit. + void throwIfCancelled() const; + + /// Whether the query was killed or exceeded its time limit. Does not throw. + bool isCancelled() const; private: const UInt64 max_input_tokens; @@ -61,6 +67,9 @@ class AIQuotaTracker const bool throw_on_quota_exceeded; const std::weak_ptr query_status; + /// Sticky: used to check for requests submitted by a cancelled query (may outlive its `QueryStatus`). + mutable std::atomic cancelled = false; + std::mutex mutex; bool quota_exceeded TSA_GUARDED_BY(mutex) = false; UInt64 input_tokens TSA_GUARDED_BY(mutex) = 0; @@ -71,9 +80,6 @@ class AIQuotaTracker /// per `throw_on_quota_exceeded`) when a token quota is met. Shared by `checkQuotas` and /// `recordApiCall` so a call is never started once a quota is known-exhausted. bool quotasExceededLocked() TSA_REQUIRES(mutex); - - /// Throws if the query was killed or exceeded its time limit. - void throwIfCancelled() const; }; } diff --git a/src/Functions/AI/AIRequestExecutor.cpp b/src/Functions/AI/AIRequestExecutor.cpp new file mode 100644 index 0000000000000..9dad0e2aad2b0 --- /dev/null +++ b/src/Functions/AI/AIRequestExecutor.cpp @@ -0,0 +1,269 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +namespace ProfileEvents +{ + extern const Event AIInputTokens; + extern const Event AIOutputTokens; + extern const Event AIAPICalls; + extern const Event AIAPICallsRetried; + extern const Event AIAPICallsFailed; + extern const Event AIAPICallsThrottled; +} + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int AI_PROVIDER_RESPONSE_TRUNCATED; + extern const int AI_PROVIDER_RESPONSE_INCOMPLETE; + extern const int CANNOT_SCHEDULE_TASK; +} + +namespace +{ + +/// Whether a failed provider request should be retried: transient network failures and +/// transient/server-side HTTP responses are retriable, deterministic argument/usage errors are not. +bool isRetriableProviderError(std::exception_ptr exception) +{ + try + { + std::rethrow_exception(exception); + } + catch (const AIProviderHTTPException & http_exception) + { + return isRetriableHTTPError(http_exception.getHTTPStatus()); + } + catch (const NetException &) + { + /// ClickHouse-level network error (e.g. a DNS failure raised by the HTTP connection pool). + return true; + } + catch (const Poco::Net::NetException &) + { + /// Connection refused/reset, TLS connect failure, or an unreachable advertised address. + return true; + } + catch (const Poco::TimeoutException &) + { + /// Connect or receive timeout. + return true; + } + catch (const Poco::IOException & io_exception) + { + /// Write-side transient I/O failure, e.g. a broken pipe (`EPIPE`) when the peer resets the + /// connection mid-request. Out-of-file-descriptors (`EMFILE`) is not retriable. + return io_exception.code() != POCO_EMFILE; + } + catch (...) + { + /// Ok: any other exception is a deterministic and non-retrieable error, e.g. a malformed + /// provider response, bad configuration, JSON parse failure, etc. + return false; + } +} + +bool isThrottledProviderError(std::exception_ptr exception) +{ + try + { + std::rethrow_exception(exception); + } + catch (const AIProviderHTTPException & http_exception) + { + return http_exception.getHTTPStatus() == Poco::Net::HTTPResponse::HTTP_TOO_MANY_REQUESTS; + } + catch (...) + { + /// Ok: any other exception is not a rate limit. + return false; + } +} + +/// Reject a response the model did not finish. Throws a plain `Exception`, which +/// `isRetriableProviderError` classifies as non-retriable. +void checkResponseIsComplete(const AIResponse & response) +{ + /// `raw_finish_reason` is provider-controlled text; sanitize control characters before + /// interpolating it into an exception message that reaches the logs and `system.query_log`. + const String safe_finish_reason = sanitize(response.raw_finish_reason); + + switch (response.finish_reason) + { + case FinishReason::Complete: + case FinishReason::Unknown: /// Don't throw on Unknown, could be new valid reason in new API version + return; + case FinishReason::Truncated: + /// Differentiate between model hitting our output cap and exhausting its context window + throw Exception( + ErrorCodes::AI_PROVIDER_RESPONSE_TRUNCATED, + "AI provider returned a truncated response (finish_reason='{}'): {}", + safe_finish_reason, + response.raw_finish_reason == "model_context_window_exceeded" + ? "the model ran out of context window before completing its answer. " + "Reduce the input or use a model with a larger context window." + : "the model hit the output token limit before completing its answer. " + "Increase max_tokens or reduce the input."); + case FinishReason::ContentFilter: + throw Exception( + ErrorCodes::AI_PROVIDER_RESPONSE_INCOMPLETE, + "AI provider withheld or filtered the response (finish_reason='{}'): the returned answer " + "is incomplete.", + safe_finish_reason); + case FinishReason::RequiresAction: + throw Exception( + ErrorCodes::AI_PROVIDER_RESPONSE_INCOMPLETE, + "AI provider stopped expecting further caller action (finish_reason='{}') instead of " + "returning a completed answer.", + safe_finish_reason); + } +} + +/// The attempt loop every request kind shares: reserve an API-call slot, run one attempt, retry a +/// transient failure after a backoff. Returns nothing when the API-call quota is exhausted, or when +/// the request failed (after all retries) and `throw_on_error` is disabled. +template +std::optional runRequest(const AIRequestPolicy & policy, AIQuotaTracker & quota, Attempt && attempt_fn) +{ + for (UInt64 attempt = 0; attempt <= policy.max_retries; ++attempt) + { + /// Reserve an API-call slot before each request; this also performs a quota check. + if (!quota.recordApiCall()) + return {}; + + try + { + /// Count the call before issuing it, so a failed request is still counted. + ProfileEvents::increment(ProfileEvents::AIAPICalls); + return attempt_fn(); + } + catch (...) + { + if (isThrottledProviderError(std::current_exception())) + ProfileEvents::increment(ProfileEvents::AIAPICallsThrottled); + + if (attempt < policy.max_retries && isRetriableProviderError(std::current_exception())) + { + ProfileEvents::increment(ProfileEvents::AIAPICallsRetried); + /// Exponential backoff. `makeRequestPolicy` bounds both settings, so the longest sleep is 160 seconds. + /// Sleep in short steps, so the retry of a cancelled query does not hold the worker. + using Clock = std::chrono::steady_clock; + const Clock::time_point deadline = Clock::now() + std::chrono::milliseconds(policy.retry_initial_delay_ms << attempt); + while (!quota.isCancelled()) + { + const Clock::time_point now = Clock::now(); + if (now >= deadline) + break; + std::this_thread::sleep_for(std::min(deadline - now, std::chrono::milliseconds(100))); + } + continue; + } + + ProfileEvents::increment(ProfileEvents::AIAPICallsFailed); + + if (!policy.throw_on_error) + return {}; + + throw; + } + } + + return {}; +} + +void runAttempt(IAIProvider & provider, const AIChatRequest & request, const ConnectionTimeouts & timeouts, AIResponse & response) +{ + response = provider.call(request, timeouts); + checkResponseIsComplete(response); +} + +void runAttempt(IAIProvider & provider, const AIEmbeddingRequest & request, const ConnectionTimeouts & timeouts, AIEmbeddingResponse & response) +{ + response = provider.embed(request, timeouts); +} + +template +std::future> submit( + std::shared_ptr provider, Request request, AIRequestPolicy policy, AIQuotaTrackerPtr quota) +{ + auto task = [my_provider = std::move(provider), + my_request = std::move(request), + my_policy = std::move(policy), + my_quota = quota] + { + return runRequest(my_policy, *my_quota, [&] + { + Response response; + + /// A response rejected by `checkResponseIsComplete` still reports the usage the provider billed for. + SCOPE_EXIT({ + my_quota->recordTokens(response.input_tokens, response.output_tokens); + ProfileEvents::increment(ProfileEvents::AIInputTokens, response.input_tokens); + ProfileEvents::increment(ProfileEvents::AIOutputTokens, response.output_tokens); + }); + + runAttempt(*my_provider, my_request, my_policy.timeouts, response); + return response; + }); + }; + + using Task = detail::CallbackRunnerTask, decltype(task)>; + auto runner_task = std::make_shared(getCurrentThreadGroup(), ThreadName::AI_REQUEST, std::move(task)); + auto future = runner_task->promise.get_future(); + + /// Wait for a free slot in the queue in short steps, so a killed or timed-out query stops waiting. + constexpr Int64 wait_microseconds = 100'000; + auto & pool = getAIRequestThreadPool().get(); + while (true) + { + Stopwatch watch; + try + { + pool.scheduleOrThrow([runner_task] { runner_task->run(); }, {}, wait_microseconds); + break; + } + catch (const Exception & e) + { + /// Only throw if we hit an unrelated error or it happened before the timeout + if (e.code() != ErrorCodes::CANNOT_SCHEDULE_TASK || watch.elapsedMicroseconds() < static_cast(wait_microseconds)) + throw; + } + quota->throwIfCancelled(); + } + + return future; +} + +} + +std::future> submitAIRequest( + std::shared_ptr provider, AIChatRequest request, AIRequestPolicy policy, AIQuotaTrackerPtr quota) +{ + return submit(std::move(provider), std::move(request), std::move(policy), std::move(quota)); +} + +std::future> submitAIRequest( + std::shared_ptr provider, AIEmbeddingRequest request, AIRequestPolicy policy, AIQuotaTrackerPtr quota) +{ + return submit(std::move(provider), std::move(request), std::move(policy), std::move(quota)); +} + +} diff --git a/src/Functions/AI/AIRequestExecutor.h b/src/Functions/AI/AIRequestExecutor.h new file mode 100644 index 0000000000000..438ab43bef879 --- /dev/null +++ b/src/Functions/AI/AIRequestExecutor.h @@ -0,0 +1,41 @@ +#pragma once + +#include +#include + +#include +#include +#include + +namespace DB +{ + +class AIQuotaTracker; +using AIQuotaTrackerPtr = std::shared_ptr; + +/// How one AI provider request is executed. Resolved once per query from the `ai_function_*` +/// settings and shared by every request that query issues. +struct AIRequestPolicy +{ + ConnectionTimeouts timeouts; + + /// Number of extra attempts after a retriable failure, and the delay before the first of them. + UInt64 max_retries = 0; + UInt64 retry_initial_delay_ms = 0; + + /// `ai_function_throw_on_error`. When disabled, a request that failed (after retries) does not + /// throw, and the caller outputs a default value. + bool throw_on_error = true; +}; + +/// Issue one chat-completion request on the AI request thread pool (`getAIRequestThreadPool`) and +/// check that the model produced a complete answer. Also handles retries with backoff, API-call +/// reservation, and token accounting. +std::future> submitAIRequest( + std::shared_ptr provider, AIChatRequest request, AIRequestPolicy policy, AIQuotaTrackerPtr quota); + +/// The same, for one batch of texts sent to an embeddings endpoint. +std::future> submitAIRequest( + std::shared_ptr provider, AIEmbeddingRequest request, AIRequestPolicy policy, AIQuotaTrackerPtr quota); + +} diff --git a/src/Functions/AI/IAIProvider.h b/src/Functions/AI/IAIProvider.h index c5ce5937b8c13..bae33011fc9d4 100644 --- a/src/Functions/AI/IAIProvider.h +++ b/src/Functions/AI/IAIProvider.h @@ -134,6 +134,9 @@ struct AIEmbeddingResponse /// Number of tokens in the input, as reported by the provider. Used for quota tracking. UInt64 input_tokens = 0; + + /// Always 0: embedding endpoints produce no output tokens. + static constexpr UInt64 output_tokens = 0; }; /** Abstract interface for AI provider HTTP clients. diff --git a/src/Functions/FunctionBaseAI.cpp b/src/Functions/FunctionBaseAI.cpp index 1751184e8e2ec..63a4f15ebb471 100644 --- a/src/Functions/FunctionBaseAI.cpp +++ b/src/Functions/FunctionBaseAI.cpp @@ -1,37 +1,36 @@ #include + #include #include -#include +#include +#include #include -#include -#include -#include -#include -#include -#include -#include -#include #include +#include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include #include +#include #include -#include -#include #include -#include -#include +#include +#include + +#include +#include + +#include +#include +#include #include +#include +#include namespace ProfileEvents { - extern const Event AIInputTokens; - extern const Event AIOutputTokens; - extern const Event AIAPICalls; extern const Event AIRowsProcessed; extern const Event AIRowsSkipped; } @@ -43,6 +42,7 @@ namespace Setting { extern const SettingsUInt64 ai_function_request_timeout_sec; extern const SettingsUInt64 ai_function_max_retries; + extern const SettingsNonZeroUInt64 ai_function_max_concurrent_requests_per_thread; extern const SettingsUInt64 ai_function_retry_initial_delay_ms; extern const SettingsBool ai_function_throw_on_error; extern const SettingsString ai_function_text_default_credentials; @@ -58,8 +58,6 @@ namespace ErrorCodes { extern const int ACCESS_DENIED; extern const int BAD_ARGUMENTS; - extern const int AI_PROVIDER_RESPONSE_TRUNCATED; - extern const int AI_PROVIDER_RESPONSE_INCOMPLETE; } namespace @@ -181,6 +179,33 @@ AIParamSpecs FunctionBaseAI::allParams() const namespace { +constexpr auto cancellation_check_interval = std::chrono::milliseconds(100); + +/// Waits for a submitted request. Throws once the query is killed or exceeds its time limit, +/// so a request queued behind other queries' requests does not hold the query. +template +std::optional waitForRequest(std::future> & request, const AIQuotaTracker & quota) +{ + while (request.wait_for(cancellation_check_interval) != std::future_status::ready) + quota.throwIfCancelled(); + return request.get(); +} + +/// Waits for the submitted requests to finish, until the query is killed or exceeds its time limit. +/// The tasks own everything they use, so they may outlive the query. +template +void drainRequests(VectorWithMemoryTracking>> & requests, const AIQuotaTracker & quota) +{ + for (auto & request : requests) + { + if (!request.valid()) + continue; + while (request.wait_for(cancellation_check_interval) != std::future_status::ready) + if (quota.isCancelled()) + return; + } +} + const Field & getResolvedAIParam(const AIParamValues & values, std::string_view key) { auto it = values.find(key); @@ -316,52 +341,28 @@ FunctionBaseAI::AIParams FunctionBaseAI::resolveAIParams( return params; } -UInt64 FunctionBaseAI::computeRetryBackoffMs(UInt64 initial_delay_ms, UInt64 attempt) +AIRequestPolicy FunctionBaseAI::makeRequestPolicy(const ContextPtr & context) { - constexpr UInt64 max_retry_delay_ms = 60'000; - UInt64 delay_ms = std::min(initial_delay_ms, max_retry_delay_ms); - for (UInt64 i = 0; i < attempt && delay_ms < max_retry_delay_ms; ++i) - delay_ms = std::min(delay_ms * 2, max_retry_delay_ms); - return delay_ms; -} + const auto & settings = context->getSettingsRef(); + + AIRequestPolicy policy; + policy.timeouts = ConnectionTimeouts::getHTTPTimeouts(settings, context->getServerSettings()); + policy.timeouts.receive_timeout + = Poco::Timespan(static_cast(settings[Setting::ai_function_request_timeout_sec].value) /*s*/, 0 /*us*/); + policy.max_retries = settings[Setting::ai_function_max_retries].value; + policy.retry_initial_delay_ms = settings[Setting::ai_function_retry_initial_delay_ms].value; + policy.throw_on_error = settings[Setting::ai_function_throw_on_error].value; + + constexpr UInt64 max_retries_limit = 5; + constexpr UInt64 retry_initial_delay_ms_limit = 10'000; + if (policy.max_retries > max_retries_limit) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Setting 'ai_function_max_retries' is {}, the maximum is {}", policy.max_retries, max_retries_limit); + if (policy.retry_initial_delay_ms > retry_initial_delay_ms_limit) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Setting 'ai_function_retry_initial_delay_ms' is {}, the maximum is {}", policy.retry_initial_delay_ms, retry_initial_delay_ms_limit); -bool FunctionBaseAI::isRetriableProviderError(std::exception_ptr exception) -{ - try - { - std::rethrow_exception(exception); - } - catch (const AIProviderHTTPException & exception) - { - return isRetriableHTTPError(exception.getHTTPStatus()); - } - catch (const NetException &) - { - /// ClickHouse-level network error (e.g. a DNS failure raised by the HTTP connection pool). - return true; - } - catch (const Poco::Net::NetException &) - { - /// Connection refused/reset, TLS connect failure, or an unreachable advertised address. - return true; - } - catch (const Poco::TimeoutException &) - { - /// Connect or receive timeout. - return true; - } - catch (const Poco::IOException & exception) - { - /// Write-side transient I/O failure, e.g. a broken pipe (`EPIPE`) when the peer resets the - /// connection mid-request. Out-of-file-descriptors (`EMFILE`) is not retriable. - return exception.code() != POCO_EMFILE; - } - catch (...) - { - /// Ok: any other exception is a deterministic and non-retrieable error, e.g. a malformed - /// provider response, bad configuration, JSON parse failure, etc. - return false; - } + return policy; } void FunctionBaseAI::insertProcessedResult(IColumn & column, const String & processed) const @@ -377,8 +378,8 @@ AIParamSpecs FunctionBaseAI::embeddingParams() }; } -FunctionBaseAI::EmbeddingResult FunctionBaseAI::embedTexts( - IAIProvider & provider, +void FunctionBaseAI::embedTexts( + const std::shared_ptr & provider, const String & model, UInt64 dimensions, const String & function_name, @@ -386,85 +387,86 @@ FunctionBaseAI::EmbeddingResult FunctionBaseAI::embedTexts( const String & initial_query_id, const VectorWithMemoryTracking & inputs, size_t max_batch_size, - UInt64 max_retries, - UInt64 retry_delay_ms, - bool throw_on_error, - AIQuotaTracker & quota, - const ConnectionTimeouts & timeouts) + size_t max_concurrent_requests, + const AIRequestPolicy & policy, + const AIQuotaTrackerPtr & quota, + EmbeddingResult & result) { - EmbeddingResult result; result.embeddings.resize(inputs.size()); - for (size_t batch_start = 0; batch_start < inputs.size(); batch_start += max_batch_size) + /// Equivalent to (n + max_batch_size - 1) / max_batch_size but can't overflow + const size_t num_batches = inputs.empty() ? 0 : 1 + (inputs.size() - 1) / max_batch_size; + const size_t concurrency = std::min(max_concurrent_requests, num_batches); + + /// Batches go out in waves of size `concurrency` and each completed wave is applied before the next + /// one starts, so at most that many of this call's requests are in flight at a time. + VectorWithMemoryTracking>> wave; + wave.reserve(concurrency); + + /// Wait for all waves to complete on error, so that API-call and request counts are recorded + /// They were already dispatched and billed either way. A cancelled query does not wait. + SCOPE_EXIT_SAFE({ drainRequests(wave, *quota); }); + + for (size_t wave_begin = 0; wave_begin < num_batches; wave_begin += concurrency) { - if (quota.checkQuotas()) - { - result.texts_skipped += inputs.size() - batch_start; - break; - } + const size_t wave_end = std::min(wave_begin + concurrency, num_batches); - size_t batch_end = std::min(batch_start + max_batch_size, inputs.size()); - - AIEmbeddingRequest ai_embedding_request; - ai_embedding_request.model = model; - ai_embedding_request.dimensions = dimensions; - ai_embedding_request.function_name = function_name; - ai_embedding_request.query_id = query_id; - ai_embedding_request.initial_query_id = initial_query_id; - ai_embedding_request.inputs.reserve(batch_end - batch_start); - for (size_t k = batch_start; k < batch_end; ++k) - ai_embedding_request.inputs.emplace_back(inputs[k]); - - AIEmbeddingResponse ai_embedding_response; - bool batch_ok = false; - for (UInt64 attempt = 0; attempt <= max_retries; ++attempt) + wave.clear(); + /// fire off `concurrency` batches at a time + for (size_t batch = wave_begin; batch < wave_end; ++batch) { - /// Reserve an API-call slot before each request; this also performs a quota check. - /// Kept outside the `try` so a `throw_on_quota_exceeded` exception isn't caught by the retry handler. - if (!quota.recordApiCall()) - break; - - try + /// Once the quota is exhausted nothing more is issued, so the batch's slot stays an + /// empty future and its inputs stay empty. + if (quota->checkQuotas()) { - ++result.api_calls; - ai_embedding_response = provider.embed(ai_embedding_request, timeouts); - result.input_tokens += ai_embedding_response.input_tokens; - quota.recordTokens(ai_embedding_response.input_tokens, 0); - batch_ok = true; - break; + wave.emplace_back(); + continue; } - catch (...) - { - if (attempt < max_retries && isRetriableProviderError(std::current_exception())) - { - std::this_thread::sleep_for(std::chrono::milliseconds(computeRetryBackoffMs(retry_delay_ms, attempt))); - continue; - } - if (!throw_on_error) /// Skip to next batch, this batch's inputs stay empty. - break; + const size_t begin = batch * max_batch_size; + const size_t end = std::min(begin + max_batch_size, inputs.size()); - throw; - } + AIEmbeddingRequest ai_embedding_request; + ai_embedding_request.model = model; + ai_embedding_request.dimensions = dimensions; + ai_embedding_request.function_name = function_name; + ai_embedding_request.query_id = query_id; + ai_embedding_request.initial_query_id = initial_query_id; + ai_embedding_request.inputs.reserve(end - begin); + for (size_t k = begin; k < end; ++k) + ai_embedding_request.inputs.emplace_back(inputs[k]); + + wave.push_back(submitAIRequest(provider, std::move(ai_embedding_request), policy, quota)); } - if (!batch_ok) + /// get the results for our `concurrency` batches + for (size_t k = 0; k < wave.size(); ++k) { - result.texts_skipped += batch_end - batch_start; - continue; - } + const size_t begin = (wave_begin + k) * max_batch_size; + const size_t end = std::min(begin + max_batch_size, inputs.size()); - chassert(ai_embedding_response.embeddings.size() == ai_embedding_request.inputs.size(), - "Number of inputs does not match number of output embeddings"); + /// Nothing when no request was issued for this batch, or when it failed and + /// `ai_function_throw_on_error` is disabled; either way its inputs stay empty. + std::optional ai_embedding_response; + if (wave[k].valid()) + ai_embedding_response = waitForRequest(wave[k], *quota); - for (size_t k = 0; k < ai_embedding_response.embeddings.size(); ++k) - { - result.embeddings[batch_start + k] = std::move(ai_embedding_response.embeddings[k]); - ++result.texts_embedded; + if (!ai_embedding_response) + { + result.texts_skipped += end - begin; + continue; + } + + chassert(ai_embedding_response->embeddings.size() == end - begin, + "Number of inputs does not match number of output embeddings"); + + for (size_t j = 0; j < ai_embedding_response->embeddings.size(); ++j) + { + result.embeddings[begin + j] = std::move(ai_embedding_response->embeddings[j]); + ++result.texts_embedded; + } } } - - return result; } ColumnPtr FunctionBaseAI::executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const @@ -481,7 +483,10 @@ ColumnPtr FunctionBaseAI::executeImpl(const ColumnsWithTypeAndName & arguments, checkSanityBeforeExecuteImpl(arguments, result_type, input_rows_count); String system_prompt = sanitizeForModel(buildSystemPrompt(arguments, params)); auto response_format = buildResponseFormat(arguments); - auto provider = createAIProvider( + + /// Shared with the submitted requests so each one is self-contained and nothing dangles even + /// if this call stops waiting for a future it handed out. + std::shared_ptr provider = createAIProvider( getContext(), params.collection.provider, params.collection.endpoint, params.collection.api_key, params.collection.api_version); if (input_rows_count == 0) @@ -497,17 +502,10 @@ ColumnPtr FunctionBaseAI::executeImpl(const ColumnsWithTypeAndName & arguments, prompt_nullable = typeid_cast(prompt_column.get()); } - UInt64 timeout_sec = settings[Setting::ai_function_request_timeout_sec].value; - UInt64 max_retries = settings[Setting::ai_function_max_retries].value; - UInt64 retry_delay_ms = settings[Setting::ai_function_retry_initial_delay_ms].value; - - bool throw_on_error = settings[Setting::ai_function_throw_on_error].value; - /// Shared across every AI function call in the query auto quota_tracker = getContext()->getAIQuotaTracker(); - auto timeouts = ConnectionTimeouts::getHTTPTimeouts(settings, getContext()->getServerSettings()); - timeouts.receive_timeout = Poco::Timespan(static_cast(timeout_sec) /*s*/, 0 /*us*/); + const auto policy = makeRequestPolicy(getContext()); const String query_id = getContext()->getCurrentQueryId(); const String & initial_query_id = getContext()->getClientInfo().initial_query_id; @@ -515,133 +513,88 @@ ColumnPtr FunctionBaseAI::executeImpl(const ColumnsWithTypeAndName & arguments, auto result_col = removeNullable(result_type)->createColumn(); auto null_map_col = prompt_nullable ? ColumnUInt8::create(input_rows_count, static_cast(0)) : nullptr; - UInt64 total_api_calls = 0; - UInt64 total_input_tokens = 0; - UInt64 total_output_tokens = 0; UInt64 rows_processed = 0; UInt64 rows_skipped = 0; - /// Increment ProfileEvents counters upon destruction, to avoid underreporting on error + /// Increment ProfileEvents counters upon destruction, to avoid underreporting on error. + /// The per-request counters (`AIAPICalls`, `AIInputTokens`, `AIOutputTokens`) are incremented by + /// the submitted requests as they complete. SCOPE_EXIT({ - ProfileEvents::increment(ProfileEvents::AIAPICalls, total_api_calls); - ProfileEvents::increment(ProfileEvents::AIInputTokens, total_input_tokens); - ProfileEvents::increment(ProfileEvents::AIOutputTokens, total_output_tokens); ProfileEvents::increment(ProfileEvents::AIRowsProcessed, rows_processed); ProfileEvents::increment(ProfileEvents::AIRowsSkipped, rows_skipped); }); - for (size_t i = 0; i < input_rows_count; ++i) + const size_t concurrency = std::min(settings[Setting::ai_function_max_concurrent_requests_per_thread].value, input_rows_count); + + /// Requests go out in waves of `concurrency` rows, and each completed wave is applied to the + /// result column in row order. A wave waits for its slowest request before the next one starts, + /// which gives up a little throughput next to a sliding window, but keeps row ordering, quota + /// accounting and error propagation identical to issuing the requests one at a time. + VectorWithMemoryTracking>> wave; + wave.reserve(concurrency); + + /// Wait for all waves to complete on error, so that API-call and request counts are recorded + /// They were already dispatched and billed either way. A cancelled query does not wait. + SCOPE_EXIT_SAFE({ drainRequests(wave, *quota_tracker); }); + + for (size_t wave_begin = 0; wave_begin < input_rows_count; wave_begin += concurrency) { - if (prompt_nullable && prompt_nullable->getNullMapData()[i]) - { - result_col->insertDefault(); - null_map_col->getData()[i] = 1; - continue; - } + const size_t wave_end = std::min(wave_begin + concurrency, input_rows_count); - if (quota_tracker->checkQuotas()) + wave.clear(); + for (size_t row = wave_begin; row < wave_end; ++row) { - result_col->insertDefault(); - ++rows_skipped; - continue; - } + /// A NULL prompt produces NULL, and once the quota is exhausted a row keeps its default + /// value. Neither issues a request, so the row's slot stays an empty future. + if ((prompt_nullable && prompt_nullable->getNullMapData()[row]) || quota_tracker->checkQuotas()) + { + wave.emplace_back(); + continue; + } - String user_message = sanitizeForModel(buildUserMessage(arguments, i)); - String result; - bool success = false; + AIChatRequest ai_request; + ai_request.system_prompt = system_prompt; + ai_request.user_message = sanitizeForModel(buildUserMessage(arguments, row)); + ai_request.response_format = response_format; + ai_request.model = model; + ai_request.temperature = temperature; + ai_request.max_tokens = max_tokens; + ai_request.function_name = getName(); + ai_request.query_id = query_id; + ai_request.initial_query_id = initial_query_id; + + wave.push_back(submitAIRequest(provider, std::move(ai_request), policy, quota_tracker)); + } - for (UInt64 attempt = 0; attempt <= max_retries; ++attempt) + for (size_t k = 0; k < wave.size(); ++k) { - /// Reserve an API-call slot before each request; this also performs a quota check. - /// Kept outside the `try` so a `throw_on_quota_exceeded` exception isn't caught by the retry handler. - if (!quota_tracker->recordApiCall()) - break; + const size_t row = wave_begin + k; - try + /// A NULL prompt produces NULL without a request. + if (prompt_nullable && prompt_nullable->getNullMapData()[row]) { - AIChatRequest ai_request; - ai_request.system_prompt = system_prompt; - ai_request.user_message = user_message; - ai_request.response_format = response_format; - ai_request.model = model; - ai_request.temperature = temperature; - ai_request.max_tokens = max_tokens; - ai_request.function_name = getName(); - ai_request.query_id = query_id; - ai_request.initial_query_id = initial_query_id; - - ++total_api_calls; - - auto ai_response = provider->call(ai_request, timeouts); - - quota_tracker->recordTokens(ai_response.input_tokens, ai_response.output_tokens); - total_input_tokens += ai_response.input_tokens; - total_output_tokens += ai_response.output_tokens; - - /// `raw_finish_reason` is provider-controlled text; sanitize control characters before - /// interpolating it into an exception message that reaches the logs and `system.query_log`. - const String safe_finish_reason = sanitize(ai_response.raw_finish_reason); - - /// Reject incomplete responses, throw plain DB::Exception so it is classified as non-retriable - switch (ai_response.finish_reason) - { - case FinishReason::Complete: - case FinishReason::Unknown: /// Don't throw on Unknown, could be new valid reason in new API version - break; - case FinishReason::Truncated: - /// Differentiate between model hitting our output cap and exhausting its context window - throw Exception( - ErrorCodes::AI_PROVIDER_RESPONSE_TRUNCATED, - "AI provider returned a truncated response (finish_reason='{}'): {}", - safe_finish_reason, - ai_response.raw_finish_reason == "model_context_window_exceeded" - ? "the model ran out of context window before completing its answer. " - "Reduce the input or use a model with a larger context window." - : "the model hit the output token limit before completing its answer. " - "Increase max_tokens or reduce the input."); - case FinishReason::ContentFilter: - throw Exception( - ErrorCodes::AI_PROVIDER_RESPONSE_INCOMPLETE, - "AI provider withheld or filtered the response (finish_reason='{}'): the returned answer " - "is incomplete.", - safe_finish_reason); - case FinishReason::RequiresAction: - throw Exception( - ErrorCodes::AI_PROVIDER_RESPONSE_INCOMPLETE, - "AI provider stopped expecting further caller action (finish_reason='{}') instead of " - "returning a completed answer.", - safe_finish_reason); - } - - result = postProcessResponse(ai_response.result); - success = true; - break; + result_col->insertDefault(); + null_map_col->getData()[row] = 1; + continue; } - catch (...) - { - if (attempt < max_retries && isRetriableProviderError(std::current_exception())) - { - std::this_thread::sleep_for(std::chrono::milliseconds(computeRetryBackoffMs(retry_delay_ms, attempt))); - continue; - } - if (!throw_on_error) - break; + /// Nothing when no request was issued because the API-call quota was exhausted, or when + /// the request failed and `ai_function_throw_on_error` is disabled; either way the row + /// keeps its default value. + std::optional ai_response; + if (wave[k].valid()) + ai_response = waitForRequest(wave[k], *quota_tracker); - throw; + if (!ai_response) + { + result_col->insertDefault(); + ++rows_skipped; + continue; } - } - if (success) - { - insertProcessedResult(*result_col, result); + insertProcessedResult(*result_col, postProcessResponse(ai_response->result)); ++rows_processed; } - else - { - result_col->insertDefault(); - ++rows_skipped; - } } if (result_type->isNullable()) diff --git a/src/Functions/FunctionBaseAI.h b/src/Functions/FunctionBaseAI.h index c04f1889ba798..140c677ff5d70 100644 --- a/src/Functions/FunctionBaseAI.h +++ b/src/Functions/FunctionBaseAI.h @@ -4,11 +4,11 @@ #include #include #include +#include #include #include #include -#include #include #include #include @@ -129,14 +129,9 @@ class FunctionBaseAI : public IFunction /// Parameters common to every AI function. Function-specific params are appended by `functionParams`. static AIParamSpecs commonParams(); - /// Exponential backoff delay capped at one minute, so adversarial values of - /// `ai_function_retry_initial_delay_ms` or `ai_function_max_retries` cannot produce a multi-hour - /// sleep or overflow `std::chrono::milliseconds`. - static UInt64 computeRetryBackoffMs(UInt64 initial_delay_ms, UInt64 attempt); - - /// Whether a failed provider request should be retried: transient network failures and - /// transient/server-side HTTP responses are retriable, deterministic argument/usage errors are not. - static bool isRetriableProviderError(std::exception_ptr exception); + /// Read the `ai_function_*` settings that govern how one provider request runs. The same policy + /// applies to every request the query issues. + static AIRequestPolicy makeRequestPolicy(const ContextPtr & context); /// Parameters accepted by the embedding functions (`aiEmbed`, `aiSimilarity`) in their optional trailing /// `Map(String, String)` argument. Unlike the text functions, they take no `max_tokens`, and `model` is a @@ -146,20 +141,20 @@ class FunctionBaseAI : public IFunction /// Result of `embedTexts`. `embeddings` is aligned 1:1 with the `inputs` argument of `embedTexts`. An /// entry is empty when that input was not embedded (quota exceeded, or a failed request with - /// `ai_function_throw_on_error` disabled). The counters feed the AI ProfileEvents. + /// `ai_function_throw_on_error` disabled). struct EmbeddingResult { VectorWithMemoryTracking> embeddings; - UInt64 api_calls = 0; - UInt64 input_tokens = 0; UInt64 texts_embedded = 0; UInt64 texts_skipped = 0; }; - /// Embed a flat list of already-filtered (non-null, non-empty) texts, reusing the shared batching, - /// retry/backoff, and quota logic. Inputs are grouped into batches of up to `max_batch_size` per HTTP call. - static EmbeddingResult embedTexts( - IAIProvider & provider, + /// Embed a flat list of already-filtered (non-null, non-empty) texts, reusing the shared batching + /// and quota logic. Inputs are grouped into batches of up to `max_batch_size` per HTTP call, and + /// up to `max_concurrent_requests` of those calls are in flight at once. + /// Accumulates into `result`, so the waves completed before a throw stay visible to the caller. + static void embedTexts( + const std::shared_ptr & provider, const String & model, UInt64 dimensions, const String & function_name, @@ -167,11 +162,10 @@ class FunctionBaseAI : public IFunction const String & initial_query_id, const VectorWithMemoryTracking & inputs, size_t max_batch_size, - UInt64 max_retries, - UInt64 retry_delay_ms, - bool throw_on_error, - AIQuotaTracker & quota, - const ConnectionTimeouts & timeouts); + size_t max_concurrent_requests, + const AIRequestPolicy & policy, + const AIQuotaTrackerPtr & quota, + EmbeddingResult & result); protected: ContextPtr context; diff --git a/src/Functions/aiEmbed.cpp b/src/Functions/aiEmbed.cpp index b227ed597e27f..b7d686147381f 100644 --- a/src/Functions/aiEmbed.cpp +++ b/src/Functions/aiEmbed.cpp @@ -19,17 +19,13 @@ #include #include -#include #include -#include #include #include namespace ProfileEvents { - extern const Event AIInputTokens; - extern const Event AIAPICalls; extern const Event AIRowsProcessed; extern const Event AIRowsSkipped; } @@ -39,11 +35,8 @@ namespace DB namespace Setting { - extern const SettingsUInt64 ai_function_request_timeout_sec; - extern const SettingsUInt64 ai_function_max_retries; - extern const SettingsUInt64 ai_function_retry_initial_delay_ms; - extern const SettingsBool ai_function_throw_on_error; extern const SettingsNonZeroUInt64 ai_function_embedding_max_batch_size; + extern const SettingsNonZeroUInt64 ai_function_max_concurrent_requests_per_thread; extern const SettingsString ai_function_embedding_default_credentials; } @@ -107,12 +100,8 @@ class FunctionAiEmbed final : public IFunction UInt64 dimensions = params.getUInt("dimensions"); String model(arguments[model_arg_index].column->getDataAt(0)); - auto provider = createAIProvider( - getContext(), - params.collection.provider, - params.collection.endpoint, - params.collection.api_key, - params.collection.api_version); + std::shared_ptr provider = createAIProvider( + getContext(), params.collection.provider, params.collection.endpoint, params.collection.api_key, params.collection.api_version); if (!provider->supportsEmbeddings()) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "AI provider '{}' does not support embeddings", params.collection.provider); @@ -120,17 +109,13 @@ class FunctionAiEmbed final : public IFunction if (input_rows_count == 0) return result_type->createColumn(); - UInt64 timeout_sec = settings[Setting::ai_function_request_timeout_sec].value; - UInt64 max_retries = settings[Setting::ai_function_max_retries].value; - UInt64 retry_delay_ms = settings[Setting::ai_function_retry_initial_delay_ms].value; - bool throw_on_error = settings[Setting::ai_function_throw_on_error].value; size_t max_batch_size = static_cast(settings[Setting::ai_function_embedding_max_batch_size].value); + size_t max_concurrent_requests = static_cast(settings[Setting::ai_function_max_concurrent_requests_per_thread].value); /// Shared across every AI function call in the query auto quota_tracker = getContext()->getAIQuotaTracker(); - auto timeouts = ConnectionTimeouts::getHTTPTimeouts(settings, getContext()->getServerSettings()); - timeouts.receive_timeout = Poco::Timespan(static_cast(timeout_sec) /*s*/, 0 /*us*/); + const auto policy = FunctionBaseAI::makeRequestPolicy(getContext()); /// `isNullAt` and `getDataAt` are virtual on `IColumn`, so a single path covers `ColumnString`, /// `ColumnConst(ColumnString)`, `ColumnNullable` and `ColumnConst(ColumnNullable)` (e.g. @@ -161,11 +146,10 @@ class FunctionAiEmbed final : public IFunction inputs.push_back(value); } - auto embedding_result = FunctionBaseAI::embedTexts( - *provider, model, dimensions, - getName(), getContext()->getCurrentQueryId(), getContext()->getClientInfo().initial_query_id, - inputs, max_batch_size, max_retries, - retry_delay_ms, throw_on_error, *quota_tracker, timeouts); + FunctionBaseAI::EmbeddingResult embedding_result; + FunctionBaseAI::embedTexts( + provider, model, dimensions, getName(), getContext()->getCurrentQueryId(), getContext()->getClientInfo().initial_query_id, + inputs, max_batch_size, max_concurrent_requests, policy, quota_tracker, embedding_result); auto data_col = ColumnVector::create(); /// float32 is standard embedding API output auto offsets_col = ColumnArray::ColumnOffsets::create(); @@ -197,8 +181,6 @@ class FunctionAiEmbed final : public IFunction for (; cursor < input_rows_count; ++cursor) offsets_vec.push_back(current_offset); - ProfileEvents::increment(ProfileEvents::AIAPICalls, embedding_result.api_calls); - ProfileEvents::increment(ProfileEvents::AIInputTokens, embedding_result.input_tokens); ProfileEvents::increment(ProfileEvents::AIRowsProcessed, embedding_result.texts_embedded); ProfileEvents::increment(ProfileEvents::AIRowsSkipped, embedding_result.texts_skipped); diff --git a/src/Functions/aiSimilarity.cpp b/src/Functions/aiSimilarity.cpp index d6907497dc60d..0536b0035d608 100644 --- a/src/Functions/aiSimilarity.cpp +++ b/src/Functions/aiSimilarity.cpp @@ -17,9 +17,7 @@ #include #include -#include #include -#include #include #include @@ -29,8 +27,6 @@ namespace ProfileEvents { - extern const Event AIInputTokens; - extern const Event AIAPICalls; extern const Event AIRowsProcessed; extern const Event AIRowsSkipped; } @@ -40,11 +36,8 @@ namespace DB namespace Setting { - extern const SettingsUInt64 ai_function_request_timeout_sec; - extern const SettingsUInt64 ai_function_max_retries; - extern const SettingsUInt64 ai_function_retry_initial_delay_ms; - extern const SettingsBool ai_function_throw_on_error; extern const SettingsNonZeroUInt64 ai_function_embedding_max_batch_size; + extern const SettingsNonZeroUInt64 ai_function_max_concurrent_requests_per_thread; extern const SettingsString ai_function_embedding_default_credentials; } @@ -135,12 +128,10 @@ class FunctionAiSimilarity final : public IFunction UInt64 dimensions = params.getUInt("dimensions"); String model(arguments[model_arg_index].column->getDataAt(0)); - auto provider = createAIProvider( - getContext(), - params.collection.provider, - params.collection.endpoint, - params.collection.api_key, - params.collection.api_version); + /// Shared with the submitted requests so each one is self-contained and nothing dangles + /// even if this call stops waiting for a future it handed out. + std::shared_ptr provider = createAIProvider( + getContext(), params.collection.provider, params.collection.endpoint, params.collection.api_key, params.collection.api_version); if (!provider->supportsEmbeddings()) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "AI provider '{}' does not support embeddings", params.collection.provider); @@ -148,16 +139,13 @@ class FunctionAiSimilarity final : public IFunction if (input_rows_count == 0) return result_type->createColumn(); - UInt64 max_retries = settings[Setting::ai_function_max_retries].value; - UInt64 retry_delay_ms = settings[Setting::ai_function_retry_initial_delay_ms].value; - bool throw_on_error = settings[Setting::ai_function_throw_on_error].value; size_t max_batch_size = static_cast(settings[Setting::ai_function_embedding_max_batch_size].value); + size_t max_concurrent_requests = static_cast(settings[Setting::ai_function_max_concurrent_requests_per_thread].value); /// Shared across every AI function call in the query auto quota_tracker = getContext()->getAIQuotaTracker(); - auto timeouts = ConnectionTimeouts::getHTTPTimeouts(settings, getContext()->getServerSettings()); - timeouts.receive_timeout = Poco::Timespan(static_cast(settings[Setting::ai_function_request_timeout_sec].value) /*s*/, 0 /*us*/); + const auto policy = FunctionBaseAI::makeRequestPolicy(getContext()); /// `isNullAt` and `getDataAt` are virtual on `IColumn`, so a single path covers `ColumnString`, /// `ColumnConst(ColumnString)`, `ColumnNullable` and `ColumnConst(ColumnNullable)`. A constant @@ -199,14 +187,10 @@ class FunctionAiSimilarity final : public IFunction } } - auto embedding_result = FunctionBaseAI::embedTexts( - *provider, model, dimensions, getName(), - getContext()->getCurrentQueryId(), getContext()->getClientInfo().initial_query_id, - inputs, max_batch_size, max_retries, - retry_delay_ms, throw_on_error, *quota_tracker, timeouts); - - ProfileEvents::increment(ProfileEvents::AIAPICalls, embedding_result.api_calls); - ProfileEvents::increment(ProfileEvents::AIInputTokens, embedding_result.input_tokens); + FunctionBaseAI::EmbeddingResult embedding_result; + FunctionBaseAI::embedTexts( + provider, model, dimensions, getName(), getContext()->getCurrentQueryId(), getContext()->getClientInfo().initial_query_id, + inputs, max_batch_size, max_concurrent_requests, policy, quota_tracker, embedding_result); const auto & embeddings = embedding_result.embeddings; diff --git a/src/IO/SharedThreadPools.h b/src/IO/SharedThreadPools.h index 07e76d8e12f70..5982a3dc29ad9 100644 --- a/src/IO/SharedThreadPools.h +++ b/src/IO/SharedThreadPools.h @@ -71,7 +71,8 @@ class StaticThreadPool M(MergeTreePrefixesDeserialization, "MergeTreePrefixesDeserializationThreadPool", MergeTreeSubcolumnsReader) \ M(DropDistributedCache, "DropDistributedCacheThreadPool", DropDistributedCache) \ M(FormatParsing, "FormatParsingThreadPool", FormatParsing) \ - M(IcebergManifestDecode, "IcebergManifestDecodeThreadPool", IcebergManifestDecode) + M(IcebergManifestDecode, "IcebergManifestDecodeThreadPool", IcebergManifestDecode) \ + M(AIRequest, "AIRequestThreadPool", AIRequest) #define DECLARE_STATIC_THREAD_POOL_GETTER(SUFFIX, NAME, METRIC) StaticThreadPool & get##SUFFIX##ThreadPool(); APPLY_FOR_STATIC_THREAD_POOLS(DECLARE_STATIC_THREAD_POOL_GETTER) diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index f8c548748a064..26a03ab3a6dbb 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -66,6 +66,7 @@ #include #include #include +#include #include #include #include @@ -952,6 +953,10 @@ struct ContextSharedPart : boost::noncopyable tryLogCurrentException(__PRETTY_FUNCTION__); } } + + /// A cancelled query does not wait for its AI requests. Their memory is tracked under the user trackers in + /// `process_list`, which is destroyed after this destructor body. + getAIRequestThreadPool().shutdown(); } void setConfig(const ConfigurationPtr & config_value) diff --git a/tests/integration/test_ai_functions/mock_ai_server.py b/tests/integration/test_ai_functions/mock_ai_server.py index d14ba6bbeb8d4..859bc38e84ff4 100644 --- a/tests/integration/test_ai_functions/mock_ai_server.py +++ b/tests/integration/test_ai_functions/mock_ai_server.py @@ -9,11 +9,18 @@ `Authorization` header is omitted when the named collection has no `api_key`). Header names are lower-cased for case-insensitive lookup. GET /concurrency — returns JSON `{"requests": N, "max_concurrency": M}` describing - the requests the slow endpoint below has served since the last `/reset-concurrency`, where - `max_concurrency` is the highest number that were ever being served at the same moment. + the requests the slow endpoints below have served since the last `/reset-concurrency`, where + `max_concurrency` is the highest number that were ever being served at the same moment. Used + to assert that `ai_function_max_concurrent_requests_per_thread` controls how many requests are in flight. GET /reset-concurrency — zeroes the counters above POST /v1/chat/slow — like `/v1/chat/completions`, but sleeps SLOW_RESPONSE_SECONDS - before answering. + before answering, so overlapping requests are observable in `/concurrency`. + POST /v1/embeddings_slow — like `/v1/embeddings`, but slow in the same way. + POST /v1/chat/hang — like `/v1/chat/slow`, but sleeps HANG_RESPONSE_SECONDS, so a + request is still in flight when the server shuts down. + POST /v1/chat/jitter — like `/v1/chat/completions` without a schema (echoes the user + message), but sleeps a random 0 to JITTER_MAX_SECONDS first, so concurrent requests complete out + of order. GET /set-flaky?count=N — arm the flaky endpoints below to fail their next N requests with a simulated transient network error (used to exercise retries). `count=0` disarms. POST /v1/chat/flaky — like `/v1/chat/completions`, but drops the connection without @@ -55,6 +62,7 @@ the opposite of the `max_tokens` case. POST /v1/anthropic/tool_use — Anthropic-shaped HTTP 200 with `stop_reason="tool_use"`, a successful structured-output (forced tool call) response that must NOT be rejected. + POST /v1/rate_limited — always returns HTTP 429, a provider rate limit. POST /v1/error — always returns HTTP 500, a transient/server-side error that the url table function (and so the AI functions) retries. POST /v1/bad_request — always returns HTTP 400, a deterministic client error that @@ -64,6 +72,7 @@ import http.server import json +import random import threading import time from urllib.parse import urlparse, parse_qs @@ -81,10 +90,17 @@ # Set via `GET /set-flaky?count=N`. Used to exercise the network-error retry path. FLAKY = {"fails_remaining": 0} -# How long the slow endpoint takes to answer. +# How long the slow endpoints take to answer. Long enough that concurrent requests overlap +# observably, short enough not to slow the test down. SLOW_RESPONSE_SECONDS = 0.5 -# Requests the slow endpoint has served, and the high-water mark of how many it was serving +# How long `/v1/chat/hang` takes to answer. +HANG_RESPONSE_SECONDS = 5 + +# Upper bound of the random delay of `/v1/chat/jitter`. +JITTER_MAX_SECONDS = 0.3 + +# Requests the slow endpoints have served, and the high-water mark of how many they were serving # simultaneously, since the last `/reset-concurrency`. Guarded by `_LOCK`. CONCURRENCY = {"requests": 0, "in_flight": 0, "max_concurrency": 0} @@ -296,6 +312,19 @@ def do_POST(self): self._serve_slowly(lambda: make_success_response(extract_user_message(body))) return + if parsed.path == "/v1/embeddings_slow": + self._serve_slowly(lambda: make_embeddings_response(body)) + return + + if parsed.path == "/v1/chat/hang": + self._serve_slowly(lambda: make_success_response(extract_user_message(body)), HANG_RESPONSE_SECONDS) + return + + if parsed.path == "/v1/chat/jitter": + time.sleep(random.uniform(0, JITTER_MAX_SECONDS)) + self._send_json(200, make_success_response(extract_user_message(body))) + return + if parsed.path in ("/v1/chat/flaky", "/v1/embeddings_flaky"): with _LOCK: should_fail = FLAKY["fails_remaining"] > 0 @@ -404,6 +433,10 @@ def do_POST(self): self._send_json(200, make_anthropic_tool_use_response(body)) return + if parsed.path == "/v1/rate_limited": + self._send_json(429, make_error_response("rate limit exceeded", error_type="rate_limit_error")) + return + if parsed.path == "/v1/error": self._send_json(500, make_error_response("permanent failure")) return @@ -450,8 +483,10 @@ def do_POST(self): self.send_response(404) self.end_headers() - def _serve_slowly(self, make_response): - """Sleep, then answer, counting the request as in flight for the duration of the sleep.""" + def _serve_slowly(self, make_response, seconds=SLOW_RESPONSE_SECONDS): + """Sleep, then answer, counting the request as in flight for the duration of the sleep. + The window closes before the response is written, so a handler that has already answered + can never overlap in `max_concurrency` with the request that answer unblocks.""" with _LOCK: CONCURRENCY["requests"] += 1 CONCURRENCY["in_flight"] += 1 @@ -459,7 +494,7 @@ def _serve_slowly(self, make_response): CONCURRENCY["max_concurrency"], CONCURRENCY["in_flight"] ) try: - time.sleep(SLOW_RESPONSE_SECONDS) + time.sleep(seconds) response = make_response() finally: with _LOCK: diff --git a/tests/integration/test_ai_functions/test.py b/tests/integration/test_ai_functions/test.py index a84b2af3cd87d..98fd6cf9f10cb 100644 --- a/tests/integration/test_ai_functions/test.py +++ b/tests/integration/test_ai_functions/test.py @@ -19,7 +19,7 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance("node") +instance = cluster.add_instance("node", stay_alive=True) def run_mock_server(): @@ -98,7 +98,7 @@ def started_cluster() -> typing.Generator[ClickHouseCluster, None, None]: f"model = 'test-model', " f"api_key = 'test-key'" ) - # Slow endpoint, used to observe how many requests an AI function sends before it is stopped. + # Slow endpoints, used to observe how many requests an AI function has in flight at once. instance.query( f"CREATE NAMED COLLECTION ai_slow AS " f"provider = 'openai', " @@ -106,6 +106,26 @@ def started_cluster() -> typing.Generator[ClickHouseCluster, None, None]: f"model = 'test-model', " f"api_key = 'test-key'" ) + instance.query( + f"CREATE NAMED COLLECTION ai_hang AS " + f"provider = 'openai', " + f"endpoint = 'http://localhost:{MOCK_PORT}/v1/chat/hang', " + f"model = 'test-model', " + f"api_key = 'test-key'" + ) + instance.query( + f"CREATE NAMED COLLECTION ai_jitter AS " + f"provider = 'openai', " + f"endpoint = 'http://localhost:{MOCK_PORT}/v1/chat/jitter', " + f"model = 'test-model', " + f"api_key = 'test-key'" + ) + instance.query( + f"CREATE NAMED COLLECTION ai_embed_slow AS " + f"provider = 'openai', " + f"endpoint = 'http://localhost:{MOCK_PORT}/v1/embeddings_slow', " + f"api_key = 'test-key'" + ) instance.query( f"CREATE NAMED COLLECTION ai_error AS " f"provider = 'openai', " @@ -244,6 +264,14 @@ def started_cluster() -> typing.Generator[ClickHouseCluster, None, None]: f"endpoint = 'http://localhost:{MOCK_PORT}/v1/embeddings_wrong_count', " f"api_key = 'test-key'" ) + # Endpoint that always replies HTTP 429, as a rate-limited provider would. + instance.query( + f"CREATE NAMED COLLECTION ai_rate_limited AS " + f"provider = 'openai', " + f"endpoint = 'http://localhost:{MOCK_PORT}/v1/rate_limited', " + f"model = 'test-model', " + f"api_key = 'test-key'" + ) # Endpoints that drop the connection for the first N requests (armed via /set-flaky), # used to test that transient network failures are retried like the url table function. instance.query( @@ -417,6 +445,8 @@ def test_generate_truncated_response_counts_tokens(started_cluster): result = instance.query( "SELECT aiGenerate(x, map('credentials', 'ai_truncated')) FROM test_input", settings={ + # One request at a time, so the token quota is seen before the next request. + "ai_function_max_concurrent_requests_per_thread": 1, "ai_function_throw_on_error": 0, "ai_function_max_output_tokens_per_query": 5, "ai_function_throw_on_quota_exceeded": 0, @@ -1417,6 +1447,8 @@ def test_embed_quota_input_tokens_exceeded(started_cluster): result = instance.query( "SELECT aiEmbed(x, 'test-embed-model', map('credentials', 'ai_embed')) FROM test_input", settings={ + # One request at a time, so the token quota is seen before the next request. + "ai_function_max_concurrent_requests_per_thread": 1, "ai_function_embedding_max_batch_size": 1, "ai_function_max_input_tokens_per_query": 5, "ai_function_throw_on_quota_exceeded": 0, @@ -1467,6 +1499,62 @@ def test_generate_retries_on_network_error(started_cluster): assert int(events["rows_processed"]) == 1 +def _request_outcome_events(query_id): + instance.query("SYSTEM FLUSH LOGS") + return [ + int(v) + for v in instance.query( + f"SELECT ProfileEvents['AIAPICallsRetried'], ProfileEvents['AIAPICallsFailed'], " + f"ProfileEvents['AIAPICallsThrottled'] FROM system.query_log " + f"WHERE query_id = '{query_id}' AND type = 'QueryFinish'" + ).split() + ] + + +def _server_event(name): + return int(instance.query(f"SELECT sum(value) FROM system.events WHERE event = '{name}'").strip() or 0) + + +def test_request_outcome_profile_events(started_cluster): + """`AIAPICallsRetried`, `AIAPICallsFailed` and `AIAPICallsThrottled` count request outcomes per query + and for the whole server, including failures swallowed by `ai_function_throw_on_error = 0`.""" + settings = {"ai_function_throw_on_error": 0, "ai_function_retry_initial_delay_ms": 1} + + # Two dropped connections, then success: two retries, no failure. + set_flaky(2) + qid = unique_query_id("outcome_recovered") + instance.query( + "SELECT aiGenerate('x', map('credentials', 'ai_flaky')) FORMAT Null", + settings={**settings, "ai_function_max_retries": 5}, + query_id=qid, + ) + assert _request_outcome_events(qid) == [2, 0, 0] + + # Every attempt dropped: one retry, then the request fails and the row gets a default value. + set_flaky(10) + try: + qid = unique_query_id("outcome_failed") + instance.query( + "SELECT aiGenerate('x', map('credentials', 'ai_flaky')) FORMAT Null", + settings={**settings, "ai_function_max_retries": 1}, + query_id=qid, + ) + finally: + set_flaky(0) + assert _request_outcome_events(qid) == [1, 1, 0] + + # HTTP 429 on both attempts: both are throttled, one is retried, the request fails. + throttled_before = _server_event("AIAPICallsThrottled") + qid = unique_query_id("outcome_throttled") + instance.query( + "SELECT aiGenerate('x', map('credentials', 'ai_rate_limited')) FORMAT Null", + settings={**settings, "ai_function_max_retries": 1}, + query_id=qid, + ) + assert _request_outcome_events(qid) == [1, 1, 2] + assert _server_event("AIAPICallsThrottled") - throttled_before == 2 + + def test_generate_network_error_not_retried_when_disabled(started_cluster): """With `ai_function_max_retries = 0`, a network failure is surfaced rather than retried.""" set_flaky(10) @@ -2199,6 +2287,8 @@ def test_input_token_quota_is_per_query(started_cluster): instance.query( f"SELECT {CHAT_CALL} FROM quota_tokens FORMAT Null", settings={ + # One request at a time, so the token quota is seen before the next request. + "ai_function_max_concurrent_requests_per_thread": 1, **_QUOTA_SCOPE_SETTINGS, "ai_function_max_input_tokens_per_query": limit, "ai_function_throw_on_quota_exceeded": 0, @@ -2305,6 +2395,7 @@ def test_api_call_quota_ignores_subquery_settings(started_cluster): SLOW_CHAT_CALL = "aiClassify(toString(number), ['positive','negative'], map('credentials', 'ai_slow'))" +SLOW_EMBED_CALL = "aiEmbed(toString(number), 'test-model', map('credentials', 'ai_embed_slow'))" def _concurrency_stats(): @@ -2321,6 +2412,157 @@ def _reset_concurrency(): ) +def _observe_concurrency(call, concurrency, rows, expected_requests, extra_settings=None): + """Run `call` over `rows` rows at the given `ai_function_max_concurrent_requests_per_thread` and return + the high-water mark of requests the mock saw in flight at the same time.""" + _reset_concurrency() + qid = unique_query_id(f"ai_concurrency_{concurrency}") + instance.query( + f"SELECT {call} FROM numbers({rows}) FORMAT Null", + settings={ + "ai_function_max_concurrent_requests_per_thread": concurrency, + **(extra_settings or {}), + }, + query_id=qid, + ) + stats = _concurrency_stats() + assert stats["requests"] == expected_requests, ( + f"expected {expected_requests} requests, got {stats['requests']} at " + f"ai_function_max_concurrent_requests_per_thread={concurrency}" + ) + assert int(get_profile_events(qid)["api_calls"]) == expected_requests + return stats["max_concurrency"] + + +def test_max_concurrent_requests_controls_requests_in_flight(started_cluster): + """`ai_function_max_concurrent_requests_per_thread` decides how many provider requests a text AI function + has in flight, and nothing else does: the same query shape and the same data give one request + at a time at 1, and several at 8.""" + one = _observe_concurrency(SLOW_CHAT_CALL, concurrency=1, rows=16, expected_requests=16) + many = _observe_concurrency(SLOW_CHAT_CALL, concurrency=8, rows=16, expected_requests=16) + + assert one == 1, ( + f"ai_function_max_concurrent_requests_per_thread=1 must issue requests one at a time, saw {one} in flight" + ) + assert 2 <= many <= 8, ( + f"ai_function_max_concurrent_requests_per_thread=8 must overlap requests without exceeding the " + f"limit, saw {many} in flight" + ) + + +def test_max_concurrent_requests_applies_to_embeddings(started_cluster): + """The same setting governs the embedding path, where the unit in flight is a batch of texts + rather than a row: 16 rows at a batch size of 2 are 8 requests.""" + batching = {"ai_function_embedding_max_batch_size": 2} + one = _observe_concurrency( + SLOW_EMBED_CALL, concurrency=1, rows=16, expected_requests=8, extra_settings=batching + ) + many = _observe_concurrency( + SLOW_EMBED_CALL, concurrency=4, rows=16, expected_requests=8, extra_settings=batching + ) + + assert one == 1, ( + f"ai_function_max_concurrent_requests_per_thread=1 must issue batches one at a time, saw {one} in flight" + ) + assert 2 <= many <= 4, ( + f"ai_function_max_concurrent_requests_per_thread=4 must overlap batches without exceeding the " + f"limit, saw {many} in flight" + ) + + +def test_max_concurrent_requests_keeps_api_call_quota_exact(started_cluster): + """Concurrency must not let the API-call quota drift: a slot is reserved before each request is + dispatched, so the cap stays exact no matter how many requests are in flight.""" + _reset_concurrency() + qid = unique_query_id("ai_concurrency_quota") + instance.query( + f"SELECT {SLOW_CHAT_CALL} FROM numbers(64) FORMAT Null", + settings={ + "ai_function_max_concurrent_requests_per_thread": 8, + "ai_function_max_api_calls_per_query": 5, + "ai_function_throw_on_quota_exceeded": 0, + }, + query_id=qid, + ) + stats = _concurrency_stats() + assert stats["requests"] == 5, ( + f"the provider saw {stats['requests']} requests, but ai_function_max_api_calls_per_query = 5 " + "is an exact cap even with concurrent requests" + ) + assert int(get_profile_events(qid)["api_calls"]) == 5 + + +def test_concurrent_responses_map_to_their_rows(started_cluster): + """With several requests in flight and responses arriving in random order, every row still gets its + own response. The mock echoes the prompt after a random delay, so each output must equal its input. + Four blocks of 25 rows over two threads also run several function calls at once.""" + result = instance.query( + "SELECT countIf(answer = prompt), count() FROM " + "(SELECT concat('row-', toString(number)) AS prompt, " + "aiGenerate(prompt, map('credentials', 'ai_jitter')) AS answer FROM numbers_mt(100))", + settings={ + "ai_function_max_concurrent_requests_per_thread": 10, + "max_block_size": 25, + "max_threads": 2, + }, + ).strip() + assert result == "100\t100", f"expected every row to get its own response, got (matching, total) = {result}" + + +MULTI_CALL_QUERY = ( + "SELECT countIf(a = pa), countIf(b = pb), count(), sum(cityHash64(pa, e)) FROM " + "(SELECT concat('a-', toString(number)) AS pa, concat('b-', toString(number)) AS pb, " + "aiGenerate(pa, map('credentials', 'ai_jitter')) AS a, " + "aiGenerate(pb, map('credentials', 'ai_jitter')) AS b, " + "aiEmbed(pa, 'test-embed-model', map('credentials', 'ai_embed')) AS e " + "FROM numbers_mt(100))" +) +MULTI_CALL_SETTINGS = { + "max_block_size": 25, + "max_threads": 2, + "ai_function_embedding_max_batch_size": 1, +} + + +def test_multiple_ai_calls_in_one_query(started_cluster): + """Three AI function calls in one query share the request pool and the query's quota tracker. + Each call's outputs stay with its own rows, the embeddings match a run with one request at a time, + and `AIAPICalls` counts the requests of all three calls: 100 + 100 + 100 (one text per batch).""" + serial = instance.query( + MULTI_CALL_QUERY, settings={**MULTI_CALL_SETTINGS, "ai_function_max_concurrent_requests_per_thread": 1} + ).split("\t") + + qid = unique_query_id("ai_multi_call") + concurrent = instance.query( + MULTI_CALL_QUERY, + settings={**MULTI_CALL_SETTINGS, "ai_function_max_concurrent_requests_per_thread": 10}, + query_id=qid, + ).split("\t") + + assert concurrent[:3] == ["100", "100", "100"], ( + f"expected each aiGenerate call to answer its own rows, got (a matching, b matching, total) = {concurrent[:3]}" + ) + assert concurrent[3] == serial[3], "aiEmbed returned different row/embedding pairs with concurrent requests" + assert int(get_profile_events(qid)["api_calls"]) == 300 + + +def test_api_call_quota_is_shared_by_ai_calls_in_one_query(started_cluster): + """`ai_function_max_api_calls_per_query` is one allowance for all AI function calls of the query, + and stays exact with concurrent requests from three calls drawing on it at once.""" + qid = unique_query_id("ai_multi_call_quota") + instance.query( + MULTI_CALL_QUERY, + settings={ + **MULTI_CALL_SETTINGS, + "ai_function_max_concurrent_requests_per_thread": 10, + "ai_function_max_api_calls_per_query": 50, + "ai_function_throw_on_quota_exceeded": 0, + }, + query_id=qid, + ) + assert int(get_profile_events(qid)["api_calls"]) == 50 + + def test_kill_query_stops_issuing_requests(started_cluster): """`KILL QUERY` stops an AI function mid-block: no new request is sent after the kill. The 400 rows form one block, which at 0.5s per request would otherwise take 200s and send all 400 requests.""" @@ -2328,6 +2570,7 @@ def test_kill_query_stops_issuing_requests(started_cluster): qid = unique_query_id("ai_kill") request = instance.get_query_request( f"SELECT {SLOW_CHAT_CALL} FROM numbers(400) FORMAT Null", + settings={"ai_function_max_concurrent_requests_per_thread": 1}, query_id=qid, ) @@ -2355,7 +2598,7 @@ def test_max_execution_time_stops_issuing_requests(started_cluster): _reset_concurrency() error = instance.query_and_get_error( f"SELECT countIf(r != '') FROM (SELECT {SLOW_CHAT_CALL} AS r FROM numbers(400))", - settings={"max_execution_time": 2}, + settings={"max_execution_time": 2, "ai_function_max_concurrent_requests_per_thread": 1}, ) assert "TIMEOUT_EXCEEDED" in error requests = _concurrency_stats()["requests"] @@ -2371,7 +2614,83 @@ def test_max_execution_time_break_stops_issuing_requests(started_cluster): settings={ "max_execution_time": 2, "timeout_overflow_mode": "break", + "ai_function_max_concurrent_requests_per_thread": 1, }, ) requests = _concurrency_stats()["requests"] assert requests < 40, f"the query kept sending requests past the time limit: {requests} of 400 were sent" + + +AI_THREAD_POOL_CONFIG = "/etc/clickhouse-server/config.d/ai_thread_pool.xml" + + +def _set_ai_thread_pool_size(size): + instance.replace_config( + AI_THREAD_POOL_CONFIG, + f"{size}", + ) + instance.query("SYSTEM RELOAD CONFIG") + + +def test_cancelled_query_does_not_wait_for_other_queries_requests(started_cluster): + """With a single pool thread, query A queues 64 requests. Query B's request queues behind them, and B + fails on `max_execution_time` before A's requests are served. Once A is killed, its queued requests + are not sent.""" + _set_ai_thread_pool_size(1) + qid_a = unique_query_id("ai_backlog") + try: + _reset_concurrency() + request_a = instance.get_query_request( + f"SELECT {SLOW_CHAT_CALL} FROM numbers(400) FORMAT Null", + settings={"ai_function_max_concurrent_requests_per_thread": 64}, + query_id=qid_a, + ) + wait_condition( + _concurrency_stats, lambda stats: stats["requests"] >= 1, max_attempts=100, delay=0.1 + ) + + error = instance.query_and_get_error( + f"SELECT {SLOW_CHAT_CALL} FROM numbers(1) FORMAT Null", + settings={"max_execution_time": 1}, + ) + assert "TIMEOUT_EXCEEDED" in error + served = _concurrency_stats()["requests"] + assert served < 64, f"the timed-out query waited for {served} requests of the other query" + + instance.query(f"KILL QUERY WHERE query_id = '{qid_a}' SYNC") + assert "Query was cancelled" in request_a.get_error() + sent = _concurrency_stats()["requests"] + wait_condition( + lambda: instance.query( + "SELECT value FROM system.metrics WHERE metric = 'AIRequestThreadsScheduled'" + ).strip(), + lambda value: value == "0", + max_attempts=100, + delay=0.1, + ) + # A request that passed its cancellation check before the kill may still arrive. + assert _concurrency_stats()["requests"] <= sent + 1 + finally: + # Fails with `NOTHING_TO_KILL` when the query was already killed above. + instance.query_and_get_answer_with_error(f"KILL QUERY WHERE query_id = '{qid_a}' SYNC") + _set_ai_thread_pool_size(1000) + + +def test_shutdown_after_cancelling_query_with_request_in_flight(started_cluster): + """A killed query does not wait for its request to `/v1/chat/hang`. The server shuts down right after, + while the request is still in flight, and must drain the AI request pool before destroying the context.""" + _reset_concurrency() + qid = unique_query_id("ai_hang") + request = instance.get_query_request( + "SELECT aiGenerate(toString(number), map('credentials', 'ai_hang')) FROM numbers(1) FORMAT Null", + settings={"ai_function_max_retries": 0}, + query_id=qid, + ) + wait_condition( + _concurrency_stats, lambda stats: stats["requests"] >= 1, max_attempts=100, delay=0.1 + ) + instance.query(f"KILL QUERY WHERE query_id = '{qid}' SYNC") + assert "Query was cancelled" in request.get_error() + + instance.restart_clickhouse() + assert instance.query("SELECT aiGenerate('x', map('credentials', 'ai_slow'))").strip() != "" diff --git a/tests/queries/0_stateless/03300_ai_functions.reference b/tests/queries/0_stateless/03300_ai_functions.reference index 7fd7da0270370..146bff8b777d6 100644 --- a/tests/queries/0_stateless/03300_ai_functions.reference +++ b/tests/queries/0_stateless/03300_ai_functions.reference @@ -47,7 +47,7 @@ ai_function_embedding_max_batch_size 100 ai_function_max_api_calls_per_query 0 ai_function_max_input_tokens_per_query 0 ai_function_max_output_tokens_per_query 0 -ai_function_max_retries 1 +ai_function_max_retries 3 ai_function_request_timeout_sec 60 ai_function_retry_initial_delay_ms 1000 ai_function_text_default_credentials diff --git a/tests/queries/0_stateless/03300_ai_functions.sql b/tests/queries/0_stateless/03300_ai_functions.sql index 5ed5773b0f8e2..f787e53a51b32 100644 --- a/tests/queries/0_stateless/03300_ai_functions.sql +++ b/tests/queries/0_stateless/03300_ai_functions.sql @@ -15,6 +15,9 @@ -- `ai_function_text_default_credentials` / `ai_function_embedding_default_credentials`. -- ============================================================================= +-- The requests below go to unreachable endpoints. One retry keeps each failing row fast. +SET ai_function_max_retries = 1; + -- Helper table: a String column with zero rows, used to test function behavior -- without triggering actual HTTP calls. A non-constant column prevents the -- optimizer from constant-folding the AI function during analysis. diff --git a/tests/queries/0_stateless/04142_ai_functions_named_collection_access.sh b/tests/queries/0_stateless/04142_ai_functions_named_collection_access.sh index 017b8258c6d40..dcafa45c1b4de 100755 --- a/tests/queries/0_stateless/04142_ai_functions_named_collection_access.sh +++ b/tests/queries/0_stateless/04142_ai_functions_named_collection_access.sh @@ -31,7 +31,7 @@ CREATE USER $user_name IDENTIFIED WITH plaintext_password BY 'password'; # when no row reaches the function — otherwise an empty input could be used to bypass the grant. function check_access_both() { - $CLICKHOUSE_CLIENT --user "$user_name" --password "password" --multiquery --ignore-error -q " + $CLICKHOUSE_CLIENT --user "$user_name" --password "password" --ai_function_max_retries 1 --multiquery --ignore-error -q " SELECT aiGenerate('hi', map('credentials', '$collection_name')) FORMAT Null; SELECT 'SEP'; SELECT aiEmbed('hi', 'test-model', map('credentials', '$collection_name')) FORMAT Null; @@ -56,7 +56,7 @@ function check_access_both() # for the embedding functions) that must enforce the same NAMED_COLLECTION grant. function check_access_both_default() { - $CLICKHOUSE_CLIENT --user "$user_name" --password "password" --multiquery --ignore-error -q " + $CLICKHOUSE_CLIENT --user "$user_name" --password "password" --ai_function_max_retries 1 --multiquery --ignore-error -q " SET ai_function_text_default_credentials = '$collection_name'; SET ai_function_embedding_default_credentials = '$collection_name'; SELECT aiGenerate('hi') FORMAT Null; diff --git a/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.reference b/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.reference index 078656c7dc9d3..62d1ad01856ce 100644 --- a/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.reference +++ b/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.reference @@ -1,5 +1,5 @@ -- Current defaults -false 0 1 +false 0 3 8 -- compatibility = 26.6 restores the legacy defaults -true 0 0 +true 0 0 1 0 diff --git a/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.sql b/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.sql index 1fe196405fc4c..cc2a7875a35e9 100644 --- a/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.sql +++ b/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.sql @@ -3,10 +3,11 @@ -- no-replicated-database: named collections are server-global, not database-scoped -- ============================================================================= --- Two AI function defaults were flipped in 26.8: `ai_function_allow_insecure_endpoint` from 1 --- to 0 and `ai_function_max_retries` from 0 to 1. `ai_function_max_api_calls_per_query` stays 0 --- (unlimited). `compatibility = 26.6` predates the flips and restores them, which pins the --- previous_value/new_value pairs in `SettingsChangesHistory`. +-- Three AI function settings changed their defaults in 26.8: `ai_function_allow_insecure_endpoint` +-- from 1 to 0, `ai_function_max_retries` from 0 to 3, and the new +-- `ai_function_max_concurrent_requests_per_thread` is 8 where requests used to go out one at a time (1). +-- `ai_function_max_api_calls_per_query` stays 0 (unlimited). `compatibility = 26.6` predates the +-- changes and restores them, which pins the previous_value/new_value pairs in `SettingsChangesHistory`. -- -- The endpoint check runs in `resolveAIParams`, before the zero-row early return -- in `executeImpl`, so an empty source table exercises it without any real HTTP @@ -21,12 +22,12 @@ CREATE NAMED COLLECTION ai_compat_remote_http AS provider = 'openai', endpoint = 'http://ai.example.com/v1/chat/completions', model = 'chat-model', api_key = 'fake-key'; SELECT '-- Current defaults'; -SELECT getSetting('ai_function_allow_insecure_endpoint'), getSetting('ai_function_max_api_calls_per_query'), getSetting('ai_function_max_retries'); +SELECT getSetting('ai_function_allow_insecure_endpoint'), getSetting('ai_function_max_api_calls_per_query'), getSetting('ai_function_max_retries'), getSetting('ai_function_max_concurrent_requests_per_thread'); SELECT aiGenerate(x, map('credentials', 'ai_compat_remote_http')) FROM tab; -- { serverError BAD_ARGUMENTS } SELECT '-- compatibility = 26.6 restores the legacy defaults'; SET compatibility = '26.6'; -SELECT getSetting('ai_function_allow_insecure_endpoint'), getSetting('ai_function_max_api_calls_per_query'), getSetting('ai_function_max_retries'); +SELECT getSetting('ai_function_allow_insecure_endpoint'), getSetting('ai_function_max_api_calls_per_query'), getSetting('ai_function_max_retries'), getSetting('ai_function_max_concurrent_requests_per_thread'); SELECT count() FROM (SELECT aiGenerate(x, map('credentials', 'ai_compat_remote_http')) AS r FROM tab); -- ============================================================================= diff --git a/tests/queries/0_stateless/05394_ai_functions_retry_setting_limits.reference b/tests/queries/0_stateless/05394_ai_functions_retry_setting_limits.reference new file mode 100644 index 0000000000000..139597f9cb07c --- /dev/null +++ b/tests/queries/0_stateless/05394_ai_functions_retry_setting_limits.reference @@ -0,0 +1,2 @@ + + diff --git a/tests/queries/0_stateless/05394_ai_functions_retry_setting_limits.sql b/tests/queries/0_stateless/05394_ai_functions_retry_setting_limits.sql new file mode 100644 index 0000000000000..363215546ca78 --- /dev/null +++ b/tests/queries/0_stateless/05394_ai_functions_retry_setting_limits.sql @@ -0,0 +1,31 @@ +-- Tags: no-parallel, no-replicated-database +-- no-parallel: creates and drops global named collections +-- no-replicated-database: named collections are server-global, not database-scoped + +-- `ai_function_max_retries` is at most 5 and `ai_function_retry_initial_delay_ms` at most 10000. The limits are +-- checked before any request is sent, so the unreachable endpoint below is never contacted for a rejected query. + +DROP NAMED COLLECTION IF EXISTS ai_retry_limits; +DROP NAMED COLLECTION IF EXISTS ai_retry_limits_embed; +CREATE NAMED COLLECTION ai_retry_limits AS + provider = 'openai', endpoint = 'http://localhost:1/v1/chat/completions', model = 'test-model', api_key = 'fake-key'; +CREATE NAMED COLLECTION ai_retry_limits_embed AS + provider = 'openai', endpoint = 'http://localhost:1/v1/embeddings', api_key = 'fake-key'; + +DROP TABLE IF EXISTS t_retry_limits; +CREATE TABLE t_retry_limits (x String) ENGINE = Memory; +INSERT INTO t_retry_limits VALUES ('a'); + +SELECT aiGenerate(x, map('credentials', 'ai_retry_limits')) FROM t_retry_limits SETTINGS ai_function_max_retries = 6; -- { serverError BAD_ARGUMENTS } +SELECT aiGenerate(x, map('credentials', 'ai_retry_limits')) FROM t_retry_limits SETTINGS ai_function_retry_initial_delay_ms = 10001; -- { serverError BAD_ARGUMENTS } +SELECT aiEmbed(x, 'test-model', map('credentials', 'ai_retry_limits_embed')) FROM t_retry_limits SETTINGS ai_function_max_retries = 6; -- { serverError BAD_ARGUMENTS } + +-- The maximums themselves are accepted: the request fails against the unreachable endpoint and the row gets an empty result. +SELECT aiGenerate(x, map('credentials', 'ai_retry_limits')) FROM t_retry_limits +SETTINGS ai_function_max_retries = 5, ai_function_retry_initial_delay_ms = 1, ai_function_throw_on_error = 0, ai_function_request_timeout_sec = 3; +SELECT aiGenerate(x, map('credentials', 'ai_retry_limits')) FROM t_retry_limits +SETTINGS ai_function_max_retries = 0, ai_function_retry_initial_delay_ms = 10000, ai_function_throw_on_error = 0, ai_function_request_timeout_sec = 3; + +DROP TABLE t_retry_limits; +DROP NAMED COLLECTION ai_retry_limits; +DROP NAMED COLLECTION ai_retry_limits_embed; From 0c15963b47b4ff1263beffd12a5801cebf1ff338 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 1 Nov 2026 17:03:10 +0000 Subject: [PATCH 049/219] Backport #116666 to 26.8: Fix a server abort on Parquet reads that record no missing values --- src/Processors/Transforms/AddingDefaultsTransform.cpp | 3 ++- .../02302_defaults_in_columnar_formats.reference | 2 ++ .../0_stateless/02302_defaults_in_columnar_formats.sql | 9 +++++++++ 3 files changed, 13 insertions(+), 1 deletion(-) diff --git a/src/Processors/Transforms/AddingDefaultsTransform.cpp b/src/Processors/Transforms/AddingDefaultsTransform.cpp index 9e0fe0f5c260e..9650df7204bb4 100644 --- a/src/Processors/Transforms/AddingDefaultsTransform.cpp +++ b/src/Processors/Transforms/AddingDefaultsTransform.cpp @@ -157,7 +157,8 @@ void AddingDefaultsTransform::transform(Chunk & chunk) return; const auto * block_missing_values = input_format.getMissingValues(); - if (!block_missing_values) + /// `BlockMissingValues::empty` is also true when the mask holds no columns at all, where the lookups below would be out of range. + if (!block_missing_values || block_missing_values->empty()) return; const auto & header = getOutputPort().getHeader(); diff --git a/tests/queries/0_stateless/02302_defaults_in_columnar_formats.reference b/tests/queries/0_stateless/02302_defaults_in_columnar_formats.reference index 9942759a9480b..2d3bd2d2ebfd9 100644 --- a/tests/queries/0_stateless/02302_defaults_in_columnar_formats.reference +++ b/tests/queries/0_stateless/02302_defaults_in_columnar_formats.reference @@ -1,3 +1,5 @@ 1 10 42 53 1 10 42 53 1 10 42 53 +1 +1 0 0 0 diff --git a/tests/queries/0_stateless/02302_defaults_in_columnar_formats.sql b/tests/queries/0_stateless/02302_defaults_in_columnar_formats.sql index c49bf392895bb..eeea8bc121cea 100644 --- a/tests/queries/0_stateless/02302_defaults_in_columnar_formats.sql +++ b/tests/queries/0_stateless/02302_defaults_in_columnar_formats.sql @@ -6,3 +6,12 @@ insert into function file(currentDatabase() || '_data_02302.orc') select 1 as x, select * from file(currentDatabase() || '_data_02302.orc', auto, 'x UInt8, xx UInt8 default 10, y default 42, z default x + xx + y') settings input_format_orc_allow_missing_columns=1; insert into function file(currentDatabase() || '_data_02302.arrow') select 1 as x, null::Nullable(UInt8) as xx settings engine_file_truncate_on_insert=1; select * from file(currentDatabase() || '_data_02302.arrow', auto, 'x UInt8, xx UInt8 default 10, y default 42, z default x + xx + y') settings input_format_arrow_allow_missing_columns=1; + +-- A count-only read reads just the single smallest declared column, so `d` must stay strictly +-- narrower than every other column for this to exercise a defaulted one. The count cache would +-- answer before a reader exists, and the two count-only optimizations are randomized off in some runs, so all three are pinned. +-- Missing columns are rejected so that a read which is not count-only fails here instead of +-- synthesizing `d` into a correctly sized mask and returning the same count with no guard in place. +select count() from file(currentDatabase() || '_data_02302.parquet', auto, 'x UInt64, d UInt8 default 10') settings use_cache_for_count_from_files = 0, optimize_count_from_files = 1, optimize_trivial_count_query = 1, input_format_parquet_allow_missing_columns = 0; +-- With the setting off no `default` expression is applied, so the columns keep their type defaults. +select * from file(currentDatabase() || '_data_02302.parquet', auto, 'x UInt8, xx UInt8 default 10, y default 42, z default x + xx + y') settings input_format_parquet_allow_missing_columns=1, input_format_defaults_for_omitted_fields=0; From 2de47f1473e002ac8d8993f24152f9506e54da3d Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 2 Nov 2026 09:52:12 +0000 Subject: [PATCH 050/219] Backport #127630 to 26.8: Fix server failing to start once after an upgrade when all_... system log tables are replaced on startup --- src/Interpreters/DatabaseCatalog.cpp | 16 +++++++++++++ .../configs/prepare_on_startup.xml | 4 ++++ .../test.py | 24 +++++++++++++++++++ 3 files changed, 44 insertions(+) create mode 100644 tests/integration/test_create_union_system_log_tables/configs/prepare_on_startup.xml diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index a5289588bb1c9..dcb7839e0b61b 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -1297,6 +1297,16 @@ void DatabaseCatalog::loadMarkedAsDroppedTables() metadata_disk_list.insert(std::move(db_disk)); } + /// Startup may already have dropped tables, e.g. `SystemLog::prepareUnionTable` replaces union tables. + /// They are queued with their storage and their UUID mapping is still registered, so only drops of previous runs are loaded here. + auto is_drop_queued = [this](const DiskPtr & disk, const String & metadata_path) + { + std::lock_guard lock(tables_marked_dropped_mutex); + return std::ranges::any_of(tables_marked_dropped, [&](const TableMarkedAsDropped & dropped) + { + return dropped.metadata_path == metadata_path && dropped.db_disk->getName() == disk->getName(); + }); + }; /// /clickhouse_root/metadata_dropped/ contains files with metadata of tables, /// which where marked as dropped by Atomic databases. @@ -1337,6 +1347,12 @@ void DatabaseCatalog::loadMarkedAsDroppedTables() dropped_id.uuid = parse(sub_path_filename.substr(prev_dot_pos + 1, dot_pos - prev_dot_pos - 1)); String full_path = path + sub_path_filename; + if (is_drop_queued(db_disk, full_path)) + { + LOG_DEBUG(log, "Table {} was dropped during this startup and is already queued for removal", dropped_id.getNameForLogs()); + continue; + } + dropped_metadata.emplace(std::move(full_path), std::pair{std::move(dropped_id), db_disk}); } } diff --git a/tests/integration/test_create_union_system_log_tables/configs/prepare_on_startup.xml b/tests/integration/test_create_union_system_log_tables/configs/prepare_on_startup.xml new file mode 100644 index 0000000000000..af9d6b3ee639e --- /dev/null +++ b/tests/integration/test_create_union_system_log_tables/configs/prepare_on_startup.xml @@ -0,0 +1,4 @@ + + + 1 + diff --git a/tests/integration/test_create_union_system_log_tables/test.py b/tests/integration/test_create_union_system_log_tables/test.py index c170b9b90a0fc..87628d14645fe 100644 --- a/tests/integration/test_create_union_system_log_tables/test.py +++ b/tests/integration/test_create_union_system_log_tables/test.py @@ -34,6 +34,17 @@ stay_alive=True, ) +# Starts with the first release that has union tables and is upgraded. The log tables are prepared +# during startup, and drops are asynchronous (the default). +node4 = cluster.add_instance( + "node4", + main_configs=["configs/union_merge.xml", "configs/prepare_on_startup.xml"], + image="clickhouse/clickhouse-server", + tag="26.8", + with_installed_binary=True, + stay_alive=True, +) + @pytest.fixture(scope="module", autouse=True) def start_cluster(): @@ -152,6 +163,19 @@ def test_stable_across_restarts(start_cluster): assert uuid_before == uuid_after +def test_replaced_during_upgrade(start_cluster): + node4.query("SYSTEM FLUSH LOGS trace_log") + old_create_query = node4.query("SHOW CREATE TABLE system.all_trace_log FORMAT TSVRaw") + + # `all_trace_log` contains the build id, so the first start of another build replaces it, and it does so + # before loading the tables left in `metadata_dropped` by previous runs. The replaced table is dropped + # by this run and must not be loaded as one of them. + node4.restart_with_latest_version() + + assert node4.query("SHOW CREATE TABLE system.all_trace_log FORMAT TSVRaw") != old_create_query + assert not node4.contains_in_log("UUID collision", from_host=True) + + def test_cluster(start_cluster): node1.query("SELECT 'test_cluster_marker_node1'") node1.query("SYSTEM FLUSH LOGS query_log") From c773ce4ba3b09faad0e604eb9967737c7e7dfaf0 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 1 Nov 2026 22:17:30 +0000 Subject: [PATCH 051/219] Backport #128476 to 26.8: Check `LowCardinality` indexes against the dictionary when reading data parts --- src/Columns/ColumnLowCardinality.cpp | 10 +- src/Columns/ColumnLowCardinality.h | 1 + .../SerializationLowCardinality.cpp | 7 + .../gtest_low_cardinality_serialization.cpp | 179 ++++++++++++++++++ 4 files changed, 192 insertions(+), 5 deletions(-) create mode 100644 src/DataTypes/Serializations/tests/gtest_low_cardinality_serialization.cpp diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp index b40c289a53c69..febbfee4ac5fd 100644 --- a/src/Columns/ColumnLowCardinality.cpp +++ b/src/Columns/ColumnLowCardinality.cpp @@ -254,7 +254,7 @@ void ColumnLowCardinality::insertRangeFromFullColumn(const IColumn & src, size_t idx.insertIndexesRange(*inserted_indexes, 0, length); } -static void checkIndexesAreLimited(const IColumn & indexes, UInt64 limit) +void ColumnLowCardinality::checkIndexesAreLimited(const IColumn & indexes, UInt64 limit) { auto check_for_type = [&](auto type) { @@ -268,14 +268,14 @@ static void checkIndexesAreLimited(const IColumn & indexes, UInt64 limit) size_t num_rows = data.size(); if (num_rows == 0) return true; - UInt64 max_position = 0; + ColumnType max_position = 0; for (size_t i = 0; i < num_rows; ++i) - max_position = std::max(max_position, data[i]); + max_position = std::max(max_position, data[i]); - if (max_position >= limit) + if (static_cast(max_position) >= limit) throw Exception(ErrorCodes::INCORRECT_DATA, "Index for LowCardinality is out of range. Dictionary size is {}, " - "but found index with value {}", limit, max_position); + "but found index with value {}", limit, static_cast(max_position)); return true; }; diff --git a/src/Columns/ColumnLowCardinality.h b/src/Columns/ColumnLowCardinality.h index 701c9ce702968..f8f60b05e76f3 100644 --- a/src/Columns/ColumnLowCardinality.h +++ b/src/Columns/ColumnLowCardinality.h @@ -95,6 +95,7 @@ class ColumnLowCardinality final : public COWHelperindex_type.has_additional_keys; bool column_is_empty = low_cardinality_column.empty(); + /// Indexes come from the stream. + if (low_cardinality_state->index_type.need_global_dictionary) + { + const size_t num_additional_keys = additional_keys ? additional_keys->size() : 0; + ColumnLowCardinality::checkIndexesAreLimited(*indexes_column, global_dictionary->size() + num_additional_keys); + } + if (!low_cardinality_state->index_type.need_global_dictionary) { if (additional_keys == nullptr) diff --git a/src/DataTypes/Serializations/tests/gtest_low_cardinality_serialization.cpp b/src/DataTypes/Serializations/tests/gtest_low_cardinality_serialization.cpp new file mode 100644 index 0000000000000..9aa26ad5cd997 --- /dev/null +++ b/src/DataTypes/Serializations/tests/gtest_low_cardinality_serialization.cpp @@ -0,0 +1,179 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace DB::ErrorCodes +{ + extern const int INCORRECT_DATA; +} + +using namespace DB; + +namespace +{ + +/// The MergeTree layout: the dictionary in one stream, the indexes and the additional keys in another. +template +auto makeGetter(Buffer & keys, Buffer & indexes) +{ + return [&keys, &indexes](const ISerialization::SubstreamPath & path) -> BufferPtr + { + if (!path.empty() && path.back().type == ISerialization::Substream::DictionaryKeys) + return &keys; + return &indexes; + }; +} + +struct Streams +{ + std::string keys; + std::string indexes; +}; + +Streams serialize(const DataTypePtr & type, const IColumn & column, size_t max_dictionary_size) +{ + WriteBufferFromOwnString keys; + WriteBufferFromOwnString indexes; + auto serialization = type->getDefaultSerialization(); + ISerialization::SerializeBinaryBulkSettings settings; + settings.getter = makeGetter(keys, indexes); + settings.low_cardinality_max_dictionary_size = max_dictionary_size; + settings.low_cardinality_use_single_dictionary_for_part = false; + ISerialization::SerializeBinaryBulkStatePtr state; + serialization->serializeBinaryBulkStatePrefix(column, settings, state); + serialization->serializeBinaryBulkWithMultipleStreams(column, 0, column.size(), settings, state); + serialization->serializeBinaryBulkStateSuffix(settings, state); + keys.finalize(); + indexes.finalize(); + return {keys.str(), indexes.str()}; +} + +ColumnPtr deserialize(const DataTypePtr & type, const Streams & streams, size_t rows) +{ + ReadBufferFromString keys(streams.keys); + ReadBufferFromString indexes(streams.indexes); + auto serialization = type->getDefaultSerialization(); + ISerialization::DeserializeBinaryBulkSettings settings; + settings.getter = makeGetter(keys, indexes); + settings.native_format = false; + ISerialization::DeserializeBinaryBulkStatePtr state; + serialization->deserializeBinaryBulkStatePrefix(settings, state, nullptr); + auto column = type->createColumn(); + serialization->deserializeBinaryBulkWithMultipleStreams(*column, rows, settings, state, nullptr); + return column; +} + +void expectIncorrectData(const DataTypePtr & type, const Streams & streams, size_t rows) +{ + try + { + deserialize(type, streams, rows); + FAIL() << "accepted an index past the keys"; + } + catch (const Exception & e) + { + ASSERT_EQ(e.code(), ErrorCodes::INCORRECT_DATA) << e.message(); + } +} + +/// Calls `check` for `LowCardinality(String)` and `LowCardinality(Nullable(UInt64))`, each serialized with a dictionary +/// of 2 keys (so most values become additional keys) and with one large enough for all values (no additional keys). +template +void forEachCase(Check && check) +{ + for (bool nullable : {false, true}) + { + DataTypePtr inner = nullable ? makeNullable(std::make_shared()) : std::make_shared(); + auto type = std::make_shared(inner); + auto column = type->createColumn(); + for (size_t i = 0; i < 20; ++i) + { + if (nullable) + column->insert(i % 6 == 0 ? Field() : Field(UInt64(i % 6))); + else + column->insert(String(1, static_cast('a' + i % 5))); + } + + for (size_t max_dictionary_size : {2, 8192}) + { + SCOPED_TRACE(type->getName() + " with max dictionary size " + std::to_string(max_dictionary_size)); + auto streams = serialize(type, *column, max_dictionary_size); + + constexpr UInt64 need_global_dictionary_bit = 1ULL << 8; + constexpr UInt64 has_additional_keys_bit = 1ULL << 9; + const UInt64 index_type = unalignedLoadLittleEndian(streams.indexes.data()); + const bool has_additional_keys = (index_type & has_additional_keys_bit) != 0; + ASSERT_NE(index_type & need_global_dictionary_bit, 0u); + ASSERT_EQ(has_additional_keys, max_dictionary_size == 2); + ASSERT_LE(index_type & 0xff, 3u); + + /// The keys stream starts with the version, then the number of keys; the indexes stream starts with the index type, + /// then the number of additional keys if there are any. + const UInt64 num_keys = unalignedLoadLittleEndian(streams.keys.data() + 8) + + (has_additional_keys ? unalignedLoadLittleEndian(streams.indexes.data() + 8) : 0); + const size_t index_width = 1ULL << (index_type & 0xff); + + check(type, *column, std::move(streams), num_keys, index_width); + } + } +} + +} + +TEST(SerializationLowCardinality, ValidStreamsRoundTrip) +{ + MainThreadStatus::getInstance(); + forEachCase([](const DataTypePtr & type, const IColumn & column, Streams streams, UInt64, size_t) + { + auto result = deserialize(type, streams, column.size()); + ASSERT_EQ(result->size(), column.size()); + for (size_t i = 0; i < column.size(); ++i) + ASSERT_EQ((*result)[i], column[i]) << "row " << i; + }); +} + +/// The indexes come from the data, so they have to be checked against the keys of the dictionary and the additional keys. +TEST(SerializationLowCardinality, IndexPastTheKeysIsRejected) +{ + MainThreadStatus::getInstance(); + forEachCase([](const DataTypePtr & type, const IColumn & column, Streams streams, UInt64 num_keys, size_t index_width) + { + const size_t rows = column.size(); + const size_t indexes_begin = streams.indexes.size() - rows * index_width; + + /// The same indexes as `UInt64` (index type 3, the low byte of the index type) read back the same values... + Streams wide{streams.keys, streams.indexes.substr(0, indexes_begin)}; + wide.indexes[0] = 3; + for (size_t row = 0; row < rows; ++row) + { + UInt64 index = 0; + for (size_t i = 0; i < index_width; ++i) + index |= UInt64(static_cast(streams.indexes[indexes_begin + row * index_width + i])) << (8 * i); + for (size_t i = 0; i < 8; ++i) + wide.indexes.push_back(static_cast(index >> (8 * i))); + } + auto result = deserialize(type, wide, rows); + ASSERT_EQ(result->size(), rows); + for (size_t row = 0; row < rows; ++row) + ASSERT_EQ((*result)[row], column[row]) << "row " << row; + + /// ...and the largest possible last index is rejected before `mapIndexWithAdditionalKeys` sizes its overflow map by it. + std::fill(wide.indexes.end() - 8, wide.indexes.end(), '\xff'); + expectIncorrectData(type, wide, rows); + + /// The last index of the original stream becomes the first value past the keys. + ASSERT_LE(num_keys, std::numeric_limits::max() >> (64 - 8 * index_width)); + for (size_t i = 0; i < index_width; ++i) + streams.indexes[streams.indexes.size() - index_width + i] = static_cast(num_keys >> (8 * i)); + expectIncorrectData(type, streams, rows); + }); +} From e2c22ea52e98d6c72a52398012935ebcb5cb7910 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 2 Nov 2026 11:45:40 +0000 Subject: [PATCH 052/219] Backport #117147 to 26.8: Fix out-of-bounds read in the ReplicatedMergeTree cleanup thread when `max_replicated_logs_to_keep = 0` --- .../BuzzHouse/Generator/TableSettings.cpp | 2 +- src/Storages/MergeTree/MergeTreeSettings.cpp | 2 +- .../ReplicatedMergeTreeCleanupThread.cpp | 2 +- ...replicated_max_logs_to_keep_zero.reference | 4 ++ .../05053_replicated_max_logs_to_keep_zero.sh | 42 +++++++++++++++++++ 5 files changed, 49 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/05053_replicated_max_logs_to_keep_zero.reference create mode 100755 tests/queries/0_stateless/05053_replicated_max_logs_to_keep_zero.sh diff --git a/src/Client/BuzzHouse/Generator/TableSettings.cpp b/src/Client/BuzzHouse/Generator/TableSettings.cpp index de0bd32094c6e..d1aa71b132b6d 100644 --- a/src/Client/BuzzHouse/Generator/TableSettings.cpp +++ b/src/Client/BuzzHouse/Generator/TableSettings.cpp @@ -295,7 +295,7 @@ static std::unordered_map mergeTreeTableSettings = { {"0", "1", "5", "25"}, false)}, {"max_replicated_fetches_network_bandwidth", bytesRangeSetting}, - {"max_replicated_logs_to_keep", highRangeSetting}, + {"max_replicated_logs_to_keep", highRangeNonZeroSetting}, {"max_replicated_merges_in_queue", CHSetting( [](RandomGenerator & rg, FuzzConfig &) { return std::to_string(rg.thresholdGenerator(0.2, 0.2, 0, 100)); }, diff --git a/src/Storages/MergeTree/MergeTreeSettings.cpp b/src/Storages/MergeTree/MergeTreeSettings.cpp index e2ee832cac906..091afbbed4dd3 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -1293,7 +1293,7 @@ Deprecated alias of `deduplication_hashes_cache_update_wait_ms`, kept for one re compatibility. It is honored only when `deduplication_hashes_cache_update_wait_ms` is left at its default; this setting will be removed in a future release. )", 0) \ - DECLARE(UInt64, max_replicated_logs_to_keep, 1000, R"( + DECLARE(NonZeroUInt64, max_replicated_logs_to_keep, 1000, R"( How many records may be in the ClickHouse Keeper log if there is inactive replica. An inactive replica becomes lost when when this number exceed. diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index f3e273840ced9..82d904c7670cc 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -12,7 +12,7 @@ namespace MergeTreeSetting { extern const MergeTreeSettingsUInt64 finished_mutations_to_keep; extern const MergeTreeSettingsSeconds lock_acquire_timeout_for_background_operations; - extern const MergeTreeSettingsUInt64 max_replicated_logs_to_keep; + extern const MergeTreeSettingsNonZeroUInt64 max_replicated_logs_to_keep; extern const MergeTreeSettingsUInt64 min_replicated_logs_to_keep; extern const MergeTreeSettingsUInt64 replicated_deduplication_window; extern const MergeTreeSettingsUInt64 replicated_deduplication_window_for_async_inserts; diff --git a/tests/queries/0_stateless/05053_replicated_max_logs_to_keep_zero.reference b/tests/queries/0_stateless/05053_replicated_max_logs_to_keep_zero.reference new file mode 100644 index 0000000000000..98fb6a6865639 --- /dev/null +++ b/tests/queries/0_stateless/05053_replicated_max_logs_to_keep_zero.reference @@ -0,0 +1,4 @@ +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/05053_replicated_max_logs_to_keep_zero.sh b/tests/queries/0_stateless/05053_replicated_max_logs_to_keep_zero.sh new file mode 100755 index 0000000000000..0e451abc81517 --- /dev/null +++ b/tests/queries/0_stateless/05053_replicated_max_logs_to_keep_zero.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Tags: zookeeper, replica, no-shared-merge-tree, no-replicated-database +# no-shared-merge-tree: the setting bounds the ReplicatedMergeTree log, which SharedMergeTree does not have +# no-replicated-database: the ZooKeeper path and the replica name are literal, so they collide between the hosts of the database + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# The oldest kept log record names the threshold that log pointers of inactive replicas are compared +# with, so a window that keeps no records at all has no meaning. +ERR="A setting's value has to be greater than 0" + +$CLICKHOUSE_CLIENT --query " + CREATE TABLE max_logs_zero (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r', '1') ORDER BY x + SETTINGS max_replicated_logs_to_keep = 0; +" 2>&1 | grep -F -q "$ERR" && echo 1 || echo 0 + +# An ATTACH that carries a full definition is user input too, and reaches the same check. +uuid=$($CLICKHOUSE_CLIENT --query "SELECT generateUUIDv4()") +$CLICKHOUSE_CLIENT --query " + ATTACH TABLE max_logs_zero UUID '$uuid' (x UInt64) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r', '1') ORDER BY x + SETTINGS max_replicated_logs_to_keep = 0; +" 2>&1 | grep -F -q "$ERR" && echo 1 || echo 0 + +$CLICKHOUSE_CLIENT --query " + CREATE TABLE max_logs (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r2', '1') ORDER BY x; +" +$CLICKHOUSE_CLIENT --query " + ALTER TABLE max_logs MODIFY SETTING max_replicated_logs_to_keep = 0; +" 2>&1 | grep -F -q "$ERR" && echo 1 || echo 0 + +# A positive value is still accepted and stored. +$CLICKHOUSE_CLIENT --query "ALTER TABLE max_logs MODIFY SETTING max_replicated_logs_to_keep = 1;" +$CLICKHOUSE_CLIENT --query " + SELECT count() FROM system.tables + WHERE database = currentDatabase() AND name = 'max_logs' + AND engine_full LIKE '%max_replicated_logs_to_keep = 1%'; +" + +$CLICKHOUSE_CLIENT --query "DROP TABLE max_logs SYNC" From ade68b4e49adda8859ca507147cf792b90264dbf Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 2 Nov 2026 11:47:35 +0000 Subject: [PATCH 053/219] Backport #113673 to 26.8: Fix signed integer overflow in refreshable materialized view `RANDOMIZE FOR` --- .../MaterializedView/RefreshSchedule.cpp | 7 +++++- ...shable_mv_randomize_for_overflow.reference | 1 + ...7_refreshable_mv_randomize_for_overflow.sh | 25 +++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/04757_refreshable_mv_randomize_for_overflow.reference create mode 100755 tests/queries/0_stateless/04757_refreshable_mv_randomize_for_overflow.sh diff --git a/src/Storages/MaterializedView/RefreshSchedule.cpp b/src/Storages/MaterializedView/RefreshSchedule.cpp index 91c164133bd8c..f6a54a31068de 100644 --- a/src/Storages/MaterializedView/RefreshSchedule.cpp +++ b/src/Storages/MaterializedView/RefreshSchedule.cpp @@ -2,6 +2,8 @@ #include +#include + namespace DB { @@ -70,7 +72,10 @@ std::chrono::sys_seconds RefreshSchedule::advance(std::chrono::sys_seconds last_ std::chrono::system_clock::time_point RefreshSchedule::addRandomSpread(std::chrono::system_clock::time_point when, Int64 randomness) const { - return when + std::chrono::milliseconds(Int64(static_cast(spread.minSeconds()) * 1e3 / 2 * static_cast(randomness) / 1e9)); + /// RANDOMIZE FOR is unbounded, so compute in Int128 and cap the offset at about 95 years. + constexpr Int64 max_offset_ms = 3'000'000'000'000; + const Int128 offset_ms = Int128(spread.minSeconds()) * randomness / 2'000'000; + return when + std::chrono::milliseconds(Int64(std::clamp(offset_ms, -max_offset_ms, max_offset_ms))); } } diff --git a/tests/queries/0_stateless/04757_refreshable_mv_randomize_for_overflow.reference b/tests/queries/0_stateless/04757_refreshable_mv_randomize_for_overflow.reference new file mode 100644 index 0000000000000..d00491fd7e5bb --- /dev/null +++ b/tests/queries/0_stateless/04757_refreshable_mv_randomize_for_overflow.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/04757_refreshable_mv_randomize_for_overflow.sh b/tests/queries/0_stateless/04757_refreshable_mv_randomize_for_overflow.sh new file mode 100755 index 0000000000000..b47c10f274496 --- /dev/null +++ b/tests/queries/0_stateless/04757_refreshable_mv_randomize_for_overflow.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Tags: memory-engine, atomic-database + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# An absurd RANDOMIZE FOR caps the offset at 3e9 seconds. The first timeslot is 2170-01-01, so both +# signs of the offset stay in the future and the view is scheduled without refreshing. +$CLICKHOUSE_CLIENT -q " + create materialized view rmv refresh every 200 year randomize for 10000000000000000000 second + (x Int64) engine Memory as select 1 as x;" + +# next_refresh_time is a UInt32 DateTime, so the expected instants are compared modulo 2^32. +query=" + with toInt64(toDateTime64('2170-01-01 00:00:00', 0, 'UTC')) as slot + select status = 'Scheduled' and last_refresh_time is null + and toUInt32(next_refresh_time) in ((slot - 3000000000) % 4294967296, (slot + 3000000000) % 4294967296) + from system.view_refreshes where database = currentDatabase() and view = 'rmv'" + +for _ in {1..100}; do + [ "$($CLICKHOUSE_CLIENT -q "$query")" = 1 ] && break + sleep 0.1 +done +$CLICKHOUSE_CLIENT -q "$query" From 65c063b332966d103b4c339e6e87c2433e6f6caf Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 2 Nov 2026 13:52:23 +0000 Subject: [PATCH 054/219] Backport #117798 to 26.8: Send the definer identity to remote shards from SQL SECURITY DEFINER views without an invoker --- .../DistributedAsyncInsertBatch.cpp | 18 ++ .../DistributedAsyncInsertDirectoryQueue.cpp | 6 + .../DistributedAsyncInsertHelpers.cpp | 8 + .../DistributedAsyncInsertHelpers.h | 9 + src/Storages/Distributed/DistributedSink.cpp | 18 +- src/Storages/StorageInMemoryMetadata.cpp | 37 +++ src/Storages/StorageMaterializedView.cpp | 14 +- .../__init__.py | 0 .../configs/remote_servers.xml | 11 + .../test.py | 99 ++++++ .../__init__.py | 0 .../configs/rabbitmq.xml | 6 + .../configs/remote_servers.xml | 14 + .../configs/users.xml | 23 ++ .../test.py | 296 ++++++++++++++++++ ...uted_async_insert_batch_per_user.reference | 1 + ...distributed_async_insert_batch_per_user.sh | 41 +++ 17 files changed, 591 insertions(+), 10 deletions(-) create mode 100644 tests/integration/test_refreshable_mv_definer_interserver_secret/__init__.py create mode 100644 tests/integration/test_refreshable_mv_definer_interserver_secret/configs/remote_servers.xml create mode 100644 tests/integration/test_refreshable_mv_definer_interserver_secret/test.py create mode 100644 tests/integration/test_streaming_engines_definer_interserver_secret/__init__.py create mode 100644 tests/integration/test_streaming_engines_definer_interserver_secret/configs/rabbitmq.xml create mode 100644 tests/integration/test_streaming_engines_definer_interserver_secret/configs/remote_servers.xml create mode 100644 tests/integration/test_streaming_engines_definer_interserver_secret/configs/users.xml create mode 100644 tests/integration/test_streaming_engines_definer_interserver_secret/test.py create mode 100644 tests/queries/0_stateless/05445_distributed_async_insert_batch_per_user.reference create mode 100755 tests/queries/0_stateless/05445_distributed_async_insert_batch_per_user.sh diff --git a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp index 819576c2dabf7..8786820604c4d 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp +++ b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp @@ -21,6 +21,7 @@ #include #include +#include #include namespace CurrentMetrics @@ -234,6 +235,11 @@ bool DistributedAsyncInsertBatch::recoverBatch() } files.erase(files.begin(), first_existing_file); + /// The batch is sent with the `ClientInfo` of its first file. Older servers could put files of different + /// users (or originating hosts) into one batch, so a restored batch of mixed identities must not be resent + /// as a whole: it is dropped, and its files are batched again by identity like any other pending files. + std::optional> batch_identity; + /// A missing file inside the surviving suffix cannot be recovered safely. for (const auto & file : files) { @@ -252,6 +258,18 @@ bool DistributedAsyncInsertBatch::recoverBatch() total_rows += header.rows; total_bytes += header.bytes; } + + auto identity = getRemoteAuthenticationIdentity(header.client_info); + if (!batch_identity) + { + batch_identity = std::move(identity); + } + else if (identity != *batch_identity) + { + LOG_WARNING(parent.log, "File {} has a different user or originating host than the first file of the batch, " + "the files of the batch will be sent in new batches", file); + return false; + } } catch (Exception & e) { diff --git a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp index da636ba25c50c..469feb7731857 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp +++ b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp @@ -525,8 +525,11 @@ struct DistributedAsyncInsertDirectoryQueue::BatchHeader bool operator==(const BatchHeader & other) const { + /// The whole batch is sent with the `ClientInfo` of its first file, so files that the shard would + /// authenticate differently must never be merged into one batch. return std::tie(settings, query, client_info.query_kind, client_info.client_agent, client_info.is_internal) == std::tie(other.settings, other.query, other.client_info.query_kind, other.client_info.client_agent, other.client_info.is_internal) && + getRemoteAuthenticationIdentity(client_info) == getRemoteAuthenticationIdentity(other.client_info) && blocksHaveEqualStructure(header, other.header); } @@ -536,6 +539,9 @@ struct DistributedAsyncInsertDirectoryQueue::BatchHeader { SipHash hash_state; hash_state.update(batch_header.query.data(), batch_header.query.size()); + const auto [initial_user, initial_host] = getRemoteAuthenticationIdentity(batch_header.client_info); + hash_state.update(initial_user.data(), initial_user.size()); + hash_state.update(initial_host.data(), initial_host.size()); hash_state.update(batch_header.client_info.client_agent.data(), batch_header.client_info.client_agent.size()); hash_state.update(batch_header.client_info.is_internal); batch_header.header.updateHash(hash_state); diff --git a/src/Storages/Distributed/DistributedAsyncInsertHelpers.cpp b/src/Storages/Distributed/DistributedAsyncInsertHelpers.cpp index 8cbb8461f5ea7..7890ecb8f89e0 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertHelpers.cpp +++ b/src/Storages/Distributed/DistributedAsyncInsertHelpers.cpp @@ -68,6 +68,14 @@ static void writeAndConvert(RemoteInserter & remote, const DistributedAsyncInser } } +std::pair getRemoteAuthenticationIdentity(const ClientInfo & client_info) +{ + String initial_host; + if (client_info.initial_address) + initial_host = client_info.initial_address->host().toString(); + return {client_info.initial_user, initial_host}; +} + void writeRemoteConvert( const DistributedAsyncInsertHeader & distributed_header, RemoteInserter & remote, diff --git a/src/Storages/Distributed/DistributedAsyncInsertHelpers.h b/src/Storages/Distributed/DistributedAsyncInsertHelpers.h index 202d9ff6fff8b..3cc335c169f98 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertHelpers.h +++ b/src/Storages/Distributed/DistributedAsyncInsertHelpers.h @@ -1,12 +1,16 @@ #pragma once #include +#include + +#include namespace DB { struct DistributedAsyncInsertHeader; +class ClientInfo; class ReadBufferFromFile; class RemoteInserter; @@ -23,6 +27,11 @@ class RemoteInserter; /// and mark batch as broken. bool isDistributedSendBroken(int code, bool remote_error); +/// The identity the shard authenticates a queued insert as (with the interserver ``): `initial_user` +/// and the host of `initial_address`, which is checked against the allowed hosts of that user. +/// A batch is sent with the `ClientInfo` of its first file, so all files of a batch must have the same identity. +std::pair getRemoteAuthenticationIdentity(const ClientInfo & client_info); + void writeRemoteConvert( const DistributedAsyncInsertHeader & distributed_header, RemoteInserter & remote, diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index a94d739a18933..84f8f89df8657 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -966,17 +966,21 @@ void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const /// `client_agent` is intentionally excluded from the embedded `ClientInfo` here and written /// as a trailing header field below, so that older binaries draining these queue files read /// the embedded `ClientInfo` in the old, append-compatible layout. - if (OpenTelemetry::CurrentContext().isTraceEnabled()) { - // if the distributed tracing is enabled, use the trace context in current thread as parent of next span auto client_info = context->getClientInfo(); - client_info.client_trace_context = OpenTelemetry::CurrentContext(); + // if the distributed tracing is enabled, use the trace context in current thread as parent of next span + if (OpenTelemetry::CurrentContext().isTraceEnabled()) + client_info.client_trace_context = OpenTelemetry::CurrentContext(); + /// Background contexts (e.g. the ones of `Kafka` / `RabbitMQ` streaming into views) have + /// `query_kind = NO_QUERY`, for which `ClientInfo::write` stores nothing but the kind. The + /// replayed insert would then reach the shard with an empty `initial_user`, which a shard + /// accepting the interserver `` executes with full access, losing the identity + /// (e.g. the definer of a `SQL SECURITY DEFINER` view) that the query is bound to. + /// `RemoteInserter` sends the insert as a secondary query anyway, so persist it as such. + if (client_info.query_kind == ClientInfo::QueryKind::NO_QUERY) + client_info.query_kind = ClientInfo::QueryKind::SECONDARY_QUERY; client_info.write(header_buf, DBMS_TCP_PROTOCOL_VERSION, /*with_trailing_fields=*/ false); } - else - { - context->getClientInfo().write(header_buf, DBMS_TCP_PROTOCOL_VERSION, /*with_trailing_fields=*/ false); - } writeVarUInt(block.rows(), header_buf); writeVarUInt(block.bytes(), header_buf); diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp index 79df13dafb692..3f52ddca92d9e 100644 --- a/src/Storages/StorageInMemoryMetadata.cpp +++ b/src/Storages/StorageInMemoryMetadata.cpp @@ -229,6 +229,43 @@ ContextMutablePtr StorageInMemoryMetadata::getSQLSecurityOverriddenContext(Conte new_context->setUser(getDefinerID(context)); + /// `Context::setUser` only sets the access-control identity (`user_id`); it does not touch the + /// user names in `ClientInfo`. That is harmless when there is an invoker whose names were copied + /// above, but a scheduled refresh of a refreshable materialized view builds its context from the + /// global context, where `current_user`, `initial_user` and `authenticated_user` are empty. + /// `RemoteQueryExecutor` sends the names as-is, and a shard that accepts the interserver + /// `` treats an empty `initial_user` as "interserver mode": it skips authentication + /// (see `TCPHandler::receiveQuery`) and executes the query with full access + /// (see `Context::getAccess`, `full_access = !user_id`). Stamp the definer into the empty fields + /// so the remote leg of the query is bounded by the same identity as the local one. Non-empty + /// fields are left alone so `currentUser` inside an ordinary `DEFINER` view still returns the + /// invoker. `AsynchronousInsertQueue` and `InterpreterExecuteAsQuery` do the same. + { + const auto & new_client_info = new_context->getClientInfo(); + const bool current_user_empty = new_client_info.current_user.empty(); + const bool initial_user_empty = new_client_info.initial_user.empty(); + const bool authenticated_user_empty = new_client_info.authenticated_user.empty(); + + if (current_user_empty || initial_user_empty || authenticated_user_empty) + { + const auto definer_name = new_context->getUserName(); + if (current_user_empty) + new_context->setCurrentUserName(definer_name); + if (initial_user_empty) + { + new_context->setInitialUserName(definer_name); + /// The shard authenticates the definer by `initial_user` and checks `initial_address` + /// against the allowed hosts of that user. Without an invoker, the address is the + /// unspecified `0.0.0.0`, which only matches users with `HOST ANY`. The query is + /// produced by this server itself, so present it as coming from the local host. + if (!new_client_info.initial_address || new_client_info.initial_address->host().isWildcard()) + new_context->setInitialAddress(Poco::Net::SocketAddress(Poco::Net::IPAddress("127.0.0.1"), 0)); + } + if (authenticated_user_empty) + new_context->setAuthenticatedUserName(definer_name); + } + } + new_context->clampToSettingsConstraints(changed_settings, SettingSource::QUERY); new_context->applySettingsChanges(changed_settings); new_context->setSetting("allow_ddl", 1); diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index cb772f006fc75..5f4a0b9ffb35c 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -639,11 +639,19 @@ ContextMutablePtr StorageMaterializedView::createRefreshContext(const String & l { ContextPtr table_context = getContext(); ClientInfo client_info = table_context->getClientInfo(); - client_info.interface = ClientInfo::Interface::BACKGROUND; - client_info.client_name = "refreshable materialized view"; auto view_metadata = getInMemoryMetadataPtr(getContext(), false); auto refresh_context = view_metadata->getSQLSecurityOverriddenContext(table_context, &client_info); - refresh_context->setClientInfo(client_info); + /// For `SQL SECURITY DEFINER`, `getSQLSecurityOverriddenContext` fills the empty user names in + /// the context's `ClientInfo` with the definer. Do not assign the local `client_info` copy to + /// the context: `setClientInfo` replaces the whole struct and would wipe that identity out. + /// Re-read it instead, so that a view without SQL security (which gets a plain copy) also gets + /// these fields. + { + ClientInfo refresh_client_info = refresh_context->getClientInfo(); + refresh_client_info.interface = ClientInfo::Interface::BACKGROUND; + refresh_client_info.client_name = "refreshable materialized view"; + refresh_context->setClientInfo(refresh_client_info); + } refresh_context->setSetting("database_replicated_allow_replicated_engine_arguments", 3); refresh_context->setSetting("log_comment", log_comment); refresh_context->setQueryKind(ClientInfo::QueryKind::INITIAL_QUERY); diff --git a/tests/integration/test_refreshable_mv_definer_interserver_secret/__init__.py b/tests/integration/test_refreshable_mv_definer_interserver_secret/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_refreshable_mv_definer_interserver_secret/configs/remote_servers.xml b/tests/integration/test_refreshable_mv_definer_interserver_secret/configs/remote_servers.xml new file mode 100644 index 0000000000000..362c39c90ed8c --- /dev/null +++ b/tests/integration/test_refreshable_mv_definer_interserver_secret/configs/remote_servers.xml @@ -0,0 +1,11 @@ + + + + foo + + node2 + 9000 + + + + diff --git a/tests/integration/test_refreshable_mv_definer_interserver_secret/test.py b/tests/integration/test_refreshable_mv_definer_interserver_secret/test.py new file mode 100644 index 0000000000000..f94ef84ab15b3 --- /dev/null +++ b/tests/integration/test_refreshable_mv_definer_interserver_secret/test.py @@ -0,0 +1,99 @@ +import pytest + +from helpers.cluster import ClickHouseCluster, QueryRuntimeException + +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance( + "node1", + main_configs=["configs/remote_servers.xml"], + with_zookeeper=True, +) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/remote_servers.xml"], + with_zookeeper=True, +) + +TENANT = {"user": "tenant", "password": "tpass"} + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_scheduled_refresh_of_definer_view_carries_definer_to_remote_shard( + started_cluster, +): + # Protected data lives only on node2. The tenant exists on both nodes but has no grant on it. + node2.query( + """ + CREATE DATABASE lab; + CREATE USER tenant IDENTIFIED WITH plaintext_password BY 'tpass'; + CREATE TABLE lab.secrets (id UInt64, payload String) ENGINE = MergeTree ORDER BY id; + INSERT INTO lab.secrets VALUES (1, 'a'), (2, 'b'), (3, 'c'); + """ + ) + node1.query( + """ + CREATE DATABASE tenant_db; + CREATE USER tenant IDENTIFIED WITH plaintext_password BY 'tpass'; + GRANT CREATE TABLE, SELECT, INSERT ON tenant_db.* TO tenant; + GRANT TABLE ENGINE ON Distributed TO tenant; + """ + ) + + node1.query( + """ + CREATE TABLE tenant_db.dist (id UInt64, payload String) + ENGINE = Distributed('secure', 'lab', 'secrets', rand()); + CREATE TABLE tenant_db.target + (id UInt64, payload String, current_user String, authenticated_user String) + ENGINE = MergeTree ORDER BY id; + """, + **TENANT, + ) + + # A direct read through the Distributed table is denied on the remote shard. + assert "ACCESS_DENIED" in node1.query_and_get_error( + "SELECT * FROM tenant_db.dist", **TENANT + ) + + # The shard also checks the allowed hosts of the definer against `initial_address`. A scheduled + # refresh has no client, so it must present itself as coming from the local host of the + # initiator rather than from the unspecified address, which only `HOST ANY` users accept. + node2.query("ALTER USER tenant HOST LOCAL") + + # The refresh runs under the definer's identity and must be denied on the remote shard too. + # Before the fix, the refresh query was sent with an empty `initial_user`, which the remote + # shard treated as interserver mode and executed without any access checks. + node1.query( + """ + CREATE MATERIALIZED VIEW tenant_db.rmv REFRESH EVERY 1 YEAR APPEND TO tenant_db.target + DEFINER = CURRENT_USER SQL SECURITY DEFINER + AS SELECT id, payload, currentUser() AS current_user, authenticatedUser() AS authenticated_user + FROM tenant_db.dist; + """, + **TENANT, + ) + + with pytest.raises(QueryRuntimeException, match="ACCESS_DENIED"): + node1.query("SYSTEM WAIT VIEW tenant_db.rmv") + assert node1.query("SELECT count() FROM tenant_db.target") == "0\n" + assert "ACCESS_DENIED" in node1.query( + "SELECT exception FROM system.view_refreshes WHERE database = 'tenant_db' AND view = 'rmv'" + ) + + # Once the definer is granted access on the remote shard, the refresh succeeds and the remote + # shard sees the definer as the current and authenticated user. + node2.query("GRANT SELECT ON lab.secrets TO tenant") + node1.query("SYSTEM REFRESH VIEW tenant_db.rmv") + node1.query("SYSTEM WAIT VIEW tenant_db.rmv") + assert node1.query("SELECT * FROM tenant_db.target ORDER BY id", **TENANT) == ( + "1\ta\ttenant\ttenant\n2\tb\ttenant\ttenant\n3\tc\ttenant\ttenant\n" + ) diff --git a/tests/integration/test_streaming_engines_definer_interserver_secret/__init__.py b/tests/integration/test_streaming_engines_definer_interserver_secret/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_streaming_engines_definer_interserver_secret/configs/rabbitmq.xml b/tests/integration/test_streaming_engines_definer_interserver_secret/configs/rabbitmq.xml new file mode 100644 index 0000000000000..0b5411455b46b --- /dev/null +++ b/tests/integration/test_streaming_engines_definer_interserver_secret/configs/rabbitmq.xml @@ -0,0 +1,6 @@ + + + root + clickhouse + + diff --git a/tests/integration/test_streaming_engines_definer_interserver_secret/configs/remote_servers.xml b/tests/integration/test_streaming_engines_definer_interserver_secret/configs/remote_servers.xml new file mode 100644 index 0000000000000..af0b7ad21cab3 --- /dev/null +++ b/tests/integration/test_streaming_engines_definer_interserver_secret/configs/remote_servers.xml @@ -0,0 +1,14 @@ + + + + + foo + + node2 + 9000 + + + + diff --git a/tests/integration/test_streaming_engines_definer_interserver_secret/configs/users.xml b/tests/integration/test_streaming_engines_definer_interserver_secret/configs/users.xml new file mode 100644 index 0000000000000..ae00b3b543f9f --- /dev/null +++ b/tests/integration/test_streaming_engines_definer_interserver_secret/configs/users.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + ::/0 + + default + default + 1 + + + + + + + + diff --git a/tests/integration/test_streaming_engines_definer_interserver_secret/test.py b/tests/integration/test_streaming_engines_definer_interserver_secret/test.py new file mode 100644 index 0000000000000..6a0ed02eadf3c --- /dev/null +++ b/tests/integration/test_streaming_engines_definer_interserver_secret/test.py @@ -0,0 +1,296 @@ +# pylint: disable=unused-argument +# pylint: disable=redefined-outer-name +# pylint: disable=line-too-long + +""" +Pushes from streaming engines into a `Distributed` table must reach the shard under the definer. + +`Kafka`, `RabbitMQ` and the other streaming engines never write anywhere themselves: a consumed +batch is pushed only into the dependent materialized views, from a copy of the global context that +has no user. A view with `SQL SECURITY DEFINER` switches that push to the definer. When the view's +target is a `Distributed` table over a cluster with an interserver ``, the definer's name +must travel with the query: before the fix `Context::setUser` left the `ClientInfo` user names +empty, the shard received an empty `initial_user`, treated the query as interserver mode and +executed it with full access. A tenant that could create tables only in its own database could +chain `Kafka -> MATERIALIZED VIEW -> Distributed('secure', default, secret_data)` and write into +a table on the shard it had no `INSERT` grant on. + +The `secure` cluster consists of `node2` only, so a table on `node1` can reach `default.secret_data` +solely over the interserver connection. Every pipeline is built twice with the same statements: by +`tenant`, who has no grants on `node2` and whose rows must be rejected there, and by `writer`, who +has `INSERT` on the target and whose rows must arrive. In both cases the shard must see the +definer's name in `system.query_log`. + +The push is also checked with `distributed_foreground_insert = 0`, where the rows go through the +background directory queue of the `Distributed` table: the persisted insert must keep the definer's +name, although the streaming context has `query_kind = NO_QUERY`. +""" + +import json +import time + +import pika +import pytest + +import helpers.kafka.common as k +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance( + "node1", + main_configs=["configs/remote_servers.xml", "configs/rabbitmq.xml"], + user_configs=["configs/users.xml"], + with_kafka=True, + with_rabbitmq=True, +) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/users.xml"], +) + +WRITER = "writer" +TENANT = "tenant" +CREDENTIALS = { + WRITER: {"user": WRITER, "password": "wpass"}, + TENANT: {"user": TENANT, "password": "tpass"}, +} +ROWS_PER_PIPELINE = 2 + +# How long a streaming engine gets to consume the messages and push them to the shard. Consumer +# group setup in Kafka alone can take several seconds. +DELIVERY_WAIT_SECONDS = 60 + +users = pytest.mark.parametrize("user", [WRITER, TENANT]) + + +def database(user): + return f"{user}_db" + + +def query_as(user, sql): + return node1.query(sql, **CREDENTIALS[user]) + + +def remote_rows(): + return int(node2.query("SELECT count() FROM default.secret_data")) + + +def shard_inserts(user, condition): + """Number of `INSERT` queries into the protected table that the shard executed under `user`.""" + node2.query("SYSTEM FLUSH LOGS") + return int( + node2.query( + f""" + SELECT count() FROM system.query_log + WHERE user = '{user}' AND query LIKE 'INSERT INTO default.secret_data%' AND {condition} + """ + ) + ) + + +def shard_finished_inserts(user): + return shard_inserts(user, "type = 'QueryFinish'") + + +def shard_denied_inserts(user): + # The access check happens before the pipeline starts, so the shard logs the denial as + # `ExceptionBeforeStart`. + return shard_inserts( + user, + "type IN ('ExceptionBeforeStart', 'ExceptionWhileProcessing') AND exception_code = 497", + ) + + +def wait_until(condition, timeout=DELIVERY_WAIT_SECONDS): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if condition(): + return True + time.sleep(1) + return False + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + + # The protected table lives on the shard only. Both users exist on both nodes, but on + # the shard only `writer` may insert into it. + node2.query( + """ + CREATE TABLE default.secret_data (key Int) ENGINE = MergeTree ORDER BY key; + CREATE USER writer IDENTIFIED WITH plaintext_password BY 'wpass'; + CREATE USER tenant IDENTIFIED WITH plaintext_password BY 'tpass'; + GRANT INSERT ON default.secret_data TO writer; + """ + ) + for user in (WRITER, TENANT): + db = database(user) + # Send to the shard synchronously by default, so that a push either delivers the rows + # or fails right away, without a background directory queue in between. + node1.query( + f""" + CREATE USER {user} IDENTIFIED WITH plaintext_password BY '{CREDENTIALS[user]["password"]}' + SETTINGS distributed_foreground_insert = 1; + CREATE DATABASE {db}; + GRANT ALL ON {db}.* TO {user}; + GRANT TABLE ENGINE ON Distributed, TABLE ENGINE ON Kafka, TABLE ENGINE ON RabbitMQ TO {user}; + """ + ) + query_as( + user, + f"CREATE TABLE {db}.dist (key Int) ENGINE = Distributed(secure, default, secret_data)", + ) + + yield cluster + finally: + cluster.shutdown() + + +@pytest.fixture(autouse=True) +def clean_remote_table(): + node2.query("TRUNCATE TABLE default.secret_data") + yield + + +def drop_pipeline(user, source): + """Stop the streaming table, so that a rejected push is not retried into the next test.""" + db = database(user) + query_as(user, f"DROP TABLE {db}.{source}_to_dist") + query_as(user, f"DROP TABLE {db}.{source} SYNC") + + +def create_definer_view(user, source, target="dist"): + """A view owned by `user` that pushes the source rows into the user's `Distributed` table.""" + db = database(user) + query_as( + user, + f""" + CREATE MATERIALIZED VIEW {db}.{source}_to_dist TO {db}.{target} + DEFINER = CURRENT_USER SQL SECURITY DEFINER + AS SELECT key FROM {db}.{source} + """, + ) + + +def check_delivery(user, finished_before, denied_before): + """The shard must have executed the push under the user's name, with the outcome the user's + grants dictate: the rows arrive for `writer` and are rejected for `tenant`.""" + if user == WRITER: + assert wait_until(lambda: remote_rows() == ROWS_PER_PIPELINE) + assert shard_finished_inserts(WRITER) > finished_before + else: + assert wait_until( + lambda: shard_denied_inserts(TENANT) > denied_before or remote_rows() > 0 + ) + assert remote_rows() == 0 + assert shard_denied_inserts(TENANT) > denied_before + + +def check_direct_insert(user): + """The control case: a direct `INSERT` carries the user's name to the shard, where it is checked. + + It passes without the fix as well, so it is not a separate test: bugfix validation requires every + new test to fail on the unfixed binary. It runs before a streaming pipeline instead.""" + db = database(user) + if user == WRITER: + query_as(WRITER, f"INSERT INTO {db}.dist VALUES (1)") + assert remote_rows() == 1 + else: + error = node1.query_and_get_error( + f"INSERT INTO {db}.dist VALUES (1)", **CREDENTIALS[TENANT] + ) + assert "ACCESS_DENIED" in error + assert remote_rows() == 0 + node2.query("TRUNCATE TABLE default.secret_data") + + +@users +@pytest.mark.parametrize("foreground", [1, 0]) +def test_kafka(user, foreground): + db = database(user) + topic = f"definer_interserver_{user}_{foreground}" + k.kafka_create_topic(k.get_admin_client(cluster), topic) + + query_as( + user, + f""" + CREATE TABLE {db}.kafka (key Int) ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = '{topic}', + kafka_group_name = '{topic}', + kafka_format = 'JSONEachRow', + kafka_flush_interval_ms = 1000 + """, + ) + # A separate `Distributed` table per mode, so that a rejected file left in the background queue + # is dropped together with the table and is not retried into the next test. + target = f"dist_{foreground}" + query_as( + user, + f"CREATE TABLE {db}.{target} (key Int) ENGINE = Distributed(secure, default, secret_data)", + ) + # The definer's settings apply to the push. It is not set in the server's profile, because the + # settings of the context the push starts from would override the definer's ones. + node1.query(f"ALTER USER {user} SETTINGS distributed_foreground_insert = {foreground}") + create_definer_view(user, "kafka", target) + + try: + finished_before = shard_finished_inserts(user) + denied_before = shard_denied_inserts(user) + k.kafka_produce( + cluster, topic, [json.dumps({"key": i}) for i in range(ROWS_PER_PIPELINE)] + ) + check_delivery(user, finished_before, denied_before) + finally: + drop_pipeline(user, "kafka") + query_as(user, f"DROP TABLE {db}.{target} SYNC") + node1.query(f"ALTER USER {user} SETTINGS distributed_foreground_insert = 1") + + +@users +def test_rabbitmq(user): + check_direct_insert(user) + db = database(user) + exchange = f"definer_interserver_{user}" + + query_as( + user, + f""" + CREATE TABLE {db}.rabbitmq (key Int) ENGINE = RabbitMQ + SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', + rabbitmq_exchange_name = '{exchange}', + rabbitmq_format = 'JSONEachRow', + rabbitmq_flush_interval_ms = 1000, + rabbitmq_max_block_size = 100 + """, + ) + create_definer_view(user, "rabbitmq") + + # The exchange and the queue are declared by the table in the background; messages + # published before that are dropped by RabbitMQ. + node1.wait_for_log_line(f"StorageRabbitMQ .{db}.rabbitmq.: RabbitMQ setup completed") + node1.wait_for_log_line(f"StorageRabbitMQ .{db}.rabbitmq.: Queue .* is declared") + + try: + finished_before = shard_finished_inserts(user) + denied_before = shard_denied_inserts(user) + + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + cluster.rabbitmq_ip, cluster.rabbitmq_port, "/", credentials + ) + connection = pika.BlockingConnection(parameters) + channel = connection.channel() + for i in range(ROWS_PER_PIPELINE): + channel.basic_publish( + exchange=exchange, routing_key="", body=json.dumps({"key": i}) + ) + connection.close() + + check_delivery(user, finished_before, denied_before) + finally: + drop_pipeline(user, "rabbitmq") diff --git a/tests/queries/0_stateless/05445_distributed_async_insert_batch_per_user.reference b/tests/queries/0_stateless/05445_distributed_async_insert_batch_per_user.reference new file mode 100644 index 0000000000000..aa20b4a608862 --- /dev/null +++ b/tests/queries/0_stateless/05445_distributed_async_insert_batch_per_user.reference @@ -0,0 +1 @@ +ACCESS_DENIED diff --git a/tests/queries/0_stateless/05445_distributed_async_insert_batch_per_user.sh b/tests/queries/0_stateless/05445_distributed_async_insert_batch_per_user.sh new file mode 100755 index 0000000000000..71282f1c84378 --- /dev/null +++ b/tests/queries/0_stateless/05445_distributed_async_insert_batch_per_user.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Tags: distributed, no-fasttest +# Tag no-fasttest: the interserver `` requires SSL, which the fast test build does not have. + +# Queued `Distributed` inserts of different users must not be sent in one batch: the batch is sent with +# the `ClientInfo` of its first file, and over a cluster with an interserver `` the shard +# authenticates the whole batch as that file's `initial_user`. Before the fix, rows of a user without +# the remote `INSERT` grant were delivered under the identity of a user that had it. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Flushing the rejected file (also when the table is dropped) logs the error on the server. +CLICKHOUSE_CLIENT=$(echo "$CLICKHOUSE_CLIENT" | sed "s/--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL:-none}/--send_logs_level=none/g") + +WRITER="writer_${CLICKHOUSE_DATABASE}" +TENANT="tenant_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} -m -q " + DROP USER IF EXISTS ${WRITER}, ${TENANT}; + CREATE TABLE target (key Int) ENGINE = MergeTree ORDER BY key; + CREATE TABLE dist (key Int) ENGINE = Distributed(test_cluster_interserver_secret, currentDatabase(), target, 0) + SETTINGS background_insert_batch = 1; + SYSTEM STOP DISTRIBUTED SENDS dist; + CREATE USER ${WRITER} IDENTIFIED WITH no_password; + CREATE USER ${TENANT} IDENTIFIED WITH no_password; + GRANT INSERT ON ${CLICKHOUSE_DATABASE}.dist TO ${WRITER}, ${TENANT}; + GRANT INSERT ON ${CLICKHOUSE_DATABASE}.target TO ${WRITER}; +" + +${CLICKHOUSE_CLIENT} --user "${WRITER}" --distributed_foreground_insert 0 --prefer_localhost_replica 0 -q "INSERT INTO dist VALUES (1)" +${CLICKHOUSE_CLIENT} --user "${TENANT}" --distributed_foreground_insert 0 --prefer_localhost_replica 0 -q "INSERT INTO dist VALUES (2)" + +${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH DISTRIBUTED dist" 2>&1 | grep -o -m1 "ACCESS_DENIED" +${CLICKHOUSE_CLIENT} -q "SELECT key FROM target WHERE key = 2" + +${CLICKHOUSE_CLIENT} -m -q " + DROP TABLE dist; + DROP USER ${WRITER}, ${TENANT}; +" From 5184580807e76057406360508006e04fc45b954b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 2 Nov 2026 11:43:30 +0000 Subject: [PATCH 055/219] Backport #124644 to 26.8: Fix stack overflow in Avro schema inference on a deep chain of named types --- .../Formats/Impl/AvroRowInputFormat.cpp | 4 +- ..._inference_deep_named_type_chain.reference | 2 + ..._schema_inference_deep_named_type_chain.sh | 55 +++++++++++++++++++ 3 files changed, 59 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/05339_avro_schema_inference_deep_named_type_chain.reference create mode 100755 tests/queries/0_stateless/05339_avro_schema_inference_deep_named_type_chain.sh diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index 7b4c6afd92b31..1c3fe5395d67f 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -1419,14 +1419,14 @@ NamesAndTypesList AvroSchemaReader::readSchema() DataTypePtr AvroSchemaReader::avroNodeToDataType(avro::NodePtr node, bool allow_nullable_tuple_type) { - checkStackSize(); - std::unordered_set seen_names; return avroNodeToDataTypeImpl(node, seen_names, allow_nullable_tuple_type); } DataTypePtr AvroSchemaReader::avroNodeToDataTypeImpl(const avro::NodePtr & node, std::unordered_set & seen_names, bool allow_nullable_tuple_type) { + checkStackSize(); + switch (node->type()) { case avro::Type::AVRO_INT: diff --git a/tests/queries/0_stateless/05339_avro_schema_inference_deep_named_type_chain.reference b/tests/queries/0_stateless/05339_avro_schema_inference_deep_named_type_chain.reference new file mode 100644 index 0000000000000..3af6e7be9ccaa --- /dev/null +++ b/tests/queries/0_stateless/05339_avro_schema_inference_deep_named_type_chain.reference @@ -0,0 +1,2 @@ +42 +too deep diff --git a/tests/queries/0_stateless/05339_avro_schema_inference_deep_named_type_chain.sh b/tests/queries/0_stateless/05339_avro_schema_inference_deep_named_type_chain.sh new file mode 100755 index 0000000000000..22ac566fbaaaf --- /dev/null +++ b/tests/queries/0_stateless/05339_avro_schema_inference_deep_named_type_chain.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: the Avro format is not available in the fast test build. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +DIR="$CLICKHOUSE_TMP/${CLICKHOUSE_TEST_UNIQUE_NAME}" +rm -rf "$DIR" +mkdir -p "$DIR" + +# A chain of named records A_k { x: A_(k-1) }, each defined inside its own array field f_k, so the schema +# JSON nests only a few levels deep. Schema inference follows the references: f_k is 2k + 1 levels deep. +function gen() +{ + python3 -c " +import json, sys +n = int(sys.argv[2]) +def zz(v): + v = (v << 1) ^ (v >> 63) + v &= (1 << 64) - 1 + out = bytearray() + while True: + b = v & 0x7f + v >>= 7 + out.append(b | 0x80 if v else b) + if not v: + break + return bytes(out) +def ab(b): + return zz(len(b)) + b +fields = [{'name': 'id', 'type': 'long'}, + {'name': 'f1', 'type': {'type': 'array', 'items': {'type': 'record', 'name': 'A1', 'fields': [{'name': 'v', 'type': 'long'}]}}}] +for k in range(2, n + 1): + fields.append({'name': 'f%d' % k, 'type': {'type': 'array', 'items': {'type': 'record', 'name': 'A%d' % k, 'fields': [{'name': 'x', 'type': 'A%d' % (k - 1)}]}}}) +fields.append({'name': 'last', 'type': ['null', 'A%d' % n]}) +schema = json.dumps({'type': 'record', 'name': 'root', 'fields': fields}, separators=(',', ':')).encode() +meta = zz(2) + ab(b'avro.schema') + ab(schema) + ab(b'avro.codec') + ab(b'null') + zz(0) +sync = bytes(16) +payload = zz(0) + b'\x00' * n + zz(0) +open(sys.argv[1], 'wb').write(b'Obj\x01' + meta + sync + zz(1) + zz(len(payload)) + payload + sync) +" "$@" +} + +gen "$DIR/chain_40.avro" 40 +gen "$DIR/chain_10000.avro" 10000 + +$CLICKHOUSE_LOCAL -q "DESC file('$DIR/chain_40.avro', Avro)" | wc -l + +# Inference builds every field's type before the deepest one overflows, so its cost grows with the stack +# size: a 1 MiB stack keeps this case small on every build. +(ulimit -s 1024; $CLICKHOUSE_LOCAL -q "DESC file('$DIR/chain_10000.avro', Avro)" 2>&1) | grep -q -F 'TOO_DEEP_RECURSION' && echo 'too deep' + +rm -rf "$DIR" From ee9c0bf3ee9048e28df393518a27703c324768db Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 2 Nov 2026 12:41:50 +0000 Subject: [PATCH 056/219] Backport #116564 to 26.8: Fix a crash on an Iceberg manifest list with a huge manifest_length --- .../Common/AvroForIcebergDeserializer.cpp | 1 + .../Common/AvroForIcebergDeserializer.h | 5 + .../DataLakes/Iceberg/IcebergIterator.cpp | 3 +- .../Iceberg/IcebergMetadataFilesCache.h | 2 +- .../Iceberg/StatelessMetadataFileGetter.cpp | 9 +- .../Iceberg/StatelessMetadataFileGetter.h | 3 +- .../test_manifest_length_cache_weight.py | 99 +++++++++++++++++++ 7 files changed, 112 insertions(+), 10 deletions(-) create mode 100644 tests/integration/test_storage_iceberg_no_spark/test_manifest_length_cache_weight.py diff --git a/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp b/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp index f383336c95964..48835c0af5569 100644 --- a/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp @@ -54,6 +54,7 @@ try } metadata = manifest_file_reader->metadata(); + bytes_read = buffer->count(); parsed_column = std::move(columns[0]); parsed_column_data_type = std::dynamic_pointer_cast(data_type); parsed_manifest_file_entries.resize(parsed_column->size()); diff --git a/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.h b/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.h index ea2a54706b60f..84e867e9076eb 100644 --- a/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.h +++ b/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.h @@ -45,6 +45,8 @@ class AvroForIcebergDeserializer std::map> metadata; + size_t bytes_read = 0; + /// Shared mutex to protect mutable cache members for thread safety mutable SharedMutex cache_mutex; @@ -71,6 +73,9 @@ class AvroForIcebergDeserializer size_t rows() const; + /// The constructor consumes the whole file, so this is its size on storage. + size_t bytesRead() const { return bytes_read; } + /// Allow to access avro paths like "a.b.c" bool hasPath(const std::string & path) const; DB::TypeIndex getTypeForPath(const std::string & path) const; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp index 099f445fa0fab..5f76f328c9abe 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp @@ -389,8 +389,7 @@ Iceberg::ManifestIteratorPtr IcebergIterator::createManifestIterator(const Manif persistent_components, local_context, logger, - manifest_list_entry.manifest_file_path, - manifest_list_entry.manifest_file_byte_size); + manifest_list_entry.manifest_file_path); return Iceberg::ManifestFileIterator::create( manifest_file_cacheable_part.deserializer, diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadataFilesCache.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadataFilesCache.h index 32fcc97d435dc..ceb715eeedbe9 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadataFilesCache.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadataFilesCache.h @@ -75,7 +75,7 @@ struct IcebergMetadataFilesCacheCell : private boost::noncopyable /// - manifest list consists of cache keys which will retrieve the manifest file from cache [file_path --> ManifestFileCacheKeys] /// - manifest file [file_path --> Iceberg::ManifestFileCacheableInfo] std::variant cached_element; - Int64 memory_bytes; + size_t memory_bytes; explicit IcebergMetadataFilesCacheCell(String && metadata_json_str) : cached_element(std::move(metadata_json_str)) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.cpp index 9b3a648938b96..57645b512f92b 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.cpp @@ -75,8 +75,7 @@ Iceberg::ManifestFileCacheableInfo getManifestFile( const PersistentTableComponents & persistent_table_components, ContextPtr local_context, LoggerPtr log, - const IcebergPathFromMetadata & filename, - size_t bytes_size) + const IcebergPathFromMetadata & filename) { auto log_level = local_context->getSettingsRef()[Setting::iceberg_metadata_log_level].value; @@ -102,7 +101,8 @@ Iceberg::ManifestFileCacheableInfo getManifestFile( auto manifest_file_deserializer = std::make_unique( std::move(buffer), filename, getFormatSettings(local_context)); - return Iceberg::ManifestFileCacheableInfo{std::move(manifest_file_deserializer), bytes_size}; + const size_t manifest_file_bytes = manifest_file_deserializer->bytesRead(); + return Iceberg::ManifestFileCacheableInfo{std::move(manifest_file_deserializer), manifest_file_bytes}; }; if (use_iceberg_metadata_cache && persistent_table_components.table_uuid.has_value()) @@ -127,8 +127,7 @@ Iceberg::ManifestFileIterator::ManifestFileEntriesHandle getManifestFileEntriesH persistent_table_components, local_context, log, - cache_key.manifest_file_path, - static_cast(cache_key.manifest_file_byte_size)); + cache_key.manifest_file_path); auto iterator = Iceberg::ManifestFileIterator::create( cacheable_info.deserializer, diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.h index 2bb2edfdff13c..f78d7eb4e4749 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.h @@ -25,8 +25,7 @@ Iceberg::ManifestFileCacheableInfo getManifestFile( const PersistentTableComponents & persistent_table_components, ContextPtr local_context, LoggerPtr log, - const IcebergPathFromMetadata & filename, - size_t bytes_size); + const IcebergPathFromMetadata & filename); /// Creates a fully initialized ManifestFileIterator from a cache key. /// All entries are drained so that aggregate methods (e.g. getRowsCountInAllFilesExcludingDeleted) diff --git a/tests/integration/test_storage_iceberg_no_spark/test_manifest_length_cache_weight.py b/tests/integration/test_storage_iceberg_no_spark/test_manifest_length_cache_weight.py new file mode 100644 index 0000000000000..d90720f797bd4 --- /dev/null +++ b/tests/integration/test_storage_iceberg_no_spark/test_manifest_length_cache_weight.py @@ -0,0 +1,99 @@ +""" +An Iceberg manifest-list entry may declare any `manifest_length`. Reading such a table must not +depend on that number: it used to become the metadata cache cell weight and terminate the server. +""" +import os +import shutil +import tempfile + +from helpers.iceberg_utils import get_uuid_str +from test_storage_iceberg_no_spark.test_local_path_traversal import ( + _download_table_from_container, + _get_manifest_paths_from_chain, + _modify_avro_file, +) + +NOCACHE = { + "use_iceberg_metadata_files_cache": False, + "use_parquet_metadata_cache": False, +} +CACHE = {"use_iceberg_metadata_files_cache": True} +HUGE_MANIFEST_LENGTH = 2**62 + + +def _make_table_with_manifest_length(instance, table_name, table_path, new_length): + """Build a well-formed IcebergLocal table with the cache off, then rewrite every + manifest-list entry's `manifest_length`. new_length=None rewrites the file without + changing the value. Returns the manifest files' real total size on disk.""" + instance.query( + f"CREATE TABLE {table_name} (c0 Int) ENGINE = IcebergLocal('{table_path}/', 'Parquet');", + settings={"allow_insert_into_iceberg": 1, **NOCACHE}, + ) + instance.query( + f"INSERT INTO {table_name} VALUES (1), (2), (3)", + settings={"allow_insert_into_iceberg": 1, **NOCACHE}, + ) + assert instance.query(f"SELECT count() FROM {table_name}", settings=NOCACHE).strip() == "3" + instance.query(f"DROP TABLE {table_name}") + + temp_dir = tempfile.mkdtemp() + try: + host_path = os.path.join(temp_dir, table_name) + os.makedirs(host_path, exist_ok=True) + _download_table_from_container(instance, table_path, host_path) + manifest_list_rel, manifest_rels = _get_manifest_paths_from_chain(host_path, table_path) + local_manifest_list = os.path.join(host_path, manifest_list_rel) + _modify_avro_file( + local_manifest_list, + ["manifest_length"], + (lambda v: v) if new_length is None else (lambda _: new_length), + ) + instance.copy_file_to_container(local_manifest_list, f"{table_path}/{manifest_list_rel}") + return sum(os.path.getsize(os.path.join(host_path, rel)) for rel in manifest_rels) + finally: + shutil.rmtree(temp_dir, ignore_errors=True) + + +def _read_back(instance, table_path): + """Read with the metadata cache ON, from a cold cache: the cache-miss path is the only + one that builds the cell whose weight the declared length used to set. Returns the + query results plus the cache weight those reads produced.""" + instance.query("SYSTEM DROP ICEBERG METADATA CACHE") + count = instance.query(f"SELECT count() FROM icebergLocal('{table_path}/')", settings=CACHE).strip() + total = instance.query(f"SELECT sum(c0) FROM icebergLocal('{table_path}/')", settings=CACHE).strip() + weight = instance.query( + "SELECT value FROM system.metrics WHERE metric = 'IcebergMetadataFilesCacheBytes'" + ).strip() + return count, total, int(weight) + + +def _assert_weighed_by_real_size(weight, manifest_bytes): + """A manifest is charged three times its size on storage, so the total must clear that + bound. Reading the size from the file rather than hardcoding it keeps this stable while + still failing if the weight ever comes from something other than the real size.""" + assert weight >= 3 * manifest_bytes, f"weight {weight} < 3 * {manifest_bytes}" + + +def test_iceberg_rewritten_manifest_list_is_readable(started_cluster_iceberg_no_spark): + """Control for the test below: the same Avro round-trip with the declared length + untouched. It runs first so that a regression of the fix, which aborts the server and + makes every later query in the package fail to connect, cannot mask this result.""" + instance = started_cluster_iceberg_no_spark.instances["node1"] + table_name = "test_iceberg_manifest_list_roundtrip" + get_uuid_str() + table_path = f"/var/lib/clickhouse/user_files/iceberg_data/default/{table_name}" + manifest_bytes = _make_table_with_manifest_length(instance, table_name, table_path, None) + count, total, weight = _read_back(instance, table_path) + assert (count, total) == ("3", "6") + _assert_weighed_by_real_size(weight, manifest_bytes) + + +def test_iceberg_huge_manifest_length_is_readable(started_cluster_iceberg_no_spark): + instance = started_cluster_iceberg_no_spark.instances["node1"] + table_name = "test_iceberg_huge_manifest_length" + get_uuid_str() + table_path = f"/var/lib/clickhouse/user_files/iceberg_data/default/{table_name}" + manifest_bytes = _make_table_with_manifest_length( + instance, table_name, table_path, HUGE_MANIFEST_LENGTH + ) + count, total, weight = _read_back(instance, table_path) + assert (count, total) == ("3", "6") + _assert_weighed_by_real_size(weight, manifest_bytes) From c538d55a842f20cdfd7c60258215f982cb7acd94 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 2 Nov 2026 15:32:44 +0000 Subject: [PATCH 057/219] Backport #128580 to 26.8: Bump `sqlite-amalgamation` from 3.53.4 to 3.54.1 --- contrib/sqlite-amalgamation | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/sqlite-amalgamation b/contrib/sqlite-amalgamation index 23c7b76929611..f48768e9089bf 160000 --- a/contrib/sqlite-amalgamation +++ b/contrib/sqlite-amalgamation @@ -1 +1 @@ -Subproject commit 23c7b76929611997c5f0315b3149372b1bee1d41 +Subproject commit f48768e9089bf66717e8c393dce5dd80c86761f5 From efdfbcff8f3ecd1fdc9f1eda2f933a047a4121e1 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 2 Nov 2026 16:12:42 +0000 Subject: [PATCH 058/219] Backport #116226 to 26.8: Scope the memory limit in 04501/04500 to the query it asserts on --- .../04500_group_by_limit_pushdown_no_order_by.sql | 8 ++++---- .../04501_group_by_limit_pushdown_heap_stress.sql | 8 ++++---- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/queries/0_stateless/04500_group_by_limit_pushdown_no_order_by.sql b/tests/queries/0_stateless/04500_group_by_limit_pushdown_no_order_by.sql index f390f1617c80f..3e2569996bcbb 100644 --- a/tests/queries/0_stateless/04500_group_by_limit_pushdown_no_order_by.sql +++ b/tests/queries/0_stateless/04500_group_by_limit_pushdown_no_order_by.sql @@ -305,9 +305,9 @@ WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND log_comm -- LowCardinality keys cannot prune the hash table, but the heap still runs in -- skip-only mode and the synthesized sort discards evicted stragglers; the -- query must stay within the memory limit and external aggregation must not --- be force-disabled (~660MB working set without spilling). -SET max_memory_usage = 300000000; -SET max_bytes_before_external_group_by = 50000000; +-- be force-disabled (~660MB working set without spilling). The limits are +-- per-query, not session `SET`s: the verification query below reads +-- `system.query_log`, whose cost tracks the whole suite's log volume. SELECT 'LowCardinality key under a memory limit (skip-only heap + spill)'; SELECT count() FROM @@ -316,7 +316,7 @@ SELECT count() FROM FROM (SELECT toLowCardinality(toString(number % 3000000)) AS k FROM numbers(9000000)) GROUP BY k LIMIT 5 -) SETTINGS log_comment = '04500_lowcard_spill'; +) SETTINGS log_comment = '04500_lowcard_spill', max_memory_usage = 300000000, max_bytes_before_external_group_by = 50000000; SYSTEM FLUSH LOGS query_log; diff --git a/tests/queries/0_stateless/04501_group_by_limit_pushdown_heap_stress.sql b/tests/queries/0_stateless/04501_group_by_limit_pushdown_heap_stress.sql index afa4594c2ef67..5533ed40308f8 100644 --- a/tests/queries/0_stateless/04501_group_by_limit_pushdown_heap_stress.sql +++ b/tests/queries/0_stateless/04501_group_by_limit_pushdown_heap_stress.sql @@ -347,9 +347,9 @@ WHERE current_database = currentDatabase() -- new key and evicts an older one, so without reuse the arena grows by one -- state per distinct key seen (20M here) even though the hash table stays -- bounded, defeating the optimization's memory contract for non-`count` --- aggregates and failing the memory limit below. - -SET max_memory_usage = 100000000; +-- aggregates and failing the memory limit below. The limit is per-query, not a +-- session `SET`: the verification query below reads `system.query_log`, whose +-- cost tracks the whole suite's log volume rather than anything under test. SELECT 'arena slot reuse under eviction'; SELECT k, sum(v) FROM @@ -359,7 +359,7 @@ SELECT k, sum(v) FROM GROUP BY k ORDER BY k ASC LIMIT 10 -SETTINGS log_comment = '04501_state_arena_reuse'; +SETTINGS log_comment = '04501_state_arena_reuse', max_memory_usage = 100000000; SYSTEM FLUSH LOGS query_log; From 447aaa215cf6835562e3be2d5f1121f09f9dc397 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 2 Nov 2026 14:41:12 +0000 Subject: [PATCH 059/219] Backport #128291 to 26.8: Fix segfault in `ReverseIndex::insert` after an exception while building the index --- src/Columns/ReverseIndex.h | 99 +++++++++----- .../gtest_reverse_index_exception_safety.cpp | 126 ++++++++++++++++++ src/Common/FailPoint.cpp | 1 + 3 files changed, 196 insertions(+), 30 deletions(-) create mode 100644 src/Columns/tests/gtest_reverse_index_exception_safety.cpp diff --git a/src/Columns/ReverseIndex.h b/src/Columns/ReverseIndex.h index 5704fc12ed41c..8b13f2c1ab544 100644 --- a/src/Columns/ReverseIndex.h +++ b/src/Columns/ReverseIndex.h @@ -1,6 +1,7 @@ #pragma once #include +#include #include #include #include @@ -16,6 +17,12 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int MEMORY_LIMIT_EXCEEDED; +} + +namespace FailPoints +{ + extern const char reverse_index_build_index_throw[]; } template @@ -441,17 +448,25 @@ void ReverseIndex::buildIndex() if (!column) throw Exception(ErrorCodes::LOGICAL_ERROR, "ReverseIndex can't build index because index column wasn't set."); + /// Build everything into local variables and publish them only at the end, so that an exception + /// thrown midway (for example, `MEMORY_LIMIT_EXCEEDED` from an allocation in `calcHashes`) does not + /// leave a half-built index behind: a non-null `index` with a null `saved_hash` would make the next + /// `insert` dereference a null pointer. auto size = column->size(); - index = std::make_unique(size); + auto new_index = std::make_unique(size); + fiu_do_on(FailPoints::reverse_index_build_index_throw, + throw Exception(ErrorCodes::MEMORY_LIMIT_EXCEEDED, "Injected failure while building ReverseIndex");); + + ColumnUInt64::MutablePtr new_saved_hash; if constexpr (use_saved_hash) - saved_hash = calcHashes(); + new_saved_hash = calcHashes(); - auto & state = index->getState(); + auto & state = new_index->getState(); state.index_column = column; state.base_index = base_index; if constexpr (use_saved_hash) - state.saved_hash_column = &saved_hash->getData(); + state.saved_hash_column = &new_saved_hash->getData(); using IteratorType = typename IndexMapType::iterator; IteratorType iterator; @@ -461,15 +476,18 @@ void ReverseIndex::buildIndex() { UInt64 hash = 0; if constexpr (use_saved_hash) - hash = saved_hash->getElement(row); + hash = new_saved_hash->getElement(row); else hash = getHash(column->getDataAt(row)); - index->reverseIndexEmplace(row + base_index, iterator, inserted, hash, column->getDataAt(row)); + new_index->reverseIndexEmplace(row + base_index, iterator, inserted, hash, column->getDataAt(row)); if (!inserted) throw Exception(ErrorCodes::LOGICAL_ERROR, "Duplicating keys found in ReverseIndex."); } + + index = std::move(new_index); + saved_hash = std::move(new_saved_hash); } template @@ -493,37 +511,58 @@ UInt64 ReverseIndex::insert(std::string_view data) if (!index) buildIndex(); - using IteratorType = typename IndexMapType::iterator; - IteratorType iterator; - bool inserted = false; - - auto hash = getHash(data); - UInt64 num_rows = size(); - - if constexpr (use_saved_hash) + try { - auto & column_data = saved_hash->getData(); - if (column_data.size() <= num_rows) - column_data.resize(num_rows + 1); - column_data[num_rows] = hash; - } - else - column->insertData(data.data(), data.size()); + using IteratorType = typename IndexMapType::iterator; + IteratorType iterator; + bool inserted = false; - index->reverseIndexEmplace(num_rows + base_index, iterator, inserted, hash, data); + auto hash = getHash(data); + UInt64 num_rows = size(); - if constexpr (use_saved_hash) - { - if (inserted) + if constexpr (use_saved_hash) + { + auto & column_data = saved_hash->getData(); + if (column_data.size() <= num_rows) + column_data.resize(num_rows + 1); + column_data[num_rows] = hash; + } + else column->insertData(data.data(), data.size()); + + index->reverseIndexEmplace(num_rows + base_index, iterator, inserted, hash, data); + + if constexpr (use_saved_hash) + { + if (inserted) + column->insertData(data.data(), data.size()); + } + else + { + if (!inserted) + column->popBack(1); + } + + return iterator->getValue(); } - else + catch (...) { - if (!inserted) - column->popBack(1); - } + /// The hash table may now disagree with the column: for example, the key was already added to the + /// hash table, but `insertData` failed, so the key refers to a row that does not exist. Drop the index, + /// it is rebuilt from the column on the next use. + index = nullptr; + saved_hash = nullptr; - return iterator->getValue(); + if constexpr (std::is_same_v) + { + /// `ColumnString::insertData` appends to `chars` before it grows `offsets`. If the latter failed, the bytes + /// of the row that was never added are still there and would become a prefix of the next inserted value. + const auto & offsets = column->getOffsets(); + column->getChars().resize_assume_reserved(offsets.empty() ? 0 : offsets.back()); + } + + throw; + } } template diff --git a/src/Columns/tests/gtest_reverse_index_exception_safety.cpp b/src/Columns/tests/gtest_reverse_index_exception_safety.cpp new file mode 100644 index 0000000000000..59de5cc52968d --- /dev/null +++ b/src/Columns/tests/gtest_reverse_index_exception_safety.cpp @@ -0,0 +1,126 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace DB::ErrorCodes +{ + extern const int MEMORY_LIMIT_EXCEEDED; +} + +namespace DB::FailPoints +{ + extern const char reverse_index_build_index_throw[]; +} + +using namespace DB; + +/// Mimics what happens during an asynchronous insert flush under memory pressure: the first insert into a +/// `LowCardinality` dictionary throws `MEMORY_LIMIT_EXCEEDED` while the reverse index is being built, the error +/// is handled by the caller, and the same dictionary is reused for the next entry. The index must either be +/// fully built or absent, never a non-null hash table paired with a null saved-hash array. +TEST(ReverseIndex, ExceptionWhileBuildingIndexLeavesNoHalfBuiltIndex) +{ + auto column = ColumnUnique::create(DataTypeString()); + + FailPointInjection::enableFailPoint(FailPoints::reverse_index_build_index_throw); + try + { + column->uniqueInsertData("a", 1); + FAIL() << "Expected an exception from the failpoint"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::MEMORY_LIMIT_EXCEEDED); + } + FailPointInjection::disableFailPoint(FailPoints::reverse_index_build_index_throw); + + /// Index 0 is the default value (empty string). + EXPECT_EQ(column->size(), 1u); + EXPECT_EQ(column->uniqueInsertData("a", 1), 1u); + EXPECT_EQ(column->uniqueInsertData("b", 1), 2u); + EXPECT_EQ(column->uniqueInsertData("a", 1), 1u); + EXPECT_EQ(column->uniqueInsertData("", 0), 0u); + EXPECT_EQ(column->size(), 3u); + EXPECT_EQ(column->getDataAt(1), "a"); + EXPECT_EQ(column->getDataAt(2), "b"); +} + +/// The same reuse after a failure inside `insert` itself: the key is already in the hash table and the string bytes are +/// already appended to `chars` when growing `offsets` fails. The dictionary must come back to the state before the +/// insert, so that the next inserted value does not get the failed one as a prefix and lookups stay correct. +TEST(ReverseIndex, ExceptionWhileInsertingDataKeepsDictionaryConsistent) +{ + MainThreadStatus::getInstance(); + auto & thread_tracker = CurrentThread::get().memory_tracker; + + auto column = ColumnUnique::create(DataTypeString()); + ASSERT_EQ(column->uniqueInsertData("a", 1), 1u); + + /// Leave `offsets` without spare capacity and `chars` with plenty, so that growing `offsets` is the only + /// allocation of the next insert. + auto & nested = assert_cast(column->getNestedColumn()->assumeMutableRef()); + nested.getOffsets().shrink_to_fit(); + nested.getChars().reserve(nested.getChars().size() + 1024); + + const Int64 saved_untracked_limit = CurrentThread::get().untracked_memory_limit; + const Int64 saved_thread_hard_limit = thread_tracker.getHardLimit(); + const Int64 saved_total_hard_limit = total_memory_tracker.getHardLimit(); + const UInt64 saved_min_alloc_to_throw = CurrentMemoryTracker::getMinAllocationSizeBytesToThrow(); + + auto lift_limits = [&] + { + total_memory_tracker.setHardLimit(saved_total_hard_limit); + thread_tracker.setHardLimit(saved_thread_hard_limit); + CurrentThread::get().untracked_memory_limit = saved_untracked_limit; + CurrentMemoryTracker::setMinAllocationSizeBytesToThrow(saved_min_alloc_to_throw); + }; + SCOPE_EXIT_SAFE(lift_limits()); + + /// Make even a tiny allocation reach the trackers and fail: no per-thread untracked buffer, limits of one byte. + CurrentMemoryTracker::setMinAllocationSizeBytesToThrow(1); + CurrentThread::get().untracked_memory_limit = 0; + CurrentThread::flushUntrackedMemory(); + total_memory_tracker.resetCounters(); + thread_tracker.resetCounters(); + total_memory_tracker.setHardLimit(1); + thread_tracker.setHardLimit(1); + + bool thrown = false; + int code = 0; + try + { + column->uniqueInsertData("bb", 2); + } + catch (const Exception & e) + { + thrown = true; + code = e.code(); + } + + /// Lift the limits before anything else, the assertions allocate. + lift_limits(); + + ASSERT_TRUE(thrown) << "the insert was expected to fail under the memory limit"; + EXPECT_EQ(code, ErrorCodes::MEMORY_LIMIT_EXCEEDED); + + EXPECT_EQ(column->size(), 2u); + EXPECT_EQ(nested.getChars().size(), nested.getOffsets().back()); + + EXPECT_EQ(column->uniqueInsertData("c", 1), 2u); + EXPECT_EQ(column->getDataAt(2), "c"); + EXPECT_EQ(column->uniqueInsertData("bb", 2), 3u); + EXPECT_EQ(column->getDataAt(3), "bb"); + EXPECT_EQ(column->uniqueInsertData("a", 1), 1u); + EXPECT_EQ(column->uniqueInsertData("c", 1), 2u); + EXPECT_EQ(column->size(), 4u); +} diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index abeebe2da0a6f..a32f91d56b7b9 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -34,6 +34,7 @@ static struct InitFiu #define APPLY_FOR_FAILPOINTS(ONCE, REGULAR, PAUSEABLE_ONCE, PAUSEABLE) \ ONCE(replicated_merge_tree_commit_zk_fail_after_op) \ ONCE(replicated_queue_fail_next_entry) \ + ONCE(reverse_index_build_index_throw) \ REGULAR(replicated_queue_unfail_entries) \ ONCE(replicated_merge_tree_insert_quorum_fail_0) \ REGULAR(replicated_merge_tree_commit_zk_fail_when_recovering_from_hw_fault) \ From 42514981c62631da1cabbfba29a68680d8cc96cf Mon Sep 17 00:00:00 2001 From: alesapin Date: Mon, 2 Nov 2026 18:19:12 +0100 Subject: [PATCH 060/219] Hide a SAS-bearing storage account url in the Azure table functions Backport of d496566cb263 from https://github.com/ClickHouse/ClickHouse/pull/123364, which was not backported as a whole because it is a large refactoring. `azureBlobStorage`, `azureBlobStorageCluster` and the Azure data lake table functions masked only a connection string's `AccountKey`/`SharedAccessSignature` and the `account_key` argument, so a storage account url carrying its own credential (`?sig=...`, `user:key@`) was shown, positional or as a named-collection override. A user granted only `SELECT` on a view over such a table function, which implies `SHOW COLUMNS`, could read the SAS through `SHOW CREATE TABLE`, `system.tables` and `EXPLAIN AST`. Such a value, or one that is not a plain string literal, is now hidden whole, as the `AzureBlobStorage` engine and the backup destination already do. Co-Authored-By: Claude Opus 5.5 --- src/Parsers/FunctionSecretArgumentsFinder.cpp | 26 +++++++++++ src/Parsers/FunctionSecretArgumentsFinder.h | 5 +++ ...sas_url_hidden_in_table_function.reference | 13 ++++++ ..._azure_sas_url_hidden_in_table_function.sh | 43 +++++++++++++++++++ 4 files changed, 87 insertions(+) create mode 100644 tests/queries/0_stateless/05447_azure_sas_url_hidden_in_table_function.reference create mode 100755 tests/queries/0_stateless/05447_azure_sas_url_hidden_in_table_function.sh diff --git a/src/Parsers/FunctionSecretArgumentsFinder.cpp b/src/Parsers/FunctionSecretArgumentsFinder.cpp index edbf0a93d06b9..51e2bc8387edf 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.cpp +++ b/src/Parsers/FunctionSecretArgumentsFinder.cpp @@ -631,6 +631,7 @@ void FunctionSecretArgumentsFinder::findAzureBlobStorageFunctionSecretArguments( /// azureBlobStorage(named_collection, ..., account_key = 'account_key', ...) if (maskAzureConnectionString(-1, true, 1)) return; + maskUnshowableAzureConnectionOverrides(1); findSecretNamedArgument("account_key", 1); return; } @@ -639,6 +640,7 @@ void FunctionSecretArgumentsFinder::findAzureBlobStorageFunctionSecretArguments( /// azureBlobStorageCluster(cluster, named_collection, ..., account_key = 'account_key', ...) if (maskAzureConnectionString(-1, true, 2)) return; + maskUnshowableAzureConnectionOverrides(2); findSecretNamedArgument("account_key", 2); return; } @@ -646,6 +648,9 @@ void FunctionSecretArgumentsFinder::findAzureBlobStorageFunctionSecretArguments( if (maskAzureConnectionString(url_arg_idx)) return; + if (url_arg_idx < function->arguments->size()) + maskUnshowableAzureConnectionValue(url_arg_idx, *function->arguments->at(url_arg_idx), /* argument_is_named= */ false); + /// We should check other arguments first because we don't need to do any replacement in case of /// azureBlobStorage(connection_string|storage_account_url, container_name, blobpath, format) -- in this case there is no account_key argument /// azureBlobStorageCluster(cluster, connection_string|storage_account_url, container_name, blobpath, format) -- in this case there is no account_key argument @@ -708,6 +713,27 @@ bool FunctionSecretArgumentsFinder::maskAzureConnectionString(ssize_t url_arg_id return false; } +void FunctionSecretArgumentsFinder::maskUnshowableAzureConnectionValue( + size_t index, const AbstractFunction::Argument & value, bool argument_is_named) +{ + String text; + if (!value.tryGetString(&text, /* allow_identifier= */ false) || classifyAzureConnectionValue(text) == AzureConnectionValue::Unmaskable) + markSecretArgument(index, argument_is_named); +} + +void FunctionSecretArgumentsFinder::maskUnshowableAzureConnectionOverrides(size_t start) +{ + for (const auto * key : {"connection_string", "storage_account_url"}) + { + for (ssize_t i = findNamedArgument(nullptr, key, start); i >= 0; i = findNamedArgument(nullptr, key, static_cast(i) + 1)) + { + const auto index = static_cast(i); + /// `findNamedArgument` returns only an `equals` function with two arguments. + maskUnshowableAzureConnectionValue(index, *function->arguments->at(index)->getFunction()->arguments->at(1), /* argument_is_named= */ true); + } + } +} + bool FunctionSecretArgumentsFinder::azureCollectionArgumentsAreShowable(size_t start, size_t positional_limit) { size_t positionals = 0; diff --git a/src/Parsers/FunctionSecretArgumentsFinder.h b/src/Parsers/FunctionSecretArgumentsFinder.h index b72c87816017c..a5e6ce8c812a0 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.h +++ b/src/Parsers/FunctionSecretArgumentsFinder.h @@ -197,6 +197,11 @@ class FunctionSecretArgumentsFinder /// to be readable enough to tell that it does not. `positional_limit` bounds the plain literals /// read beside the overrides: one filename for a backup locator, none for a table engine. bool azureCollectionArgumentsAreShowable(size_t start, size_t positional_limit); + /// Hides a connection value that no rule here can show: a URL carrying a credential of its own + /// (`user:key@`, a SAS `?sig=...`), or a value that is not a plain string literal. + void maskUnshowableAzureConnectionValue(size_t index, const AbstractFunction::Argument & value, bool argument_is_named); + /// The same, for the `connection_string` and `storage_account_url` overrides of a named collection from `start` on. + void maskUnshowableAzureConnectionOverrides(size_t start); /// Masks the secrets of every URL form (`url`/`urlCluster` table functions, the `URL` table /// engine, and their named-collection variants): the userinfo password of the url positional or a /// named `url = ...` override, and the `headers(...)` values at any position. `url` is at diff --git a/tests/queries/0_stateless/05447_azure_sas_url_hidden_in_table_function.reference b/tests/queries/0_stateless/05447_azure_sas_url_hidden_in_table_function.reference new file mode 100644 index 0000000000000..65bc13dc12ac7 --- /dev/null +++ b/tests/queries/0_stateless/05447_azure_sas_url_hidden_in_table_function.reference @@ -0,0 +1,13 @@ +SELECT * FROM azureBlobStorage('[HIDDEN]', 'container', 'blob') +SELECT * FROM azureBlobStorage('[HIDDEN]', 'container', 'blob') +SELECT * FROM azureBlobStorage('[HIDDEN]', 'container', 'blob') +SELECT * FROM azureBlobStorageCluster('cluster', '[HIDDEN]', 'container', 'blob') +SELECT * FROM icebergAzure('[HIDDEN]', 'container', 'path') +SELECT * FROM azureBlobStorage(azure_conf, storage_account_url = '[HIDDEN]', container = 'container', blob_path = 'blob') +SELECT * FROM azureBlobStorageCluster('cluster', azure_conf, connection_string = '[HIDDEN]', container = 'container') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', 'CSV') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', 'account', '[HIDDEN]') +SELECT * FROM azureBlobStorage(azure_conf, storage_account_url = 'https://account.blob.core.windows.net', container = 'container') +HIDDEN +0 1 +0 1 diff --git a/tests/queries/0_stateless/05447_azure_sas_url_hidden_in_table_function.sh b/tests/queries/0_stateless/05447_azure_sas_url_hidden_in_table_function.sh new file mode 100755 index 0000000000000..8ef143807a791 --- /dev/null +++ b/tests/queries/0_stateless/05447_azure_sas_url_hidden_in_table_function.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: the fast test build has no `azureBlobStorage` table function + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A storage account url carrying a credential of its own (a SAS `?sig=...`, `user:key@`), or a value that is +# not a plain string literal, is hidden whole in the Azure table functions, positional or as a named-collection +# override, as the `AzureBlobStorage` engine and the backup destination already do. +while read -r query; do + $CLICKHOUSE_FORMAT --oneline --query "$query" +done <<'EOF' +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/?sp=r&sig=plain_sas_signature', 'container', 'blob') +SELECT * FROM azureBlobStorage('https://user:plain_password@account.blob.core.windows.net', 'container', 'blob') +SELECT * FROM azureBlobStorage(concat('https://account.blob.core.windows.net/?sig=', 'plain_sas_signature'), 'container', 'blob') +SELECT * FROM azureBlobStorageCluster('cluster', 'https://account.blob.core.windows.net/?sp=r&sig=plain_sas_signature', 'container', 'blob') +SELECT * FROM icebergAzure('https://account.blob.core.windows.net/?sp=r&sig=plain_sas_signature', 'container', 'path') +SELECT * FROM azureBlobStorage(azure_conf, storage_account_url = 'https://account.blob.core.windows.net/?sp=r&sig=plain_sas_signature', container = 'container', blob_path = 'blob') +SELECT * FROM azureBlobStorageCluster('cluster', azure_conf, connection_string = 'https://account.blob.core.windows.net/?sp=r&sig=plain_sas_signature', container = 'container') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', 'CSV') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', 'account', 'plain_account_key') +SELECT * FROM azureBlobStorage(azure_conf, storage_account_url = 'https://account.blob.core.windows.net', container = 'container') +EOF + +# `SELECT` on a view implies `SHOW COLUMNS`, so a user granted only that sees the view definition through +# `SHOW CREATE TABLE` and `system.tables`, but not the SAS in it. +user="user_${CLICKHOUSE_TEST_UNIQUE_NAME}" +$CLICKHOUSE_CLIENT -q "DROP USER IF EXISTS ${user}" +$CLICKHOUSE_CLIENT -q "CREATE VIEW ${CLICKHOUSE_DATABASE}.v_sas AS SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/?sp=r&sig=plain_sas_signature', 'container', 'file.csv', 'CSV', 'none', 'id UInt8')" +$CLICKHOUSE_CLIENT -q "CREATE USER ${user} IDENTIFIED WITH plaintext_password BY 'hello'" +$CLICKHOUSE_CLIENT -q "GRANT SELECT ON ${CLICKHOUSE_DATABASE}.v_sas TO ${user}" + +$CLICKHOUSE_CLIENT --user "${user}" --password hello -q "SHOW CREATE TABLE ${CLICKHOUSE_DATABASE}.v_sas" | grep -o -e 'plain_sas_signature' -e 'HIDDEN' +$CLICKHOUSE_CLIENT --user "${user}" --password hello -q "SELECT countIf(create_table_query LIKE '%plain_sas_signature%'), countIf(create_table_query LIKE '%[HIDDEN]%') FROM system.tables WHERE database = '${CLICKHOUSE_DATABASE}' AND name = 'v_sas'" + +# `EXPLAIN AST optimize = 1` expands the view, which executes the table function and needs more than `SELECT` +# on the view, so it is run as the default user, who sees no secrets either without +# `format_display_secrets_in_show_and_select`. +$CLICKHOUSE_CLIENT --format_display_secrets_in_show_and_select 0 -q "SELECT countIf(explain LIKE '%plain_sas_signature%'), countIf(explain LIKE '%[HIDDEN]%') FROM (EXPLAIN AST optimize = 1 SELECT * FROM ${CLICKHOUSE_DATABASE}.v_sas)" + +$CLICKHOUSE_CLIENT -q "DROP USER ${user}" From 752c2f381954499e206155d44083d43612ce2212 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 2 Nov 2026 18:35:41 +0000 Subject: [PATCH 061/219] Backport #126491 to 26.8: Estimate an array join as 100 rows per input row in join reordering --- .../QueryPlan/Optimizations/optimizeJoin.cpp | 31 +++++++++- ...06_join_swap_array_join_estimate.reference | 9 +++ .../05406_join_swap_array_join_estimate.sql | 60 +++++++++++++++++++ 3 files changed, 99 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05406_join_swap_array_join_estimate.reference create mode 100644 tests/queries/0_stateless/05406_join_swap_array_join_estimate.sql diff --git a/src/Processors/QueryPlan/Optimizations/optimizeJoin.cpp b/src/Processors/QueryPlan/Optimizations/optimizeJoin.cpp index 6820bf9b3593d..2fd31ceb62579 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeJoin.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeJoin.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -325,6 +326,18 @@ struct RuntimeHashStatisticsContext } }; +/// The sizes of the arrays are unknown, so an array join is assumed to produce this many rows per input row. +constexpr Float64 array_join_rows_multiplier = 100; + +static UInt64 toUInt64Saturating(Float64 estimate) +{ + if (!(estimate > 0)) + return 0; + if (estimate >= static_cast(std::numeric_limits::max())) + return std::numeric_limits::max(); + return static_cast(estimate); +} + static RelationStats estimateAggregatingStepStats(const AggregatingStep & aggregating_step, const RelationStats & input_stats) { const auto & aggregator_params = aggregating_step.getAggregatorParameters(); @@ -508,7 +521,23 @@ RelationStats estimateReadRowsCount(QueryPlan::Node & node, const ActionsDAG::No return estimated; } - if (const auto * expression_step = typeid_cast(step); expression_step && !expression_step->getExpression().hasArrayJoin()) + const auto * expression_step = typeid_cast(step); + const auto * array_join_step = typeid_cast(step); + if (array_join_step || (expression_step && expression_step->getExpression().hasArrayJoin())) + { + /// A filter above the array join can read its output, which the statistics of the input do not describe. + auto stats = estimateReadRowsCount(*node.children.front(), nullptr); + if (stats.estimated_rows) + stats.estimated_rows = toUInt64Saturating(Float64(*stats.estimated_rows) * array_join_rows_multiplier); + if (expression_step) + remapColumnStats(stats.column_stats, expression_step->getExpression()); + else + for (const auto & column : array_join_step->getColumns()) + stats.column_stats.erase(column); + return stats; + } + + if (expression_step) { auto stats = estimateReadRowsCount(*node.children.front(), filter); remapColumnStats(stats.column_stats, expression_step->getExpression()); diff --git a/tests/queries/0_stateless/05406_join_swap_array_join_estimate.reference b/tests/queries/0_stateless/05406_join_swap_array_join_estimate.reference new file mode 100644 index 0000000000000..a19d957fd1bd0 --- /dev/null +++ b/tests/queries/0_stateless/05406_join_swap_array_join_estimate.reference @@ -0,0 +1,9 @@ +arrayJoin, empty input 1 0 +lowered arrayJoin, empty input, LEFT JOIN 1 0 +ARRAY JOIN, empty input 1 0 +lowered arrayJoin, 5 rows 1 0 +arrayJoin, 100 rows 0 1 +ARRAY JOIN, 100 rows 0 1 +0 +10 +2000 diff --git a/tests/queries/0_stateless/05406_join_swap_array_join_estimate.sql b/tests/queries/0_stateless/05406_join_swap_array_join_estimate.sql new file mode 100644 index 0000000000000..9be52cdbe674e --- /dev/null +++ b/tests/queries/0_stateless/05406_join_swap_array_join_estimate.sql @@ -0,0 +1,60 @@ +-- Random settings limits: query_plan_optimize_join_order_randomize=(0, 0); query_plan_optimize_join_order_limit=(1, None) + +-- A hash join compares the sides by their row estimates, and an array join input is estimated +-- as 100 rows per row of its input. + +SET enable_parallel_replicas = 0, join_algorithm = 'hash', query_plan_join_swap_table = 'auto'; + +DROP TABLE IF EXISTS aj_l; +DROP TABLE IF EXISTS aj_s; +DROP TABLE IF EXISTS aj_r; + +CREATE TABLE aj_l (id UInt64, arr Array(UInt64)) ENGINE = MergeTree ORDER BY id SETTINGS auto_statistics_types = ''; +CREATE TABLE aj_s (id UInt64, arr Array(UInt64)) ENGINE = MergeTree ORDER BY id SETTINGS auto_statistics_types = ''; +CREATE TABLE aj_r (id UInt64) ENGINE = MergeTree ORDER BY id SETTINGS auto_statistics_types = ''; + +INSERT INTO aj_l SELECT number, range(number, number + 20) FROM numbers(100); +INSERT INTO aj_s SELECT number, [number, number + 1] FROM numbers(5); +INSERT INTO aj_r SELECT number FROM numbers(1000); + +-- Each line counts the plans whose join reads `aj_r` first (`s` is the build side) and those reading `s` first. + +SELECT 'arrayJoin, empty input', countIf(explain ILIKE '%Join: aj_r%'), countIf(explain ILIKE '%Join: s%') +FROM (EXPLAIN actions = 1, keep_logical_steps = 1 + SELECT count() FROM (SELECT arrayJoin(arr) AS k FROM aj_l WHERE id > 1000) AS s INNER JOIN aj_r ON aj_r.id = s.k) +WHERE explain ILIKE '%Join:%'; + +SELECT 'lowered arrayJoin, empty input, LEFT JOIN', countIf(explain ILIKE '%Join: aj_r%'), countIf(explain ILIKE '%Join: s%') +FROM (EXPLAIN actions = 1, keep_logical_steps = 1 + SELECT count() FROM (SELECT arrayJoin(arr) AS k FROM aj_l WHERE id > 1000) AS s LEFT JOIN aj_r ON aj_r.id = s.k) +WHERE explain ILIKE '%Join:%'; + +SELECT 'ARRAY JOIN, empty input', countIf(explain ILIKE '%Join: aj_r%'), countIf(explain ILIKE '%Join: s%') +FROM (EXPLAIN actions = 1, keep_logical_steps = 1 + SELECT count() FROM (SELECT k FROM aj_l ARRAY JOIN arr AS k WHERE id > 1000) AS s INNER JOIN aj_r ON aj_r.id = s.k) +WHERE explain ILIKE '%Join:%'; + +-- 5 input rows are estimated as 500 rows, fewer than the 1000 rows of `aj_r`. +SELECT 'lowered arrayJoin, 5 rows', countIf(explain ILIKE '%Join: aj_r%'), countIf(explain ILIKE '%Join: s%') +FROM (EXPLAIN actions = 1, keep_logical_steps = 1 + SELECT count() FROM (SELECT arrayJoin(arr) AS k FROM aj_s) AS s INNER JOIN aj_r ON aj_r.id = s.k) +WHERE explain ILIKE '%Join:%'; + +-- 100 input rows are estimated as 10000 rows (2000 in fact), more than the 1000 rows of `aj_r`. +SELECT 'arrayJoin, 100 rows', countIf(explain ILIKE '%Join: aj_r%'), countIf(explain ILIKE '%Join: s%') +FROM (EXPLAIN actions = 1, keep_logical_steps = 1 + SELECT count() FROM (SELECT arrayJoin(arr) AS k FROM aj_l) AS s INNER JOIN aj_r ON aj_r.id = s.k) +WHERE explain ILIKE '%Join:%'; + +SELECT 'ARRAY JOIN, 100 rows', countIf(explain ILIKE '%Join: aj_r%'), countIf(explain ILIKE '%Join: s%') +FROM (EXPLAIN actions = 1, keep_logical_steps = 1 + SELECT count() FROM (SELECT k FROM aj_l ARRAY JOIN arr AS k) AS s INNER JOIN aj_r ON aj_r.id = s.k) +WHERE explain ILIKE '%Join:%'; + +SELECT count() FROM (SELECT arrayJoin(arr) AS k FROM aj_l WHERE id > 1000) AS s LEFT JOIN aj_r ON aj_r.id = s.k; +SELECT count() FROM (SELECT arrayJoin(arr) AS k FROM aj_s) AS s INNER JOIN aj_r ON aj_r.id = s.k; +SELECT count() FROM (SELECT k FROM aj_l ARRAY JOIN arr AS k) AS s INNER JOIN aj_r ON aj_r.id = s.k; + +DROP TABLE aj_l; +DROP TABLE aj_s; +DROP TABLE aj_r; From 4e50a55e7fa8585e5fe0751de8076bd186fd54fb Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 2 Nov 2026 19:45:50 +0000 Subject: [PATCH 062/219] Backport #127242 to 26.8: Fix use-after-free in `CPUSlotsAllocation` wait timer --- src/Common/Scheduler/CPUSlotsAllocation.cpp | 7 +++- src/Common/Scheduler/CPUSlotsAllocation.h | 4 ++ .../tests/gtest_workload_resource_manager.cpp | 38 +++++++++++++++++++ 3 files changed, 48 insertions(+), 1 deletion(-) diff --git a/src/Common/Scheduler/CPUSlotsAllocation.cpp b/src/Common/Scheduler/CPUSlotsAllocation.cpp index a8965c3990c94..729030c4004f7 100644 --- a/src/Common/Scheduler/CPUSlotsAllocation.cpp +++ b/src/Common/Scheduler/CPUSlotsAllocation.cpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace ProfileEvents { @@ -72,6 +73,10 @@ CPUSlotsAllocation::CPUSlotsAllocation(SlotCount master_slots_, SlotCount worker for (CPUSlotRequest & request : requests) request.allocation = this; + wait_thread_group = CurrentThread::getGroup(); + if (wait_thread_group) + wait_counters = &wait_thread_group->performance_counters; + std::unique_lock lock{schedule_mutex}; while (allocated < total_slots) { @@ -79,7 +84,7 @@ CPUSlotsAllocation::CPUSlotsAllocation(SlotCount master_slots_, SlotCount worker { queue->enqueueRequest(current_request); scheduled_slot_increment.emplace(CurrentMetrics::ConcurrencyControlScheduled); - wait_timer.emplace(CurrentThread::getProfileEvents().timer(ProfileEvents::ConcurrencyControlWaitMicroseconds)); + wait_timer.emplace(wait_counters->timer(ProfileEvents::ConcurrencyControlWaitMicroseconds)); break; } else // noncompeting slot - provide for free diff --git a/src/Common/Scheduler/CPUSlotsAllocation.h b/src/Common/Scheduler/CPUSlotsAllocation.h index c7f94f2557d9d..03e18a4c8a116 100644 --- a/src/Common/Scheduler/CPUSlotsAllocation.h +++ b/src/Common/Scheduler/CPUSlotsAllocation.h @@ -18,6 +18,7 @@ namespace DB { class CPUSlotsAllocation; +class ThreadGroup; // Represents a resource request for a cpu slot for a single thread class CPUSlotRequest final : public ResourceRequest @@ -114,6 +115,9 @@ class CPUSlotsAllocation final : public ISlotAllocation std::vector requests; // Requests per every slot CPUSlotRequest * current_request; std::optional scheduled_slot_increment; + /// `wait_timer` may end on another thread after the creating one has exited, so it counts into the query's group, kept alive here. + std::shared_ptr wait_thread_group; + ProfileEvents::Counters * wait_counters = &ProfileEvents::global_counters; std::optional wait_timer; }; diff --git a/src/Common/Scheduler/Nodes/tests/gtest_workload_resource_manager.cpp b/src/Common/Scheduler/Nodes/tests/gtest_workload_resource_manager.cpp index 3ea85d499877f..8c00ef44a95b7 100644 --- a/src/Common/Scheduler/Nodes/tests/gtest_workload_resource_manager.cpp +++ b/src/Common/Scheduler/Nodes/tests/gtest_workload_resource_manager.cpp @@ -13,6 +13,8 @@ #include #include #include +#include +#include #include #include #include @@ -65,6 +67,7 @@ namespace ProfileEvents { extern const Event ConcurrencyControlUpscales; extern const Event ConcurrencyControlDownscales; + extern const Event ConcurrencyControlWaitMicroseconds; } namespace CurrentMetrics @@ -1331,6 +1334,41 @@ TEST(SchedulerWorkloadResourceManager, CPUSchedulingRoundRobin) t.wait(); } +/// The wait for a CPU slot is accounted to the query even when the allocation outlives the thread that created it +/// (here: destroyed on another thread while its request is still queued, like a `KILL QUERY` of a waiting query). +TEST(SchedulerWorkloadResourceManager, CPUSlotsWaitTimerOutlivesCreatingThread) +{ + ResourceTest t; + t.query("CREATE RESOURCE cpu (MASTER THREAD, WORKER THREAD)"); + t.query("CREATE WORKLOAD all SETTINGS max_concurrent_threads = 1"); + ClassifierPtr c = t.manager->acquire("all"); + ResourceLink link = c->get("cpu"); + + /// Takes the only slot, so the next allocation keeps its request queued. + auto blocker = std::make_shared(1, 0, link, link); + auto blocker_slot = blocker->acquire(); + + std::weak_ptr weak_group; + SlotAllocationPtr allocation; + std::thread([&] + { + DB::ThreadStatus thread_status; + auto group = std::make_shared(Context::getGlobalContextInstance(), 0); + weak_group = group; + CurrentThread::attachToGroupIfDetached(group); + allocation = std::make_shared(1, 0, link, link); + CurrentThread::detachFromGroupIfNotDetached(); + }).join(); + + ASSERT_TRUE(allocation->isRequesting()); + /// The creating thread is gone, so only the allocation can still own the group. + ThreadGroupPtr group = weak_group.lock(); + ASSERT_NE(group, nullptr); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + allocation.reset(); + EXPECT_GE(group->performance_counters[ProfileEvents::ConcurrencyControlWaitMicroseconds], 10'000); +} + TEST(SchedulerWorkloadResourceManager, CPUSchedulingFairness) { ResourceTest t; From 0803d8aef6bf646b456c1e41388c01d6a616dd2e Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 2 Nov 2026 20:15:40 +0000 Subject: [PATCH 063/219] Backport #127401 to 26.8: Fix int96 overflow in parquet reader --- .../Formats/Impl/Parquet/Decoding.cpp | 21 ++++++++++-------- .../Formats/Impl/Parquet/Decoding.h | 3 +++ .../Formats/Impl/Parquet/SchemaConverter.cpp | 4 +++- .../0_stateless/00900_long_parquet_load_2.sh | 4 ++++ ...nt96_date_time_overflow_behavior.reference | 13 +++++++++++ ...rquet_int96_date_time_overflow_behavior.sh | 19 ++++++++++++++++ ...444_parquet_int96_lower_boundary.reference | 2 ++ .../05444_parquet_int96_lower_boundary.sh | 16 +++++++++++++ .../05443_int96_out_of_range.parquet | Bin 0 -> 700 bytes .../05444_int96_lower_boundary.parquet | Bin 0 -> 535 bytes 10 files changed, 72 insertions(+), 10 deletions(-) create mode 100644 tests/queries/0_stateless/05443_parquet_int96_date_time_overflow_behavior.reference create mode 100755 tests/queries/0_stateless/05443_parquet_int96_date_time_overflow_behavior.sh create mode 100644 tests/queries/0_stateless/05444_parquet_int96_lower_boundary.reference create mode 100755 tests/queries/0_stateless/05444_parquet_int96_lower_boundary.sh create mode 100644 tests/queries/0_stateless/data_parquet/05443_int96_out_of_range.parquet create mode 100644 tests/queries/0_stateless/data_parquet/05444_int96_lower_boundary.parquet diff --git a/src/Processors/Formats/Impl/Parquet/Decoding.cpp b/src/Processors/Formats/Impl/Parquet/Decoding.cpp index ca7b8f5057a35..87924b120ed70 100644 --- a/src/Processors/Formats/Impl/Parquet/Decoding.cpp +++ b/src/Processors/Formats/Impl/Parquet/Decoding.cpp @@ -1961,6 +1961,8 @@ Int96Converter::Int96Converter() void Int96Converter::convertColumn(std::span data, size_t num_values, IColumn & col) const { + const bool saturate = overflow_behavior == FormatSettings::DateTimeOverflowBehavior::Saturate; + std::span to_span = col.insertRawUninitialized(num_values); Int64 * to = reinterpret_cast(to_span.data()); for (size_t i = 0; i < num_values; ++i) @@ -1985,16 +1987,17 @@ void Int96Converter::convertColumn(std::span data, size_t num_values /// This interpretation is consistent with the arrow reader code (DecodeInt96Timestamp in /// arrow/cpp/src/parquet/types.h) and with a test file written by spark /// (tests/queries/0_stateless/02998_native_parquet_reader.sh). - bool overflow = false; - Int64 x = 0; - overflow |= common::subOverflow(julian_day, static_cast(2440588l), x); // unix day number - overflow |= common::mulOverflow(x, day_nanos, x); // unix nanoseconds - overflow |= common::addOverflow(x, nanos, x); - - if (overflow) - throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "INT96 timestamp out of range: julian day {}, time of day {} ns", julian_day, nanos); - to[i] = x; + /// julian_day comes from Int32, so neither the subtraction nor the product can overflow Int128. + const Int128 total = Int128(julian_day - 2440588l) * day_nanos + nanos; + if (total < std::numeric_limits::min() || total > std::numeric_limits::max()) + { + if (!saturate) + throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "INT96 timestamp out of range: julian day {}, time of day {} ns", julian_day, nanos); + to[i] = julian_day < 2440588l ? std::numeric_limits::min() : std::numeric_limits::max(); + } + else + to[i] = static_cast(total); } } diff --git a/src/Processors/Formats/Impl/Parquet/Decoding.h b/src/Processors/Formats/Impl/Parquet/Decoding.h index e0533fd295073..88f0cc76411d1 100644 --- a/src/Processors/Formats/Impl/Parquet/Decoding.h +++ b/src/Processors/Formats/Impl/Parquet/Decoding.h @@ -397,6 +397,9 @@ extern template struct BigEndianDecimalStringConverter; struct Int96Converter : public FixedSizeConverter { + /// A value that does not fit into Int64 nanoseconds is clamped to the `DateTime64(9)` range with Saturate, otherwise throws. + FormatSettings::DateTimeOverflowBehavior overflow_behavior = FormatSettings::DateTimeOverflowBehavior::Ignore; + Int96Converter(); void convertColumn(std::span data, size_t num_values, IColumn & col) const override; diff --git a/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp b/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp index bf5cd131f135e..d764b100b5c7e 100644 --- a/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp +++ b/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp @@ -1530,7 +1530,9 @@ void SchemaConverter::processPrimitiveColumn( case parq::Type::INT96: { out_inferred_type = std::make_shared(9, "UTC"); - out_decoder.fixed_size_converter = std::make_shared(); + auto converter = std::make_shared(); + converter->overflow_behavior = options.format.date_time_overflow_behavior; + out_decoder.fixed_size_converter = std::move(converter); /// (Leaving allow_stats == false because INT96 sort order is undefined.) return; } diff --git a/tests/queries/0_stateless/00900_long_parquet_load_2.sh b/tests/queries/0_stateless/00900_long_parquet_load_2.sh index 459655fda1ddc..35b74b839dea2 100755 --- a/tests/queries/0_stateless/00900_long_parquet_load_2.sh +++ b/tests/queries/0_stateless/00900_long_parquet_load_2.sh @@ -65,6 +65,10 @@ EXCLUDE=( # malformed on purpose, so loading it here would print an exception. 05035_delta_byte_array_zero_values.parquet 05035_delta_byte_array_decimal.parquet + # `INT96` timestamps outside the `DateTime64(9)` range (throws with the default `date_time_overflow_behavior`), tested by 05443. + 05443_int96_out_of_range.parquet + # `INT96` timestamp on the lower boundary day of the `DateTime64(9)` range, tested by 05444. + 05444_int96_lower_boundary.parquet ) for NAME in $(find "$DATA_DIR" -type f \( -iname '*.parquet' -o -iname '*.parquet.gz' \) -print0 | xargs -0 -n 1 basename | LC_ALL=C sort | grep -vFf <(printf '%s\n' "${EXCLUDE[@]}")); do diff --git a/tests/queries/0_stateless/05443_parquet_int96_date_time_overflow_behavior.reference b/tests/queries/0_stateless/05443_parquet_int96_date_time_overflow_behavior.reference new file mode 100644 index 0000000000000..f104f80c58911 --- /dev/null +++ b/tests/queries/0_stateless/05443_parquet_int96_date_time_overflow_behavior.reference @@ -0,0 +1,13 @@ +VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE +VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE +0 1677-09-21 00:12:43.145224192 Nullable(DateTime64(9, \'UTC\')) +1 1800-01-01 12:30:00.000000000 Nullable(DateTime64(9, \'UTC\')) +2 2020-05-17 10:20:30.123456000 Nullable(DateTime64(9, \'UTC\')) +3 2262-04-11 23:47:16.854775807 Nullable(DateTime64(9, \'UTC\')) +4 \N Nullable(DateTime64(9, \'UTC\')) +0 1677-09-21 00:12:43.145225 Nullable(DateTime64(6, \'UTC\')) +1 1800-01-01 12:30:00.000000 Nullable(DateTime64(6, \'UTC\')) +2 2020-05-17 10:20:30.123456 Nullable(DateTime64(6, \'UTC\')) +3 2262-04-11 23:47:16.854775 Nullable(DateTime64(6, \'UTC\')) +4 \N Nullable(DateTime64(6, \'UTC\')) +2 diff --git a/tests/queries/0_stateless/05443_parquet_int96_date_time_overflow_behavior.sh b/tests/queries/0_stateless/05443_parquet_int96_date_time_overflow_behavior.sh new file mode 100755 index 0000000000000..184859b094300 --- /dev/null +++ b/tests/queries/0_stateless/05443_parquet_int96_date_time_overflow_behavior.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: the fast test build has no Parquet support + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# INT96 timestamps 0001-01-01, 1800-01-01 12:30, 2020-05-17 10:20:30.123456, 9999-12-31 23:59:59 and NULL, written by +# pyarrow with `use_deprecated_int96_timestamps=True`. The first and the fourth do not fit into Int64 nanoseconds. +DATA_FILE="${CUR_DIR}/data_parquet/05443_int96_out_of_range.parquet" + +${CLICKHOUSE_LOCAL} --query "SELECT * FROM file('${DATA_FILE}', Parquet) ORDER BY id" 2>&1 | grep -o -m1 'VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE' +${CLICKHOUSE_LOCAL} --query "SELECT * FROM file('${DATA_FILE}', Parquet) ORDER BY id SETTINGS date_time_overflow_behavior = 'throw'" 2>&1 | grep -o -m1 'VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE' + +${CLICKHOUSE_LOCAL} --query " + SELECT id, ts, toTypeName(ts) FROM file('${DATA_FILE}', Parquet) ORDER BY id SETTINGS date_time_overflow_behavior = 'saturate'; + SELECT id, ts, toTypeName(ts) FROM file('${DATA_FILE}', Parquet, 'id Int32, ts Nullable(DateTime64(6, \'UTC\'))') ORDER BY id SETTINGS date_time_overflow_behavior = 'saturate'; + SELECT count() FROM file('${DATA_FILE}', Parquet) WHERE ts > '2000-01-01' SETTINGS date_time_overflow_behavior = 'saturate';" diff --git a/tests/queries/0_stateless/05444_parquet_int96_lower_boundary.reference b/tests/queries/0_stateless/05444_parquet_int96_lower_boundary.reference new file mode 100644 index 0000000000000..4d2fdda745397 --- /dev/null +++ b/tests/queries/0_stateless/05444_parquet_int96_lower_boundary.reference @@ -0,0 +1,2 @@ +0 1677-09-21 12:00:00.000000000 -9223329600000000000 +0 1677-09-21 12:00:00.000000000 -9223329600000000000 diff --git a/tests/queries/0_stateless/05444_parquet_int96_lower_boundary.sh b/tests/queries/0_stateless/05444_parquet_int96_lower_boundary.sh new file mode 100755 index 0000000000000..39e02424f38a4 --- /dev/null +++ b/tests/queries/0_stateless/05444_parquet_int96_lower_boundary.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: the fast test build has no Parquet support + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A single INT96 timestamp 1677-09-21 12:00:00 (julian day 2333836, time of day 43200000000000 ns), written by pyarrow +# with `use_deprecated_int96_timestamps=True`. Midnight of that day does not fit into Int64 nanoseconds, but the full value +# -9223329600000000000 does, so it must be read exactly instead of being saturated to the Int64 minimum or rejected. +DATA_FILE="${CUR_DIR}/data_parquet/05444_int96_lower_boundary.parquet" + +${CLICKHOUSE_LOCAL} --query " + SELECT id, ts, toUnixTimestamp64Nano(ts) FROM file('${DATA_FILE}', Parquet) SETTINGS date_time_overflow_behavior = 'saturate'; + SELECT id, ts, toUnixTimestamp64Nano(ts) FROM file('${DATA_FILE}', Parquet) SETTINGS date_time_overflow_behavior = 'throw';" diff --git a/tests/queries/0_stateless/data_parquet/05443_int96_out_of_range.parquet b/tests/queries/0_stateless/data_parquet/05443_int96_out_of_range.parquet new file mode 100644 index 0000000000000000000000000000000000000000..d3a66f8168d429f51dab8e773c8a66d3884683c2 GIT binary patch literal 700 zcmaJ<-D(q25S~37maqsW(pmP%UPLfU3fq5LMcRwm{MeFM(}0xTb=ihZC~eauZJt2z z0i;h-f)`%+2tI(9-s-j9h%={&^+FuZnZGmNH#2bB)LFwERG`*BQ+>fo1s*rPY z<~tIo>C52xd@=P*T`HPO=fa~AC5KX$QO)2}nRh1jyG=~AvjcuNo8b}B*Y_dmqTM+s z%Sgh-{5jA?(yFE`wR{4hIO~K=Wjl7=B}Xf}W{@pajjY)=+iq##pbu_Q=h(2_ti3>u z9J)?2=+GRx-X5s=SumAbf+5?n4%qkartR%4)<&I(Tt*DJ7X=wHr|d<1%^YzzE3xM> z-%7E|JP?a`XAz0(J?42JA3%DAus`Z&$y9Xaij(78A=2Oj@9du!#a=!NEc0aI_oNwy zL6aGK>2?l~?dJ7(AxI8I0lE`VGY3E}E@o%dTVTviD!?e2g#g2kAR) Date: Tue, 3 Nov 2026 00:09:02 +0000 Subject: [PATCH 064/219] Backport #124477 to 26.8: Keep the population of a Replicated database DDL entry on the replica that executes it --- src/Interpreters/InterpreterCreateQuery.cpp | 27 +++- .../configs/remote_servers.xml | 13 ++ .../test.py | 120 ++++++++++++++++++ 3 files changed, 158 insertions(+), 2 deletions(-) diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 47fad29cdac17..694a135e8f14f 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -3023,6 +3023,17 @@ BlockIO InterpreterCreateQuery::doCreateOrReplaceTemporaryTable(ASTCreateQuery & return fillTableIfNeeded(create); } +namespace +{ +/// The population's own `SETTINGS` are reapplied on top of its context under the names they were written with. +constexpr std::array settings_sending_population_to_other_servers +{ + "allow_experimental_parallel_reading_from_replicas", + "enable_parallel_replicas", + "parallel_distributed_insert_select", +}; +} + BlockIO InterpreterCreateQuery::fillTableIfNeeded(const ASTCreateQuery & create, bool skip_target_insert_access_check) { /// If the query is a CREATE SELECT, insert the data into the table. @@ -3038,10 +3049,22 @@ BlockIO InterpreterCreateQuery::fillTableIfNeeded(const ASTCreateQuery & create, else insert->select = create.select->clone(); + ContextMutablePtr insert_context = getContext(); + /// Every replica of a `Replicated` database executes this entry and fills its own copy of the table, so the population + /// must neither read through parallel replicas nor be sent to other servers as a distributed `INSERT`. + if (getContext()->getZooKeeperMetadataTransaction()) + { + auto local_context = Context::createCopy(getContext()); + local_context->setSetting("parallel_distributed_insert_select", Field{0}); + local_context->setSetting("enable_parallel_replicas", Field{0}); + removeSettingsFromQuery(insert->select, settings_sending_population_to_other_servers); + insert_context = local_context; + } + InterpreterInsertQuery interpreter( insert, - getContext(), - getContext()->getSettingsRef()[Setting::insert_allow_materialized_columns], + insert_context, + insert_context->getSettingsRef()[Setting::insert_allow_materialized_columns], /* no_squash */ false, /* no_destination */ false, /* async_isnert */ false); diff --git a/tests/integration/test_parallel_replicas_insert_select/configs/remote_servers.xml b/tests/integration/test_parallel_replicas_insert_select/configs/remote_servers.xml index 89438a1ee76c9..35f7e776d71fc 100644 --- a/tests/integration/test_parallel_replicas_insert_select/configs/remote_servers.xml +++ b/tests/integration/test_parallel_replicas_insert_select/configs/remote_servers.xml @@ -34,6 +34,19 @@ + + + true + + node1 + 9000 + + + node2 + 1234 + + + diff --git a/tests/integration/test_parallel_replicas_insert_select/test.py b/tests/integration/test_parallel_replicas_insert_select/test.py index d92065f46bd02..c56ce0759dcf2 100644 --- a/tests/integration/test_parallel_replicas_insert_select/test.py +++ b/tests/integration/test_parallel_replicas_insert_select/test.py @@ -328,3 +328,123 @@ def test_insert_select_where(start_cluster, max_parallel_replicas, parallel_repl settings={"skip_unavailable_shards": 1}, ) assert (int(number_of_queries) > 1) + + +def create_replicated_database(db): + for i, node in enumerate([node1, node2, node3], start=1): + node.query( + f"CREATE DATABASE {db} ENGINE = Replicated('/clickhouse/databases/{db}', 'shard1', 'node{i}')" + ) + + +def sync_database(db): + for node in [node1, node2, node3]: + node.query(f"SYSTEM SYNC DATABASE REPLICA {db}") + + +def drop_database(db): + for node in [node1, node2, node3]: + node.query(f"DROP DATABASE IF EXISTS {db} SYNC") + + +def create_replicated_source(db): + node1.query(f"CREATE TABLE {db}.src (id UInt64) ENGINE = ReplicatedMergeTree ORDER BY id") + sync_database(db) + node1.query(f"INSERT INTO {db}.src SELECT number FROM numbers(1000)") + for node in [node2, node3]: + node.query(f"SYSTEM SYNC REPLICA {db}.src") + + +def populate_settings(cluster_name, max_parallel_replicas, parallel_distributed_insert_select=2): + return { + "enable_parallel_replicas": 1, + "max_parallel_replicas": max_parallel_replicas, + "parallel_distributed_insert_select": parallel_distributed_insert_select, + "parallel_replicas_local_plan": 0, + "cluster_for_parallel_replicas": cluster_name, + } + + +def query_per_node(query): + return [node.query(query) for node in [node1, node2, node3]] + + +# Every replica of a Replicated database populates its own view, not the view of another replica. +def test_replicated_database_populate(start_cluster): + db = f"db_{uuid.uuid4().hex}" + try: + create_replicated_database(db) + create_replicated_source(db) + node1.query( + f"CREATE MATERIALIZED VIEW {db}.mv ENGINE = MergeTree ORDER BY id POPULATE AS SELECT id FROM {db}.src", + settings=populate_settings("test_1_shard_2_replicas_1_unavailable", 2), + ) + sync_database(db) + assert query_per_node(f"SELECT count(), uniqExact(id), sum(id) FROM {db}.mv") == ["1000\t1000\t499500\n"] * 3 + finally: + drop_database(db) + + +# The initiator populates its view while the other replicas have not created theirs yet. +def test_replicated_database_populate_before_other_replicas(start_cluster): + db = f"db_{uuid.uuid4().hex}" + try: + create_replicated_database(db) + create_replicated_source(db) + for node in [node2, node3]: + node.query("SYSTEM ENABLE FAILPOINT database_replicated_stop_entry_execution") + node1.query( + f"CREATE MATERIALIZED VIEW {db}.mv ENGINE = MergeTree ORDER BY id POPULATE AS SELECT id FROM {db}.src", + settings={**populate_settings("test_1_shard_3_replicas", 3), "distributed_ddl_task_timeout": 0}, + ) + for node in [node2, node3]: + node.query("SYSTEM DISABLE FAILPOINT database_replicated_stop_entry_execution") + sync_database(db) + assert query_per_node(f"SELECT count(), uniqExact(id), sum(id) FROM {db}.mv") == ["1000\t1000\t499500\n"] * 3 + finally: + for node in [node2, node3]: + node.query("SYSTEM DISABLE FAILPOINT database_replicated_stop_entry_execution") + drop_database(db) + + +# `CREATE TABLE ... AS SELECT` fills the table of every replica the same way. +def test_replicated_database_create_as_select(start_cluster): + db = f"db_{uuid.uuid4().hex}" + try: + create_replicated_database(db) + create_replicated_source(db) + node1.query( + f"CREATE TABLE {db}.t ENGINE = Memory AS SELECT id FROM {db}.src", + settings=populate_settings("test_1_shard_2_replicas_1_unavailable", 2), + ) + sync_database(db) + assert query_per_node(f"SELECT count(), uniqExact(id), sum(id) FROM {db}.t") == ["1000\t1000\t499500\n"] * 3 + finally: + drop_database(db) + + +# The population reads the source of the replica executing it, also when the view's own `SELECT` enables parallel replicas. +def test_replicated_database_populate_reads_local_source(start_cluster): + db = f"db_{uuid.uuid4().hex}" + try: + create_replicated_database(db) + node1.query(f"CREATE TABLE {db}.src_local (id UInt64) ENGINE = MergeTree ORDER BY id") + sync_database(db) + for i, node in enumerate([node1, node2, node3]): + node.query(f"INSERT INTO {db}.src_local SELECT {i * 1000} + number FROM numbers(1000)") + node1.query( + f"CREATE MATERIALIZED VIEW {db}.mv ENGINE = MergeTree ORDER BY id POPULATE AS SELECT id FROM {db}.src_local " + "SETTINGS allow_experimental_parallel_reading_from_replicas = 1", + settings={ + **populate_settings("test_1_shard_2_replicas_1_unavailable", 2, parallel_distributed_insert_select=0), + "parallel_replicas_for_non_replicated_merge_tree": 1, + }, + ) + sync_database(db) + assert query_per_node(f"SELECT count(), min(id), max(id) FROM {db}.mv") == [ + "1000\t0\t999\n", + "1000\t1000\t1999\n", + "1000\t2000\t2999\n", + ] + finally: + drop_database(db) From 5fcc7ed030b94f75033596dbcc8f0c62b8981ede Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 1 Nov 2026 08:50:52 +0000 Subject: [PATCH 065/219] Backport #128048 to 26.8: Restore JSONExtract DateTime parsing under old compatibility --- src/Core/Settings.cpp | 7 +++++++ src/Functions/FunctionsJSON.cpp | 5 ++++- ...n_extract_datetime_compatibility.reference | 5 +++++ ...57_json_extract_datetime_compatibility.sql | 19 +++++++++++++++++++ 4 files changed, 35 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05457_json_extract_datetime_compatibility.reference create mode 100644 tests/queries/0_stateless/05457_json_extract_datetime_compatibility.sql diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 332099c64c708..ea5426bc0dc1e 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -4551,6 +4551,13 @@ Reject patterns which will likely be expensive to evaluate with hyperscan (due t DECLARE(Bool, allow_simdjson, true, R"( Allow using simdjson library in 'JSON*' functions if AVX2 instructions are available. If disabled rapidjson will be used. )", 0) \ + DECLARE(Bool, json_extract_use_cast_string_to_date_time_mode, true, R"( +Controls which setting the `JSONExtract` family of functions follows when parsing a JSON string value into `DateTime` or `DateTime64`. When enabled, the parsing mode is taken from [cast_string_to_date_time_mode](#cast_string_to_date_time_mode), consistently with `CAST`. When disabled, it is taken from [date_time_input_format](/reference/settings/formats/input-format#date_time_input_format), as before version 26.8. + +When the chosen parser cannot read the value, the function returns the default value of the type (`1970-01-01 00:00:00`) without an error, so a mismatch between the two settings silently changes the extracted values. +)", 0, \ + {"26.11", true, true, "New setting to choose whether the `JSONExtract` family parses `DateTime` values according to `cast_string_to_date_time_mode` (since 26.8) or `date_time_input_format` (before 26.8)."}, \ + {"26.8", false, true, "`JSONExtract` parses `DateTime` values according to `cast_string_to_date_time_mode` instead of `date_time_input_format`. `compatibility` with versions before 26.8 restores the previous behavior."}) \ DECLARE(Bool, allow_introspection_functions, false, R"( Enables or disables [introspection functions](/reference/functions/regular-functions/introspection) for query profiling. diff --git a/src/Functions/FunctionsJSON.cpp b/src/Functions/FunctionsJSON.cpp index 891d0c8c846ef..9ab5f242d2c3d 100644 --- a/src/Functions/FunctionsJSON.cpp +++ b/src/Functions/FunctionsJSON.cpp @@ -51,6 +51,7 @@ namespace Setting { extern const SettingsBool allow_simdjson; extern const SettingsDateTimeInputFormat cast_string_to_date_time_mode; + extern const SettingsBool json_extract_use_cast_string_to_date_time_mode; } namespace ErrorCodes @@ -698,7 +699,9 @@ class JSONOverloadResolver final : public IFunctionOverloadResolver { /// Extracting a string JSON value into a DateTime/DateTime64 column is a string-to-type /// cast, so we honour `cast_string_to_date_time_mode` (rather than `date_time_input_format`). - format_settings.date_time_input_format = context->getSettingsRef()[Setting::cast_string_to_date_time_mode]; + /// Before 26.8 it followed `date_time_input_format`, which `compatibility` restores. + if (context->getSettingsRef()[Setting::json_extract_use_cast_string_to_date_time_mode]) + format_settings.date_time_input_format = context->getSettingsRef()[Setting::cast_string_to_date_time_mode]; } bool isVariadic() const override { return true; } diff --git a/tests/queries/0_stateless/05457_json_extract_datetime_compatibility.reference b/tests/queries/0_stateless/05457_json_extract_datetime_compatibility.reference new file mode 100644 index 0000000000000..414d8128901e7 --- /dev/null +++ b/tests/queries/0_stateless/05457_json_extract_datetime_compatibility.reference @@ -0,0 +1,5 @@ +1970-01-01 00:00:00.000 +2026-10-19 07:30:26.743 +2026-10-19 07:30:26 +2026-10-19 07:30:26.743 +1970-01-01 00:00:00.000 diff --git a/tests/queries/0_stateless/05457_json_extract_datetime_compatibility.sql b/tests/queries/0_stateless/05457_json_extract_datetime_compatibility.sql new file mode 100644 index 0000000000000..0d3234ea3e1b4 --- /dev/null +++ b/tests/queries/0_stateless/05457_json_extract_datetime_compatibility.sql @@ -0,0 +1,19 @@ +-- `compatibility` with versions before 26.8 must restore the old behavior of `JSONExtract`, which parsed +-- `DateTime` values according to `date_time_input_format` rather than `cast_string_to_date_time_mode`. + +SET date_time_input_format = 'best_effort', cast_string_to_date_time_mode = 'basic'; + +SET json_extract_use_cast_string_to_date_time_mode = 1; +SELECT JSONExtract('{"createdAt":"2026-10-19T07:30:26.743Z"}', 'createdAt', 'DateTime64(3, \'UTC\')'); + +SET json_extract_use_cast_string_to_date_time_mode = 0; +SELECT JSONExtract('{"createdAt":"2026-10-19T07:30:26.743Z"}', 'createdAt', 'DateTime64(3, \'UTC\')'); +SELECT JSONExtract('{"createdAt":"2026-10-19T07:30:26Z"}', 'createdAt', 'DateTime(\'UTC\')'); + +-- `compatibility` resets the setting, but does not touch `date_time_input_format` set explicitly. +SET json_extract_use_cast_string_to_date_time_mode = DEFAULT; +SET compatibility = '23.9', date_time_input_format = 'best_effort'; +SELECT JSONExtract('{"createdAt":"2026-10-19T07:30:26.743Z"}', 'createdAt', 'DateTime64(3, \'UTC\')'); + +SET compatibility = '26.8', cast_string_to_date_time_mode = 'basic'; +SELECT JSONExtract('{"createdAt":"2026-10-19T07:30:26.743Z"}', 'createdAt', 'DateTime64(3, \'UTC\')'); From 7fec11b076d5c5d2577f73f21040b9788922bd01 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Tue, 3 Nov 2026 09:20:26 +0000 Subject: [PATCH 066/219] Add json_extract_use_cast_string_to_date_time_mode to SettingsChangesHistory for 26.8 The backport added the setting to Settings.cpp (with master's inline history), but on 26.8 both the `compatibility` mechanism and the 03999 guard read src/Core/SettingsChangesHistory.cpp, which had no entry for it. Add the 26.8 entry {false, true} matching master's inline `{"26.8", false, true}`. Fixes two stateless failures: - 03999_stateless_settings_history: "PLEASE ADD THE NEW SETTING TO SettingsChangesHistory.cpp". - 05457_json_extract_datetime_compatibility: `SET compatibility = '23.9'` now resets the setting to its pre-26.8 value (false), restoring `date_time_input_format` parsing as the test expects. https://github.com/ClickHouse/ClickHouse/pull/128467 https://github.com/ClickHouse/ClickHouse/pull/128048 https://s3.amazonaws.com/clickhouse-test-reports/json.html?PR=128467&sha=4b790cb064df2c7c5e1712d22b15f34aa9a832af&name_0=BackportPR Co-Authored-By: Claude Opus 4.8 --- src/Core/SettingsChangesHistory.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index b6ee1b4351242..997ba249e3bcf 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -47,6 +47,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() }); addSettingsChanges(settings_changes_history, "26.8", { + {"json_extract_use_cast_string_to_date_time_mode", false, true, "`JSONExtract` parses `DateTime` values according to `cast_string_to_date_time_mode` instead of `date_time_input_format`. `compatibility` with versions before 26.8 restores the previous behavior."}, {"backup_s3_expired_token_retry_timeout_ms", 0, 120000, "Retry S3 backup and restore requests after an expired token; 0 restores the previous behavior."}, {"allow_named_collection_override_by_default", true, true, "Obsolete. Overriding named collection keys requires `SHOW NAMED COLLECTIONS SECRETS`."}, {"use_aggregation_memory_tracker", false, false, "New setting. In this release it is disabled by default, which restores the two-level conversion of `GROUP BY` from before 26.7 (https://github.com/ClickHouse/ClickHouse/pull/107490)."}, From 496d7353b12307d5a88f1f07da75c6fb9b7dad88 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 2 Nov 2026 14:43:22 +0000 Subject: [PATCH 067/219] Backport #124468 to 26.8: Support virtual rows in reading in order with FINAL --- src/Core/BaseSettings.h | 10 +- src/Core/BaseSettingsProgramOptions.h | 4 +- src/Core/Settings.cpp | 5 + src/Core/Settings.h | 1 + src/Core/SettingsFields.cpp | 77 ++++++ src/Core/SettingsFields.h | 38 +++ .../QueryPlanOptimizationSettings.cpp | 5 +- .../QueryPlanOptimizationSettings.h | 3 +- .../Optimizations/optimizeReadInOrder.cpp | 42 +-- .../QueryPlan/Optimizations/optimizeTree.cpp | 2 +- src/Processors/QueryPlan/PartsSplitter.cpp | 1 + src/Processors/QueryPlan/PartsSplitter.h | 3 + .../QueryPlan/ReadFromMergeTree.cpp | 260 +++++++++++++++--- src/Processors/QueryPlan/ReadFromMergeTree.h | 10 + .../performance/final_read_in_order_limit.xml | 12 +- ...d_filters_after_final_in_explain.reference | 8 + ...eferred_filters_after_final_in_explain.sql | 1 + ..._virtual_row_disabled_with_final.reference | 7 - .../04489_virtual_row_disabled_with_final.sql | 46 ---- ..._read_in_order_final_virtual_row.reference | 167 +++++++++++ .../05325_read_in_order_final_virtual_row.sql | 171 ++++++++++++ ...407_bool_force_setting_shorthand.reference | 4 + .../05407_bool_force_setting_shorthand.sh | 28 ++ 23 files changed, 777 insertions(+), 128 deletions(-) delete mode 100644 tests/queries/0_stateless/04489_virtual_row_disabled_with_final.reference delete mode 100644 tests/queries/0_stateless/04489_virtual_row_disabled_with_final.sql create mode 100644 tests/queries/0_stateless/05325_read_in_order_final_virtual_row.reference create mode 100644 tests/queries/0_stateless/05325_read_in_order_final_virtual_row.sql create mode 100644 tests/queries/0_stateless/05407_bool_force_setting_shorthand.reference create mode 100755 tests/queries/0_stateless/05407_bool_force_setting_shorthand.sh diff --git a/src/Core/BaseSettings.h b/src/Core/BaseSettings.h index 149fe72671f21..c3028b8eb6d17 100644 --- a/src/Core/BaseSettings.h +++ b/src/Core/BaseSettings.h @@ -497,17 +497,17 @@ SettingsChanges BaseSettings::changes() const template void BaseSettings::checkShorthandChange(const SettingChange & change) const { - /// `SET name` without a value means `SET name = true`, which only makes sense for a Bool - /// setting. This is where the settings schema is known, so this is where it is checked. + /// `SET name` without a value means `SET name = true`, which only makes sense for a Bool* + /// settings. This is where the settings schema is known, so this is where it is checked. if (!change.shorthand) return; - if (std::string_view type = getTypeName(change.name); type != "Bool") + if (std::string_view type = getTypeName(change.name); !type.starts_with("Bool")) BaseSettingsHelpers::throwValuelessSettingIsNotBool(change.name, type); - /// The type alone is not enough. The parser writes Bool `true` for the valueless form, but a + /// The type alone is not enough. The parser writes `Bool` `true` for the valueless form, but a /// `SettingChange` can also arrive from the AST JSON dialect, which is free to pair the flag - /// with any other value - and for a `Bool` setting the check above lets that through. Such a + /// with any other value - and for a `Bool*` setting the check above lets that through. Such a /// change would execute with the carried value while `ASTSetQuery::formatImpl` renders the bare /// name, so `system.query_log` and every other formatter would under-report what ran. This is /// the first point after deserialization where an exception is logged with the AST masked diff --git a/src/Core/BaseSettingsProgramOptions.h b/src/Core/BaseSettingsProgramOptions.h index e1bf34f060bf1..f7b2cd213cf36 100644 --- a/src/Core/BaseSettingsProgramOptions.h +++ b/src/Core/BaseSettingsProgramOptions.h @@ -11,7 +11,7 @@ template void addProgramOptionAsMultitoken(T &cmd_settings, boost::program_options::options_description & options, std::string_view name, const typename T::SettingFieldRef & field) { auto on_program_option = boost::function1([&cmd_settings, name](const Strings & values) { cmd_settings.set(name, values.back()); }); - if (field.getTypeName() == "Bool") + if (std::string_view type = field.getTypeName(); type.starts_with("Bool")) { options.add( boost::shared_ptr(new boost::program_options::option_description( @@ -54,7 +54,7 @@ template void addProgramOption(T & cmd_settings, boost::program_options::options_description & options, std::string_view name, const typename T::SettingFieldRef & field) { auto on_program_option = boost::function1([&cmd_settings, name](const std::string & value) { cmd_settings.set(name, value); }); - if (field.getTypeName() == "Bool") + if (std::string_view type = field.getTypeName(); type.starts_with("Bool")) { options.add(boost::shared_ptr(new boost::program_options::option_description( name.data(), boost::program_options::value()->composing()->implicit_value("1")->notifier(on_program_option), field.getDescription().data()))); // NOLINT diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index ea5426bc0dc1e..11b179fd82655 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -6861,6 +6861,11 @@ Possible values: - 0 - Disable - 1 - Enable )", 0) \ + DECLARE(BoolForce, query_plan_read_in_order_through_join, true, R"( +Keep reading in order from the left table in JOIN operations, which can be utilized by subsequent steps. + +For a join other than `LEFT`, the read stays in order only when every part of the left table announces a virtual row (see [`read_in_order_use_virtual_row`](#read_in_order_use_virtual_row)); without the virtual rows the merge after the join may read far more than the limit needs. The value `force` keeps the read in order regardless. +)", 0, \ DECLARE(Bool, query_plan_remove_redundant_sorting, true, R"( Toggles a query-plan-level optimization which removes redundant sorting steps, e.g. in subqueries. Only takes effect if setting [`query_plan_enable_optimizations`](#query_plan_enable_optimizations) is 1. diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 449589b2431fd..18f53168d5418 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -52,6 +52,7 @@ class WriteBuffer; M(CLASS_NAME, ArrowFlightDescriptorType) \ M(CLASS_NAME, Bool) \ M(CLASS_NAME, BoolAuto) \ + M(CLASS_NAME, BoolForce) \ M(CLASS_NAME, CapnProtoEnumComparingMode) \ M(CLASS_NAME, Char) \ M(CLASS_NAME, DateTimeInputFormat) \ diff --git a/src/Core/SettingsFields.cpp b/src/Core/SettingsFields.cpp index 001edbfeb6290..faeaadc691e0a 100644 --- a/src/Core/SettingsFields.cpp +++ b/src/Core/SettingsFields.cpp @@ -562,6 +562,83 @@ void SettingFieldURI::readBinary(ReadBuffer & in) } +static bool isTriBoolKeyword(const Field & f, std::string_view keyword) +{ + return f.getType() == Field::Types::String && f.safeGet() == keyword; +} + +template +SettingFieldTriBool::SettingFieldTriBool(const Field & f) : is_keyword(isTriBoolKeyword(f, keyword)) +{ + if (!is_keyword) + base = SettingFieldBool(f); +} + +template +SettingFieldTriBool & SettingFieldTriBool::operator=(const Field & f) +{ + changed = true; + is_keyword = isTriBoolKeyword(f, keyword); + if (!is_keyword) + base = f; + return *this; +} + +template +SettingFieldTriBool::operator Field() const +{ + return is_keyword ? Field(String(keyword)) : Field(base); +} + +template +String SettingFieldTriBool::toString() const +{ + return is_keyword ? String(keyword) : base.toString(); +} + +template +void SettingFieldTriBool::parseFromString(const String & str) +{ + changed = true; + is_keyword = (str == keyword); + if (!is_keyword) + base.parseFromString(str); +} + +template +void SettingFieldTriBool::writeBinary(WriteBuffer & out) const +{ + if (is_keyword) + writeStringBinary(keyword, out); + else + base.writeBinary(out); +} + +template +void SettingFieldTriBool::readBinary(ReadBuffer & in) +{ + changed = true; + + /// A Bool is a varint 0/1 (see `SettingFieldNumber`); the keyword string starts with its length, which is longer. + UInt64 x = 0; + readVarUInt(x, in); + is_keyword = x > 1; + if (!is_keyword) + { + base = SettingFieldBool(x != 0); + return; + } + + if (x != keyword.size()) + throw Exception(ErrorCodes::CANNOT_PARSE_BOOL, "Cannot parse a bool or '{}' from the binary value of a setting", keyword); + String str(x, '\0'); + in.readStrict(str.data(), x); + if (str != keyword) + throw Exception(ErrorCodes::CANNOT_PARSE_BOOL, "Cannot parse a bool or '{}' from the binary value '{}' of a setting", keyword, str); +} + +template struct SettingFieldTriBool; + void SettingFieldEnumHelpers::writeBinary(std::string_view str, WriteBuffer & out) { writeStringBinary(str, out); diff --git a/src/Core/SettingsFields.h b/src/Core/SettingsFields.h index 0c06a6e8e3efd..c217c594b85da 100644 --- a/src/Core/SettingsFields.h +++ b/src/Core/SettingsFields.h @@ -200,6 +200,44 @@ using SettingFieldUInt32Auto = SettingAutoWrapper; using SettingFieldInt32Auto = SettingAutoWrapper; using SettingFieldDoubleAuto = SettingAutoWrapper; +/// A Bool setting with a third, named value: accepts everything a Bool accepts plus `Keyword::value`. +/// Mirrors `SettingAutoWrapper`: `base` holds the Bool, `is_keyword` tells that the third value is set. +/// The members are defined in SettingsFields.cpp, which instantiates the struct for every keyword. +template +struct SettingFieldTriBool +{ + constexpr static std::string_view keyword = Keyword::value; + /// In the binary format a Bool is a varint 0/1 and a string starts with its length (see `readBinary`). + static_assert(keyword.size() > 1); + + SettingFieldBool base; + bool is_keyword = false; + bool changed = false; + + explicit SettingFieldTriBool(bool x = false) : base(x) {} + explicit SettingFieldTriBool(const Field & f); + SettingFieldTriBool(const SettingFieldTriBool &) = default; + SettingFieldTriBool & operator=(const SettingFieldTriBool &) = default; + SettingFieldTriBool & operator=(const Field & f); + + bool isChanged() const { return changed; } + void setChanged(bool changed_) { changed = changed_; } + + explicit operator Field() const; + String toString() const; + void parseFromString(const String & str); + + void writeBinary(WriteBuffer & out) const; + void readBinary(ReadBuffer & in); +}; + +struct SettingFieldKeywordForce +{ + constexpr static std::string_view value = "force"; +}; + +using SettingFieldBoolForce = SettingFieldTriBool; + /* Similar to SettingFieldUInt64Auto with small differences to behave like regular UInt64, supported to compatibility. * When setting to 'auto' it becomes equal to the number of processor cores without taking into account SMT. * A value of 0 is also treated as 'auto', so 'auto' is parsed and serialized in the same way as 0. diff --git a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp index ee68091b256df..c7cf243021544 100644 --- a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp +++ b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp @@ -78,7 +78,7 @@ namespace Setting extern const SettingsBool query_plan_push_down_limit; extern const SettingsBool query_plan_push_limit_by_into_sort; extern const SettingsBool query_plan_top_k_through_join; - extern const SettingsBool query_plan_read_in_order_through_join; + extern const SettingsBoolForce query_plan_read_in_order_through_join; extern const SettingsBool optimize_aggregation_in_order_limit; extern const SettingsBool query_plan_read_in_order; extern const SettingsBool query_plan_remove_redundant_distinct; @@ -247,7 +247,8 @@ QueryPlanOptimizationSettings::QueryPlanOptimizationSettings( && from[Setting::query_plan_optimize_count_from_text_index] && from[Setting::optimize_trivial_count_query]; enable_full_text_index = from[Setting::enable_full_text_index]; - read_in_order_through_join = from[Setting::query_plan_read_in_order_through_join]; + const auto & through_join = from[Setting::query_plan_read_in_order_through_join]; + read_in_order_through_join = through_join.is_keyword ? 2 : through_join.base.value; /// In-memory buffer for correlated subqueries uses a non-serializable ChunkBuffer, /// incompatible with distributed execution. When make_distributed_plan is enabled, /// always use the materialization path (materializeQueryPlanReferences) instead. diff --git a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h index 7923e0cced7db..fafa82d0719b7 100644 --- a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h +++ b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h @@ -106,7 +106,8 @@ struct QueryPlanOptimizationSettings bool optimize_projection; bool use_query_condition_cache; bool use_query_condition_cache_for_top_k; - bool read_in_order_through_join; + /// 0 - off, 1 - on, 2 - on even when the parts have no virtual rows (`force`). + UInt64 read_in_order_through_join; bool optimize_aggregation_in_order_limit; bool correlated_subqueries_use_in_memory_buffer; bool push_limit_by_into_sort; diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp index fab0836a8d946..c5e2f3bf05fb7 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp @@ -1151,7 +1151,7 @@ InputOrderInfoPtr buildInputOrderInfo( { FindReadingStepContext find_reading_ctx{ .allow_existing_order = false, - .read_in_order_through_join = optimization_settings.read_in_order_through_join, + .read_in_order_through_join = optimization_settings.read_in_order_through_join != 0, }; QueryPlan::Node * reading_node = findReadingStep(node, find_reading_ctx); if (!reading_node) @@ -1196,22 +1196,24 @@ InputOrderInfoPtr buildInputOrderInfo( virtual_row_reader = reading; } - if (!uses_virtual_row) - { - /// Virtual row is required for INNER JOIN to maintain read-in-order optimization. - /// Without it, the sorting step after join cannot correctly determine - /// which input stream to read from when most rows are filtered out, - /// potentially reading excessive amount of data. - for (const auto * join_step : find_reading_ctx.joins_to_keep_in_order) + /// Joins that can discard left rows need virtual rows from every selected part to avoid + /// excessive reading by the merge after the join. `LEFT ALL` and `LEFT ANY` preserve left rows. + /// `query_plan_read_in_order_through_join = 'force'` accepts the over-read. + const bool join_requires_virtual_rows = std::ranges::any_of( + find_reading_ctx.joins_to_keep_in_order, + [](const auto * join_step) { const auto & table_join = join_step->getJoin()->getTableJoin(); - auto strictness = table_join.strictness(); - if (table_join.kind() != JoinKind::Left || (strictness != JoinStrictness::All && strictness != JoinStrictness::Any)) - { - LOG_DEBUG(getLogger("optimizeReadInOrder"), "Skip using read in order for inner join without virtual row optimization"); - return nullptr; - } - } + const auto strictness = table_join.strictness(); + return table_join.kind() != JoinKind::Left + || (strictness != JoinStrictness::All && strictness != JoinStrictness::Any); + }); + if (join_requires_virtual_rows && optimization_settings.read_in_order_through_join != 2 + && (!uses_virtual_row || !reading->everyPartHasVirtualRowBound(order_info.input_order->direction < 0))) + { + LOG_DEBUG(getLogger("optimizeReadInOrder"), "Skip using read in order for inner join without virtual row optimization"); + reading->resetVirtualRowConversions(); + return nullptr; } bool can_read = reading->requestReadingInOrder( @@ -1221,7 +1223,11 @@ InputOrderInfoPtr buildInputOrderInfo( query_limit); if (!can_read) + { + /// E.g. a reverse `FINAL` read the engine does not support: the read stays as it was. + reading->resetVirtualRowConversions(); return nullptr; + } for (auto * join_step : find_reading_ctx.joins_to_keep_in_order) join_step->keepLeftPipelineInOrder(/* disable_squashing */ true); @@ -1276,7 +1282,7 @@ InputOrder buildInputOrderInfo(AggregatingStep & aggregating, QueryPlan::Node & { FindReadingStepContext find_reading_ctx { .allow_existing_order = false, - .read_in_order_through_join = optimization_settings.read_in_order_through_join, + .read_in_order_through_join = optimization_settings.read_in_order_through_join != 0, }; QueryPlan::Node * reading_node = findReadingStep(node, find_reading_ctx); if (!reading_node) @@ -1397,7 +1403,7 @@ InputOrder buildInputOrderInfo(DistinctStep & distinct, QueryPlan::Node & node, FindReadingStepContext find_reading_ctx { .allow_existing_order = true, - .read_in_order_through_join = optimization_settings.read_in_order_through_join, + .read_in_order_through_join = optimization_settings.read_in_order_through_join != 0, }; QueryPlan::Node * reading_node = findReadingStep(node, find_reading_ctx); if (!reading_node) @@ -1488,7 +1494,7 @@ InputOrder buildInputOrderInfo(LimitByStep & limit_by, QueryPlan::Node & node, c FindReadingStepContext find_reading_ctx{ .allow_existing_order = true, - .read_in_order_through_join = optimization_settings.read_in_order_through_join, + .read_in_order_through_join = optimization_settings.read_in_order_through_join != 0, }; QueryPlan::Node * reading_node = findReadingStep(node, find_reading_ctx); if (!reading_node) diff --git a/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp b/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp index 9556da9e95500..7b572302a599a 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp @@ -92,7 +92,7 @@ void optimizeTreeFirstPass(const QueryPlanOptimizationSettings & optimization_se optimization_settings.max_limit_for_top_k_optimization, optimization_settings.use_skip_indexes_on_data_read, optimization_settings.read_in_order, - optimization_settings.read_in_order_through_join, + optimization_settings.read_in_order_through_join != 0, optimization_settings.join_swap_table, optimization_settings.enable_group_by_top_k_optimization, optimization_settings.top_k_optimization_observation_rows, diff --git a/src/Processors/QueryPlan/PartsSplitter.cpp b/src/Processors/QueryPlan/PartsSplitter.cpp index 84629ef81b69c..cda59ee7e8427 100644 --- a/src/Processors/QueryPlan/PartsSplitter.cpp +++ b/src/Processors/QueryPlan/PartsSplitter.cpp @@ -1221,6 +1221,7 @@ SplitPartsWithRangesByPrimaryKeyResult splitPartsWithRangesByPrimaryKey( auto split_ranges = splitIntersectingPartsRangesIntoLayers( intersecting_parts_ranges, max_layers, primary_key.column_names.size(), in_reverse_order, logger); + result.borders = split_ranges.borders; result.merging_pipes = readByLayers(std::move(split_ranges), primary_key, create_merging_pipe, context); return result; } diff --git a/src/Processors/QueryPlan/PartsSplitter.h b/src/Processors/QueryPlan/PartsSplitter.h index ccab952da0a08..ba6a40d6ccd2a 100644 --- a/src/Processors/QueryPlan/PartsSplitter.h +++ b/src/Processors/QueryPlan/PartsSplitter.h @@ -35,6 +35,9 @@ struct SplitPartsWithRangesByPrimaryKeyResult { RangesInDataParts non_intersecting_parts_ranges; Pipes merging_pipes; + /// Borders between consecutive `merging_pipes` in the table's key order (see `SplitPartsByRanges::borders`); + /// empty when the intersecting ranges were not split into layers. + std::vector> borders; }; /** Splits parts ranges into: diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 38c99f94d19bc..48c294aff4acc 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -787,6 +787,95 @@ Pipe ReadFromMergeTree::readFromPool( return pipe; } +/// The index of `part` if it has the first `num_key_columns` PK columns, `nullptr` otherwise. +/// `optimizeIndexColumns` may drop trailing PK columns, and a reverse read needs the final mark. +static IMergeTreeDataPart::IndexPtr indexWithVirtualRowBound(const IMergeTreeDataPart & part, size_t num_key_columns, bool read_in_reverse) +{ + if (read_in_reverse && !part.index_granularity->hasFinalMark()) + return nullptr; + auto index = part.getIndex(); + if (index->size() < num_key_columns) + return nullptr; + return index; +} + +/// PK values at the first selected mark, or the end mark for a reverse read. +/// Returns `std::nullopt` if the index cannot provide them. +static std::optional getVirtualRowBound(const RangesInDataPart & part_with_ranges, const Block & pk_header, bool read_in_reverse) +{ + const auto index = indexWithVirtualRowBound(*part_with_ranges.data_part, pk_header.columns(), read_in_reverse); + if (!index) + return std::nullopt; + + const size_t mark_pos = read_in_reverse ? part_with_ranges.ranges.back().end : part_with_ranges.ranges.front().begin; + for (size_t j = 0; j < pk_header.columns(); ++j) + if ((*index)[j]->size() <= mark_pos) + return std::nullopt; + + /// Insert via `Field`: the index may have the type before an `ALTER MODIFY COLUMN` + /// (e.g. `String` vs `LowCardinality(String)`), because a mutation can keep the source part's index. + auto pk_columns = pk_header.cloneEmptyColumns(); + for (size_t j = 0; j < pk_columns.size(); ++j) + pk_columns[j]->insert((*(*index)[j])[mark_pos]); + return pk_header.cloneWithColumns(std::move(pk_columns)); +} + +/// Whether `lhs` comes before `rhs` in the table's order, reversed for a reverse read. +static bool virtualRowPrecedes(const Block & lhs, const Block & rhs, const std::vector & reverse_flags, bool read_in_reverse) +{ + for (size_t j = 0; j < lhs.columns(); ++j) + { + int cmp = lhs.getByPosition(j).column->compareAt(0, 0, *rhs.getByPosition(j).column, /*nan_direction_hint=*/ 1); + if (!reverse_flags.empty() && reverse_flags[j]) + cmp = -cmp; + if (cmp != 0) + return read_in_reverse ? cmp > 0 : cmp < 0; + } + return false; +} + +/// The virtual row of a `FINAL` merge over `parts`: the least of the parts' bounds (the greatest for a reverse read). +/// Returns `std::nullopt` if the index of some part does not have the bound. +static std::optional getFinalMergeVirtualRow( + const RangesInDataParts & parts, const Block & pk_header, const std::vector & reverse_flags, bool read_in_reverse) +{ + std::optional bound; + for (const auto & part : parts) + { + auto part_bound = getVirtualRowBound(part, pk_header, read_in_reverse); + if (!part_bound) + return std::nullopt; + if (!bound || virtualRowPrecedes(*part_bound, *bound, reverse_flags, read_in_reverse)) + bound = std::move(part_bound); + } + return bound; +} + +/// With `FINAL`, intersecting ranges are split into layers which are read in parallel. Borders between layers +/// are PK values which can be in the middle of a granule, so a border is a tighter bound than the index. +/// Use the border as the virtual row if it has all the needed PK columns. +static std::optional tightenFinalMergeVirtualRow( + std::optional virtual_row, + const std::vector & border, + const Block & pk_header, + const std::vector & reverse_flags, + bool read_in_reverse) +{ + /// A border has as many values as the parts' indexes share, which may be fewer than the virtual row needs. + if (border.size() < pk_header.columns()) + return virtual_row; + + auto columns = pk_header.cloneEmptyColumns(); + for (size_t j = 0; j < columns.size(); ++j) + columns[j]->insert(border[j]); + Block border_row = pk_header.cloneWithColumns(std::move(columns)); + + /// The border should always be at least as tight, compare just in case. + if (!virtual_row || virtualRowPrecedes(*virtual_row, border_row, reverse_flags, read_in_reverse)) + return border_row; + return virtual_row; +} + Pipe ReadFromMergeTree::readInOrder( RangesInDataParts parts_with_ranges, const MergeTreeIndexBuildContextPtr & index_build_context, @@ -801,7 +890,10 @@ Pipe ReadFromMergeTree::readInOrder( const bool has_hard_limit_below_one_block = read_type != ReadType::Default && read_limit && read_limit < block_size.max_block_size_rows; const bool has_soft_limit_below_one_block = read_type != ReadType::Default && query_task_size_limit && query_task_size_limit < block_size.max_block_size_rows; - const bool use_virtual_row = virtual_row_conversion && (read_type == ReadType::InOrder || read_type == ReadType::InReverseOrder); + /// The `FINAL` merges do not consume virtual rows: a `FINAL` read gets one per merge instead (see + /// `spreadMarkRangesAmongStreamsFinal`). + const bool use_virtual_row = virtual_row_conversion && !isQueryWithFinal() + && (read_type == ReadType::InOrder || read_type == ReadType::InReverseOrder); const bool use_virtual_row_per_block = use_virtual_row && context->getSettingsRef()[Setting::read_in_order_use_virtual_row_per_block]; if (use_virtual_row_per_block && read_type == ReadType::InReverseOrder) @@ -908,6 +1000,8 @@ Pipe ReadFromMergeTree::readInOrder( const UInt64 in_order_limit = query_info.input_order_info ? query_info.input_order_info->limit : 0; const bool set_total_rows_approx = !is_parallel_reading_from_replicas || isParallelReplicasLocalPlanForInitiator(); + const Block pk_header = use_virtual_row ? makeVirtualRowHeader() : Block{}; + Pipes pipes; for (size_t i = 0; i < parts_with_ranges.size(); ++i) { @@ -958,23 +1052,8 @@ Pipe ReadFromMergeTree::readInOrder( processor->addPartLevelToChunk(isQueryWithFinal()); - Block pk_header; - if (use_virtual_row) - { - const auto & primary_key = storage_snapshot->metadata->primary_key; - size_t num_pk_columns_required = virtual_row_conversion->getRequiredColumnsWithTypes().size(); - - ColumnsWithTypeAndName pk_header_columns; - pk_header_columns.reserve(num_pk_columns_required); - for (size_t j = 0; j < num_pk_columns_required; ++j) - pk_header_columns.push_back( - {primary_key.data_types[j]->createColumn(), primary_key.data_types[j], primary_key.column_names[j]}); - - pk_header = Block(std::move(pk_header_columns)); - - if (use_virtual_row_per_block) - processor->setVirtualRowConversions(virtual_row_conversion, pk_header, read_type == ReadType::InReverseOrder); - } + if (use_virtual_row_per_block) + processor->setVirtualRowConversions(virtual_row_conversion, pk_header, read_type == ReadType::InReverseOrder); auto source = std::make_shared(std::move(processor), data.getLogName()); if (set_total_rows_approx) @@ -984,26 +1063,11 @@ Pipe ReadFromMergeTree::readInOrder( if (use_virtual_row) { - const auto & index = part_with_ranges.data_part->getIndex(); - - bool has_final_mark = part_with_ranges.data_part->index_granularity->hasFinalMark(); - bool read_in_direct_order = read_type == ReadType::InOrder; - size_t mark_range_pos = read_in_direct_order ? part_with_ranges.ranges.front().begin : part_with_ranges.ranges.back().end; - bool has_pk_value = (read_in_direct_order || has_final_mark) && std::ranges::all_of(*index, [&](const auto & col) { return col->size() > mark_range_pos; }); - - /// The index may have fewer columns than the primary key if suffix columns were - /// removed by optimizeIndexColumns (controlled by primary_key_ratio_of_unique_prefix_values_to_skip_suffix_columns). - /// In that case, we cannot apply virtual row optimization because we don't have all required columns. - auto pk_columns = pk_header.cloneEmptyColumns(); - if (index->size() >= pk_columns.size() && has_pk_value) + if (auto pk_block = getVirtualRowBound(part_with_ranges, pk_header, read_type == ReadType::InReverseOrder)) { - for (size_t j = 0; j < pk_columns.size(); ++j) - pk_columns[j]->insert((*(*index)[j])[mark_range_pos]); - - Block pk_block = pk_header.cloneWithColumns(std::move(pk_columns)); pipe.addSimpleTransform([&](const SharedHeader & header) { - return std::make_shared(header, pk_block, virtual_row_conversion); + return std::make_shared(header, *pk_block, virtual_row_conversion); }); } } @@ -2279,6 +2343,18 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( chassert(num_streams == requested_num_streams); num_streams = std::min(num_streams, settings[Setting::max_final_threads]); + /// Always false on this branch (`requestReadingInOrder` rejects reverse order with FINAL), + /// kept as a runtime value so the virtual-row helpers get the real flag. + const bool read_in_reverse = reader_settings.read_in_order && query_info.input_order_info && query_info.input_order_info->direction < 0; + + /// A virtual row after each `FINAL` merge announces the key its layer starts from, so the merge downstream + /// defers the merge and the reading of its parts. Only when a step requested reading in order: the parts + /// of a `FINAL` read are in order for its merges anyway, but without the request nothing downstream + /// consumes virtual rows, and vertical `FINAL` may be on. + const bool use_virtual_row = virtual_row_conversion && reader_settings.read_in_order; + const Block virtual_row_header = use_virtual_row ? makeVirtualRowHeader() : Block{}; + const auto reverse_flags = storage_snapshot->metadata->getSortingKeyReverseFlags(); + /// If do_not_merge_across_partitions_select_final is true than we won't merge parts from different partitions. /// We have all parts in parts vector, where parts with same partition are nearby. /// So we will store iterators pointed to the beginning of each partition range (and parts.end()), @@ -2385,6 +2461,8 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( } Pipes pipes; + /// The virtual row of each of `pipes`, when its parts have one. + std::vector> pipes_virtual_rows; { RangesInDataParts new_parts; size_t current_ranges_marks = 0; @@ -2410,9 +2488,18 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( if (storage_snapshot->metadata->hasPrimaryKey()) { + /// `splitPartsWithRangesByPrimaryKey` calls the getter once per merging pipe, in their order. + std::vector> merging_pipes_virtual_rows; + // Let's split parts into non intersecting parts ranges and layers to ensure data parallelism of FINAL. - auto in_order_reading_step_getter = [this, &index_build_context, &column_names, &info](auto parts) + auto in_order_reading_step_getter + = [this, &index_build_context, &column_names, &info, read_in_reverse, + use_virtual_row, &virtual_row_header, &reverse_flags, &merging_pipes_virtual_rows](auto parts) + { + if (use_virtual_row) + merging_pipes_virtual_rows.push_back(getFinalMergeVirtualRow(parts, virtual_row_header, reverse_flags, read_in_reverse)); + return this->read( std::move(parts), index_build_context, @@ -2498,10 +2585,29 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( /// reach `createProjection` or `addMergingFinal` below. Dropping it is safe here: /// unlike the join-by-shards path, the per-layer pipes are simply united, so their /// positions carry no meaning. - for (auto && merging_pipe : split_ranges_result.merging_pipes) + for (size_t i = 0; i < split_ranges_result.merging_pipes.size(); ++i) { - if (!merging_pipe.empty()) - pipes.push_back(std::move(merging_pipe)); + auto & merging_pipe = split_ranges_result.merging_pipes[i]; + if (merging_pipe.empty()) + continue; + + pipes.push_back(std::move(merging_pipe)); + if (use_virtual_row) + { + /// The parts' index bounds a layer loosely: a first granule of a sparse part spans many keys. + /// The border the layer starts from bounds it exactly; the layers run in the table's order, + /// so a forward read starts layer `i` after `borders[i - 1]`, a reverse one after `borders[i]`. + auto virtual_row = std::move(merging_pipes_virtual_rows.at(i)); + const auto & borders = split_ranges_result.borders; + const bool has_start_border = read_in_reverse ? (i < borders.size()) : (i > 0); + if (has_start_border) + { + const auto & border = borders[read_in_reverse ? i : (i - 1)]; + virtual_row = tightenFinalMergeVirtualRow( + std::move(virtual_row), border, virtual_row_header, reverse_flags, read_in_reverse); + } + pipes_virtual_rows.push_back(std::move(virtual_row)); + } } } else @@ -2517,6 +2623,9 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( pipes.back().addSimpleTransform([sorting_expr](const SharedHeader & header) { return std::make_shared(header, sorting_expr); }); + /// Without a primary key there is nothing to build a virtual row from. + if (use_virtual_row) + pipes_virtual_rows.emplace_back(); } /// Drop temporary columns, added by 'sorting_key_expr' @@ -2528,7 +2637,6 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( continue; Names sort_columns = storage_snapshot->metadata->getSortingKeyColumns(); - std::vector reverse_flags = storage_snapshot->metadata->getSortingKeyReverseFlags(); SortDescription sort_description; sort_description.compile_sort_description = settings[Setting::compile_sort_description]; sort_description.min_count_to_compile_sort_description = settings[Setting::min_count_to_compile_sort_description]; @@ -2544,15 +2652,25 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( sort_description.emplace_back(sort_columns[i], 1); } - for (auto & pipe : pipes) + for (size_t i = 0; i < pipes.size(); ++i) + { addMergingFinal( - pipe, + pipes[i], sort_description, data.merging_params, storage_snapshot->metadata, block_size.max_block_size_rows, enable_vertical_final); + if (use_virtual_row && pipes_virtual_rows[i]) + { + pipes[i].addSimpleTransform([&](const SharedHeader & header) + { + return std::make_shared(header, *pipes_virtual_rows[i], virtual_row_conversion); + }); + } + } + merging_pipes.emplace_back(Pipe::unitePipes(std::move(pipes))); } @@ -3810,14 +3928,68 @@ bool ReadFromMergeTree::requestReadingInOrder(size_t prefix_size, int direction, bool ReadFromMergeTree::setVirtualRowConversions(ActionsDAG virtual_row_conversion_) { - /// Disable virtual row for FINAL. - if (isQueryWithFinal() || !context->getSettingsRef()[Setting::read_in_order_use_virtual_row]) + if (!context->getSettingsRef()[Setting::read_in_order_use_virtual_row]) + return false; + + if (isQueryWithFinal() && !canUseVirtualRowWithFinal()) return false; virtual_row_conversion = std::make_shared(std::move(virtual_row_conversion_)); return true; } +bool ReadFromMergeTree::canUseVirtualRowWithFinal() const +{ + /// The virtual row of a `FINAL` merge announces the key its parts start from (see + /// `spreadMarkRangesAmongStreamsFinal`). That bounds the output only for the engines that never + /// emit a key below their inputs: `Graphite` rounds the time column down. + switch (data.merging_params.mode) + { + case MergeTreeData::MergingParams::Replacing: + case MergeTreeData::MergingParams::Collapsing: + case MergeTreeData::MergingParams::VersionedCollapsing: + case MergeTreeData::MergingParams::Summing: + case MergeTreeData::MergingParams::Aggregating: + case MergeTreeData::MergingParams::Coalescing: + break; + case MergeTreeData::MergingParams::Ordinary: + case MergeTreeData::MergingParams::Graphite: + return false; + } + + /// The lanes of a parallel-replicas `FINAL` are laid out by the coordinator (see `buildDistributedFinalPipe`); not covered yet. + if (is_parallel_reading_from_replicas) + return false; + + return true; +} + +bool ReadFromMergeTree::everyPartHasVirtualRowBound(bool read_in_reverse) const +{ + /// Without part filters or an existing analysis, checking bounds would load indexes of unrelated parts. + if (!analyzed_result_ptr && !indexes) + return false; + + const size_t num_key_columns = virtual_row_conversion->getRequiredColumnsWithTypes().size(); + for (const auto & part : getPartsForPrewhere()) + if (!indexWithVirtualRowBound(*part.data_part, num_key_columns, read_in_reverse)) + return false; + return true; +} + +Block ReadFromMergeTree::makeVirtualRowHeader() const +{ + /// The leading primary key columns the conversions read: the merge downstream compares nothing else. + const auto & primary_key = storage_snapshot->metadata->primary_key; + const size_t num_columns = virtual_row_conversion->getRequiredColumnsWithTypes().size(); + + ColumnsWithTypeAndName columns; + columns.reserve(num_columns); + for (size_t j = 0; j < num_columns; ++j) + columns.push_back({primary_key.data_types[j]->createColumn(), primary_key.data_types[j], primary_key.column_names[j]}); + return Block(std::move(columns)); +} + bool ReadFromMergeTree::readsInOrder() const { diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index d7c306b5f9017..32c776dc8e844 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -360,6 +360,10 @@ class ReadFromMergeTree final : public SourceStepWithFilter bool requestReadingInOrder(size_t prefix_size, int direction, size_t read_limit, size_t query_limit = 0); bool setVirtualRowConversions(ActionsDAG virtual_row_conversion_); void resetVirtualRowConversions() { virtual_row_conversion = nullptr; } + + /// Whether every part surviving pruning has the value of its virtual row in the index. + /// Loads only those parts' primary indexes; returns false if part filters are unavailable. + bool everyPartHasVirtualRowBound(bool read_in_reverse) const; bool readsInOrder() const; const InputOrderInfoPtr & getInputOrder() const { return query_info.input_order_info; } const SortDescription & getSortDescription() const override { return result_sort_description; } @@ -617,6 +621,12 @@ class ReadFromMergeTree final : public SourceStepWithFilter /// a single pool reads the whole table (no splitting). std::optional split_index = std::nullopt); + /// Whether the `FINAL` merges of this read can be followed by a virtual row (see `spreadMarkRangesAmongStreamsFinal`). + bool canUseVirtualRowWithFinal() const; + + /// The columns of a virtual row: the leading primary key columns the conversions read. + Block makeVirtualRowHeader() const; + Pipe spreadMarkRanges( RangesInDataParts && parts_with_ranges, const MergeTreeIndexBuildContextPtr & index_build_context, diff --git a/tests/performance/final_read_in_order_limit.xml b/tests/performance/final_read_in_order_limit.xml index e543444482307..7d96fb223413c 100644 --- a/tests/performance/final_read_in_order_limit.xml +++ b/tests/performance/final_read_in_order_limit.xml @@ -9,7 +9,13 @@ no filtering) skips the split. The others are controls that must keep the layer split: with a filter the query consumes limit / selectivity rows, which is unknown at planning time (for the rare-value filter it is the - whole table), and the last query consumes most of the ordered stream. + whole table), and the `LIMIT 4000000` query consumes most of the ordered + stream. + + With `read_in_order_use_virtual_row` each layer announces the key it starts + from, so the merge after the sort defers the layers the limit never reaches. + The reverse read with a mild filter is answered by the first layer alone; + the same query with the setting off is a control that must not move. --> CREATE TABLE final_inorder_limit (k UInt64, v UInt64, payload String) @@ -25,10 +31,12 @@ INSERT INTO final_inorder_limit SELECT number, cityHash64(number, 2) % 1000, toString(cityHash64(number, 102)) FROM numbers_mt(4000000) INSERT INTO final_inorder_limit SELECT number, cityHash64(number, 3) % 1000, toString(cityHash64(number, 103)) FROM numbers_mt(4000000) + 0 + SELECT k, payload FROM final_inorder_limit FINAL ORDER BY k LIMIT 100 FORMAT Null SELECT k, payload FROM final_inorder_limit FINAL WHERE v = 7 ORDER BY k LIMIT 100 FORMAT Null SELECT k, payload FROM final_inorder_limit FINAL WHERE v = 1000 ORDER BY k LIMIT 100 FORMAT Null SELECT k, payload FROM final_inorder_limit FINAL ORDER BY k LIMIT 4000000 FORMAT Null - + SELECT k, payload FROM final_inorder_limit FINAL WHERE v < 500 ORDER BY k DESC LIMIT 100 FORMAT Null DROP TABLE IF EXISTS final_inorder_limit diff --git a/tests/queries/0_stateless/03928_deferred_filters_after_final_in_explain.reference b/tests/queries/0_stateless/03928_deferred_filters_after_final_in_explain.reference index 297db7472339e..f7a363d0eab41 100644 --- a/tests/queries/0_stateless/03928_deferred_filters_after_final_in_explain.reference +++ b/tests/queries/0_stateless/03928_deferred_filters_after_final_in_explain.reference @@ -40,6 +40,10 @@ Positions: 3 0 1 Deferred filters (applied after FINAL) Deferred row level filter column: notEquals(y, \'ccc\'_String) Deferred prewhere filter column: notEquals(y, \'ccc\'_String) + Virtual row conversions + Actions: INPUT : 0 -> x UInt32 : 0 + ALIAS x :: 0 -> __table1.x UInt32 : 1 + Positions: 1 = full plan: nothing deferred = Expression (Project names) Actions: INPUT : 0 -> __table1.x UInt32 : 0 @@ -79,6 +83,10 @@ Positions: 3 0 1 COLUMN Const(String) -> \'ccc\'_String String : 1 FUNCTION notEquals(y : 0, \'ccc\'_String :: 1) -> notEquals(y, \'ccc\'_String) UInt8 : 2 Positions: 2 0 + Virtual row conversions + Actions: INPUT : 0 -> x UInt32 : 0 + ALIAS x :: 0 -> __table1.x UInt32 : 1 + Positions: 1 = row policy deferred = Deferred filters (applied after FINAL) Deferred row level filter column: notEquals(y, \'ccc\'_String) diff --git a/tests/queries/0_stateless/03928_deferred_filters_after_final_in_explain.sql b/tests/queries/0_stateless/03928_deferred_filters_after_final_in_explain.sql index 8b4b08f97b66d..352ad8ef5f8b3 100644 --- a/tests/queries/0_stateless/03928_deferred_filters_after_final_in_explain.sql +++ b/tests/queries/0_stateless/03928_deferred_filters_after_final_in_explain.sql @@ -14,6 +14,7 @@ INSERT INTO tab SELECT 1, 'ccc', 2; CREATE ROW POLICY pol1 ON tab USING y != 'ccc' TO ALL; SET enable_analyzer = 1; +SET read_in_order_use_virtual_row = 1; SELECT '= full plan: both deferred ='; EXPLAIN actions=1 SELECT * FROM tab FINAL PREWHERE y != 'ccc' ORDER BY x diff --git a/tests/queries/0_stateless/04489_virtual_row_disabled_with_final.reference b/tests/queries/0_stateless/04489_virtual_row_disabled_with_final.reference deleted file mode 100644 index 3aa5aa91e7cca..0000000000000 --- a/tests/queries/0_stateless/04489_virtual_row_disabled_with_final.reference +++ /dev/null @@ -1,7 +0,0 @@ -8192 16384 -0 2 -1 2 -2 2 -16384 -1 -1 diff --git a/tests/queries/0_stateless/04489_virtual_row_disabled_with_final.sql b/tests/queries/0_stateless/04489_virtual_row_disabled_with_final.sql deleted file mode 100644 index 31d2673b9f179..0000000000000 --- a/tests/queries/0_stateless/04489_virtual_row_disabled_with_final.sql +++ /dev/null @@ -1,46 +0,0 @@ --- Tags: no-random-merge-tree-settings - --- SELECT ... FINAL must not lose rows when the read-in-order virtual row is enabled. --- ORDER BY with a LIMIT is required: without it the read-in-order plan is not built at all. - -DROP TABLE IF EXISTS t_vrow_select_final; - -CREATE TABLE t_vrow_select_final (x UInt64, y UInt64) -ENGINE = ReplacingMergeTree(y) -ORDER BY x; - -SYSTEM STOP MERGES t_vrow_select_final; - -INSERT INTO t_vrow_select_final SELECT number, 1 FROM numbers(8192); -INSERT INTO t_vrow_select_final SELECT number, 2 FROM numbers(8192); - --- Every SETTINGS clause below sits on the outermost statement: one attached to an inner subquery --- loses to the same setting arriving as a client option. -SELECT count(), sum(y) -FROM (SELECT x, y FROM t_vrow_select_final FINAL ORDER BY x LIMIT 20000) -SETTINGS optimize_read_in_order = 1, read_in_order_use_virtual_row = 1, enable_parallel_replicas = 0; - -SELECT x, y FROM t_vrow_select_final FINAL ORDER BY x LIMIT 3 -SETTINGS optimize_read_in_order = 1, read_in_order_use_virtual_row = 1, enable_parallel_replicas = 0; - -SELECT count() -FROM (SELECT x FROM t_vrow_select_final ORDER BY x LIMIT 20000) -SETTINGS optimize_read_in_order = 1, read_in_order_use_virtual_row = 1, enable_parallel_replicas = 0; - --- The row counts above stay correct whenever the virtual row is never attempted, so assert as a --- pair that the in-order FINAL read carries no virtual row conversion while the same fixture --- without FINAL does build one. `ReadType:` is matched whitespace-insensitively: the label prints --- as `Read type: ` or `ReadType: ` depending on `pretty`. -SELECT countIf(replaceAll(explain, ' ', '') ILIKE '%ReadType:InOrder%') > 0 - AND countIf(explain ILIKE '%Virtual row conversions%') = 0 -FROM (EXPLAIN actions = 1 - SELECT count(), sum(y) FROM (SELECT x, y FROM t_vrow_select_final FINAL ORDER BY x LIMIT 20000) - SETTINGS optimize_read_in_order = 1, read_in_order_use_virtual_row = 1, enable_parallel_replicas = 0); - -SELECT countIf(replaceAll(explain, ' ', '') ILIKE '%ReadType:InOrder%') > 0 - AND countIf(explain ILIKE '%Virtual row conversions%') > 0 -FROM (EXPLAIN actions = 1 - SELECT count() FROM (SELECT x FROM t_vrow_select_final ORDER BY x LIMIT 20000) - SETTINGS optimize_read_in_order = 1, read_in_order_use_virtual_row = 1, enable_parallel_replicas = 0); - -DROP TABLE t_vrow_select_final; diff --git a/tests/queries/0_stateless/05325_read_in_order_final_virtual_row.reference b/tests/queries/0_stateless/05325_read_in_order_final_virtual_row.reference new file mode 100644 index 0000000000000..22f0a97e44978 --- /dev/null +++ b/tests/queries/0_stateless/05325_read_in_order_final_virtual_row.reference @@ -0,0 +1,167 @@ +--- ReplacingMergeTree: plan +-- INNER JOIN +Prefix sort description: k ASC +Read type: InOrder +Virtual row conversions +Read type: Default +-- INNER JOIN, reverse +Prefix sort description: k DESC +Read type: InReverseOrder +Virtual row conversions +Read type: Default +--- ReplacingMergeTree: results +0 p4 +3 p1 +6 p1 +9 p1 +12 p1 +15 p1 +18 p1 +198 p3 +195 p3 +192 p3 +189 p3 +186 p3 +45 p1 +48 p1 +51 p2 +54 p2 +57 p2 +60 p4 +63 p2 +48 p1 +49 p1 +50 p4 +151 p3 +150 p4 +149 p3 +--- CollapsingMergeTree +Prefix sort description: k ASC +Read type: InOrder +Virtual row conversions +Read type: Default +0 b +3 a +9 a +12 b +15 a +21 a +--- VersionedCollapsingMergeTree +Prefix sort description: k ASC +Read type: InOrder +Virtual row conversions +Read type: Default +0 b +3 a +9 a +12 b +15 a +21 a +--- SummingMergeTree +Prefix sort description: k ASC +Read type: InOrder +Virtual row conversions +Read type: Default +45 11 +48 11 +51 111 +54 111 +57 111 +60 111 +--- AggregatingMergeTree +Prefix sort description: k ASC +Read type: InOrder +Virtual row conversions +Read type: Default +45 10 +48 10 +51 10 +54 10 +57 10 +60 10 +--- CoalescingMergeTree +Prefix sort description: k ASC +Read type: InOrder +Virtual row conversions +Read type: Default +45 1 x +48 1 x +51 2 x +54 2 x +57 2 x +60 2 x +--- layers +Prefix sort description: k ASC +Read type: InOrder +Virtual row conversions +Read type: Default +-- several FINAL merges, each followed by a virtual row +1 1 1 +0 p4 +3 p1 +6 p1 +9 p1 +12 p1 +15 p1 +18 p1 +1998 p3 +1995 p3 +1992 p3 +1989 p3 +1986 p3 +735 p2 +738 p2 +741 p2 +744 p2 +747 p2 +750 p4 +1248 p3 +1249 p3 +1250 p4 +1251 p3 +1252 p3 +1253 p3 +1254 p3 +1255 p3 +738 p2 +739 p2 +740 p4 +741 p2 +742 p2 +1254 p3 +1253 p3 +1252 p3 +1251 p3 +1250 p4 +--- ORDER BY a prefix of the primary key, which is a prefix of the sorting key +-- ORDER BY a +Prefix sort description: a ASC +Read type: InOrder +Virtual row conversions +Read type: Default +0 15 2 3 -1 1 +3 15 2 0 -1 1 +-- ORDER BY a, b +Prefix sort description: a ASC, b ASC +Read type: InOrder +Virtual row conversions +Read type: Default +0 0 3 0 3 -1 1 +0 1 3 0 0 -1 1 +0 2 3 1 0 -1 1 +-- ORDER BY a, b, c: beyond the primary key, no virtual row +Read type: Default +Read type: Default +0 0 -1 p3 +0 0 0 p3 +0 0 1 p3 +0 1 -1 p1 +0 1 0 p1 +0 1 1 p1 +0 2 -1 p1 +0 2 0 p2 +0 2 1 p1 +0 3 -1 p1 +--- GraphiteMergeTree: no virtual row +Prefix sort description: Path ASC, Time ASC +Read type: InOrder diff --git a/tests/queries/0_stateless/05325_read_in_order_final_virtual_row.sql b/tests/queries/0_stateless/05325_read_in_order_final_virtual_row.sql new file mode 100644 index 0000000000000..04a82041ed5bf --- /dev/null +++ b/tests/queries/0_stateless/05325_read_in_order_final_virtual_row.sql @@ -0,0 +1,171 @@ +-- Tags: no-parallel-replicas + +-- Virtual rows with FINAL: each FINAL merge announces the key its parts start from, so the read in +-- order through an INNER JOIN, which requires a virtual row, applies to FINAL as well, and the merge +-- after the join defers the FINAL merges the limit does not reach. + +SET optimize_read_in_order = 1, read_in_order_use_virtual_row = 1, read_in_order_use_virtual_row_per_block = 0; +SET query_plan_read_in_order_through_join = 1, max_bytes_ratio_before_external_join = 0, max_bytes_before_external_join = 0; +SET query_plan_optimize_join_order_limit = 1, query_plan_optimize_join_order_randomize = 0, query_plan_join_swap_table = 0; +SET join_algorithm = 'hash', enable_join_runtime_filters = 0, enable_parallel_replicas = 0; +SET max_threads = 3, max_final_threads = 3, optimize_on_insert = 0, optimize_read_in_reverse_order_final = 1; +SET do_not_merge_across_partitions_select_final = 0; + +DROP TABLE IF EXISTS r; +CREATE TABLE r (k UInt64) ENGINE = MergeTree ORDER BY k; +INSERT INTO r SELECT number * 3 FROM numbers(700); + +SELECT '--- ReplacingMergeTree: plan'; +DROP TABLE IF EXISTS t_rep; +CREATE TABLE t_rep (k UInt64, ver UInt8, v String) ENGINE = ReplacingMergeTree(ver) ORDER BY k; +INSERT INTO t_rep SELECT number, 1, 'p1' FROM numbers(100); +INSERT INTO t_rep SELECT 50 + number, 2, 'p2' FROM numbers(100); +INSERT INTO t_rep SELECT 100 + number, 3, 'p3' FROM numbers(100); +INSERT INTO t_rep SELECT number * 10, 4, 'p4' FROM numbers(20); + +SELECT '-- INNER JOIN'; +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 + SELECT l.k, l.v FROM t_rep AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k LIMIT 7) WHERE e != ''; + +SELECT '-- INNER JOIN, reverse'; +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 + SELECT l.k, l.v FROM t_rep AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k DESC LIMIT 5) WHERE e != ''; + +SELECT '--- ReplacingMergeTree: results'; +SELECT l.k, l.v FROM t_rep AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k LIMIT 7; +SELECT l.k, l.v FROM t_rep AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k DESC LIMIT 5; +SELECT l.k, l.v FROM t_rep AS l FINAL INNER JOIN r ON l.k = r.k WHERE l.k >= 45 ORDER BY l.k LIMIT 7; +SELECT k, v FROM t_rep FINAL ORDER BY k LIMIT 3 OFFSET 48; +SELECT k, v FROM t_rep FINAL ORDER BY k DESC LIMIT 3 OFFSET 48; + +SELECT '--- CollapsingMergeTree'; +DROP TABLE IF EXISTS t_col; +CREATE TABLE t_col (k UInt64, sign Int8, v String) ENGINE = CollapsingMergeTree(sign) ORDER BY k; +INSERT INTO t_col SELECT number, 1, 'a' FROM numbers(100); +INSERT INTO t_col SELECT number * 2, -1, 'a' FROM numbers(50); +INSERT INTO t_col SELECT number * 4, 1, 'b' FROM numbers(25); +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 + SELECT l.k, l.v FROM t_col AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k LIMIT 6) WHERE e != ''; +SELECT l.k, l.v FROM t_col AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k LIMIT 6; + +SELECT '--- VersionedCollapsingMergeTree'; +DROP TABLE IF EXISTS t_vcol; +CREATE TABLE t_vcol (k UInt64, sign Int8, ver UInt8, v String) ENGINE = VersionedCollapsingMergeTree(sign, ver) ORDER BY k; +INSERT INTO t_vcol SELECT number, 1, 1, 'a' FROM numbers(100); +INSERT INTO t_vcol SELECT number * 2, -1, 1, 'a' FROM numbers(50); +INSERT INTO t_vcol SELECT number * 4, 1, 2, 'b' FROM numbers(25); +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 + SELECT l.k, l.v FROM t_vcol AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k LIMIT 6) WHERE e != ''; +SELECT l.k, l.v FROM t_vcol AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k LIMIT 6; + +SELECT '--- SummingMergeTree'; +DROP TABLE IF EXISTS t_sum; +CREATE TABLE t_sum (k UInt64, s UInt64) ENGINE = SummingMergeTree ORDER BY k; +INSERT INTO t_sum SELECT number, 1 FROM numbers(100); +INSERT INTO t_sum SELECT number, 10 FROM numbers(100); +INSERT INTO t_sum SELECT 50 + number, 100 FROM numbers(100); +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 + SELECT l.k, l.s FROM t_sum AS l FINAL INNER JOIN r ON l.k = r.k WHERE l.k >= 45 ORDER BY l.k LIMIT 6) WHERE e != ''; +SELECT l.k, l.s FROM t_sum AS l FINAL INNER JOIN r ON l.k = r.k WHERE l.k >= 45 ORDER BY l.k LIMIT 6; + +SELECT '--- AggregatingMergeTree'; +DROP TABLE IF EXISTS t_agg; +CREATE TABLE t_agg (k UInt64, m SimpleAggregateFunction(max, UInt64), c AggregateFunction(count)) ENGINE = AggregatingMergeTree ORDER BY k; +INSERT INTO t_agg SELECT number, 1, countState() FROM numbers(100) GROUP BY number; +INSERT INTO t_agg SELECT number, 10, countState() FROM numbers(100) GROUP BY number; +INSERT INTO t_agg SELECT 50 + number, 5, countState() FROM numbers(100) GROUP BY number; +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 + SELECT l.k, l.m FROM t_agg AS l FINAL INNER JOIN r ON l.k = r.k WHERE l.k >= 45 ORDER BY l.k LIMIT 6) WHERE e != ''; +SELECT l.k, l.m FROM t_agg AS l FINAL INNER JOIN r ON l.k = r.k WHERE l.k >= 45 ORDER BY l.k LIMIT 6; + +SELECT '--- CoalescingMergeTree'; +DROP TABLE IF EXISTS t_coal; +CREATE TABLE t_coal (k UInt64, a Nullable(UInt64), b Nullable(String)) ENGINE = CoalescingMergeTree ORDER BY k; +INSERT INTO t_coal SELECT number, 1, NULL FROM numbers(100); +INSERT INTO t_coal SELECT number, NULL, 'x' FROM numbers(100); +INSERT INTO t_coal SELECT 50 + number, 2, NULL FROM numbers(100); +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 + SELECT l.k, l.a, l.b FROM t_coal AS l FINAL INNER JOIN r ON l.k = r.k WHERE l.k >= 45 ORDER BY l.k LIMIT 6) WHERE e != ''; +SELECT l.k, l.a, l.b FROM t_coal AS l FINAL INNER JOIN r ON l.k = r.k WHERE l.k >= 45 ORDER BY l.k LIMIT 6; + +-- Parts of several granules are split into PK-range layers, each merged by its own FINAL merge +-- and trimmed to its range. Each layer announces the key its parts start from, so the merge after +-- the join defers the layers the limit does not reach. +SELECT '--- layers'; +DROP TABLE IF EXISTS t_layers; +CREATE TABLE t_layers (k UInt64, ver UInt8, v String) ENGINE = ReplacingMergeTree(ver) ORDER BY k SETTINGS index_granularity = 8; +INSERT INTO t_layers SELECT number, 1, 'p1' FROM numbers(1000); +INSERT INTO t_layers SELECT 500 + number, 2, 'p2' FROM numbers(1000); +INSERT INTO t_layers SELECT 1000 + number, 3, 'p3' FROM numbers(1000); +INSERT INTO t_layers SELECT number * 10, 4, 'p4' FROM numbers(200); +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 + SELECT l.k, l.v FROM t_layers AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k LIMIT 7) WHERE e != ''; +SELECT '-- several FINAL merges, each followed by a virtual row'; +SELECT countIf(explain LIKE '%filter values in%') > 1, countIf(explain LIKE '%ReplacingSorted%') > 0, countIf(explain LIKE '%VirtualRowTransform%') > 0 +FROM (EXPLAIN PIPELINE SELECT l.k, l.v FROM t_layers AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k LIMIT 7); +-- the limit is reached within the first layer: far fewer rows read than the table has +SELECT l.k, l.v FROM t_layers AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k LIMIT 7 SETTINGS max_rows_to_read = 3200; +SELECT l.k, l.v FROM t_layers AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k DESC LIMIT 5 SETTINGS max_rows_to_read = 3200; +-- limits crossing a layer border (740 and 1252 for these parts with max_final_threads = 3) +SELECT l.k, l.v FROM t_layers AS l FINAL INNER JOIN r ON l.k = r.k WHERE l.k >= 735 ORDER BY l.k LIMIT 6; +SELECT l.k, l.v FROM t_layers AS l FINAL LEFT JOIN r ON l.k = r.k WHERE l.k >= 1248 ORDER BY l.k LIMIT 8; +SELECT k, v FROM t_layers FINAL ORDER BY k LIMIT 5 OFFSET 738; +SELECT k, v FROM t_layers FINAL ORDER BY k DESC LIMIT 5 OFFSET 745; + +-- The query orders by a prefix of the primary key, itself a prefix of the sorting key. The virtual +-- row covers the primary key columns the sort reads; the FINAL merge compares the whole sorting key +-- and a sort beyond the primary key gets no virtual row, so the INNER JOIN keeps no order for it. +SELECT '--- ORDER BY a prefix of the primary key, which is a prefix of the sorting key'; +DROP TABLE IF EXISTS t_prefix; +CREATE TABLE t_prefix (a UInt64, b UInt64, c Int32, ver UInt8, v String) + ENGINE = ReplacingMergeTree(ver) ORDER BY (a, b, c) PRIMARY KEY (a, b) SETTINGS index_granularity = 4; +INSERT INTO t_prefix SELECT number % 10, intDiv(number, 10) % 5, toInt32(intDiv(number, 50)) - 1, 1, 'p1' FROM numbers(150); +INSERT INTO t_prefix SELECT number % 10, intDiv(number, 10) % 5, toInt32(intDiv(number, 50)) - 1, 2, 'p2' FROM numbers(150) WHERE number % 7 = 0; +INSERT INTO t_prefix SELECT number % 10, intDiv(number, 10) % 5, toInt32(intDiv(number, 50)) - 1, 3, 'p3' FROM numbers(150) WHERE number % 50 = 0; +SELECT '-- ORDER BY a'; +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 + SELECT l.a FROM t_prefix AS l FINAL INNER JOIN r ON l.a = r.k ORDER BY l.a LIMIT 30) WHERE e != ''; +SELECT a, count(), countIf(v = 'p2'), countIf(v = 'p3'), min(c), max(c) +FROM (SELECT l.a, l.b, l.c, l.v FROM t_prefix AS l FINAL INNER JOIN r ON l.a = r.k ORDER BY l.a LIMIT 30) +GROUP BY a ORDER BY a; +SELECT '-- ORDER BY a, b'; +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 + SELECT l.a, l.b FROM t_prefix AS l FINAL INNER JOIN r ON l.a = r.k ORDER BY l.a, l.b LIMIT 9) WHERE e != ''; +SELECT a, b, count(), countIf(v = 'p2'), countIf(v = 'p3'), min(c), max(c) +FROM (SELECT l.a, l.b, l.c, l.v FROM t_prefix AS l FINAL INNER JOIN r ON l.a = r.k ORDER BY l.a, l.b LIMIT 9) +GROUP BY a, b ORDER BY a, b; +SELECT '-- ORDER BY a, b, c: beyond the primary key, no virtual row'; +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 + SELECT l.a, l.b, l.c FROM t_prefix AS l FINAL INNER JOIN r ON l.a = r.k ORDER BY l.a, l.b, l.c LIMIT 10) WHERE e != ''; +SELECT l.a, l.b, l.c, l.v FROM t_prefix AS l FINAL INNER JOIN r ON l.a = r.k ORDER BY l.a, l.b, l.c LIMIT 10; + +SELECT '--- GraphiteMergeTree: no virtual row'; +DROP TABLE IF EXISTS t_graphite; +CREATE TABLE t_graphite (Path String, Time DateTime, Value Float64, Version UInt32, k UInt64 ALIAS Version) + ENGINE = GraphiteMergeTree('graphite_rollup') ORDER BY (Path, Time); +INSERT INTO t_graphite SELECT 'one_min.x', toDateTime('2020-01-01 00:00:00') + number, number, 1 FROM numbers(10); +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 + SELECT Path, Time FROM t_graphite FINAL ORDER BY Path, Time LIMIT 3) WHERE e != ''; + +DROP TABLE t_rep; +DROP TABLE t_col; +DROP TABLE t_vcol; +DROP TABLE t_sum; +DROP TABLE t_agg; +DROP TABLE t_coal; +DROP TABLE t_layers; +DROP TABLE t_prefix; +DROP TABLE t_graphite; +DROP TABLE r; diff --git a/tests/queries/0_stateless/05407_bool_force_setting_shorthand.reference b/tests/queries/0_stateless/05407_bool_force_setting_shorthand.reference new file mode 100644 index 0000000000000..98fb6a6865639 --- /dev/null +++ b/tests/queries/0_stateless/05407_bool_force_setting_shorthand.reference @@ -0,0 +1,4 @@ +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/05407_bool_force_setting_shorthand.sh b/tests/queries/0_stateless/05407_bool_force_setting_shorthand.sh new file mode 100755 index 0000000000000..a7f3ea6a2171b --- /dev/null +++ b/tests/queries/0_stateless/05407_bool_force_setting_shorthand.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -e + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `SET` shorthand changes both `false` and `force` to `true`. +$CLICKHOUSE_CLIENT --multiquery --query " + SET query_plan_read_in_order_through_join = 0; + SET query_plan_read_in_order_through_join; + SELECT value FROM system.settings WHERE name = 'query_plan_read_in_order_through_join'; + SET query_plan_read_in_order_through_join = 'force'; + SET query_plan_read_in_order_through_join; + SELECT value FROM system.settings WHERE name = 'query_plan_read_in_order_through_join'; +" + +# Query-level `SETTINGS` shorthand overrides a disabled session setting. +$CLICKHOUSE_CLIENT --query_plan_read_in_order_through_join=0 --query " + SELECT value FROM system.settings WHERE name = 'query_plan_read_in_order_through_join' + SETTINGS query_plan_read_in_order_through_join +" + +# A client option without a value means `true`. +$CLICKHOUSE_CLIENT --query_plan_read_in_order_through_join --query " + SELECT value FROM system.settings WHERE name = 'query_plan_read_in_order_through_join' +" From 8a43b19e13ae09103d164ef1af17aeea87821026 Mon Sep 17 00:00:00 2001 From: vdimir Date: Mon, 2 Nov 2026 18:52:42 +0000 Subject: [PATCH 068/219] Fix the backport of #124468 to 26.8 after conflict resolution - `Settings.cpp`: drop the old `Bool` declaration of `query_plan_read_in_order_through_join` left by the conflict resolution and terminate the new `BoolForce` one - `optimizeTree.cpp`: convert `read_in_order_through_join` at the second-pass call site too - `ReadFromMergeTree`: add `getPartsForPrewhere` and `MergeTreeDataSelectExecutor::filterParts`, which master has and 26.8 does not - `SettingsChangesHistory.cpp`: record the new `force` value in the 26.8 section - `ReadFromMergeTree.cpp`: remove a whitespace-only line in a lambda capture Build failure: https://s3.amazonaws.com/clickhouse-test-reports/json.html?PR=128683&sha=d096f2332cff46ae424b5df8a698eb42c356c437&name_0=BackportPR&name_1=Build%20%28amd_debug%29 PR: https://github.com/ClickHouse/ClickHouse/pull/128683 Co-Authored-By: Claude Fable 5.1 --- src/Core/Settings.cpp | 3 +-- src/Core/SettingsChangesHistory.cpp | 1 + .../QueryPlan/Optimizations/optimizeTree.cpp | 2 +- .../QueryPlan/ReadFromMergeTree.cpp | 19 ++++++++++++++++--- src/Processors/QueryPlan/ReadFromMergeTree.h | 2 ++ .../MergeTree/MergeTreeDataSelectExecutor.cpp | 18 ++++++++++++++++++ .../MergeTree/MergeTreeDataSelectExecutor.h | 13 +++++++++++++ 7 files changed, 52 insertions(+), 6 deletions(-) diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 11b179fd82655..a80163cb14b0d 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -6847,7 +6847,6 @@ Possible values: - 0 - Disable - 1 - Enable )", 0) \ - DECLARE(Bool, query_plan_read_in_order_through_join, true, "Keep reading in order from the left table in JOIN operations, which can be utilized by subsequent steps.", 0) \ DECLARE(Bool, query_plan_aggregation_in_order, true, R"( Toggles the aggregation in-order query-plan-level optimization. Only takes effect if setting [`query_plan_enable_optimizations`](#query_plan_enable_optimizations) is 1. @@ -6865,7 +6864,7 @@ Possible values: Keep reading in order from the left table in JOIN operations, which can be utilized by subsequent steps. For a join other than `LEFT`, the read stays in order only when every part of the left table announces a virtual row (see [`read_in_order_use_virtual_row`](#read_in_order_use_virtual_row)); without the virtual rows the merge after the join may read far more than the limit needs. The value `force` keeps the read in order regardless. -)", 0, \ +)", 0) \ DECLARE(Bool, query_plan_remove_redundant_sorting, true, R"( Toggles a query-plan-level optimization which removes redundant sorting steps, e.g. in subqueries. Only takes effect if setting [`query_plan_enable_optimizations`](#query_plan_enable_optimizations) is 1. diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 997ba249e3bcf..4a0d5fe98e44c 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -172,6 +172,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"output_format_arrow_record_batch_size", 0, 0, "New setting to combine small blocks in `Arrow` and `ArrowStream` output using a target row count. The default `0` preserves one record batch per block."}, {"output_format_arrow_record_batch_size_bytes", 0, 0, "New setting to combine small blocks in `Arrow` and `ArrowStream` output using a target size in bytes of accumulated data. The default `0` preserves one record batch per block."}, {"query_plan_filter_push_down_below_limit_by", false, true, "New setting to control pushing a filter on the `LIMIT BY` key columns below the `LIMIT BY` step. Set it to false to keep the filter above the `LIMIT BY`."}, + {"query_plan_read_in_order_through_join", true, true, "Added the value `force`: keep reading in order through a join even when the parts announce no virtual rows."}, }); addSettingsChanges(settings_changes_history, "26.7", { diff --git a/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp b/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp index 7b572302a599a..0d3d9a4cca666 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp @@ -223,7 +223,7 @@ void optimizeTreeSecondPass( optimization_settings.max_limit_for_top_k_optimization, optimization_settings.use_skip_indexes_on_data_read, optimization_settings.read_in_order, - optimization_settings.read_in_order_through_join, + optimization_settings.read_in_order_through_join != 0, optimization_settings.join_swap_table, optimization_settings.enable_group_by_top_k_optimization, optimization_settings.top_k_optimization_observation_rows, diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 48c294aff4acc..8e9295562d9e4 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -2343,8 +2343,8 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( chassert(num_streams == requested_num_streams); num_streams = std::min(num_streams, settings[Setting::max_final_threads]); - /// Always false on this branch (`requestReadingInOrder` rejects reverse order with FINAL), - /// kept as a runtime value so the virtual-row helpers get the real flag. + /// Always false (`requestReadingInOrder` rejects reverse order with FINAL), kept as a value + /// so the virtual-row helpers take the real flag. const bool read_in_reverse = reader_settings.read_in_order && query_info.input_order_info && query_info.input_order_info->direction < 0; /// A virtual row after each `FINAL` merge announces the key its layer starts from, so the merge downstream @@ -2495,7 +2495,6 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( auto in_order_reading_step_getter = [this, &index_build_context, &column_names, &info, read_in_reverse, use_virtual_row, &virtual_row_header, &reverse_flags, &merging_pipes_virtual_rows](auto parts) - { if (use_virtual_row) merging_pipes_virtual_rows.push_back(getFinalMergeVirtualRow(parts, virtual_row_header, reverse_flags, read_in_reverse)); @@ -3964,6 +3963,20 @@ bool ReadFromMergeTree::canUseVirtualRowWithFinal() const return true; } +RangesInDataParts ReadFromMergeTree::getPartsForPrewhere() const +{ + if (analyzed_result_ptr || !indexes) + return getParts(); + + /// Share all part filters with `selectRangesToRead`, including the snapshot boundary and statistics. + /// Keep this snapshot temporary: `PREWHERE` optimization can still change filters, + /// so execution must filter again with the final conditions. + IndexStats unused_stats; + return MergeTreeDataSelectExecutor::filterParts( + getParts(), *indexes, getStorageMetadata(), data, query_info, mutations_snapshot, getContext(), + max_block_numbers_to_read.get(), log, unused_stats); +} + bool ReadFromMergeTree::everyPartHasVirtualRowBound(bool read_in_reverse) const { /// Without part filters or an existing analysis, checking bounds would load indexes of unrelated parts. diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index 32c776dc8e844..2c6ad2410aaea 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -439,6 +439,8 @@ class ReadFromMergeTree final : public SourceStepWithFilter void createReadTasksForTextIndex(const UsefulSkipIndexes & skip_indexes, const IndexReadColumns & added_columns, const Names & removed_columns, bool is_final); const std::optional & getIndexes() const { return indexes; } + /// A temporary part snapshot for PREWHERE costs; does not publish range analysis. + RangesInDataParts getPartsForPrewhere() const; ConditionSelectivityEstimatorPtr getConditionSelectivityEstimator(const Names & required_columns) const; /// Compose statistics over the part set of the given partition/PK analysis result /// instead of all prepared parts. Passing nullptr falls back to getParts(). diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index ff799d6aeabdc..f7b12e9f78026 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -691,6 +691,24 @@ std::optional> MergeTreeDataSelectExecutor::filterPar return result; } +RangesInDataParts MergeTreeDataSelectExecutor::filterParts( + const RangesInDataParts & parts, + const ReadFromMergeTree::Indexes & indexes, + const StorageMetadataPtr & metadata_snapshot, + const MergeTreeData & data, + const SelectQueryInfo & query_info, + const MergeTreeData::MutationsSnapshotPtr & mutations_snapshot, + const ContextPtr & context, + const PartitionIdToMaxBlock * max_block_numbers_to_read, + LoggerPtr log, + ReadFromMergeTree::IndexStats & index_stats) +{ + auto res = filterPartsByPartition( + parts, indexes.partition_pruner, indexes.minmax_idx_condition, indexes.part_values, + metadata_snapshot, data, context, max_block_numbers_to_read, log, index_stats); + return filterPartsByStatistics(res, metadata_snapshot, query_info, mutations_snapshot, context, log, index_stats); +} + RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPartition( const RangesInDataParts & parts, const std::optional & partition_pruner, diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index 5c169c583900f..c925b34dbdaaf 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -202,6 +202,19 @@ class MergeTreeDataSelectExecutor const ActionsDAG::Node * predicate, ContextPtr context); + /// Apply snapshot, virtual-column, min-max, partition and statistics filters before range analysis. + static RangesInDataParts filterParts( + const RangesInDataParts & parts, + const ReadFromMergeTree::Indexes & indexes, + const StorageMetadataPtr & metadata_snapshot, + const MergeTreeData & data, + const SelectQueryInfo & query_info, + const MergeTreeData::MutationsSnapshotPtr & mutations_snapshot, + const ContextPtr & context, + const PartitionIdToMaxBlock * max_block_numbers_to_read, + LoggerPtr log, + ReadFromMergeTree::IndexStats & index_stats); + /// Filter parts using minmax index and partition key. static RangesInDataParts filterPartsByPartition( const RangesInDataParts & parts, From 40118159746d6e0cc7df543e34da053a6a4ac153 Mon Sep 17 00:00:00 2001 From: vdimir Date: Tue, 3 Nov 2026 10:27:43 +0000 Subject: [PATCH 069/219] fix test --- ..._read_in_order_final_virtual_row.reference | 23 ------------------- .../05325_read_in_order_final_virtual_row.sql | 11 +-------- 2 files changed, 1 insertion(+), 33 deletions(-) diff --git a/tests/queries/0_stateless/05325_read_in_order_final_virtual_row.reference b/tests/queries/0_stateless/05325_read_in_order_final_virtual_row.reference index 22f0a97e44978..61aa4150afa3d 100644 --- a/tests/queries/0_stateless/05325_read_in_order_final_virtual_row.reference +++ b/tests/queries/0_stateless/05325_read_in_order_final_virtual_row.reference @@ -4,11 +4,6 @@ Prefix sort description: k ASC Read type: InOrder Virtual row conversions Read type: Default --- INNER JOIN, reverse -Prefix sort description: k DESC -Read type: InReverseOrder -Virtual row conversions -Read type: Default --- ReplacingMergeTree: results 0 p4 3 p1 @@ -17,11 +12,6 @@ Read type: Default 12 p1 15 p1 18 p1 -198 p3 -195 p3 -192 p3 -189 p3 -186 p3 45 p1 48 p1 51 p2 @@ -32,9 +22,6 @@ Read type: Default 48 p1 49 p1 50 p4 -151 p3 -150 p4 -149 p3 --- CollapsingMergeTree Prefix sort description: k ASC Read type: InOrder @@ -104,11 +91,6 @@ Read type: Default 12 p1 15 p1 18 p1 -1998 p3 -1995 p3 -1992 p3 -1989 p3 -1986 p3 735 p2 738 p2 741 p2 @@ -128,11 +110,6 @@ Read type: Default 740 p4 741 p2 742 p2 -1254 p3 -1253 p3 -1252 p3 -1251 p3 -1250 p4 --- ORDER BY a prefix of the primary key, which is a prefix of the sorting key -- ORDER BY a Prefix sort description: a ASC diff --git a/tests/queries/0_stateless/05325_read_in_order_final_virtual_row.sql b/tests/queries/0_stateless/05325_read_in_order_final_virtual_row.sql index 04a82041ed5bf..41b6bc3e8f130 100644 --- a/tests/queries/0_stateless/05325_read_in_order_final_virtual_row.sql +++ b/tests/queries/0_stateless/05325_read_in_order_final_virtual_row.sql @@ -8,7 +8,7 @@ SET optimize_read_in_order = 1, read_in_order_use_virtual_row = 1, read_in_order SET query_plan_read_in_order_through_join = 1, max_bytes_ratio_before_external_join = 0, max_bytes_before_external_join = 0; SET query_plan_optimize_join_order_limit = 1, query_plan_optimize_join_order_randomize = 0, query_plan_join_swap_table = 0; SET join_algorithm = 'hash', enable_join_runtime_filters = 0, enable_parallel_replicas = 0; -SET max_threads = 3, max_final_threads = 3, optimize_on_insert = 0, optimize_read_in_reverse_order_final = 1; +SET max_threads = 3, max_final_threads = 3, optimize_on_insert = 0; SET do_not_merge_across_partitions_select_final = 0; DROP TABLE IF EXISTS r; @@ -28,17 +28,10 @@ SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conver FROM (EXPLAIN PLAN actions = 1, indexes = 0 SELECT l.k, l.v FROM t_rep AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k LIMIT 7) WHERE e != ''; -SELECT '-- INNER JOIN, reverse'; -SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e -FROM (EXPLAIN PLAN actions = 1, indexes = 0 - SELECT l.k, l.v FROM t_rep AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k DESC LIMIT 5) WHERE e != ''; - SELECT '--- ReplacingMergeTree: results'; SELECT l.k, l.v FROM t_rep AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k LIMIT 7; -SELECT l.k, l.v FROM t_rep AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k DESC LIMIT 5; SELECT l.k, l.v FROM t_rep AS l FINAL INNER JOIN r ON l.k = r.k WHERE l.k >= 45 ORDER BY l.k LIMIT 7; SELECT k, v FROM t_rep FINAL ORDER BY k LIMIT 3 OFFSET 48; -SELECT k, v FROM t_rep FINAL ORDER BY k DESC LIMIT 3 OFFSET 48; SELECT '--- CollapsingMergeTree'; DROP TABLE IF EXISTS t_col; @@ -113,12 +106,10 @@ SELECT countIf(explain LIKE '%filter values in%') > 1, countIf(explain LIKE '%Re FROM (EXPLAIN PIPELINE SELECT l.k, l.v FROM t_layers AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k LIMIT 7); -- the limit is reached within the first layer: far fewer rows read than the table has SELECT l.k, l.v FROM t_layers AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k LIMIT 7 SETTINGS max_rows_to_read = 3200; -SELECT l.k, l.v FROM t_layers AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k DESC LIMIT 5 SETTINGS max_rows_to_read = 3200; -- limits crossing a layer border (740 and 1252 for these parts with max_final_threads = 3) SELECT l.k, l.v FROM t_layers AS l FINAL INNER JOIN r ON l.k = r.k WHERE l.k >= 735 ORDER BY l.k LIMIT 6; SELECT l.k, l.v FROM t_layers AS l FINAL LEFT JOIN r ON l.k = r.k WHERE l.k >= 1248 ORDER BY l.k LIMIT 8; SELECT k, v FROM t_layers FINAL ORDER BY k LIMIT 5 OFFSET 738; -SELECT k, v FROM t_layers FINAL ORDER BY k DESC LIMIT 5 OFFSET 745; -- The query orders by a prefix of the primary key, itself a prefix of the sorting key. The virtual -- row covers the primary key columns the sort reads; the FINAL merge compares the whole sorting key From 6aa8ebc4401515151d4e50a33095d522ef849a2b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 3 Nov 2026 10:33:27 +0000 Subject: [PATCH 070/219] Backport #128704 to 26.8: Fix flaky test `test_ttl_move/test.py::test_moves_after_merges` --- tests/integration/test_ttl_move/test.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/tests/integration/test_ttl_move/test.py b/tests/integration/test_ttl_move/test.py index 926079afcaf09..8c003af149e26 100644 --- a/tests/integration/test_ttl_move/test.py +++ b/tests/integration/test_ttl_move/test.py @@ -1346,12 +1346,15 @@ def test_moves_after_merges(started_cluster, engine, request): """) # we want to control over when to MOVE to eliminate flakinness node1.query(f"SYSTEM STOP MOVES {table_name}") + # SYSTEM STOP MOVES does not stop a merge from applying the TTL move, so the TTL must outlive the OPTIMIZE below. + ttl_ts = int(time.time()) + 10 + f_ttl_datetime = lambda _: f'toDateTime({ttl_ts})' populate_with_records( node=node1, table_name=table_name, generators=[ - gen_pair_records(10, F_SMALL_STRING, F_SOON_TO_EXPIRE_DATETIME), - gen_pair_records(10, F_SMALL_STRING, F_SOON_TO_EXPIRE_DATETIME), + gen_pair_records(10, F_SMALL_STRING, f_ttl_datetime), + gen_pair_records(10, F_SMALL_STRING, f_ttl_datetime), ] ) @@ -1365,7 +1368,7 @@ def test_moves_after_merges(started_cluster, engine, request): # 2. Waiting for the data to expire and then expecting it at the target location node1.query(f"SYSTEM START MOVES {table_name}") - time.sleep(SOON_SECONDS) + time.sleep(max(0, ttl_ts - time.time())) wait_if_moving_then_assert_disks(node1, table_name, target_state={ 'external' : {'rows': 20} }) From 935c169244774b71aa5d3e90e84ccc97031282d8 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 3 Nov 2026 11:24:45 +0000 Subject: [PATCH 071/219] Backport #127673 to 26.8: Throw an exception instead of aborting when an Azure endpoint omits the `ETag` header --- .../AzureBlobStorageCommon.cpp | 8 ++ .../AzureBlobStorage/AzureBlobStorageCommon.h | 4 + .../AzureBlobStorage/AzureObjectStorage.cpp | 6 +- .../IO/ReadBufferFromAzureBlobStorage.cpp | 2 +- .../gtest_azure_read_buffer_missing_etag.cpp | 106 ++++++++++++++++++ 5 files changed, 122 insertions(+), 4 deletions(-) create mode 100644 src/Disks/tests/gtest_azure_read_buffer_missing_etag.cpp diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp index 02a5af057f7a7..5729df511e008 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp @@ -88,6 +88,7 @@ namespace ServerSetting namespace ErrorCodes { + extern const int AZURE_BLOB_STORAGE_ERROR; extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; } @@ -376,6 +377,13 @@ std::unique_ptr getContainerClient(const ConnectionParams & par } } +String getETag(const Azure::ETag & etag, const String & blob_path) +{ + if (!etag.HasValue()) + throw Exception(ErrorCodes::AZURE_BLOB_STORAGE_ERROR, "Azure Blob Storage endpoint did not return an ETag for blob {}", blob_path); + return etag.ToString(); +} + AuthMethod getAuthMethod(const Poco::Util::AbstractConfiguration & config, const String & config_prefix) { if (config.has(config_prefix + ".account_key") && config.has(config_prefix + ".account_name")) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h b/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h index 79ff8caeaedb8..f4a6a896102c3 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h @@ -166,6 +166,10 @@ BlobClientOptions getClientOptions( AuthMethod getAuthMethod(const Poco::Util::AbstractConfiguration & config, const String & config_prefix); +/// `ETag` is an optional response header, and `Azure::ETag::ToString` aborts the process when the tag is absent +/// (also in release builds). Use this function instead: it throws an exception when the endpoint did not send the tag. +String getETag(const Azure::ETag & etag, const String & blob_path); + #endif Endpoint processEndpoint(const Poco::Util::AbstractConfiguration & config, const String & config_prefix); diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp index 4e37f48d50392..ecc285029a5d4 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp @@ -106,7 +106,7 @@ class AzureIteratorAsync final : public IObjectStorageIteratorAsync .last_modified = Poco::Timestamp::fromEpochTime( std::chrono::duration_cast( static_cast(blob.Details.LastModified).time_since_epoch()).count()), - .etag = blob.Details.ETag.ToString(), + .etag = AzureBlobStorage::getETag(blob.Details.ETag, blob.Name), .tags = {}, .attributes = {}, })); @@ -217,7 +217,7 @@ void AzureObjectStorage::listObjects(const std::string & path, RelativePathsWith .last_modified = Poco::Timestamp::fromEpochTime( std::chrono::duration_cast( static_cast(blob.Details.LastModified).time_since_epoch()).count()), - .etag = blob.Details.ETag.ToString(), + .etag = AzureBlobStorage::getETag(blob.Details.ETag, blob.Name), .tags = {}, .attributes = {}, })); @@ -651,7 +651,7 @@ ObjectMetadata AzureObjectStorage::getObjectMetadata(const std::string & path, b ObjectMetadata result; result.size_bytes = properties.BlobSize; - result.etag = properties.ETag.ToString(); + result.etag = AzureBlobStorage::getETag(properties.ETag, path); if (!properties.Metadata.empty()) { result.attributes.emplace(); diff --git a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp index 5724a16deef8b..68a794c9a4a47 100644 --- a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp @@ -569,7 +569,7 @@ void ReadBufferFromAzureBlobStorage::setMetadataFromResponse(const Azure::Storag { ObjectMetadata new_metadata; new_metadata.size_bytes = blob_size; - new_metadata.etag = details.ETag.ToString(); + new_metadata.etag = AzureBlobStorage::getETag(details.ETag, path); new_metadata.last_modified = static_cast(details.LastModified).time_since_epoch().count(); if (!details.Metadata.empty()) { diff --git a/src/Disks/tests/gtest_azure_read_buffer_missing_etag.cpp b/src/Disks/tests/gtest_azure_read_buffer_missing_etag.cpp new file mode 100644 index 0000000000000..161256caba5ba --- /dev/null +++ b/src/Disks/tests/gtest_azure_read_buffer_missing_etag.cpp @@ -0,0 +1,106 @@ +#include "config.h" + +#if USE_AZURE_BLOB_STORAGE + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +namespace DB::ErrorCodes +{ + extern const int AZURE_BLOB_STORAGE_ERROR; +} + +namespace +{ + +class OwningBodyStream : public Azure::Core::IO::BodyStream +{ +public: + explicit OwningBodyStream(std::vector data_) : data(std::move(data_)) { } + + int64_t Length() const override { return static_cast(data.size()); } + + void Rewind() override { position = 0; } + +private: + size_t OnRead(uint8_t * buffer, size_t count, const Azure::Core::Context &) override + { + const size_t to_read = std::min(count, data.size() - position); + if (to_read != 0) + memcpy(buffer, data.data() + position, to_read); + position += to_read; + return to_read; + } + + std::vector data; + size_t position = 0; +}; + +/// A fake Azure endpoint that serves a 10-byte blob in a response without the optional `ETag` header. +class EndpointWithoutETag : public Azure::Core::Http::HttpTransport +{ +public: + std::unique_ptr Send(Azure::Core::Http::Request &, const Azure::Core::Context &) override + { + constexpr size_t blob_size = 10; + auto response = std::make_unique( + 1, 1, Azure::Core::Http::HttpStatusCode::PartialContent, "Partial Content"); + response->SetHeader("Content-Length", std::to_string(blob_size)); + response->SetHeader("Content-Range", "bytes 0-" + std::to_string(blob_size - 1) + "/" + std::to_string(blob_size)); + response->SetHeader("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"); + response->SetHeader("x-ms-blob-type", "BlockBlob"); + response->SetBodyStream(std::make_unique(std::vector(blob_size, 'a'))); + return response; + } +}; + +} + +/// `Azure::ETag::ToString` aborts the process on an absent tag, also in release builds. +/// A download response without `ETag` must fail the read with an exception instead. +TEST(AzureReadBuffer, MissingETagIsAnException) +{ + Azure::Storage::Blobs::BlobClientOptions client_options; + client_options.Retry.MaxRetries = 0; + client_options.Transport.Transport = std::make_shared(); + + auto container_client = std::make_shared( + Azure::Storage::Blobs::BlobContainerClient("http://azure.invalid/container", client_options), /* blob_prefix */ ""); + + DB::ReadBufferFromAzureBlobStorage buffer( + container_client, + "blob", + DB::ReadSettings{}, + /* max_single_read_retries */ 1, + /* max_single_download_retries */ 1); + + std::optional error_code; + try + { + std::string data; + DB::readStringUntilEOF(data, buffer); + } + catch (const DB::Exception & e) + { + error_code = e.code(); + } + ASSERT_EQ(error_code, std::optional(DB::ErrorCodes::AZURE_BLOB_STORAGE_ERROR)); +} + +#endif From f7fe923faf2713520d89d696994808e291ca838f Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 3 Nov 2026 12:13:00 +0000 Subject: [PATCH 072/219] Backport #127180 to 26.8: Check grants and allowed hosts for the move destination of S3Queue and AzureQueue --- src/Backups/BackupMetadataFinder.cpp | 11 + src/Backups/RestorerFromBackup.cpp | 14 +- src/Interpreters/InterpreterAlterQuery.cpp | 48 +++ src/Interpreters/InterpreterCreateQuery.cpp | 32 ++ src/Interpreters/InterpreterCreateQuery.h | 4 + ...bjectStorageQueueMoveDestinationAccess.cpp | 119 ++++++ .../ObjectStorageQueueMoveDestinationAccess.h | 37 ++ .../ObjectStorageQueuePostProcessor.cpp | 22 ++ .../ObjectStorageQueueSettings.cpp | 6 +- .../StorageObjectStorageQueue.cpp | 66 +++- .../registerQueueStorage.cpp | 9 + src/Storages/StorageAlias.cpp | 22 ++ src/Storages/StorageAlias.h | 2 +- src/Storages/StorageFactory.cpp | 36 ++ src/Storages/StorageFactory.h | 15 + .../test_allowed_url_from_config/test.py | 37 ++ ..._s3queue_move_destination_access.reference | 35 ++ .../05441_s3queue_move_destination_access.sh | 374 ++++++++++++++++++ 18 files changed, 881 insertions(+), 8 deletions(-) create mode 100644 src/Storages/ObjectStorageQueue/ObjectStorageQueueMoveDestinationAccess.cpp create mode 100644 src/Storages/ObjectStorageQueue/ObjectStorageQueueMoveDestinationAccess.h create mode 100644 tests/queries/0_stateless/05441_s3queue_move_destination_access.reference create mode 100755 tests/queries/0_stateless/05441_s3queue_move_destination_access.sh diff --git a/src/Backups/BackupMetadataFinder.cpp b/src/Backups/BackupMetadataFinder.cpp index cfc3c257784de..803a89bbe2b65 100644 --- a/src/Backups/BackupMetadataFinder.cpp +++ b/src/Backups/BackupMetadataFinder.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -285,6 +286,16 @@ void BackupMetadataFinder::findTableInBackupImpl( ParserCreateQuery create_parser; ASTPtr create_table_query = parseQuery(create_parser, create_query_str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS); + + /// `BACKUP` writes the full definition, without `AS` (see `cleanupObjectDefinitionFromTemporaryFlags`). A definition that + /// inherits its engine with `AS` would let the access check of the restore see no engine, so it is rejected. + if (const auto & create = create_table_query->as(); !create.as_table.empty()) + throw Exception( + ErrorCodes::CANNOT_RESTORE_TABLE, + "The definition of {} in the backup uses AS {}, which BACKUP never writes", + tableNameWithTypeToString(table_name_in_backup.database, table_name_in_backup.table, false), + backQuoteIfNeed(create.as_table)); + applyCustomStoragePolicy(create_table_query); renameDatabaseAndTableNameInCreateQuery(create_table_query, renaming_map, context->getGlobalContext()); String create_table_query_str = create_table_query->formatWithSecretsOneLine(); diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index 49078930ef869..a047b66a4ff59 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -331,7 +332,17 @@ void RestorerFromBackup::checkAccessForObjectsFoundInBackup() const continue; } - if (table_name.database == DatabaseCatalog::TEMPORARY_DATABASE) + const ASTCreateQuery & create = table_info.create_table_query->as(); + const bool is_temporary = (table_name.database == DatabaseCatalog::TEMPORARY_DATABASE); + + /// The definition comes from the backup, and the table is created by an internal query that checks + /// nothing, so settings that send the data elsewhere are authorized here, as for `CREATE`. + /// An external engine replaced by `Null` sends nothing, but temporary tables keep their engine. + if (restore_settings.create_table != RestoreTableCreationMode::kMustExist + && (is_temporary || !context->getSettingsRef()[Setting::restore_replace_external_engines_to_null])) + required_access.append_range(StorageFactory::instance().getExportAccess(create)); + + if (is_temporary) { if (restore_settings.create_table != RestoreTableCreationMode::kMustExist) required_access.emplace_back(AccessType::CREATE_TEMPORARY_TABLE); @@ -339,7 +350,6 @@ void RestorerFromBackup::checkAccessForObjectsFoundInBackup() const } AccessFlags flags; - const ASTCreateQuery & create = table_info.create_table_query->as(); if (restore_settings.create_table != RestoreTableCreationMode::kMustExist) { diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index 016cda373bac0..d49f401deadd6 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -30,9 +31,11 @@ #include #include #include +#include #include #include #include +#include #include #include #include @@ -78,6 +81,7 @@ namespace ServerSetting namespace ErrorCodes { + extern const int ACCESS_DENIED; extern const int LOGICAL_ERROR; extern const int NOT_IMPLEMENTED; extern const int TABLE_IS_PERMANENTLY_READ_ONLY; @@ -406,6 +410,31 @@ BlockIO runCommandSegments(CommandSegments & segments, const StoragePtr & table, return res; } +/// A setting that sends the table's data elsewhere copies the data as it is stored, so nothing can filter the rows +/// that a row policy hides (as `StorageSet::checkNoRowPolicy`). It reads only the query and the access entities, +/// not the table, so it runs on the initiator. +void checkNoRowPolicyForExport(const ASTAlterQuery & alter, const String & database, const ContextPtr & context) +{ + for (const auto & child : alter.command_list->children) + { + const auto & command = child->as(); + if (command.type != ASTAlterCommand::MODIFY_SETTING || !command.settings_changes) + continue; + + const auto & changes = command.settings_changes->as().changes; + if (StorageFactory::instance().getExportAccessOfAnyEngine(changes).empty()) + continue; + + auto row_policy_filter = context->getRowPolicyFilter(database, alter.getTable(), RowPolicyFilterType::SELECT_FILTER); + if (row_policy_filter && !row_policy_filter->isAlwaysTrue()) + throw Exception( + ErrorCodes::ACCESS_DENIED, + "Cannot change the settings of table {}.{} that send its data elsewhere because a row policy applies to it. " + "The data is copied as it is stored, so its rows cannot be filtered", + backQuoteIfNeed(database), backQuoteIfNeed(alter.getTable())); + } +} + } InterpreterAlterQuery::InterpreterAlterQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) @@ -486,10 +515,15 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) DDLQueryOnClusterParams params; params.access_to_check = getRequiredAccess(table); + /// The hosts run the entry without the user, so the row policy is checked here before the entry is enqueued. + checkNoRowPolicyForExport( + alter, alter.getDatabase().empty() ? getContext()->getCurrentDatabase() : alter.getDatabase(), getContext()); + return executeDDLQueryOnCluster(query_ptr, getContext(), params); } getContext()->checkAccess(getRequiredAccess(table)); + checkNoRowPolicyForExport(alter, getContext()->resolveDatabase(alter.getDatabase()), getContext()); if (!table_id) throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database {} does not exist", backQuoteIfNeed(alter.getDatabase())); @@ -888,6 +922,20 @@ AccessRightsElements InterpreterAlterQuery::getRequiredAccessForCommand( case ASTAlterCommand::MODIFY_SETTING: { required_access.emplace_back(AccessType::ALTER_SETTINGS, database, table); + + /// A setting that sends the table's data elsewhere, e.g. the move destination of a queue, is the same data flow as + /// `INSERT INTO FUNCTION s3(...) SELECT * FROM `. The creator of the table supplied its source and credentials, + /// so the reading part is `SELECT` on the table. The grants do not depend on the engine of the table, so the initiator + /// of `ON CLUSTER` can check them without the table. + if (command.type == ASTAlterCommand::MODIFY_SETTING && command.settings_changes) + { + auto export_access = StorageFactory::instance().getExportAccessOfAnyEngine(command.settings_changes->as().changes); + if (!export_access.empty()) + { + required_access.append_range(std::move(export_access)); + required_access.emplace_back(AccessType::SELECT, database, table); + } + } break; } case ASTAlterCommand::ATTACH_PARTITION: diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 694a135e8f14f..10d1aec703852 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -1976,6 +1976,11 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) /// Set and retrieve list of columns, indices and constraints. Set table engine if needed. Rewrite query in canonical way. TableProperties properties = getTablePropertiesAndNormalizeCreateQuery(create, mode); + /// Only here is an engine inherited through `AS` known, and nothing is enqueued or created yet. + /// A replay of a stored definition was checked when it was created. + if (isFreshTableDefinition(mode, create.attach_short_syntax)) + checkExportAccess(create); + DatabasePtr database; bool need_add_to_database = !create.isTemporary(); // In case of an ON CLUSTER query, the database may not be present on the initiator node @@ -3565,6 +3570,21 @@ BlockIO InterpreterCreateQuery::execute() /// getRequiredAccess sees no engine, because this query holds none of its own if (inherited.storage && inherited.storage->engine) getContext()->checkAccess(AccessType::TABLE_ENGINE, inherited.storage->engine->name); + + /// setEngine copies only the outer engine of a `TimeSeries` source, and the worker copies + /// its inner engines while normalizing, so add the ones this query does not give itself + if (inherited.is_time_series_table) + { + auto as_create_ptr = DatabaseCatalog::instance().getDatabase(inherited.as_database)->getCreateTableQuery(inherited.as_table, getContext()); + const auto & as_create = as_create_ptr->as(); + if (as_create.targets) + for (const auto & target : as_create.targets->targets) + if (target.inner_engine && !inherited.getTargetInnerEngine(target.kind)) + inherited.setTargetInnerEngine(target.kind, target.inner_engine->clone()); + } + + /// the worker inherits the move destination of the source too, and runs with no user + checkExportAccess(inherited); } return executeQueryOnCluster(create); @@ -3649,9 +3669,21 @@ AccessRightsElements InterpreterCreateQuery::getRequiredAccess() const if (create.storage && create.storage->engine) required_access.emplace_back(AccessType::TABLE_ENGINE, create.storage->engine->name); + /// Settings that send the table's data elsewhere, e.g. the move destination of a queue, need the grants of that place. + required_access.append_range(StorageFactory::instance().getExportAccess(create)); + return required_access; } +void InterpreterCreateQuery::checkExportAccess(const ASTCreateQuery & create) const +{ + /// Internal queries (initiated by the server itself) always have access to everything. + if (internal) + return; + + getContext()->checkAccess(StorageFactory::instance().getExportAccess(create)); +} + void InterpreterCreateQuery::extendQueryLogElemImpl(QueryLogElement & elem, const ASTPtr &, ContextPtr) const { if (!as_table_saved.empty()) diff --git a/src/Interpreters/InterpreterCreateQuery.h b/src/Interpreters/InterpreterCreateQuery.h index 9adba483ee71b..9baa88d61bbd5 100644 --- a/src/Interpreters/InterpreterCreateQuery.h +++ b/src/Interpreters/InterpreterCreateQuery.h @@ -117,6 +117,10 @@ class InterpreterCreateQuery : public IInterpreter, WithMutableContext void setEngine(ASTCreateQuery & create) const; AccessRightsElements getRequiredAccess() const; + /// The part of `getRequiredAccess` for settings that send the table's data elsewhere, checked again once + /// `setEngine` has resolved an engine inherited through `AS`, which `getRequiredAccess` cannot see. + void checkExportAccess(const ASTCreateQuery & create) const; + /// Create IStorage and add it to database. If table already exists and IF NOT EXISTS specified, do nothing and return false. bool doCreateTable(ASTCreateQuery & create, const TableProperties & properties, DDLGuardPtr & ddl_guard, LoadingStrictnessLevel mode); BlockIO doCreateOrReplaceTable(ASTCreateQuery & create, const InterpreterCreateQuery::TableProperties & properties, LoadingStrictnessLevel mode); diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMoveDestinationAccess.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMoveDestinationAccess.cpp new file mode 100644 index 0000000000000..e845fa34a6c0d --- /dev/null +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMoveDestinationAccess.cpp @@ -0,0 +1,119 @@ +#include + +#include "config.h" + +#include +#include +#include +#include + +#include + +#include +#include +#include + + +namespace DB +{ + +namespace +{ + +enum class MoveDestinationKind : uint8_t +{ + S3, + Azure, +}; + +std::optional getMoveDestinationKind(std::string_view name) +{ + /// The prefix is accepted for every queue setting, see `normalizeSetting` in `StorageObjectStorageQueue.cpp`. + static constexpr std::string_view legacy_prefix = "s3queue_"; + if (name.starts_with(legacy_prefix)) + name.remove_prefix(legacy_prefix.size()); + + if (name == "after_processing_move_uri") + return MoveDestinationKind::S3; + if (name == "after_processing_move_connection_string" || name == "after_processing_move_container") + return MoveDestinationKind::Azure; + return std::nullopt; +} + +AccessTypeObjects::Source getAccessSource(MoveDestinationKind kind) +{ + switch (kind) + { + case MoveDestinationKind::S3: + return AccessTypeObjects::Source::S3; + case MoveDestinationKind::Azure: + return AccessTypeObjects::Source::AZURE; + } +} + +/// A value that is not a `String` is never empty here: it is checked like a destination that cannot be parsed. +bool isEmptyValue(const Field & value) +{ + return value.getType() == Field::Types::String && value.safeGet().empty(); +} + +/// A table moves files only to the destination of its own storage type, see `ObjectStorageQueuePostProcessor::process`. +AccessRightsElements getMoveDestinationAccess(const SettingsChanges & changes, MoveDestinationKind table_kind) +{ + const bool sets_destination = std::ranges::any_of( + changes, + [&](const SettingChange & change) { return getMoveDestinationKind(change.name) == table_kind && !isEmptyValue(change.value); }); + if (!sets_destination) + return {}; + + /// Unfiltered on purpose: a move has no single destination path to match a filter against. + /// For S3, the move writes keys built from `after_processing_move_prefix` and the source keys, and uses only + /// the bucket of the URI, not its key. For Azure, the `azureBlobStorage` function matches source filters against + /// the blob path (`StorageAzureConfiguration::getRawURI` returns `blob_path.path`). + const auto source = getAccessSource(table_kind); + AccessRightsElements res; + res.emplace_back(AccessType::WRITE, AccessTypeObjects::toStringSource(source)); + return res; +} + +} + +AccessRightsElements getS3QueueMoveDestinationAccess(const SettingsChanges & changes) +{ + return getMoveDestinationAccess(changes, MoveDestinationKind::S3); +} + +AccessRightsElements getAzureQueueMoveDestinationAccess(const SettingsChanges & changes) +{ + return getMoveDestinationAccess(changes, MoveDestinationKind::Azure); +} + +void checkObjectStorageQueueS3MoveDestinationHost( + [[maybe_unused]] const String & move_uri, + [[maybe_unused]] const RemoteHostFilter & filter) +{ +#if USE_AWS_S3 + if (!move_uri.empty()) + S3::URI(move_uri).checkRemoteHostFilter(filter); +#endif +} + +void checkObjectStorageQueueAzureMoveDestinationHost( + [[maybe_unused]] const String & move_connection_string, + [[maybe_unused]] const String & move_container, + [[maybe_unused]] const RemoteHostFilter & filter) +{ +#if USE_AZURE_BLOB_STORAGE + if (!move_connection_string.empty()) + { + AzureBlobStorage::ConnectionParams connection_params; + AzureBlobStorage::processURL(move_connection_string, move_container, connection_params.endpoint, connection_params.auth_method); + /// Without the query: a connection string can carry a SAS, which the message of `checkURL` must not show. + String url = connection_params.getConnectionURL(); + url = url.substr(0, url.find_first_of("?#")); + filter.checkURL(Poco::URI(url)); + } +#endif +} + +} diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMoveDestinationAccess.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMoveDestinationAccess.h new file mode 100644 index 0000000000000..40601b4b11458 --- /dev/null +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMoveDestinationAccess.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include + +namespace DB +{ + +class RemoteHostFilter; + +/// The move destination of an `S3Queue` or `AzureQueue` table is the same data flow as +/// `INSERT INTO FUNCTION s3('') SELECT * FROM
`: the table reads every processed file +/// with its own source credentials and writes it to the destination. So setting the destination needs +/// `WRITE` on the `S3` (or `AZURE`) source without a URL filter: the move writes keys built from +/// `after_processing_move_prefix` and the source keys, not from the URI, so a filtered grant cannot authorize them. +/// For `ALTER` it also needs `SELECT` on the table, because the creator of the table, not the user of the `ALTER`, +/// supplied the source and its credentials. + +/// The `StorageFactory::GetExportAccessFn` of `S3Queue` and `AzureQueue`: unfiltered `WRITE` if a setting moves files outside +/// the source bucket or container. Only these settings choose such a destination: `after_processing_move_uri` for S3, +/// `after_processing_move_connection_string` and `after_processing_move_container` for Azure, with or without the legacy +/// `s3queue_` prefix. Changing only the move keys or prefix stays inside a destination an authorized user set. +/// An empty value disables the move destination, so it is not a destination and needs no grant. +AccessRightsElements getS3QueueMoveDestinationAccess(const SettingsChanges & changes); +AccessRightsElements getAzureQueueMoveDestinationAccess(const SettingsChanges & changes); + +/// Throws `UNACCEPTABLE_URL` if `remote_url_allow_hosts` does not allow the destination of a move to another S3 bucket. +/// The move builds its client directly, so the destination never passes `StorageS3Configuration::check`. +/// Does nothing when `move_uri` is empty. +void checkObjectStorageQueueS3MoveDestinationHost(const String & move_uri, const RemoteHostFilter & filter); + +/// The same for a move to another Azure container (see `StorageAzureConfiguration::check`). +/// Does nothing when `move_connection_string` is empty. +void checkObjectStorageQueueAzureMoveDestinationHost( + const String & move_connection_string, const String & move_container, const RemoteHostFilter & filter); + +} diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueuePostProcessor.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueuePostProcessor.cpp index c520d97da2bb0..6f351f28f8841 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueuePostProcessor.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueuePostProcessor.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -51,6 +52,12 @@ namespace S3AuthSetting extern const S3AuthSettingsString google_adc_client_id; extern const S3AuthSettingsString google_adc_client_secret; extern const S3AuthSettingsString google_adc_refresh_token; + extern const S3AuthSettingsS3UriStyle uri_style; +} + +namespace S3RequestSetting +{ + extern const S3RequestSettingsBool allow_native_copy; } #endif @@ -307,6 +314,10 @@ void ObjectStorageQueuePostProcessor::moveS3Objects(const StoredObjects & object throw Exception(ErrorCodes::BAD_ARGUMENTS, "Not enough settings to move S3 objects"); } + /// The settings are checked at `CREATE` and `ALTER` too. This is defense in depth for a table that was created before + /// those checks existed, or before the list of allowed hosts was tightened. If it throws, nothing is copied and nothing is removed. + checkObjectStorageQueueS3MoveDestinationHost(move_uri, getContext()->getRemoteHostFilter()); + if (auto * s3_storage = dynamic_cast(object_storage.get()); s3_storage != nullptr) { auto src_client = s3_storage->getS3StorageClient(); @@ -337,6 +348,10 @@ void ObjectStorageQueuePostProcessor::moveS3Objects(const StoredObjects & object /// headers and SSE-C/SSE-KMS keys) merged from the server `` config: otherwise the server's /// headers or encryption keys would be sent to the user-supplied move destination. s3_settings->auth_settings.clearServerManagedRequestAuth(); + /// The host check and `dst_uri` below parse `move_uri` with automatic style detection. The client must use the same + /// parse, so the URI style of the server `` config or of the `s3_uri_style` query setting must not apply here: + /// otherwise the requests can go to a host that `checkObjectStorageQueueS3MoveDestinationHost` did not check. + s3_settings->auth_settings[S3AuthSetting::uri_style] = S3UriStyle::AUTO; std::shared_ptr dst_client = getClient( move_uri, *s3_settings, @@ -344,6 +359,10 @@ void ObjectStorageQueuePostProcessor::moveS3Objects(const StoredObjects & object /* for_disk_s3 */ false ); auto dst_uri = S3::URI(move_uri); + /// Native copy sends `CopyObject` through the source client, so it writes to the source endpoint. That is the checked + /// destination only if both clients use the same endpoint, otherwise use the buffered copy, which uploads with `dst_client`. + if (src_client->getInitialEndpoint() != dst_client->getInitialEndpoint()) + s3_settings->request_settings[S3RequestSetting::allow_native_copy] = false; auto read_settings = getReadSettings(); const auto read_settings_to_use = s3_storage->patchSettings(read_settings); auto scheduler = threadPoolCallbackRunnerUnsafe( @@ -430,6 +449,9 @@ void ObjectStorageQueuePostProcessor::moveAzureBlobs(const StoredObjects & objec throw Exception(ErrorCodes::BAD_ARGUMENTS, "Not enough settings to move Azure blobs"); } + /// See the comment in `moveS3Objects`: if it throws, nothing is copied and nothing is removed. + checkObjectStorageQueueAzureMoveDestinationHost(move_connection_string, move_container, getContext()->getRemoteHostFilter()); + if (auto * azure_storage = dynamic_cast(object_storage.get()); azure_storage != nullptr) { auto contextPtr = getContext(); diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp index 2678a4a03f540..ff1148a959da0 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp @@ -59,15 +59,15 @@ namespace ErrorCodes DECLARE(String, partition_component, "", "Name of the capture group from partition_regex to use as partition key. Required when using partitioning_mode='regex'. Example: 'hostname'", 0) \ DECLARE(ObjectStorageQueueBucketingMode, bucketing_mode, ObjectStorageQueueBucketingMode::PATH, "Bucketing strategy for Ordered mode parallel processing: PATH (hash full file path, default), PARTITION (hash partition key, requires partitioning_mode != NONE)", 0) \ DECLARE(UInt32, after_processing_retries, 10, "Number of retries for the after_processing action before giving up", 0) \ - DECLARE(String, after_processing_move_uri, "", "S3 bucket URL to move processed files to", 0) \ + DECLARE(String, after_processing_move_uri, "", "S3 bucket URL to move processed files to. Setting it requires the `WRITE` grant on the `S3` source (not limited to a URL filter), and `ALTER` also requires `SELECT` on the table. The URL must be allowed by `remote_url_allow_hosts`", 0) \ DECLARE(String, after_processing_move_prefix, "", "Path prefix to move processed files to", 0) \ DECLARE(Bool, after_processing_move_preserve_path, false, "If true, preserve the full source path under after_processing_move_prefix instead of using only the file name", 0) \ DECLARE(String, after_processing_tag_key, "", "Tag key to tag processed files in the storage", 0) \ DECLARE(String, after_processing_tag_value, "", "Tag value to tag processed files in the storage", 0) \ DECLARE(String, after_processing_move_access_key_id, "", "S3 Access Key ID accompanying after_processing_move_uri", 0) \ DECLARE(String, after_processing_move_secret_access_key, "", "S3 Secret Access Key accompanying after_processing_move_uri", 0) \ - DECLARE(String, after_processing_move_connection_string, "", "Azure connection string to move processed files to", 0) \ - DECLARE(String, after_processing_move_container, "", "Azure container accompanying after_processing_move_connection_string", 0) \ + DECLARE(String, after_processing_move_connection_string, "", "Azure connection string to move processed files to. Setting it requires the `WRITE` grant on the `AZURE` source (not limited to a URL filter), and `ALTER` also requires `SELECT` on the table. The endpoint must be allowed by `remote_url_allow_hosts`", 0) \ + DECLARE(String, after_processing_move_container, "", "Azure container accompanying after_processing_move_connection_string. Setting it requires the `WRITE` grant on the `AZURE` source (not limited to a URL filter), and `ALTER` also requires `SELECT` on the table", 0) \ DECLARE(Bool, use_hive_partitioning, false, "DEPRECATED: Use partitioning_mode='hive' instead. Whether path contains hive partitioning and this engine should process such files with a separate processed files tracking per partition.", 0) \ #define LIST_OF_OBJECT_STORAGE_QUEUE_SETTINGS(M, ALIAS) \ diff --git a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp index 093276d481ac5..6692ac9ccc268 100644 --- a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp +++ b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp @@ -1,3 +1,5 @@ +#include +#include #include #include @@ -28,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -40,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -171,9 +175,26 @@ namespace ErrorCodes namespace { + /// The move uses only the destination of the storage type of the table, so only that one is checked. + void checkMoveDestinationHost( + ObjectStorageType type, + const String & move_uri, + const String & move_connection_string, + const String & move_container, + const RemoteHostFilter & filter) + { + if (type == ObjectStorageType::S3) + checkObjectStorageQueueS3MoveDestinationHost(move_uri, filter); + else if (type == ObjectStorageType::Azure) + checkObjectStorageQueueAzureMoveDestinationHost(move_connection_string, move_container, filter); + } + void validateSettings( ObjectStorageQueueSettings & queue_settings, - bool is_attach) + ObjectStorageType type, + const RemoteHostFilter & remote_host_filter, + bool is_attach, + bool is_loading_from_existing_metadata) { if (!is_attach && !queue_settings[ObjectStorageQueueSetting::mode].changed) { @@ -232,6 +253,21 @@ namespace "Move of processed objects requires specifying target bucket/container or prefix"); } } + + /// The host filter of `configuration` covers the source only, so the move destination is checked here. + /// It is checked whenever it is set, not only for `after_processing = 'move'`: otherwise a table created with + /// `after_processing = 'keep'` and a destination that is not allowed could be switched to `move` by an `ALTER` + /// that does not change the destination. A table loaded from existing metadata must never fail to load. + if (!is_loading_from_existing_metadata) + { + checkMoveDestinationHost( + type, + queue_settings[ObjectStorageQueueSetting::after_processing_move_uri].value, + queue_settings[ObjectStorageQueueSetting::after_processing_move_connection_string].value, + queue_settings[ObjectStorageQueueSetting::after_processing_move_container].value, + remote_host_filter); + } + if (queue_settings[ObjectStorageQueueSetting::after_processing] == ObjectStorageQueueAction::TAG) { if (queue_settings[ObjectStorageQueueSetting::after_processing_tag_key].toString().empty()) @@ -340,7 +376,7 @@ StorageObjectStorageQueue::StorageObjectStorageQueue( } const bool is_attach = mode > LoadingStrictnessLevel::CREATE; - validateSettings(*queue_settings_, is_attach); + validateSettings(*queue_settings_, type, context_->getRemoteHostFilter(), is_attach, isLoadingFromExistingMetadata(mode)); /// The object storage S3 client is built once here and reused by background threads, so the effective /// per-session credential restriction must be captured now from the CREATE query (its @@ -1567,6 +1603,32 @@ void StorageObjectStorageQueue::checkAlterIsPossible(const AlterCommands & comma } } } + + /// The host filter of `configuration` covers the source only, so a new move destination is checked here. + /// `checkAlterIsPossible` runs on every host and the check depends only on the server config, so it is correct everywhere. + static constexpr std::array move_destination_settings{ + "after_processing_move_uri", "after_processing_move_connection_string", "after_processing_move_container"}; + + auto get_move_destination_value = [](const SettingsChanges * settings, std::string_view name) -> String + { + const Field * value = settings ? settings->tryGet(name) : nullptr; + return value ? convertFieldToString(*value) : String{}; + }; + + const bool move_destination_changed = std::ranges::any_of( + move_destination_settings, + [&](std::string_view name) + { return get_move_destination_value(old_settings, name) != get_move_destination_value(&new_settings, name); }); + + if (move_destination_changed) + { + checkMoveDestinationHost( + type, + get_move_destination_value(&new_settings, "after_processing_move_uri"), + get_move_destination_value(&new_settings, "after_processing_move_connection_string"), + get_move_destination_value(&new_settings, "after_processing_move_container"), + local_context->getRemoteHostFilter()); + } } void StorageObjectStorageQueue::alter( diff --git a/src/Storages/ObjectStorageQueue/registerQueueStorage.cpp b/src/Storages/ObjectStorageQueue/registerQueueStorage.cpp index 0ec97a2f1402b..6de88a0c3f836 100644 --- a/src/Storages/ObjectStorageQueue/registerQueueStorage.cpp +++ b/src/Storages/ObjectStorageQueue/registerQueueStorage.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -168,6 +169,7 @@ void registerStorageS3Queue(StorageFactory & factory) .supports_schema_inference = true, .source_access_type = AccessTypeObjects::Source::S3, .has_builtin_setting_fn = ObjectStorageQueueSettings::hasBuiltin, + .get_export_access_fn = getS3QueueMoveDestinationAccess, }, Documentation{ .description = R"DOCS_MD( @@ -358,6 +360,8 @@ Default value: empty string. URI of S3 bucket to move successfully processed files to, if the destination is another S3 bucket. +Setting it is the same data flow as `INSERT INTO FUNCTION s3('') SELECT * FROM
`, so it requires the `WRITE` grant on the `S3` source. A grant limited by a URL filter is not enough, because the move writes keys that are built from `after_processing_move_prefix` and the source keys, not from the URI. `ALTER TABLE ... MODIFY SETTING` also requires `SELECT` on the table. The destination must be allowed by `remote_url_allow_hosts`. The move always parses the URI with automatic style detection, as the `s3` table function does with the default `s3_uri_style`, so the `uri_style` server setting and the `s3_uri_style` query setting do not apply to it. + Possible values: - String. @@ -758,6 +762,7 @@ void registerStorageAzureQueue(StorageFactory & factory) .supports_schema_inference = true, .source_access_type = AccessTypeObjects::Source::AZURE, .has_builtin_setting_fn = ObjectStorageQueueSettings::hasBuiltin, + .get_export_access_fn = getAzureQueueMoveDestinationAccess, }, Documentation{ .description = R"DOCS_MD( @@ -804,6 +809,8 @@ Below are the settings only compatible with AzureQueue and not applicable for S3 Connection string for Azure Blob Storage to move successfully processed files to, if the destination is another Azure container. +Setting it requires the `WRITE` grant on the `AZURE` source. A grant limited by a URL filter is not enough, because a move has no single blob path and writes keys that are built from `after_processing_move_prefix` and the source keys. `ALTER TABLE ... MODIFY SETTING` also requires `SELECT` on the table. The destination must be allowed by `remote_url_allow_hosts`. + Possible values: - String. @@ -814,6 +821,8 @@ Default value: empty string. Container name to move successfully processed files to, if the destination is another Azure container. +Changing it redirects the move to another container of the same account, so it needs the same grants as `after_processing_move_connection_string`: the `WRITE` grant on the `AZURE` source without a URL filter, and `SELECT` on the table for `ALTER TABLE ... MODIFY SETTING`. + Possible values: - String. diff --git a/src/Storages/StorageAlias.cpp b/src/Storages/StorageAlias.cpp index eff18fef021cf..dff2af5530826 100644 --- a/src/Storages/StorageAlias.cpp +++ b/src/Storages/StorageAlias.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -19,6 +20,7 @@ #include #include #include +#include namespace DB @@ -250,6 +252,26 @@ SinkToStoragePtr StorageAlias::write( return std::make_shared(*this, local_context, target_metadata, async_insert); } +void StorageAlias::checkAlterIsPossible(const AlterCommands & commands, ContextPtr local_context) const +{ + /// `InterpreterAlterQuery` checks the export grants (`SELECT` on the table and its row policy) on the name in the query, + /// which is the alias, not the target. The refusal does not need the user, so it also holds on the hosts of `ON CLUSTER`. + for (const auto & command : commands) + { + if (command.type == AlterCommand::MODIFY_SETTING && !StorageFactory::instance().getExportAccessOfAnyEngine(command.settings_changes).empty()) + { + throw Exception( + ErrorCodes::NOT_IMPLEMENTED, + "Changing the settings that send the data of a table elsewhere through an alias is not supported. " + "Execute the ALTER directly on the target table: {}.{}", + backQuoteIfNeed(target_database), + backQuoteIfNeed(target_table)); + } + } + + getTargetTable()->checkAlterIsPossible(commands, local_context); +} + void StorageAlias::alter( const AlterCommands & params, ContextPtr local_context, diff --git a/src/Storages/StorageAlias.h b/src/Storages/StorageAlias.h index 3fa188702e069..219707a565637 100644 --- a/src/Storages/StorageAlias.h +++ b/src/Storages/StorageAlias.h @@ -68,7 +68,7 @@ class StorageAlias final : public IStorage, WithContext /// Distributed write to target table std::optional distributedWrite(const ASTInsertQuery & query, ContextPtr local_context) override; - void checkAlterIsPossible(const AlterCommands & commands, ContextPtr local_context) const override { getTargetTable()->checkAlterIsPossible(commands, local_context); } + void checkAlterIsPossible(const AlterCommands & commands, ContextPtr local_context) const override; /// Alter target table void alter( diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp index ad3d71376f151..bef934fa0c04e 100644 --- a/src/Storages/StorageFactory.cpp +++ b/src/Storages/StorageFactory.cpp @@ -1,9 +1,12 @@ #include +#include #include #include #include #include +#include #include +#include #include #include #include @@ -277,4 +280,37 @@ const StorageFactory::StorageFeatures & StorageFactory::getStorageFeatures(const return it->second.features; } +AccessRightsElements StorageFactory::getExportAccess(const ASTCreateQuery & create) const +{ + AccessRightsElements res; + auto add = [&](const ASTStorage & storage) + { + if (!storage.engine || !storage.settings) + return; + auto it = storages.find(storage.engine->name); + if (it != storages.end() && it->second.features.get_export_access_fn) + res.append_range(it->second.features.get_export_access_fn(storage.settings->changes)); + }; + + if (create.storage) + add(*create.storage); + if (create.targets) + { + for (const ASTStorage * inner_engine : create.targets->getInnerEngines()) + add(*inner_engine); + } + return res; +} + +AccessRightsElements StorageFactory::getExportAccessOfAnyEngine(const SettingsChanges & changes) const +{ + AccessRightsElements res; + for (const auto & [_, creator] : storages) + { + if (creator.features.get_export_access_fn) + res.append_range(creator.features.get_export_access_fn(changes)); + } + return res; +} + } diff --git a/src/Storages/StorageFactory.h b/src/Storages/StorageFactory.h index f5085d18b827a..e8859f00cec95 100644 --- a/src/Storages/StorageFactory.h +++ b/src/Storages/StorageFactory.h @@ -17,6 +17,8 @@ namespace DB class Context; class ASTCreateQuery; class ASTStorage; +class AccessRightsElements; +class SettingsChanges; struct StorageID; struct ConstraintsDescription; @@ -34,6 +36,10 @@ class StorageFactory : private boost::noncopyable, public IHints<> /// Used to validate if table settings belong to the engine or the query before the start of the query interpretation using HasBuiltinSettingFn = bool(std::string_view); + /// The grants needed by settings that make a table send its data to another place, e.g. the move destination of a queue. + /// Returns nothing if `changes` set no such place. It reads only the changes, so the table need not exist on this host. + using GetExportAccessFn = AccessRightsElements(const SettingsChanges & changes); + struct Arguments { const String & engine_name; @@ -82,6 +88,7 @@ class StorageFactory : private boost::noncopyable, public IHints<> std::optional source_access_type = std::nullopt; HasBuiltinSettingFn * has_builtin_setting_fn = nullptr; + GetExportAccessFn * get_export_access_fn = nullptr; }; using CreatorFn = std::function; @@ -120,6 +127,7 @@ class StorageFactory : private boost::noncopyable, public IHints<> .supports_sql_security = false, .source_access_type = std::nullopt, .has_builtin_setting_fn = nullptr, + .get_export_access_fn = nullptr, }, Documentation documentation = {}); const Storages & getAllStorages() const @@ -149,6 +157,13 @@ class StorageFactory : private boost::noncopyable, public IHints<> const StorageFeatures & getStorageFeatures(const String & storage_name) const; + /// See `GetExportAccessFn`. For the engine of the table and the inner engines of a materialized view or a `TimeSeries` + /// table. An unknown engine needs nothing here: creating it fails later. + AccessRightsElements getExportAccess(const ASTCreateQuery & create) const; + /// The grants of every engine for which `changes` send the data elsewhere. It does not depend on the engine of the table, + /// so `ALTER ... ON CLUSTER` can be authorized on an initiator that does not have the table or has another engine. + AccessRightsElements getExportAccessOfAnyEngine(const SettingsChanges & changes) const; + private: Storages storages; }; diff --git a/tests/integration/test_allowed_url_from_config/test.py b/tests/integration/test_allowed_url_from_config/test.py index 25c07fc5744fa..d30410845a40f 100644 --- a/tests/integration/test_allowed_url_from_config/test.py +++ b/tests/integration/test_allowed_url_from_config/test.py @@ -81,6 +81,43 @@ def test_azure_host_filter_before_client_creation(start_cluster, query): assert node5.query(container_requests) == before +def test_object_storage_queue_move_destination_host_filter(start_cluster): + # The source is on an allowed host (`host:80`), so only the move destination can be rejected. `node1` has no + # Keeper, so a rejection with "not allowed" must come from the settings validation, which runs before any + # Keeper access. + s3_queue = ( + "CREATE TABLE s3queue_move_host (x UInt8) " + "ENGINE = S3Queue('http://host:80/bucket/data/', 'key', 'secret', 'CSV') " + "SETTINGS mode = 'unordered', after_processing = 'move', " + "after_processing_move_uri = '{}', " + "after_processing_move_access_key_id = 'key', after_processing_move_secret_access_key = 'secret'" + ) + azure_queue = ( + "CREATE TABLE azurequeue_move_host (x UInt8) " + "ENGINE = AzureQueue('http://host:80', 'container', '*', 'account', 'YQ==', 'CSV') " + "SETTINGS mode = 'unordered', after_processing = 'move', " + "after_processing_move_connection_string = " + "'DefaultEndpointsProtocol=http;AccountName=account;AccountKey=YQ==;BlobEndpoint=http://host:123/account;', " + "after_processing_move_container = 'dst'" + ) + + try: + error = node1.query_and_get_error(s3_queue.format("http://host:123/bucket/")) + assert "not allowed" in error, error + + error = node1.query_and_get_error( + azure_queue, settings={"azure_sdk_max_retries": 0} + ) + assert "not allowed" in error, error + + # Control: an allowed destination passes the host check and only fails later, on the missing Keeper. + error = node1.query_and_get_error(s3_queue.format("http://host:80/bucket/")) + assert "not allowed" not in error, error + finally: + node1.query("DROP TABLE IF EXISTS s3queue_move_host SYNC") + node1.query("DROP TABLE IF EXISTS azurequeue_move_host SYNC") + + def test_config_with_only_primary_hosts(start_cluster): assert ( node2.query( diff --git a/tests/queries/0_stateless/05441_s3queue_move_destination_access.reference b/tests/queries/0_stateless/05441_s3queue_move_destination_access.reference new file mode 100644 index 0000000000000..e5046dab0a101 --- /dev/null +++ b/tests/queries/0_stateless/05441_s3queue_move_destination_access.reference @@ -0,0 +1,35 @@ +table created +-- ALTER with only ALTER SETTINGS +settings that are not a destination: allowed +-- ALTER with SELECT on the table, but no WRITE on S3 +-- ALTER ON CLUSTER is rejected by the initiator +-- ALTER ON CLUSTER of a table that does not exist on the initiator +-- ALTER with WRITE on S3 limited by a URL filter +-- ALTER with WRITE on S3 +destination with WRITE on S3: allowed +empty destination without WRITE on S3: allowed +-- ALTER with WRITE on S3, restricted by a row policy +empty destination with a row policy: allowed +-- ALTER through an Alias +settings that are not a destination through an alias: allowed +-- CREATE without WRITE on S3 or AZURE +table without a destination: created +-- CREATE AS with a destination, without WRITE on S3 +-- CREATE MATERIALIZED VIEW with a queue inner engine, without WRITE on S3 +-- CREATE AS ON CLUSTER with the legacy entry format, without WRITE on S3 +-- CREATE AS ON CLUSTER with the default entry format, without WRITE on S3 +-- CREATE ON CLUSTER with a queue engine and a destination, without WRITE on S3 +-- ATTACH with a full definition and a destination, without WRITE on S3 +-- RESTORE of a queue with a destination, without WRITE on S3 +0 +0 +0 +-- CREATE AS and CREATE MATERIALIZED VIEW with WRITE on S3 +CREATE AS with WRITE on S3: created +materialized view with WRITE on S3: created +RESTORE with WRITE on S3: restored +RESTORE ON CLUSTER with WRITE on S3: restored +CREATE ON CLUSTER with WRITE on S3: created +CREATE AS ON CLUSTER with WRITE on S3: created +ATTACH with WRITE on S3: attached +RESTORE TEMPORARY with WRITE on S3: restored 1 diff --git a/tests/queries/0_stateless/05441_s3queue_move_destination_access.sh b/tests/queries/0_stateless/05441_s3queue_move_destination_access.sh new file mode 100755 index 0000000000000..5474fde6a4fd2 --- /dev/null +++ b/tests/queries/0_stateless/05441_s3queue_move_destination_access.sh @@ -0,0 +1,374 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, zookeeper, no-replicated-database +# Tag no-fasttest: the `S3Queue` and `AzureQueue` engines are not built into the fast test build. +# Tag zookeeper: `S3Queue` keeps its state in Keeper, and `ON CLUSTER` queries use the distributed DDL queue. +# Tag no-replicated-database: with `access_control_improvements.enable_read_write_grants` disabled (the default), +# a replicated access storage round-trips the user through serialized SQL, which rewrites `WRITE ON S3('')` +# into the old-style whole-source `S3` grant without the filter. + +# The move destination of an `S3Queue` or `AzureQueue` table is the same data flow as +# `INSERT INTO FUNCTION s3('') SELECT * FROM
`: the table reads the processed files with its +# own credentials and writes them to the destination. So setting `after_processing_move_uri`, +# `after_processing_move_connection_string` or `after_processing_move_container` needs `WRITE` on the `S3` (`AZURE`) +# source without a URL filter, because the move writes keys built from the prefix and the source keys, not from the URI. +# `ALTER` needs `SELECT` on the table too. Settings that do not choose a destination need no extra grant. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +db="${CLICKHOUSE_DATABASE}" +user_alter="s3queue_move_alter_${db}" +user_create="s3queue_move_create_${db}" +keeper_path="/s3queue_move_destination_access_${db}" +src="http://localhost:11111/test/${db}_move_src/" +dst="http://localhost:11111/test/${db}_move_dst/" +backup="Disk('backups', '05441_${CLICKHOUSE_TEST_UNIQUE_NAME}')" +backup_tmp="Disk('backups', '05441_${CLICKHOUSE_TEST_UNIQUE_NAME}_tmp')" +keys="after_processing_move_access_key_id = 'clickhouse', after_processing_move_secret_access_key = 'clickhouse'" +attach_uuid=$($CLICKHOUSE_CLIENT -q "SELECT generateUUIDv4()") +azure_connection_string="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;" + +$CLICKHOUSE_CLIENT -q " + DROP TABLE IF EXISTS ${db}.t_alias SYNC; + DROP TABLE IF EXISTS ${db}.t SYNC; + DROP TABLE IF EXISTS ${db}.t_create SYNC; + DROP TABLE IF EXISTS ${db}.t_as SYNC; + DROP TABLE IF EXISTS ${db}.t_as_cluster SYNC; + DROP TABLE IF EXISTS ${db}.t_as_cluster_new SYNC; + DROP TABLE IF EXISTS ${db}.t_create_cluster SYNC; + DROP TABLE IF EXISTS ${db}.t_attach SYNC; + DROP TABLE IF EXISTS ${db}.t_backup SYNC; + DROP TABLE IF EXISTS ${db}.t_backup_cluster SYNC; + DROP TABLE IF EXISTS ${db}.mv SYNC; + DROP ROW POLICY IF EXISTS p_${db} ON ${db}.t; + DROP USER IF EXISTS ${user_alter}, ${user_create}; +" + +$CLICKHOUSE_CLIENT -q " + CREATE TABLE ${db}.t (x UInt8) ENGINE = S3Queue('${src}', 'clickhouse', 'clickhouse', 'CSV') + SETTINGS mode = 'unordered', keeper_path = '${keeper_path}', after_processing = 'keep' +" +echo "table created" + +$CLICKHOUSE_CLIENT -q " + CREATE USER ${user_alter}; + GRANT ALTER SETTINGS ON ${db}.t TO ${user_alter}; + CREATE USER ${user_create}; + GRANT CREATE TABLE ON ${db}.* TO ${user_create}; + GRANT TABLE ENGINE ON S3Queue TO ${user_create}; + GRANT TABLE ENGINE ON AzureQueue TO ${user_create}; +" + +echo "-- ALTER with only ALTER SETTINGS" + +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t MODIFY SETTING after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t MODIFY SETTING s3queue_after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" + +# Changing only the container redirects the move to another container of the same account. +# The check reads only the query, so it fails before the storage is involved, although `t` is an `S3Queue`. +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t MODIFY SETTING after_processing_move_container = 'dst' + -- { serverError ACCESS_DENIED } +" + +# Settings that do not choose a destination need only ALTER SETTINGS. +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t MODIFY SETTING polling_max_timeout_ms = 20000, after_processing_move_prefix = 'prefix_${db}' +" && echo "settings that are not a destination: allowed" + +echo "-- ALTER with SELECT on the table, but no WRITE on S3" + +$CLICKHOUSE_CLIENT -q "GRANT SELECT ON ${db}.t TO ${user_alter}" +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t MODIFY SETTING after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t MODIFY SETTING s3queue_after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" + +# Changing only the container is a destination change that needs `WRITE` on `AZURE`. +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t MODIFY SETTING after_processing_move_container = 'dst' + -- { serverError ACCESS_DENIED } +" + +echo "-- ALTER ON CLUSTER is rejected by the initiator" + +$CLICKHOUSE_CLIENT -q "GRANT CLUSTER ON *.* TO ${user_alter}" +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t ON CLUSTER test_shard_localhost MODIFY SETTING after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" + +echo "-- ALTER ON CLUSTER of a table that does not exist on the initiator" + +# The grants come from the settings in the query, not from the table, so the initiator checks them without the table. +# With the grant, the query reaches the host and fails there. +$CLICKHOUSE_CLIENT -q "GRANT ALTER SETTINGS, SELECT ON ${db}.t_missing TO ${user_alter}" +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t_missing ON CLUSTER test_shard_localhost MODIFY SETTING after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" +$CLICKHOUSE_CLIENT -q "GRANT WRITE ON S3 TO ${user_alter}" +$CLICKHOUSE_CLIENT --user "${user_alter}" --distributed_ddl_output_mode none -q " + ALTER TABLE ${db}.t_missing ON CLUSTER test_shard_localhost MODIFY SETTING after_processing_move_uri = '${dst}', ${keys} + -- { serverError UNKNOWN_TABLE } +" +$CLICKHOUSE_CLIENT -q "REVOKE WRITE ON S3 FROM ${user_alter}" + +echo "-- ALTER with WRITE on S3 limited by a URL filter" + +# The move writes keys built from the prefix and the source keys, so a URL filter cannot authorize them. +$CLICKHOUSE_CLIENT -q "GRANT WRITE ON S3('${dst}.*') TO ${user_alter}" +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t MODIFY SETTING after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" +$CLICKHOUSE_CLIENT -q "REVOKE WRITE ON S3('${dst}.*') FROM ${user_alter}" + +echo "-- ALTER with WRITE on S3" + +$CLICKHOUSE_CLIENT -q "GRANT WRITE ON S3 TO ${user_alter}" +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t MODIFY SETTING after_processing_move_uri = '${dst}', ${keys} +" && echo "destination with WRITE on S3: allowed" + +# Resetting the destination disables the move, so it needs no grant for a destination. +$CLICKHOUSE_CLIENT -q "REVOKE WRITE ON S3 FROM ${user_alter}" +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t MODIFY SETTING after_processing_move_uri = '' +" && echo "empty destination without WRITE on S3: allowed" + +echo "-- ALTER with WRITE on S3, restricted by a row policy" + +# The move copies whole files, so a user whose row policy hides rows of the table cannot set a destination. +$CLICKHOUSE_CLIENT -q " + GRANT WRITE ON S3 TO ${user_alter}; + CREATE ROW POLICY p_${db} ON ${db}.t USING x = 1 TO ${user_alter}; +" +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t MODIFY SETTING after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t ON CLUSTER test_shard_localhost MODIFY SETTING after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" +# Turning the move off needs no check. +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t MODIFY SETTING after_processing_move_uri = '' +" && echo "empty destination with a row policy: allowed" +$CLICKHOUSE_CLIENT -q " + DROP ROW POLICY p_${db} ON ${db}.t; + REVOKE WRITE ON S3 FROM ${user_alter}; +" + +echo "-- ALTER through an Alias" + +# The export checks see only the name in the query, which is the alias, so the change is refused for every user. +$CLICKHOUSE_CLIENT --allow_experimental_alias_table_engine 1 -q "CREATE TABLE ${db}.t_alias ENGINE = Alias('${db}', 't')" +$CLICKHOUSE_CLIENT -q " + ALTER TABLE ${db}.t_alias MODIFY SETTING after_processing_move_uri = '${dst}', ${keys} + -- { serverError NOT_IMPLEMENTED } +" +$CLICKHOUSE_CLIENT -q " + ALTER TABLE ${db}.t_alias MODIFY SETTING polling_max_timeout_ms = 20000 +" && echo "settings that are not a destination through an alias: allowed" +$CLICKHOUSE_CLIENT -q "DROP TABLE ${db}.t_alias SYNC" + +echo "-- CREATE without WRITE on S3 or AZURE" + +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + CREATE TABLE ${db}.t_create (x UInt8) ENGINE = S3Queue('${src}', 'clickhouse', 'clickhouse', 'CSV') + SETTINGS mode = 'unordered', keeper_path = '${keeper_path}_create', after_processing = 'move', + after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + CREATE TABLE ${db}.t_create (x UInt8) ENGINE = S3Queue('${src}', 'clickhouse', 'clickhouse', 'CSV') + SETTINGS mode = 'unordered', keeper_path = '${keeper_path}_create', after_processing = 'move', + s3queue_after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + CREATE TABLE ${db}.t_create (x UInt8) ENGINE = AzureQueue('https://account.blob.core.windows.net', 'container', '*', 'CSV') + SETTINGS mode = 'unordered', keeper_path = '${keeper_path}_create', after_processing = 'move', + after_processing_move_connection_string = '${azure_connection_string}', after_processing_move_container = 'dst' + -- { serverError ACCESS_DENIED } +" + +# The same CREATE without a destination, or with an empty one, needs no grant for it. +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + CREATE TABLE ${db}.t_create (x UInt8) ENGINE = S3Queue('${src}', 'clickhouse', 'clickhouse', 'CSV') + SETTINGS mode = 'unordered', keeper_path = '${keeper_path}_create', after_processing = 'keep', + after_processing_move_uri = '' +" && echo "table without a destination: created" + +echo "-- CREATE AS with a destination, without WRITE on S3" + +# The engine and the settings of `AS` are inherited after the query is parsed, so the check has to see the final definition. +# `SELECT` is required because the definition of the source has secret parts. +$CLICKHOUSE_CLIENT -q "GRANT SHOW COLUMNS, SELECT ON ${db}.t TO ${user_create}" +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + CREATE TABLE ${db}.t_as AS ${db}.t + SETTINGS keeper_path = '${keeper_path}_as', after_processing = 'move', after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" + +echo "-- CREATE MATERIALIZED VIEW with a queue inner engine, without WRITE on S3" + +# The engine is an inner engine of the view, and the inner table is created as an internal query, which checks nothing. +$CLICKHOUSE_CLIENT -q "GRANT CREATE VIEW ON ${db}.* TO ${user_create}" +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + CREATE MATERIALIZED VIEW ${db}.mv ENGINE = S3Queue('${src}', 'clickhouse', 'clickhouse', 'CSV') + SETTINGS mode = 'unordered', keeper_path = '${keeper_path}_mv', after_processing = 'move', + after_processing_move_uri = '${dst}', ${keys} + AS SELECT 1::UInt8 AS x + -- { serverError ACCESS_DENIED } +" + +echo "-- CREATE AS ON CLUSTER with the legacy entry format, without WRITE on S3" + +# This format ships the query as written and the workers resolve `AS` with no user. +$CLICKHOUSE_CLIENT -q "GRANT CLUSTER ON *.* TO ${user_create}" +$CLICKHOUSE_CLIENT --user "${user_create}" --distributed_ddl_entry_format_version 2 -q " + CREATE TABLE ${db}.t_as_cluster ON CLUSTER test_shard_localhost AS ${db}.t + SETTINGS keeper_path = '${keeper_path}_as_cluster', after_processing = 'move', after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" + +echo "-- CREATE AS ON CLUSTER with the default entry format, without WRITE on S3" + +# This format builds the definition on the initiator. +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + CREATE TABLE ${db}.t_as_cluster_new ON CLUSTER test_shard_localhost AS ${db}.t + SETTINGS keeper_path = '${keeper_path}_as_cluster_new', after_processing = 'move', after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" + +echo "-- CREATE ON CLUSTER with a queue engine and a destination, without WRITE on S3" + +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + CREATE TABLE ${db}.t_create_cluster ON CLUSTER test_shard_localhost (x UInt8) ENGINE = S3Queue('${src}', 'clickhouse', 'clickhouse', 'CSV') + SETTINGS mode = 'unordered', keeper_path = '${keeper_path}_create_cluster', after_processing = 'move', + after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" + +echo "-- ATTACH with a full definition and a destination, without WRITE on S3" + +# A full definition in `ATTACH` is a new definition, like `CREATE`. An `Atomic` database needs a `UUID` for it. +# The server warns that a full definition in `ATTACH` is not recommended, so only errors are sent to the client. +$CLICKHOUSE_CLIENT --user "${user_create}" --send_logs_level=error -q " + ATTACH TABLE ${db}.t_attach UUID '${attach_uuid}' (x UInt8) ENGINE = S3Queue('${src}', 'clickhouse', 'clickhouse', 'CSV') + SETTINGS mode = 'unordered', keeper_path = '${keeper_path}_attach', after_processing = 'move', + after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" + +echo "-- RESTORE of a queue with a destination, without WRITE on S3" + +# The restored definition comes from the backup, and the table is created by an internal query, +# so the restore has to authorize the destination before it creates anything. +$CLICKHOUSE_CLIENT -q " + CREATE TABLE ${db}.t_backup (x UInt8) ENGINE = S3Queue('${src}', 'clickhouse', 'clickhouse', 'CSV') + SETTINGS mode = 'unordered', keeper_path = '${keeper_path}_backup', after_processing = 'move', + after_processing_move_uri = '${dst}', ${keys} +" +$CLICKHOUSE_CLIENT -q "BACKUP TABLE ${db}.t_backup TO ${backup} FORMAT Null" +$CLICKHOUSE_CLIENT -q "DROP TABLE ${db}.t_backup SYNC" +$CLICKHOUSE_CLIENT -q "GRANT READ ON DISK TO ${user_create}" +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + RESTORE TABLE ${db}.t_backup FROM ${backup} SETTINGS structure_only = 1 FORMAT Null + -- { serverError ACCESS_DENIED } +" +$CLICKHOUSE_CLIENT -q "SELECT count() FROM system.tables WHERE database = '${db}' AND name = 't_backup'" + +# The hosts create the tables with no user, so the initiator must authorize the destination. +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + RESTORE TABLE ${db}.t_backup AS ${db}.t_backup_cluster ON CLUSTER test_shard_localhost FROM ${backup} + SETTINGS structure_only = 1 FORMAT Null + -- { serverError ACCESS_DENIED } +" +$CLICKHOUSE_CLIENT -q "SELECT count() FROM system.tables WHERE database = '${db}' AND name = 't_backup_cluster'" + +# A temporary table keeps its engine, even with `restore_replace_external_engines_to_null`, so the same check applies. +# Temporary tables live in a session, so the table is created and backed up in one client. +$CLICKHOUSE_CLIENT -m -q " + CREATE TEMPORARY TABLE t_tmp_backup (x UInt8) ENGINE = S3Queue('${src}', 'clickhouse', 'clickhouse', 'CSV') + SETTINGS mode = 'unordered', keeper_path = '${keeper_path}_tmp_backup', after_processing = 'move', + after_processing_move_uri = '${dst}', ${keys}; + BACKUP TEMPORARY TABLE t_tmp_backup TO ${backup_tmp} FORMAT Null; +" +$CLICKHOUSE_CLIENT -q "GRANT CREATE TEMPORARY TABLE ON *.* TO ${user_create}" +$CLICKHOUSE_CLIENT --user "${user_create}" -m -q " + RESTORE TEMPORARY TABLE t_tmp_backup FROM ${backup_tmp} SETTINGS structure_only = 1 FORMAT Null; -- { serverError ACCESS_DENIED } + RESTORE TEMPORARY TABLE t_tmp_backup FROM ${backup_tmp} + SETTINGS structure_only = 1, restore_replace_external_engines_to_null = 1 FORMAT Null; -- { serverError ACCESS_DENIED } + SELECT count() FROM system.tables WHERE is_temporary AND name = 't_tmp_backup'; +" + +echo "-- CREATE AS and CREATE MATERIALIZED VIEW with WRITE on S3" + +# The same queries succeed with the grant, so `WRITE` on `S3` was the only missing one. +$CLICKHOUSE_CLIENT -q "GRANT WRITE ON S3 TO ${user_create}" +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + CREATE TABLE ${db}.t_as AS ${db}.t + SETTINGS keeper_path = '${keeper_path}_as', after_processing = 'move', after_processing_move_uri = '${dst}', ${keys} +" && echo "CREATE AS with WRITE on S3: created" +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + CREATE MATERIALIZED VIEW ${db}.mv ENGINE = S3Queue('${src}', 'clickhouse', 'clickhouse', 'CSV') + SETTINGS mode = 'unordered', keeper_path = '${keeper_path}_mv', after_processing = 'move', + after_processing_move_uri = '${dst}', ${keys} + AS SELECT 1::UInt8 AS x +" && echo "materialized view with WRITE on S3: created" +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + RESTORE TABLE ${db}.t_backup FROM ${backup} SETTINGS structure_only = 1 FORMAT Null +" && echo "RESTORE with WRITE on S3: restored" +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + RESTORE TABLE ${db}.t_backup AS ${db}.t_backup_cluster ON CLUSTER test_shard_localhost FROM ${backup} + SETTINGS structure_only = 1 FORMAT Null +" && echo "RESTORE ON CLUSTER with WRITE on S3: restored" +$CLICKHOUSE_CLIENT --user "${user_create}" --distributed_ddl_output_mode none -q " + CREATE TABLE ${db}.t_create_cluster ON CLUSTER test_shard_localhost (x UInt8) ENGINE = S3Queue('${src}', 'clickhouse', 'clickhouse', 'CSV') + SETTINGS mode = 'unordered', keeper_path = '${keeper_path}_create_cluster', after_processing = 'move', + after_processing_move_uri = '${dst}', ${keys} +" && echo "CREATE ON CLUSTER with WRITE on S3: created" +$CLICKHOUSE_CLIENT --user "${user_create}" --distributed_ddl_output_mode none -q " + CREATE TABLE ${db}.t_as_cluster_new ON CLUSTER test_shard_localhost AS ${db}.t + SETTINGS keeper_path = '${keeper_path}_as_cluster_new', after_processing = 'move', after_processing_move_uri = '${dst}', ${keys} +" && echo "CREATE AS ON CLUSTER with WRITE on S3: created" +$CLICKHOUSE_CLIENT --user "${user_create}" --send_logs_level=error -q " + ATTACH TABLE ${db}.t_attach UUID '${attach_uuid}' (x UInt8) ENGINE = S3Queue('${src}', 'clickhouse', 'clickhouse', 'CSV') + SETTINGS mode = 'unordered', keeper_path = '${keeper_path}_attach', after_processing = 'move', + after_processing_move_uri = '${dst}', ${keys} +" && echo "ATTACH with WRITE on S3: attached" +$CLICKHOUSE_CLIENT --user "${user_create}" -m -q " + RESTORE TEMPORARY TABLE t_tmp_backup FROM ${backup_tmp} SETTINGS structure_only = 1 FORMAT Null; + SELECT 'RESTORE TEMPORARY with WRITE on S3: restored', count() FROM system.tables WHERE is_temporary AND name = 't_tmp_backup'; +" + +$CLICKHOUSE_CLIENT -q " + DROP TABLE IF EXISTS ${db}.t_alias SYNC; + DROP TABLE IF EXISTS ${db}.t SYNC; + DROP TABLE IF EXISTS ${db}.t_create SYNC; + DROP TABLE IF EXISTS ${db}.t_as SYNC; + DROP TABLE IF EXISTS ${db}.t_as_cluster SYNC; + DROP TABLE IF EXISTS ${db}.t_as_cluster_new SYNC; + DROP TABLE IF EXISTS ${db}.t_create_cluster SYNC; + DROP TABLE IF EXISTS ${db}.t_attach SYNC; + DROP TABLE IF EXISTS ${db}.t_backup SYNC; + DROP TABLE IF EXISTS ${db}.t_backup_cluster SYNC; + DROP TABLE IF EXISTS ${db}.mv SYNC; + DROP ROW POLICY IF EXISTS p_${db} ON ${db}.t; + DROP USER IF EXISTS ${user_alter}, ${user_create}; +" From b1965c2262d7752df4596196a100f21773cf8fbe Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 3 Nov 2026 12:22:59 +0000 Subject: [PATCH 073/219] Backport #120240 to 26.8: Fix ALTER QUOTA silently not applying when switching to KEYED BY client_key --- src/Access/AccessControl.cpp | 9 +--- src/Access/ContextAccess.cpp | 4 ++ src/Access/EnabledQuota.cpp | 17 ++++++ src/Access/EnabledQuota.h | 9 ++++ src/Access/QuotaCache.cpp | 26 ++++----- src/Access/QuotaCache.h | 7 ++- ...8_quota_client_key_stale_context.reference | 12 +++++ .../05218_quota_client_key_stale_context.sh | 54 +++++++++++++++++++ 8 files changed, 111 insertions(+), 27 deletions(-) create mode 100644 tests/queries/0_stateless/05218_quota_client_key_stale_context.reference create mode 100755 tests/queries/0_stateless/05218_quota_client_key_stale_context.sh diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index c869445e76824..5fe6573b5714f 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -859,7 +859,7 @@ std::shared_ptr AccessControl::getEnabledQuota( const String & forwarded_address, const String & custom_quota_key) const { - return quota_cache->getEnabledQuota(user_id, user_name, enabled_roles, address, forwarded_address, custom_quota_key, true); + return quota_cache->getEnabledQuota(user_id, user_name, enabled_roles, address, forwarded_address, custom_quota_key); } std::shared_ptr AccessControl::getAuthenticationQuota( @@ -872,18 +872,13 @@ std::shared_ptr AccessControl::getAuthenticationQuota( const auto new_current_roles = user->granted_roles.findGranted(user->default_roles); const auto roles_info = getEnabledRolesInfo(new_current_roles, {}); - // client_key is not received at the moment of authentication during TCP connection - // if key type is set to QuotaKeyType::CLIENT_KEY - // QuotaCache::QuotaInfo::calculateKey will throw exception without throw_if_client_key_empty = false String quota_key; - bool throw_if_client_key_empty = false; return quota_cache->getEnabledQuota(*user_id, user->getName(), roles_info->enabled_roles, std::make_shared(address), forwarded_address, - quota_key, - throw_if_client_key_empty); + quota_key); } return nullptr; } diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index c014ea12a7a46..15d1e49dd815e 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -614,6 +614,10 @@ std::shared_ptr ContextAccess::getQuota() const } } + /// Outside the branch above on purpose: `QuotaCache` refreshes the quota set in place, so an already + /// memoised object has to be re-checked on every call. + enabled_quota->checkClientKeySupplied(); + return enabled_quota; } diff --git a/src/Access/EnabledQuota.cpp b/src/Access/EnabledQuota.cpp index 8fa463a890766..d44357445b872 100644 --- a/src/Access/EnabledQuota.cpp +++ b/src/Access/EnabledQuota.cpp @@ -14,6 +14,7 @@ namespace DB namespace ErrorCodes { extern const int QUOTA_EXCEEDED; + extern const int QUOTA_REQUIRES_CLIENT_KEY; } @@ -451,6 +452,22 @@ void EnabledQuota::checkExceeded(QuotaType quota_type) const } +void EnabledQuota::checkClientKeySupplied() const +{ + auto loaded = quotas.load(); + for (const auto & quota : *loaded) + { + if (!quota->requires_client_key) + continue; + throw Exception( + ErrorCodes::QUOTA_REQUIRES_CLIENT_KEY, + "Quota {} (for user {}) requires a client supplied key.", + quota->intervals->quota_name, + getUserName()); + } +} + + void EnabledQuota::checkExceededForQuery(UInt64 normalized_query_hash, QuotaType quota_type) const { if (empty) diff --git a/src/Access/EnabledQuota.h b/src/Access/EnabledQuota.h index 431afed6ad616..ccf68d6c2a625 100644 --- a/src/Access/EnabledQuota.h +++ b/src/Access/EnabledQuota.h @@ -73,6 +73,10 @@ class EnabledQuota : public boost::noncopyable void checkExceeded() const; void checkExceeded(QuotaType quota_type) const; + /// Throws if a governing quota is keyed by `client_key` and this context supplied none. Authentication + /// is metered before a client key can exist, so only the query path calls this. + void checkClientKeySupplied() const; + /// Same as `checkExceeded(quota_type)`, but for `NORMALIZED_QUERY_HASH` quotas the check is /// performed against the intervals resolved for `normalized_query_hash`. void checkExceededForQuery(UInt64 normalized_query_hash, QuotaType quota_type) const; @@ -139,6 +143,11 @@ class EnabledQuota : public boost::noncopyable /// Non-null only for `NORMALIZED_QUERY_HASH` quotas: resolves intervals per query hash. IntervalResolver interval_resolver; + /// This quota is keyed by `client_key` and the context supplied none, so it cannot be metered. + /// Published with the set, so it is refreshed in place whenever quota definitions change; + /// see `checkClientKeySupplied`. + bool requires_client_key = false; + /// Cache of resolved intervals per normalized query hash. mutable std::mutex resolved_intervals_mutex; mutable HashMap> resolved_intervals_cache; diff --git a/src/Access/QuotaCache.cpp b/src/Access/QuotaCache.cpp index 70d878f2bbc29..4a1a84a0b5215 100644 --- a/src/Access/QuotaCache.cpp +++ b/src/Access/QuotaCache.cpp @@ -26,7 +26,6 @@ namespace DB { namespace ErrorCodes { - extern const int QUOTA_REQUIRES_CLIENT_KEY; extern const int LOGICAL_ERROR; } @@ -40,7 +39,7 @@ void QuotaCache::QuotaInfo::setQuota(const QuotaPtr & quota_, const UUID & quota } -String QuotaCache::QuotaInfo::calculateKey(const EnabledQuota & enabled, bool throw_if_client_key_empty) const +String QuotaCache::QuotaInfo::calculateKey(const EnabledQuota & enabled) const { const auto & params = enabled.params; auto mask_address = [this](const Poco::Net::IPAddress & addr) -> String @@ -128,14 +127,9 @@ String QuotaCache::QuotaInfo::calculateKey(const EnabledQuota & enabled, bool th { if (!params.client_key.empty()) return params.client_key; - - if (throw_if_client_key_empty) - throw Exception( - ErrorCodes::QUOTA_REQUIRES_CLIENT_KEY, - "Quota {} (for user {}) requires a client supplied key.", - quota->getName(), - params.user_name); - return ""; // Authentication quota has no client key at time of authentication. + /// An empty key is a legitimate state: at the time of authentication no client key exists yet. + /// The requirement is published as `SingleQuota::requires_client_key` and enforced on the query path. + return ""; } case QuotaKeyType::CLIENT_KEY_OR_USER_NAME: { @@ -256,8 +250,7 @@ std::shared_ptr QuotaCache::getEnabledQuota( const boost::container::flat_set & enabled_roles, const std::shared_ptr & client_address, const String & forwarded_address, - const String & client_key, - bool throw_if_client_key_empty) + const String & client_key) { std::lock_guard lock{mutex}; ensureAllQuotasRead(); @@ -280,7 +273,7 @@ std::shared_ptr QuotaCache::getEnabledQuota( auto res = std::shared_ptr(new EnabledQuota(params)); enabled_quotas.emplace(std::move(params), res); - chooseQuotaToConsumeFor(*res, throw_if_client_key_empty); + chooseQuotaToConsumeFor(*res); return res; } @@ -370,7 +363,7 @@ void QuotaCache::chooseQuotaToConsume() i = enabled_quotas.erase(i); else { - chooseQuotaToConsumeFor(*elem, true); + chooseQuotaToConsumeFor(*elem); ++i; } } @@ -384,7 +377,7 @@ void QuotaCache::chooseQuotaToConsume() LOG_TRACE(getLogger("QuotaCache"), "Re-chose quotas for {} enabled set(s) over {} quotas in {} ms", enabled_quotas.size(), all_quotas.size(), elapsed_ms); } -void QuotaCache::chooseQuotaToConsumeFor(EnabledQuota & enabled, bool throw_if_client_key_empty) +void QuotaCache::chooseQuotaToConsumeFor(EnabledQuota & enabled) { /// `mutex` is already locked. @@ -396,9 +389,10 @@ void QuotaCache::chooseQuotaToConsumeFor(EnabledQuota & enabled, bool throw_if_c if (!info.roles->match(enabled.params.user_id, enabled.params.enabled_roles)) continue; - String key = info.calculateKey(enabled, throw_if_client_key_empty); + String key = info.calculateKey(enabled); auto single = std::make_unique(); single->intervals = info.getOrBuildIntervals(key); + single->requires_client_key = (info.quota->key_type == QuotaKeyType::CLIENT_KEY) && enabled.params.client_key.empty(); /// For NORMALIZED_QUERY_HASH keyed quotas, set up a resolver callback /// so that EnabledQuota can lazily resolve intervals per query hash. diff --git a/src/Access/QuotaCache.h b/src/Access/QuotaCache.h index 7880049c13064..086b9e44d318d 100644 --- a/src/Access/QuotaCache.h +++ b/src/Access/QuotaCache.h @@ -28,8 +28,7 @@ class QuotaCache const boost::container::flat_set & enabled_roles, const std::shared_ptr & address, const String & forwarded_address, - const String & client_key, - bool throw_if_client_key_empty); + const String & client_key); std::vector getAllQuotasUsage() const; @@ -44,7 +43,7 @@ class QuotaCache QuotaInfo(const QuotaPtr & quota_, const UUID & quota_id_) { setQuota(quota_, quota_id_); } void setQuota(const QuotaPtr & quota_, const UUID & quota_id_); - String calculateKey(const EnabledQuota & enabled_quota, bool throw_if_client_key_empty) const; + String calculateKey(const EnabledQuota & enabled_quota) const; boost::shared_ptr getOrBuildIntervals(const String & key); boost::shared_ptr rebuildIntervals(const String & key, std::chrono::system_clock::time_point current_time); void rebuildAllIntervals(); @@ -60,7 +59,7 @@ class QuotaCache void quotaRemoved(const UUID & quota_id) TSA_REQUIRES(mutex); void chooseQuotaToConsumeIfNeeded() TSA_REQUIRES(mutex); void chooseQuotaToConsume() TSA_REQUIRES(mutex); - void chooseQuotaToConsumeFor(EnabledQuota & enabled_quota, bool throw_if_client_key_empty) TSA_REQUIRES(mutex); + void chooseQuotaToConsumeFor(EnabledQuota & enabled_quota) TSA_REQUIRES(mutex); const AccessControl & access_control; mutable std::mutex mutex; diff --git a/tests/queries/0_stateless/05218_quota_client_key_stale_context.reference b/tests/queries/0_stateless/05218_quota_client_key_stale_context.reference new file mode 100644 index 0000000000000..3fce02c191074 --- /dev/null +++ b/tests/queries/0_stateless/05218_quota_client_key_stale_context.reference @@ -0,0 +1,12 @@ +1. warm two cached quota contexts of the same user, without and with a quota key +u_05218 1000 +u_05218 1000 +2. a tightening that keeps the key type reaches the warm context +u_05218 900 +3. switching the key type to client_key must reach both warm contexts +QUOTA_REQUIRES_CLIENT_KEY +kb 800 +4. a user whose context was never cached is rejected the same way +QUOTA_REQUIRES_CLIENT_KEY +5. reverting the key type lets the never-detached context resume +u_05218 700 diff --git a/tests/queries/0_stateless/05218_quota_client_key_stale_context.sh b/tests/queries/0_stateless/05218_quota_client_key_stale_context.sh new file mode 100755 index 0000000000000..9589c358e97dc --- /dev/null +++ b/tests/queries/0_stateless/05218_quota_client_key_stale_context.sh @@ -0,0 +1,54 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# Quotas and users are server-global, so every name is suffixed with the (unique) database name to +# keep the test isolated when it runs in parallel with itself (e.g. in the flaky check). +user="u_05218_${CLICKHOUSE_DATABASE}" +control_user="uctl_05218_${CLICKHOUSE_DATABASE}" +quota="q_05218_${CLICKHOUSE_DATABASE}" + +# Server logs must not reach any client here: a quota recompute that fails is logged at error level +# from the ALTER QUOTA query's own thread, and clickhouse-test fails a shell test on any stderr. +CLICKHOUSE_CLIENT=$(echo "${CLICKHOUSE_CLIENT}" | sed "s/--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL}/--send_logs_level=none/") + +${CLICKHOUSE_CLIENT} -q "DROP QUOTA IF EXISTS ${quota}" +${CLICKHOUSE_CLIENT} -q "DROP USER IF EXISTS ${user}, ${control_user}" + +${CLICKHOUSE_CLIENT} -q "CREATE USER ${user}, ${control_user}" +${CLICKHOUSE_CLIENT} -q "GRANT SELECT ON *.* TO ${user}, ${control_user}" +${CLICKHOUSE_CLIENT} -q "GRANT SHOW QUOTAS ON *.* TO ${user}, ${control_user}" +${CLICKHOUSE_CLIENT} -q "CREATE QUOTA ${quota} KEYED BY user_name FOR INTERVAL 100 YEAR MAX queries = 1000 TO ${user}, ${control_user}" + +# Reads back the quota context of the connecting user, which also builds and caches that context. +# Prints the effective quota key and limit, or only the error name when the query is rejected. +usage() { + ${CLICKHOUSE_CLIENT} "$@" -q \ + "SELECT replaceOne(quota_key, '_${CLICKHOUSE_DATABASE}', ''), max_queries FROM system.quota_usage WHERE quota_name = '${quota}'" 2>&1 \ + | tr '\t' ' ' | grep -oE "QUOTA_REQUIRES_CLIENT_KEY|^[[:alnum:]_]+ [[:digit:]]+$" +} + +echo "1. warm two cached quota contexts of the same user, without and with a quota key" +usage --user "${user}" +usage --user "${user}" --quota_key kb + +echo "2. a tightening that keeps the key type reaches the warm context" +${CLICKHOUSE_CLIENT} -q "ALTER QUOTA ${quota} FOR INTERVAL 100 YEAR MAX queries = 900" +usage --user "${user}" + +echo "3. switching the key type to client_key must reach both warm contexts" +${CLICKHOUSE_CLIENT} -q "ALTER QUOTA ${quota} KEYED BY client_key FOR INTERVAL 100 YEAR MAX queries = 800" +usage --user "${user}" +usage --user "${user}" --quota_key kb + +echo "4. a user whose context was never cached is rejected the same way" +usage --user "${control_user}" + +echo "5. reverting the key type lets the never-detached context resume" +${CLICKHOUSE_CLIENT} -q "ALTER QUOTA ${quota} KEYED BY user_name FOR INTERVAL 100 YEAR MAX queries = 700" +usage --user "${user}" + +${CLICKHOUSE_CLIENT} -q "DROP QUOTA ${quota}" +${CLICKHOUSE_CLIENT} -q "DROP USER ${user}, ${control_user}" From 2136309d25cabd0cd06b61a8ffc8eb421d6d5bfd Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 3 Nov 2026 13:08:56 +0000 Subject: [PATCH 074/219] Backport #128525 to 26.8: Fix Iceberg `ADD COLUMN` reusing a nested field id when `last-column-id` is too small --- .../DataLakes/Iceberg/MetadataGenerator.cpp | 13 +- .../DataLakes/Iceberg/SchemaProcessor.cpp | 50 +++++++ .../DataLakes/Iceberg/SchemaProcessor.h | 3 + ..._too_small_last_column_id_128455.reference | 10 ++ ..._column_too_small_last_column_id_128455.sh | 130 ++++++++++++++++++ 5 files changed, 204 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/05480_iceberg_add_column_too_small_last_column_id_128455.reference create mode 100755 tests/queries/0_stateless/05480_iceberg_add_column_too_small_last_column_id_128455.sh diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp index 812a6c62ef038..375d7699b5b89 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp @@ -124,6 +124,16 @@ bool checkValidSchemaEvolution(Poco::Dynamic::Var old_type, Poco::Dynamic::Var n return false; } +/// `last-column-id` written by some writers does not count nested field ids, so the ids of all schemas are taken into account too. +Int32 getLastAssignedFieldId(const Poco::JSON::Object::Ptr & metadata_object) +{ + Int64 result = metadata_object->getValue(Iceberg::f_last_column_id); + auto schemas = metadata_object->getArray(Iceberg::f_schemas); + for (UInt32 i = 0; i < schemas->size(); ++i) + result = std::max(result, Iceberg::IcebergSchemaProcessor::getMaxFieldId(schemas->getObject(i)->getArray(Iceberg::f_fields))); + return static_cast(result); +} + } MetadataGenerator::MetadataGenerator(Poco::JSON::Object::Ptr metadata_object_) @@ -475,12 +485,11 @@ void MetadataGenerator::generateAddColumnMetadata(const String & column_name, Da throw Exception(ErrorCodes::BAD_ARGUMENTS, "Column {} already exists", column_name); } - auto last_column_id = metadata_object->getValue(Iceberg::f_last_column_id); /// The new top-level field takes the next id; nested tuple/array/map /// children (when `type` is complex) take the ids after it via the shared /// `iter`. After getIcebergType `iter` is the max assigned id, which is what /// last-column-id must record so every added child gets a globally unique id. - Int32 new_field_id = last_column_id + 1; + Int32 new_field_id = getLastAssignedFieldId(metadata_object) + 1; Int32 iter = new_field_id; auto new_type = Iceberg::getIcebergType(type, iter); metadata_object->set(Iceberg::f_last_column_id, iter); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp index 5978dc1a41851..ba9f49a03d8fd 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp @@ -230,6 +230,42 @@ String canonicalizeTypeSpacing(const String & s) return result; } +/// Calls `callback(id, type)` for `field_id` and for the ids of the fields, list elements and map keys/values nested in it. +/// `type` is a JSON string (primitive leaf) or object (complex type) describing `field_id`. +template +void forEachFieldId(const Poco::Dynamic::Var & type, Int64 field_id, Callback && callback) +{ + checkStackSize(); + callback(field_id, type); + if (type.isString()) + return; + + const auto & type_object = type.extract(); + auto type_str = type_object->getValue(f_type); + if (type_str == "map") + { + if (type_object->has(f_key)) + forEachFieldId(type_object->get(f_key), type_object->getValue(f_key_id), callback); + if (type_object->has(f_value)) + forEachFieldId(type_object->get(f_value), type_object->getValue(f_value_id), callback); + } + else if (type_str == "list") + { + if (type_object->has(f_element)) + forEachFieldId(type_object->get(f_element), type_object->getValue(f_element_id), callback); + } + else if (type_str == "struct") + { + auto fields = type_object->getArray(f_fields); + for (UInt32 i = 0; i < fields->size(); ++i) + { + auto field = fields->getObject(i); + if (field->has(f_type)) + forEachFieldId(field->get(f_type), field->getValue(f_id), callback); + } + } +} + /// Compare two Iceberg type descriptors for the same field. A type is either a primitive /// string ("long", "decimal(20, 0)", ...) or a nested object ("struct" with a fields array, /// or "list" / "map" wrappers whose element/key/value members are themselves types). @@ -964,6 +1000,20 @@ std::unordered_map IcebergSchemaProcessor::traverseSchema(Poco::J return result; } +Int64 IcebergSchemaProcessor::getMaxFieldId(Poco::JSON::Array::Ptr schema) +{ + Int64 result = 0; + for (UInt32 i = 0; i < schema->size(); ++i) + { + auto current_object = schema->getObject(i); + if (current_object->has(Iceberg::f_type)) + forEachFieldId( + current_object->get(Iceberg::f_type), current_object->getValue(Iceberg::f_id), + [&](Int64 field_id, const Poco::Dynamic::Var &) { result = std::max(result, field_id); }); + } + return result; +} + std::unordered_set IcebergSchemaProcessor::collectIcebergStringPaths(Poco::JSON::Array::Ptr schema) { std::unordered_set result; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h index e2353b4a6e357..5866405401f18 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h @@ -119,6 +119,9 @@ class IcebergSchemaProcessor static std::unordered_map traverseSchema(Poco::JSON::Array::Ptr schema); + /// The largest field id in `schema`, nested ids included. + static Int64 getMaxFieldId(Poco::JSON::Array::Ptr schema); + /// Paths whose Iceberg logical type is `string` (not `binary`); both read as DataTypeString. static std::unordered_set collectIcebergStringPaths(Poco::JSON::Array::Ptr schema); diff --git a/tests/queries/0_stateless/05480_iceberg_add_column_too_small_last_column_id_128455.reference b/tests/queries/0_stateless/05480_iceberg_add_column_too_small_last_column_id_128455.reference new file mode 100644 index 0000000000000..1351103cad4e9 --- /dev/null +++ b/tests/queries/0_stateless/05480_iceberg_add_column_too_small_last_column_id_128455.reference @@ -0,0 +1,10 @@ +1 [1,2] \N +1 [1,2] \N +2 [3] x +id=1 arr=2 arr.element=3 extra=4 last-column-id=4 +1 {'k':[1,2]} \N +id=1 m=2 m.key=3 m.value=4 m.value.element=5 extra=6 last-column-id=6 +1 +1 \N \N +id=1 c=5 d=6 last-column-id=6 +a=1 a.b=2 a.b=3 c=4 last-column-id=4 diff --git a/tests/queries/0_stateless/05480_iceberg_add_column_too_small_last_column_id_128455.sh b/tests/queries/0_stateless/05480_iceberg_add_column_too_small_last_column_id_128455.sh new file mode 100755 index 0000000000000..37cc7e4c38a7e --- /dev/null +++ b/tests/queries/0_stateless/05480_iceberg_add_column_too_small_last_column_id_128455.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# - no-fasttest: requires IcebergLocal (USE_AVRO build option) + +# Regression test for https://github.com/ClickHouse/ClickHouse/issues/128455 +# Tables created by 26.6 and older record a `last-column-id` that does not count nested field ids. ADD COLUMN must +# not reuse such an id: new ids start after the largest id of any schema (the current one and those of dropped columns). + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +TABLE_A="t_${CLICKHOUSE_DATABASE}_${RANDOM}_a" +TABLE_A_DIR="${USER_FILES_PATH}/${TABLE_A}" +TABLE_B="t_${CLICKHOUSE_DATABASE}_${RANDOM}_b" +TABLE_B_DIR="${USER_FILES_PATH}/${TABLE_B}" +TABLE_C="t_${CLICKHOUSE_DATABASE}_${RANDOM}_c" +TABLE_C_DIR="${USER_FILES_PATH}/${TABLE_C}" +TABLE_D="t_${CLICKHOUSE_DATABASE}_${RANDOM}_d" +TABLE_D_DIR="${USER_FILES_PATH}/${TABLE_D}" +rm -rf "${TABLE_A_DIR}" "${TABLE_B_DIR}" "${TABLE_C_DIR}" "${TABLE_D_DIR}" +cleanup() { + rm -rf "${TABLE_A_DIR}" "${TABLE_B_DIR}" "${TABLE_C_DIR}" "${TABLE_D_DIR}" + for table in ${TABLE_A} ${TABLE_B} ${TABLE_C} ${TABLE_D}; do + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${table}; DROP TABLE IF EXISTS ${table}_reopened" + done +} +trap cleanup EXIT + +# Publishes a new metadata version that differs from the latest one only by `last-column-id`, as 26.6 wrote it. +# A new file rather than an in-place edit, because the server caches parsed metadata files by path. +lower_last_column_id() { + python3 - "$1/metadata" "$2" <<'PY' +import json, os, re, sys +metadata_dir, value = sys.argv[1], int(sys.argv[2]) +latest = max(int(m.group(1)) for f in os.listdir(metadata_dir) if (m := re.fullmatch(r"v(\d+)\.metadata\.json", f))) +with open(os.path.join(metadata_dir, f"v{latest}.metadata.json")) as f: + metadata = json.load(f) +metadata["last-column-id"] = value +metadata["last-updated-ms"] += 1 +tmp = os.path.join(metadata_dir, ".tmp_next") +with open(tmp, "w") as f: + json.dump(metadata, f) +os.rename(tmp, os.path.join(metadata_dir, f"v{latest + 1}.metadata.json")) +PY +} + +# Field ids of the current schema, nested ones included, and `last-column-id` of the latest metadata version. +field_ids() { + python3 - "$1/metadata" <<'PY' +import json, os, re, sys +metadata_dir = sys.argv[1] +latest = max(int(m.group(1)) for f in os.listdir(metadata_dir) if (m := re.fullmatch(r"v(\d+)\.metadata\.json", f))) +with open(os.path.join(metadata_dir, f"v{latest}.metadata.json")) as f: + metadata = json.load(f) +schema = next(s for s in metadata["schemas"] if s["schema-id"] == metadata["current-schema-id"]) +ids = [] +def walk(node, prefix): + if not isinstance(node, dict): + return + if node["type"] == "struct": + for field in node["fields"]: + ids.append((field["id"], prefix + field["name"])) + walk(field["type"], prefix + field["name"] + ".") + elif node["type"] == "list": + ids.append((node["element-id"], prefix + "element")) + walk(node["element"], prefix + "element.") + elif node["type"] == "map": + ids.append((node["key-id"], prefix + "key")) + ids.append((node["value-id"], prefix + "value")) + walk(node["key"], prefix + "key.") + walk(node["value"], prefix + "value.") +walk(schema, "") +print(" ".join(f"{path}={field_id}" for field_id, path in sorted(ids)), f"last-column-id={metadata['last-column-id']}") +PY +} + +# The reproducer from the issue: ids id=1 arr=2 arr.element=3, and 26.6 records last-column-id=2. +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --multiquery -q " +CREATE TABLE ${TABLE_A} (id Int64, arr Array(Int64)) ENGINE = IcebergLocal('${TABLE_A_DIR}/'); +INSERT INTO ${TABLE_A} VALUES (1, [1, 2]); +" +lower_last_column_id "${TABLE_A_DIR}" 2 +# A second table over the same path starts from the lowered value, as a newer server opening an old table does. +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --iceberg_metadata_staleness_ms=0 --multiquery -q " +CREATE TABLE ${TABLE_A}_reopened ENGINE = IcebergLocal('${TABLE_A_DIR}/'); +ALTER TABLE ${TABLE_A}_reopened ADD COLUMN extra Nullable(String); +SELECT * FROM ${TABLE_A}_reopened ORDER BY id; +INSERT INTO ${TABLE_A}_reopened VALUES (2, [3], 'x'); +SELECT * FROM ${TABLE_A}_reopened ORDER BY id; +" +field_ids "${TABLE_A_DIR}" + +# The same with a `Map`: ids id=1 m=2 m.key=3 m.value=4 m.value.element=5. +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --multiquery -q " +CREATE TABLE ${TABLE_D} (id Int64, m Map(String, Array(Int64))) ENGINE = IcebergLocal('${TABLE_D_DIR}/'); +INSERT INTO ${TABLE_D} VALUES (1, map('k', [1, 2])); +" +lower_last_column_id "${TABLE_D_DIR}" 2 +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --iceberg_metadata_staleness_ms=0 --multiquery -q " +CREATE TABLE ${TABLE_D}_reopened ENGINE = IcebergLocal('${TABLE_D_DIR}/'); +ALTER TABLE ${TABLE_D}_reopened ADD COLUMN extra Nullable(String); +SELECT * FROM ${TABLE_D}_reopened ORDER BY id; +" +field_ids "${TABLE_D_DIR}" + +# A dropped column keeps its ids 2, 3, 4 in the first schema and in the data file, so they must not be reused either. +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --multiquery -q " +CREATE TABLE ${TABLE_B} (id Int64, t Tuple(a Int64, b Int64)) ENGINE = IcebergLocal('${TABLE_B_DIR}/'); +INSERT INTO ${TABLE_B} VALUES (1, (10, 20)); +ALTER TABLE ${TABLE_B} DROP COLUMN t; +" +lower_last_column_id "${TABLE_B_DIR}" 2 +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --iceberg_metadata_staleness_ms=0 --multiquery -q " +CREATE TABLE ${TABLE_B}_reopened ENGINE = IcebergLocal('${TABLE_B_DIR}/'); +SELECT * FROM ${TABLE_B}_reopened ORDER BY id; +ALTER TABLE ${TABLE_B}_reopened ADD COLUMN c Nullable(Int64); +ALTER TABLE ${TABLE_B}_reopened ADD COLUMN d Nullable(Int64); +SELECT * FROM ${TABLE_B}_reopened ORDER BY id; +" +field_ids "${TABLE_B_DIR}" + +# A top-level column named `a.b` (id 2) and the field `b` of the tuple `a` (id 3) have the same dotted path. +${CLICKHOUSE_CLIENT} --query "CREATE TABLE ${TABLE_C} (a Tuple(b Int64), \`a.b\` Int64) ENGINE = IcebergLocal('${TABLE_C_DIR}/')" +lower_last_column_id "${TABLE_C_DIR}" 2 +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --iceberg_metadata_staleness_ms=0 --multiquery -q " +CREATE TABLE ${TABLE_C}_reopened ENGINE = IcebergLocal('${TABLE_C_DIR}/'); +ALTER TABLE ${TABLE_C}_reopened ADD COLUMN c Nullable(Int64); +" +field_ids "${TABLE_C_DIR}" From 41ae98b592042071a20bdf530389b28f04c18cfc Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 3 Nov 2026 11:29:16 +0000 Subject: [PATCH 075/219] Backport #128730 to 26.8: Fix crash in `ssd_cache` dictionaries after the cache file fails to grow --- src/Dictionaries/SSDCacheDictionaryStorage.h | 3 +- ...ictionary_cache_file_cannot_grow.reference | 6 ++ ...cache_dictionary_cache_file_cannot_grow.sh | 57 +++++++++++++++++++ 3 files changed, 65 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05488_ssd_cache_dictionary_cache_file_cannot_grow.reference create mode 100755 tests/queries/0_stateless/05488_ssd_cache_dictionary_cache_file_cannot_grow.sh diff --git a/src/Dictionaries/SSDCacheDictionaryStorage.h b/src/Dictionaries/SSDCacheDictionaryStorage.h index 3edd9815022c8..9d8f2c2a48a51 100644 --- a/src/Dictionaries/SSDCacheDictionaryStorage.h +++ b/src/Dictionaries/SSDCacheDictionaryStorage.h @@ -1396,9 +1396,10 @@ class SSDCacheDictionaryStorage final : public ICacheDictionaryStorage if (memory_buffer_partitions.size() < configuration.max_partitions_count) { /// Try tro create next partition without reusing old partitions - ++current_partition_index; file_buffer.allocateSizeForNextPartition(); memory_buffer_partitions.emplace_back(configuration.block_size, configuration.write_buffer_blocks_size); + /// Advance only after both calls succeed, so the index stays valid if one of them throws. + ++current_partition_index; } else { diff --git a/tests/queries/0_stateless/05488_ssd_cache_dictionary_cache_file_cannot_grow.reference b/tests/queries/0_stateless/05488_ssd_cache_dictionary_cache_file_cannot_grow.reference new file mode 100644 index 0000000000000..799f581f2dea1 --- /dev/null +++ b/tests/queries/0_stateless/05488_ssd_cache_dictionary_cache_file_cannot_grow.reference @@ -0,0 +1,6 @@ +d_complex LOADED +d_simple LOADED +1 1 +alive +4 +0 diff --git a/tests/queries/0_stateless/05488_ssd_cache_dictionary_cache_file_cannot_grow.sh b/tests/queries/0_stateless/05488_ssd_cache_dictionary_cache_file_cannot_grow.sh new file mode 100755 index 0000000000000..65b12771f0a81 --- /dev/null +++ b/tests/queries/0_stateless/05488_ssd_cache_dictionary_cache_file_cannot_grow.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-llvm-coverage +# no-fasttest: waits 7 s for the update backoff; no-llvm-coverage: the file size limit below would also cut the coverage profile. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `ssd_cache` dictionaries whose cache file cannot grow for the next partition stay usable after the failed update, +# and the next update fails the same way. The file size limit lets the first 64 KiB of each cache file be allocated, +# but not the next 64 KiB, while 100 values of 1.2 KB need more than that. + +dir="${CLICKHOUSE_USER_FILES_UNIQUE:?}" +rm -rf "${dir:?}" +mkdir -p "$dir" +trap 'rm -rf "${dir:?}"' EXIT + +query=" +CREATE TABLE src (k UInt64, v String) ENGINE = Memory; +INSERT INTO src SELECT number, toString(number) || ':' || repeat('x', 1200) FROM numbers(100); + +CREATE DICTIONARY d_simple (k UInt64, v String DEFAULT '') PRIMARY KEY k +SOURCE(CLICKHOUSE(TABLE 'src')) LIFETIME(MIN 1000 MAX 1000) +LAYOUT(SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 65536 WRITE_BUFFER_SIZE 4096 MAX_PARTITIONS_COUNT 4 PATH '$dir/d_simple')); + +CREATE DICTIONARY d_complex (k UInt64, v String DEFAULT '') PRIMARY KEY k +SOURCE(CLICKHOUSE(TABLE 'src')) LIFETIME(MIN 1000 MAX 1000) +LAYOUT(COMPLEX_KEY_SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 65536 WRITE_BUFFER_SIZE 4096 MAX_PARTITIONS_COUNT 4 PATH '$dir/d_complex')); + +-- Both updates fail, because the cache files cannot grow. +SELECT dictGet('d_simple', 'v', number) FROM numbers(100) FORMAT Null; +SELECT dictGet('d_complex', 'v', tuple(number)) FROM numbers(100) FORMAT Null; + +-- Wait for the 5-6 s backoff after a failed update, so that the next lookups update the dictionaries again. +SELECT sleep(3) FORMAT Null; +SELECT sleep(3) FORMAT Null; +SELECT sleep(1) FORMAT Null; + +SELECT dictGet('d_simple', 'v', number) FROM numbers(100) FORMAT Null; +SELECT dictGet('d_complex', 'v', tuple(number)) FROM numbers(100) FORMAT Null; + +SELECT name, status FROM system.dictionaries ORDER BY name; +SELECT dictGet('d_simple', 'v', toUInt64(0)) = '0:' || repeat('x', 1200), dictGet('d_complex', 'v', tuple(toUInt64(0))) = '0:' || repeat('x', 1200); +SELECT 'alive'; +" + +( + cd "$dir" || exit 1 + ulimit -c 0 + trap '' XFSZ + ulimit -f 100 + $CLICKHOUSE_LOCAL --ignore-error --query "$query" 2>"$dir/stderr" +) + +# Each of the four failed updates reports the failed allocation, and none of them was refused by the backoff. +grep -o -F 'Cannot preallocate space' "$dir/stderr" | wc -l +grep -o -F 'nearest update is scheduled' "$dir/stderr" | wc -l From 7c7a3359beb53bded7507eac3c8badb5e34c091b Mon Sep 17 00:00:00 2001 From: Groene AI <270696204+groeneai@users.noreply.github.com> Date: Tue, 3 Nov 2026 12:50:44 +0000 Subject: [PATCH 076/219] Make 05488 see the failed updates when --ignore-error hides them On this release branch `clickhouse-local --ignore-error` drops the exception of a failed statement without printing it (`LocalServer::processError` returns early). Printing it came with #119572, which is in 26.10 and later only. The four failed updates still happen, but nothing reached stderr, so the test printed 0 instead of 4. With `--send_logs_level=error` each failed query is logged once by `executeQuery`, which restores the count. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit ec6408d6245d1d87c93515b2aac51a6eeac11d53) --- .../05488_ssd_cache_dictionary_cache_file_cannot_grow.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/05488_ssd_cache_dictionary_cache_file_cannot_grow.sh b/tests/queries/0_stateless/05488_ssd_cache_dictionary_cache_file_cannot_grow.sh index 65b12771f0a81..aa8ddefa83721 100755 --- a/tests/queries/0_stateless/05488_ssd_cache_dictionary_cache_file_cannot_grow.sh +++ b/tests/queries/0_stateless/05488_ssd_cache_dictionary_cache_file_cannot_grow.sh @@ -49,7 +49,7 @@ SELECT 'alive'; ulimit -c 0 trap '' XFSZ ulimit -f 100 - $CLICKHOUSE_LOCAL --ignore-error --query "$query" 2>"$dir/stderr" + $CLICKHOUSE_LOCAL --ignore-error --send_logs_level=error --query "$query" 2>"$dir/stderr" ) # Each of the four failed updates reports the failed allocation, and none of them was refused by the backoff. From f54d61cac6839bc218541d9dc7f108b8280514c1 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 3 Nov 2026 17:55:48 +0000 Subject: [PATCH 077/219] Backport #123651 to 26.8: Hide Azure credentials that were logged in clear in `system.query_log` --- .../FunctionSecretArgumentsFinderTreeNode.cpp | 6 + .../FunctionSecretArgumentsFinderTreeNode.h | 3 +- src/Backups/BackupIO_AzureBlobStorage.cpp | 17 +- src/Common/maskURIPassword.h | 65 ++++-- .../AzureBlobStorageCommon.cpp | 22 ++ .../AzureBlobStorage/AzureBlobStorageCommon.h | 2 + src/Parsers/FunctionSecretArgumentsFinder.cpp | 197 +++++++++++++----- src/Parsers/FunctionSecretArgumentsFinder.h | 20 ++ .../ObjectStorage/Azure/Configuration.cpp | 2 +- .../ObjectStorage/Azure/Configuration.h | 2 +- .../ObjectStorageQueue/AzureQueue_fwd.h | 3 +- .../test.py | 67 +++++- .../test_mask_sensitive_info/test.py | 24 +-- ...entials_hidden_in_logged_queries.reference | 39 ++++ ...re_credentials_hidden_in_logged_queries.sh | 62 ++++++ 15 files changed, 436 insertions(+), 95 deletions(-) create mode 100644 tests/queries/0_stateless/05458_azure_credentials_hidden_in_logged_queries.reference create mode 100755 tests/queries/0_stateless/05458_azure_credentials_hidden_in_logged_queries.sh diff --git a/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.cpp b/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.cpp index 345fbe80a8659..cc0b3797077ee 100644 --- a/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.cpp +++ b/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.cpp @@ -61,6 +61,12 @@ void forEachSecretArgumentNode( continue; } + if (const auto masked = secret_arguments.masked_arguments.find(n); masked != secret_arguments.masked_arguments.end()) + { + on_secret(n, masked->second ? secretValueSlot(arguments[n]) : arguments[n]); + continue; + } + if (secret_arguments.isSecretArgument(n)) on_secret(n, secretValueSlot(arguments[n])); } diff --git a/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.h b/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.h index 6afdd5d3a952b..ecd7322afc29b 100644 --- a/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.h +++ b/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.h @@ -122,7 +122,8 @@ using FunctionSecretArgumentsFinderTreeNode = FunctionSecretArgumentsFinderTreeN using TableFunctionSecretArgumentsFinderTreeNode = FunctionSecretArgumentsFinderTreeNodeImpl; /// Visits the secret value slots selected by a finder result in a resolved argument list, for the -/// query-tree surfaces (`EXPLAIN QUERY TREE`, projection names): the span members and the arguments +/// query-tree surfaces (`EXPLAIN QUERY TREE`, projection names): the span members, the individually +/// masked arguments (whole, unless masked as a named `key = value`) and the arguments /// with a partial replacement (for a `key = value` argument, its value; a tree dump cannot represent /// partial masking, so the whole value is masked: fail closed), and the values of the nested secret /// maps (`headers(..)` / `extra_credentials(..)`; a malformed child is visited whole). The callback diff --git a/src/Backups/BackupIO_AzureBlobStorage.cpp b/src/Backups/BackupIO_AzureBlobStorage.cpp index 5b6112e9f80f7..49d1ce41f4bbc 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.cpp +++ b/src/Backups/BackupIO_AzureBlobStorage.cpp @@ -2,6 +2,7 @@ #include #if USE_AZURE_BLOB_STORAGE +#include #include #include #include @@ -66,6 +67,14 @@ namespace }; } + /// An Azure disk configured with `connection_string` is described by that whole string. + String maskedDescription(const DataSourceDescription & description) + { + String result = description.description; + maskAzureConnectionStringSecrets(result); + return result; + } + #if CLICKHOUSE_CLOUD /// The recorded endpoint of a snapshot's source disk as a service URL: scheme, host, port and path, without /// query parameters and trailing slashes. A source disk that authenticates with a SAS records it in the @@ -149,7 +158,7 @@ BackupReaderAzureBlobStorage::BackupReaderAzureBlobStorage( const WriteSettings & write_settings_, const ContextPtr & context_) : BackupReaderDefault(read_settings_, write_settings_, getLogger("BackupReaderAzureBlobStorage")) - , data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::Azure, MetadataStorageType::None, connection_params_.getConnectionURL(), false, false, ""} + , data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::Azure, MetadataStorageType::None, connection_params_.getConnectionURLWithoutQuery(), false, false, ""} , connection_params(connection_params_) , blob_path(blob_path_) { @@ -202,7 +211,7 @@ void BackupReaderAzureBlobStorage::copyFileToDisk(const String & path_in_backup, DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) { auto destination_data_source_description = destination_disk->getDataSourceDescription(); - LOG_TRACE(log, "Source description {}, destination description {}", data_source_description.description, destination_data_source_description.description); + LOG_TRACE(log, "Source description {}, destination description {}", data_source_description.description, maskedDescription(destination_data_source_description)); if (destination_data_source_description.object_storage_type == ObjectStorageType::Azure && destination_data_source_description.is_encrypted == encrypted_in_backup) { @@ -249,7 +258,7 @@ BackupWriterAzureBlobStorage::BackupWriterAzureBlobStorage( const ContextPtr & context_, bool attempt_to_create_container) : BackupWriterDefault(read_settings_, write_settings_, getLogger("BackupWriterAzureBlobStorage")) - , data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::Azure, MetadataStorageType::None, connection_params_.getConnectionURL(), false, false, ""} + , data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::Azure, MetadataStorageType::None, connection_params_.getConnectionURLWithoutQuery(), false, false, ""} , connection_params(connection_params_) , blob_path(blob_path_) { @@ -279,7 +288,7 @@ void BackupWriterAzureBlobStorage::copyFileFromDisk( { /// Use the native copy as a more optimal way to copy a file from AzureBlobStorage to AzureBlobStorage if it's possible. auto source_data_source_description = src_disk->getDataSourceDescription(); - LOG_TRACE(log, "Source description {}, destination description {}", source_data_source_description.description, data_source_description.description); + LOG_TRACE(log, "Source description {}, destination description {}", maskedDescription(source_data_source_description), data_source_description.description); if (source_data_source_description.object_storage_type == ObjectStorageType::Azure && source_data_source_description.is_encrypted == copy_encrypted) { diff --git a/src/Common/maskURIPassword.h b/src/Common/maskURIPassword.h index b49fd22240075..81559987db593 100644 --- a/src/Common/maskURIPassword.h +++ b/src/Common/maskURIPassword.h @@ -1,33 +1,66 @@ #pragma once +#include #include #include #include +#include namespace DB { -/** Mask the value of `=` in a connection string, where the value runs up to the next - * ';' or to the end of the string. Only the first occurrence is masked, and nothing is masked if - * the key is absent. - * - * This replaces the regular expressions `AccountKey=.*?(;|$)` and `SharedAccessSignature=.*?(;|$)` - * that used to be applied here, so that masking a secret does not require a regex engine. +/** Hides the secrets of an Azure connection string as the Azure SDK reads it (`Azure::Storage::_internal::ParseConnectionString`, + * `contrib/azure/sdk/storage/azure-storage-common/src/storage_credential.cpp`): a key runs to '=' and its value to ';', keys + * match exactly, and the last copy of a repeated key wins. Splitting at every ';' first can only hide more than the SDK reads. + * - secret keys: `AccountName=a;AccountKey=K;SharedAccessSignature=S` -> `AccountName=a;AccountKey=[HIDDEN];SharedAccessSignature=[HIDDEN]` + * - repeated keys: `AccountKey=K1;AccountKey=K2` -> `AccountKey=[HIDDEN];AccountKey=[HIDDEN]` + * - an endpoint is a url, so a query in it is a SAS: `BlobEndpoint=https://h/?sig=S` or `BlobEndpoint=https://u@h/` -> `BlobEndpoint=[HIDDEN]` + * - the whole string is hidden when a '?', '#' or '@' is still shown, or when it has one and holds a url outside the pair + * values (':' or '/' in a key or in a part without '='; no key has them): `https://h/?sig=S` or `https://u:p=s@h/` -> `[HIDDEN]` + * Returns whether anything was hidden. */ -inline bool maskConnectionStringKey(std::string & str, std::string_view key_with_eq) +inline bool maskAzureConnectionStringSecrets(std::string & str) { - size_t key_position = str.find(key_with_eq); - if (key_position == std::string::npos) - return false; + static constexpr std::array secret_keys = {"AccountKey", "SharedAccessSignature"}; + auto is_secret_key = [](std::string_view key) { return std::ranges::find(secret_keys, key) != secret_keys.end(); }; + + const bool has_url_delimiters = str.find_first_of("?#@") != std::string::npos; + bool url_outside_values = false; + std::string result; + size_t copied = 0; + bool masked = false; + for (size_t part_begin = 0; part_begin <= str.length();) + { + const size_t part_end = std::min(str.find(';', part_begin), str.length()); + const std::string_view part = std::string_view(str).substr(part_begin, part_end - part_begin); + const size_t equals = part.find('='); + if (part.substr(0, equals).find_first_of(":/") != std::string_view::npos) + url_outside_values = true; + if (equals != std::string_view::npos + && (is_secret_key(part.substr(0, equals)) || part.substr(equals + 1).find_first_of("?@") != std::string_view::npos)) + { + const size_t value_begin = part_begin + equals + 1; + result.append(str, copied, value_begin - copied); + result.append("[HIDDEN]"); + copied = part_end; + masked = true; + } + part_begin = part_end + 1; + } - size_t value_begin = key_position + key_with_eq.length(); - size_t value_end = str.find(';', value_begin); - if (value_end == std::string::npos) - value_end = str.length(); + if (masked) + { + result.append(str, copied, std::string::npos); + str = std::move(result); + } - str.replace(value_begin, value_end - value_begin, "[HIDDEN]"); - return true; + if (str.find_first_of("?#@") != std::string::npos || (url_outside_values && has_url_delimiters)) + { + str = "[HIDDEN]"; + return true; + } + return masked; } /** Replace the password in a URI of the form `scheme://user:password@host` with `[HIDDEN]`. diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp index 5729df511e008..bf73ed4f56bfa 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp @@ -228,6 +228,28 @@ String ConnectionParams::getConnectionURL() const return endpoint.storage_account_url; } +String ConnectionParams::getConnectionURLWithoutQuery() const +{ + if (std::holds_alternative(auth_method)) + { + try + { + /// `ParseConnectionString` appends the SAS of the connection string to `BlobServiceUrl` as its query. + auto url = Azure::Storage::_internal::ParseConnectionString(endpoint.storage_account_url).BlobServiceUrl; + const auto query = url.GetQueryParameters(); + for (const auto & [key, value] : query) + url.RemoveQueryParameter(key); + return url.GetAbsoluteUrl(); + } + catch (const std::logic_error & e) + { + translateAzureSdkParseError(e); + } + } + + return endpoint.storage_account_url.substr(0, endpoint.storage_account_url.find_first_of("?#")); +} + std::unique_ptr ConnectionParams::createForService() const { try diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h b/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h index f4a6a896102c3..399183f094531 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h @@ -149,6 +149,8 @@ struct ConnectionParams String getContainer() const { return endpoint.container_name; } String getConnectionURL() const; + /// Without the query, which can be a SAS, for where the url only names the endpoint (logs, errors, cache keys). + String getConnectionURLWithoutQuery() const; std::unique_ptr createForService() const; std::unique_ptr createForContainer() const; diff --git a/src/Parsers/FunctionSecretArgumentsFinder.cpp b/src/Parsers/FunctionSecretArgumentsFinder.cpp index 51e2bc8387edf..90d584a1d0047 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.cpp +++ b/src/Parsers/FunctionSecretArgumentsFinder.cpp @@ -66,6 +66,16 @@ namespace return AzureConnectionValue::PlainStorageAccountURL; } + /// The keys an Azure named collection accepts (`StorageAzureConfiguration`). An override with any + /// other key, which can name any credential, is rejected only after the statement is logged. + bool isAzureCollectionKey(std::string_view key) + { + static const std::unordered_set keys = {"blob_path", "container", "format", "compression", "structure", + "compression_method", "account_name", "account_key", "connection_string", "storage_account_url", "partition_strategy", + "partition_columns_in_data_file", "client_id", "tenant_id"}; + return keys.contains(key); + } + /// The backup engines whose locator names a destination with no credential in it, each with the /// argument count it accepts. `BackupFactory` registers exactly these plus `S3` and `AzureBlobStorage`. std::optional credentialFreeBackupEngineArity(const String & engine_name) @@ -626,48 +636,143 @@ void FunctionSecretArgumentsFinder::findAzureBlobStorageFunctionSecretArguments( /// azureBlobStorageCluster('cluster_name', 'conn_string/storage_account_url', ...) has 'conn_string/storage_account_url' as its second argument. size_t url_arg_idx = is_cluster_function ? 1 : 0; + maskAzureExtraCredentials(); + + /// The other values are hidden before a connection string is: hiding that replaces its whole argument. if (!is_cluster_function && isNamedCollectionName(0)) { /// azureBlobStorage(named_collection, ..., account_key = 'account_key', ...) + maskPositionalsAfterAzureCollection(1); + maskUnreadAzureCollectionOverrides(1); + findSecretNamedArgument("account_key", 1); if (maskAzureConnectionString(-1, true, 1)) return; maskUnshowableAzureConnectionOverrides(1); - findSecretNamedArgument("account_key", 1); return; } if (is_cluster_function && isNamedCollectionName(1)) { /// azureBlobStorageCluster(cluster, named_collection, ..., account_key = 'account_key', ...) + maskPositionalsAfterAzureCollection(2); + maskUnreadAzureCollectionOverrides(2); + findSecretNamedArgument("account_key", 2); if (maskAzureConnectionString(-1, true, 2)) return; maskUnshowableAzureConnectionOverrides(2); - findSecretNamedArgument("account_key", 2); return; } - if (maskAzureConnectionString(url_arg_idx)) + /// Slot `n` is the raw argument `arg(n)`; past the end it is an index no argument has. + const auto slot_arguments = azureSlotArguments(url_arg_idx); + auto arg = [&](size_t slot) { return slot < slot_arguments.size() ? slot_arguments[slot] : function->arguments->size(); }; + + if (const auto credential_arg_idx = azureCredentialArgument(slot_arguments, url_arg_idx)) + markSecretArgument(*credential_arg_idx); + + if (maskAzureConnectionString(arg(url_arg_idx))) return; - if (url_arg_idx < function->arguments->size()) - maskUnshowableAzureConnectionValue(url_arg_idx, *function->arguments->at(url_arg_idx), /* argument_is_named= */ false); + if (arg(url_arg_idx) < function->arguments->size()) + maskUnshowableAzureConnectionValue(arg(url_arg_idx), *function->arguments->at(arg(url_arg_idx)), /* argument_is_named= */ false); +} + +void FunctionSecretArgumentsFinder::maskAzureExtraCredentials() +{ + for (size_t i = 0, size = function->arguments->size(); i < size; ++i) + { + if (const auto f = function->arguments->at(i)->getFunction(); f && f->name() == "extra_credentials") + markSecretArgument(i); + } +} + +std::optional FunctionSecretArgumentsFinder::tryGetAzureKey(const AbstractFunction & equals) +{ + String key; + if (equals.arguments && equals.arguments->size() == 2 && tryGetStringFromArgument(*equals.arguments->at(0), &key) + && isAzureCollectionKey(key)) + return key; + return {}; +} - /// We should check other arguments first because we don't need to do any replacement in case of - /// azureBlobStorage(connection_string|storage_account_url, container_name, blobpath, format) -- in this case there is no account_key argument - /// azureBlobStorageCluster(cluster, connection_string|storage_account_url, container_name, blobpath, format) -- in this case there is no account_key argument - size_t count = function->arguments->size(); - if ((url_arg_idx + 4 <= count) && (count <= url_arg_idx + 7)) +void FunctionSecretArgumentsFinder::maskUnreadAzureCollectionOverrides(size_t start) +{ + std::vector connection_overrides; + for (size_t i = start, size = function->arguments->size(); i < size; ++i) { - String fourth_arg; - if (tryGetStringFromArgument(url_arg_idx + 3, &fourth_arg)) + const auto f = function->arguments->at(i)->getFunction(); + if (!f || f->name() != "equals") + continue; + const auto key = tryGetAzureKey(*f); + if (!key) + markSecretArgument(i); + else if (!(tryGetStringFromArgument(*f->arguments->at(1), nullptr) || f->arguments->at(1)->tryGetLiteralText(nullptr))) + markSecretArgument(i, /* argument_is_named= */ true); + if (key == "connection_string" || key == "storage_account_url") + connection_overrides.push_back(i); + } + + if (connection_overrides.size() > 1) + for (const size_t i : connection_overrides) + markSecretArgument(i, /* argument_is_named= */ true); +} + +std::vector FunctionSecretArgumentsFinder::azureSlotArguments(size_t start) +{ + std::vector slots; + size_t extra_credentials = 0; + for (size_t i = 0, size = function->arguments->size(); i < size; ++i) + { + const auto argument = function->arguments->at(i); + if (argument->isSettings()) + continue; + const auto f = i < start ? nullptr : argument->getFunction(); + if (f && f->name() == "extra_credentials") { - if (fourth_arg == "auto" || KnownFormatNames::instance().exists(fourth_arg)) - return; + ++extra_credentials; + continue; + } + if (f && f->name() == "equals" && f->arguments && f->arguments->size() == 2) + { + const auto key = tryGetAzureKey(*f); + if (!key) + markSecretArgument(i); + else if ((key != "partition_strategy" && key != "partition_columns_in_data_file") || !f->arguments->at(1)->tryGetLiteralText(nullptr)) + markSecretArgument(i, /* argument_is_named= */ true); + continue; } + slots.push_back(i); } - /// We're going to replace 'account_key' with '[HIDDEN]' if account_key is used in the signature - if (url_arg_idx + 4 < count) - markSecretArgument(url_arg_idx + 4); + if (extra_credentials > 1) + { + maskEveryArgument(); + return {}; + } + return slots; +} + +std::optional FunctionSecretArgumentsFinder::azureCredentialArgument(const std::vector & slot_arguments, size_t url_arg_idx) const +{ + const size_t count = slot_arguments.size(); + if (count == url_arg_idx + 2) + return slot_arguments[url_arg_idx + 1]; + if (count <= url_arg_idx + 4) + return {}; + String fourth_arg; + if (count <= url_arg_idx + 7 && tryGetStringFromArgument(slot_arguments[url_arg_idx + 3], &fourth_arg) + && (fourth_arg == "auto" || KnownFormatNames::instance().exists(fourth_arg))) + return {}; + return slot_arguments[url_arg_idx + 4]; +} + +void FunctionSecretArgumentsFinder::maskPositionalsAfterAzureCollection(size_t start) +{ + for (size_t i = start, size = function->arguments->size(); i < size; ++i) + { + const auto f = function->arguments->at(i)->getFunction(); + if (!f || (f->name() != "equals" && f->name() != "extra_credentials")) + markSecretArgument(i); + } } bool FunctionSecretArgumentsFinder::maskAzureConnectionString(ssize_t url_arg_idx, bool argument_is_named, size_t start) @@ -687,27 +792,14 @@ bool FunctionSecretArgumentsFinder::maskAzureConnectionString(ssize_t url_arg_id return false; } - if (!url_arg.starts_with("http")) + if (!url_arg.starts_with("http") && maskAzureConnectionStringSecrets(url_arg)) { - if (maskConnectionStringKey(url_arg, "AccountKey=")) - { - chassert(result.count == 0); /// We shouldn't use replacement with masking other arguments - result.start = url_arg_idx; - result.are_named = argument_is_named; - result.count = 1; - result.replacement = url_arg; - return true; - } - - if (maskConnectionStringKey(url_arg, "SharedAccessSignature=")) - { - chassert(result.count == 0); /// We shouldn't use replacement with masking other arguments - result.start = url_arg_idx; - result.are_named = argument_is_named; - result.count = 1; - result.replacement = url_arg; - return true; - } + chassert(result.count == 0); /// We shouldn't use replacement with masking other arguments + result.start = url_arg_idx; + result.are_named = argument_is_named; + result.count = 1; + result.replacement = url_arg; + return true; } return false; @@ -744,8 +836,10 @@ bool FunctionSecretArgumentsFinder::azureCollectionArgumentsAreShowable(size_t s { /// A key this rule cannot read hides which credential the override carries; a value that is /// no plain literal or identifier can nest one (`headers('Authorization' = '...')`). + String key; if (argument_function->arguments && argument_function->arguments->size() == 2 - && tryGetStringFromArgument(*argument_function->arguments->at(0), nullptr) + && tryGetStringFromArgument(*argument_function->arguments->at(0), &key) + && isAzureCollectionKey(key) && (tryGetStringFromArgument(*argument_function->arguments->at(1), nullptr) || argument_function->arguments->at(1)->tryGetLiteralText(nullptr))) continue; @@ -1171,6 +1265,8 @@ void FunctionSecretArgumentsFinder::findAzureBlobStorageTableEngineSecretArgumen /// AzureBlobStorage(connection_string|storage_account_url, container_name, blobpath, format, [account_name, account_key, ...]) size_t url_arg_idx = 0; + maskAzureExtraCredentials(); + if (isNamedCollectionName(url_arg_idx)) { /// AzureBlobStorage(named_collection, ..., account_key = 'account_key', ...) @@ -1185,28 +1281,15 @@ void FunctionSecretArgumentsFinder::findAzureBlobStorageTableEngineSecretArgumen return; } - /// We should check other arguments first because we don't need to do any replacement in case of - /// AzureBlobStorage(connection_string|storage_account_url, container_name, blobpath, format) -- in this case there is no account_key argument - size_t count = function->arguments->size(); - bool fourth_argument_is_format = false; - if ((url_arg_idx + 4 <= count) && (count <= url_arg_idx + 7)) - { - String fourth_arg; - if (tryGetStringFromArgument(url_arg_idx + 3, &fourth_arg)) - fourth_argument_is_format = fourth_arg == "auto" || KnownFormatNames::instance().exists(fourth_arg); - } - /// Which argument holds a credential: the two-argument shape takes a shared access signature beside - /// the url (`endpoint.sas_auth`), the longer ones an `account_key` - unless the fourth names a format. - std::optional credential_arg_idx; - if (count == url_arg_idx + 2) - credential_arg_idx = url_arg_idx + 1; - else if (!fourth_argument_is_format && (url_arg_idx + 4 < count)) - credential_arg_idx = url_arg_idx + 4; + /// Slot `n` is the raw argument `arg(n)`; past the end it is an index no argument has. + const auto slot_arguments = azureSlotArguments(url_arg_idx); + auto arg = [&](size_t slot) { return slot < slot_arguments.size() ? slot_arguments[slot] : function->arguments->size(); }; + const auto credential_arg_idx = azureCredentialArgument(slot_arguments, url_arg_idx); /// The engine reads this argument as a connection string or as a plain account url; a value of /// another shape is read by neither rule below, and a hidden connection string replaces it whole. String connection_value; - const auto shape = tryGetStringFromArgument(url_arg_idx, &connection_value) + const auto shape = tryGetStringFromArgument(arg(url_arg_idx), &connection_value) ? classifyAzureConnectionValue(connection_value) : AzureConnectionValue::Unmaskable; if (shape == AzureConnectionValue::Unmaskable || (shape == AzureConnectionValue::ConnectionString && credential_arg_idx)) @@ -1215,7 +1298,7 @@ void FunctionSecretArgumentsFinder::findAzureBlobStorageTableEngineSecretArgumen return; } - if (maskAzureConnectionString(url_arg_idx)) + if (maskAzureConnectionString(arg(url_arg_idx))) return; if (credential_arg_idx) diff --git a/src/Parsers/FunctionSecretArgumentsFinder.h b/src/Parsers/FunctionSecretArgumentsFinder.h index a5e6ce8c812a0..7e3baa828eb0f 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.h +++ b/src/Parsers/FunctionSecretArgumentsFinder.h @@ -191,6 +191,26 @@ class FunctionSecretArgumentsFinder void findXDBCSecretArguments(); void findS3FunctionSecretArguments(bool is_cluster_function); void findAzureBlobStorageFunctionSecretArguments(bool is_cluster_function); + /// Hides every `extra_credentials(...)` whole, its keys included: a key can be written as anything, a credential too. + void maskAzureExtraCredentials(); + /// The key of a `key = value` argument if it is one of the keys an Azure named collection accepts. Any other left + /// operand can be the credential itself (`'sp=r&sig=...' = 'x'`), so its argument is hidden whole. + static std::optional tryGetAzureKey(const AbstractFunction & equals); + /// Hides every override from `start` on that this rule cannot read: one whose key an Azure named collection does not + /// accept or whose value is no plain literal (it can nest a credential), and every connection override when there are + /// several: only one is read, and only one can be partly shown. + void maskUnreadAzureCollectionOverrides(size_t start); + /// The raw indexes of the arguments the Azure parser assigns slots to, the ones before `start` (the cluster name) as + /// written: it strips the `SETTINGS` clause and the first `extra_credentials(...)`, and a `key = value` takes no slot. + /// Also hides the value of a key the explicit form does not read (all of an argument whose key is no Azure key), and + /// every argument when a second `extra_credentials(...)` shifts the slots. + std::vector azureSlotArguments(size_t start); + /// The raw index of the argument holding a credential, if any: the two-argument shape takes a shared access signature + /// beside the url (`endpoint.sas_auth`), the longer ones an `account_key` - unless the fourth names a format. + std::optional azureCredentialArgument(const std::vector & slot_arguments, size_t url_arg_idx) const; + /// After the collection name every argument must be an override or `extra_credentials(...)`; a positional one is + /// invalid but logged before validation rejects it, and can be a credential. + void maskPositionalsAfterAzureCollection(size_t start); bool maskAzureConnectionString(ssize_t url_arg_idx, bool argument_is_named = false, size_t start = 0); /// Whether the arguments an `AzureBlobStorage(named_collection, ...)` destination or table takes /// from `start` can be shown: only an argument written here can carry a credential, and each has diff --git a/src/Storages/ObjectStorage/Azure/Configuration.cpp b/src/Storages/ObjectStorage/Azure/Configuration.cpp index 4ca0c16780bd3..e83b2eed55701 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.cpp +++ b/src/Storages/ObjectStorage/Azure/Configuration.cpp @@ -69,7 +69,7 @@ const std::unordered_set optional_configuration_keys = { void StorageAzureConfiguration::check(ContextPtr context) { - auto url = Poco::URI(connection_params.getConnectionURL()); + auto url = Poco::URI(connection_params.getConnectionURLWithoutQuery()); context->getGlobalContext()->getRemoteHostFilter().checkURL(url); StorageObjectStorageConfiguration::check(context); } diff --git a/src/Storages/ObjectStorage/Azure/Configuration.h b/src/Storages/ObjectStorage/Azure/Configuration.h index c9d45d0bdb8a5..0c6d57af8e14c 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.h +++ b/src/Storages/ObjectStorage/Azure/Configuration.h @@ -106,7 +106,7 @@ class StorageAzureConfiguration : public StorageObjectStorageConfiguration void setPaths(const Paths & paths) override { blobs_paths = paths; } String getNamespace() const override { return connection_params.getContainer(); } - String getDataSourceDescription() const override { return std::filesystem::path(connection_params.getConnectionURL()) / connection_params.getContainer(); } + String getDataSourceDescription() const override { return std::filesystem::path(connection_params.getConnectionURLWithoutQuery()) / connection_params.getContainer(); } StorageObjectStorageQuerySettings getQuerySettings(const ContextPtr &) const override; String getContainer() const { return connection_params.getContainer(); } diff --git a/src/Storages/ObjectStorageQueue/AzureQueue_fwd.h b/src/Storages/ObjectStorageQueue/AzureQueue_fwd.h index db7c6314d7214..a338ae7dead0d 100644 --- a/src/Storages/ObjectStorageQueue/AzureQueue_fwd.h +++ b/src/Storages/ObjectStorageQueue/AzureQueue_fwd.h @@ -20,8 +20,7 @@ static inline std::unordered_map SETTINGS_TO_HIDE = std::string masked_value; if (!value.tryGet(masked_value)) return {}; - DB::maskConnectionStringKey(masked_value, "AccountKey="); - DB::maskConnectionStringKey(masked_value, "SharedAccessSignature="); + DB::maskAzureConnectionStringSecrets(masked_value); return fmt::format("'{}'", masked_value); }}, }; diff --git a/tests/integration/test_backup_restore_azure_blob_storage/test.py b/tests/integration/test_backup_restore_azure_blob_storage/test.py index 36e4afda81fbe..2778e4dbb6b85 100644 --- a/tests/integration/test_backup_restore_azure_blob_storage/test.py +++ b/tests/integration/test_backup_restore_azure_blob_storage/test.py @@ -1,11 +1,17 @@ #!/usr/bin/env python3 +import datetime import io import os import time import pytest -from azure.storage.blob import BlobServiceClient +from azure.storage.blob import ( + AccountSasPermissions, + BlobServiceClient, + ResourceTypes, + generate_account_sas, +) from helpers.cluster import ClickHouseCluster @@ -936,6 +942,65 @@ def test_backup_legacy_disk_endpoint_settings_loaded(cluster): azure_query(node, "DROP TABLE test_legacy_native_copy_restored") +def test_connection_string_secrets_not_logged(cluster): + # The disk is configured by an account key connection string, which is also its description. The + # backup uses a SAS connection string, which the SDK appends to the blob service url as its query. + node = cluster.instances["node_legacy_native_copy"] + sas = generate_account_sas( + "devstoreaccount1", + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + resource_types=ResourceTypes(service=True, container=True, object=True), + permission=AccountSasPermissions( + read=True, write=True, delete=True, list=True, add=True, create=True + ), + expiry=datetime.datetime.now(datetime.timezone.utc) + + datetime.timedelta(hours=1), + ) + # URL-encoded, so it has no grep metacharacters. + sig = sas.split("sig=")[1].split("&")[0] + connection_string = f"BlobEndpoint={cluster.env_variables['AZURITE_STORAGE_ACCOUNT_URL']};SharedAccessSignature={sas}" + + azure_query(node, "DROP TABLE IF EXISTS test_secrets_not_logged SYNC") + azure_query( + node, + "CREATE TABLE test_secrets_not_logged(key UInt64, data String) Engine = MergeTree() ORDER BY tuple() SETTINGS storage_policy='blob_storage_policy_legacy_native_copy'", + ) + azure_query(node, "INSERT INTO test_secrets_not_logged VALUES (1, 'a')") + + backup_destination = ( + f"AzureBlobStorage('{connection_string}', 'cont', '{new_backup_name()}')" + ) + azure_query(node, f"BACKUP TABLE test_secrets_not_logged TO {backup_destination}") + azure_query(node, "DROP TABLE IF EXISTS test_secrets_not_logged_restored") + azure_query( + node, + f"RESTORE TABLE test_secrets_not_logged AS test_secrets_not_logged_restored FROM {backup_destination}", + ) + assert ( + azure_query(node, "SELECT * from test_secrets_not_logged_restored") == "1\ta\n" + ) + + function = f"azureBlobStorage('{connection_string}', 'cont', 'test_secrets_not_logged.csv'" + azure_query( + node, + f"INSERT INTO TABLE FUNCTION {function}, 'CSV', 'auto', 'x UInt64') SELECT 42 SETTINGS azure_truncate_on_insert = 1", + ) + assert azure_query(node, f"SELECT * FROM {function})") == "42\n" + sources = azure_query( + node, + "SELECT source FROM system.schema_inference_cache WHERE source LIKE '%test_secrets_not_logged.csv'", + ) + assert sources != "" and sig not in sources + + assert node.contains_in_log("BackupWriterAzureBlobStorage: Source description") + assert node.contains_in_log("BackupReaderAzureBlobStorage: Source description") + assert not node.contains_in_log("Source description.*Eby8vdM02xNO") + assert not node.contains_in_log(f"Source description.*{sig}") + + azure_query(node, "DROP TABLE test_secrets_not_logged") + azure_query(node, "DROP TABLE test_secrets_not_logged_restored") + + def test_reload_config_keeps_azure_endpoint_settings(cluster): # use_native_copy=true must survive SYSTEM RELOAD CONFIG (settings reloaded, not dropped). node = cluster.instances["node_native_copy"] diff --git a/tests/integration/test_mask_sensitive_info/test.py b/tests/integration/test_mask_sensitive_info/test.py index 24b95f0764938..031e83f12ea19 100644 --- a/tests/integration/test_mask_sensitive_info/test.py +++ b/tests/integration/test_mask_sensitive_info/test.py @@ -725,13 +725,13 @@ def test_table_functions_object_storage_cluster(): f"icebergS3Cluster('{ch_cluster}', '{s3_url}', '{s3_access_key_id}', '{s3_secret_access_key}')", f"icebergS3Cluster('{ch_cluster}', {named_collection}, url = '{s3_url}', access_key_id = '{s3_access_key_id}', secret_access_key = '{s3_secret_access_key}')", f"azureBlobStorageCluster('{ch_cluster}', '{azure_storage_account_url}', 'test', 'test', '{azure_account_name}', '{azure_account_key}')", - f"azureBlobStorageCluster('{ch_cluster}', {named_collection}, storage_account_url = '{azure_storage_account_url}', container_name = 'test', blobpath = 'test', account_name = '{azure_account_name}', account_key = '{azure_account_key}')", + f"azureBlobStorageCluster('{ch_cluster}', {named_collection}, storage_account_url = '{azure_storage_account_url}', container = 'test', blob_path = 'test', account_name = '{azure_account_name}', account_key = '{azure_account_key}')", f"azureBlobStorageCluster('{ch_cluster}', '{azure_connection_string}', 'test', 'test')", - f"azureBlobStorageCluster('{ch_cluster}', {named_collection}, connection_string = '{azure_connection_string}', container_name = 'test', blobpath = 'test')", + f"azureBlobStorageCluster('{ch_cluster}', {named_collection}, connection_string = '{azure_connection_string}', container = 'test', blob_path = 'test')", f"icebergAzureCluster('{ch_cluster}', '{azure_storage_account_url}', 'test', 'test', '{azure_account_name}', '{azure_account_key}')", - f"icebergAzureCluster('{ch_cluster}', {named_collection}, storage_account_url = '{azure_storage_account_url}', container_name = 'test', blobpath = 'test', account_name = '{azure_account_name}', account_key = '{azure_account_key}')", + f"icebergAzureCluster('{ch_cluster}', {named_collection}, storage_account_url = '{azure_storage_account_url}', container = 'test', blob_path = 'test', account_name = '{azure_account_name}', account_key = '{azure_account_key}')", f"icebergAzureCluster('{ch_cluster}', '{azure_connection_string}', 'test', 'test')", - f"icebergAzureCluster('{ch_cluster}', {named_collection}, connection_string = '{azure_connection_string}', container_name = 'test', blobpath = 'test')", + f"icebergAzureCluster('{ch_cluster}', {named_collection}, connection_string = '{azure_connection_string}', container = 'test', blob_path = 'test')", ] # DeltaLake table functions are not available under Memory Sanitizer (Rust code is not instrumented) @@ -742,9 +742,9 @@ def test_table_functions_object_storage_cluster(): f"deltaLakeS3Cluster('{ch_cluster}', '{s3_url}', '{s3_access_key_id}', '{s3_secret_access_key}')", f"deltaLakeS3Cluster('{ch_cluster}', {named_collection}, url = '{s3_url}', access_key_id = '{s3_access_key_id}', secret_access_key = '{s3_secret_access_key}')", f"deltaLakeAzureCluster('{ch_cluster}', '{azure_storage_account_url}', 'test', 'test', '{azure_account_name}', '{azure_account_key}')", - f"deltaLakeAzureCluster('{ch_cluster}', {named_collection}, storage_account_url = '{azure_storage_account_url}', container_name = 'test', blobpath = 'test', account_name = '{azure_account_name}', account_key = '{azure_account_key}')", + f"deltaLakeAzureCluster('{ch_cluster}', {named_collection}, storage_account_url = '{azure_storage_account_url}', container = 'test', blob_path = 'test', account_name = '{azure_account_name}', account_key = '{azure_account_key}')", f"deltaLakeAzureCluster('{ch_cluster}', '{azure_connection_string}', 'test', 'test')", - f"deltaLakeAzureCluster('{ch_cluster}', {named_collection}, connection_string = '{azure_connection_string}', container_name = 'test', blobpath = 'test')", + f"deltaLakeAzureCluster('{ch_cluster}', {named_collection}, connection_string = '{azure_connection_string}', container = 'test', blob_path = 'test')", ] if has_paimon: @@ -764,13 +764,13 @@ def test_table_functions_object_storage_cluster(): f"icebergS3Cluster('{ch_cluster}', '{s3_url}', '{s3_access_key_id}', '[HIDDEN]')", f"icebergS3Cluster('{ch_cluster}', {named_collection}, url = '{s3_url}', access_key_id = '{s3_access_key_id}', secret_access_key = '[HIDDEN]')", f"azureBlobStorageCluster('{ch_cluster}', '{azure_storage_account_url}', 'test', 'test', '{azure_account_name}', '[HIDDEN]')", - f"azureBlobStorageCluster('{ch_cluster}', {named_collection}, storage_account_url = '{azure_storage_account_url}', container_name = 'test', blobpath = 'test', account_name = '{azure_account_name}', account_key = '[HIDDEN]')", + f"azureBlobStorageCluster('{ch_cluster}', {named_collection}, storage_account_url = '{azure_storage_account_url}', container = 'test', blob_path = 'test', account_name = '{azure_account_name}', account_key = '[HIDDEN]')", f"azureBlobStorageCluster('{ch_cluster}', '{masked_azure_connection_string}', 'test', 'test')", - f"azureBlobStorageCluster('{ch_cluster}', {named_collection}, connection_string = '{masked_azure_connection_string}', container_name = 'test', blobpath = 'test')", + f"azureBlobStorageCluster('{ch_cluster}', {named_collection}, connection_string = '{masked_azure_connection_string}', container = 'test', blob_path = 'test')", f"icebergAzureCluster('{ch_cluster}', '{azure_storage_account_url}', 'test', 'test', '{azure_account_name}', '[HIDDEN]')", - f"icebergAzureCluster('{ch_cluster}', {named_collection}, storage_account_url = '{azure_storage_account_url}', container_name = 'test', blobpath = 'test', account_name = '{azure_account_name}', account_key = '[HIDDEN]')", + f"icebergAzureCluster('{ch_cluster}', {named_collection}, storage_account_url = '{azure_storage_account_url}', container = 'test', blob_path = 'test', account_name = '{azure_account_name}', account_key = '[HIDDEN]')", f"icebergAzureCluster('{ch_cluster}', '{masked_azure_connection_string}', 'test', 'test')", - f"icebergAzureCluster('{ch_cluster}', {named_collection}, connection_string = '{masked_azure_connection_string}', container_name = 'test', blobpath = 'test')", + f"icebergAzureCluster('{ch_cluster}', {named_collection}, connection_string = '{masked_azure_connection_string}', container = 'test', blob_path = 'test')", ] if has_delta_lake: @@ -780,9 +780,9 @@ def test_table_functions_object_storage_cluster(): f"deltaLakeS3Cluster('{ch_cluster}', '{s3_url}', '{s3_access_key_id}', '[HIDDEN]')", f"deltaLakeS3Cluster('{ch_cluster}', {named_collection}, url = '{s3_url}', access_key_id = '{s3_access_key_id}', secret_access_key = '[HIDDEN]')", f"deltaLakeAzureCluster('{ch_cluster}', '{azure_storage_account_url}', 'test', 'test', '{azure_account_name}', '[HIDDEN]')", - f"deltaLakeAzureCluster('{ch_cluster}', {named_collection}, storage_account_url = '{azure_storage_account_url}', container_name = 'test', blobpath = 'test', account_name = '{azure_account_name}', account_key = '[HIDDEN]')", + f"deltaLakeAzureCluster('{ch_cluster}', {named_collection}, storage_account_url = '{azure_storage_account_url}', container = 'test', blob_path = 'test', account_name = '{azure_account_name}', account_key = '[HIDDEN]')", f"deltaLakeAzureCluster('{ch_cluster}', '{masked_azure_connection_string}', 'test', 'test')", - f"deltaLakeAzureCluster('{ch_cluster}', {named_collection}, connection_string = '{masked_azure_connection_string}', container_name = 'test', blobpath = 'test')", + f"deltaLakeAzureCluster('{ch_cluster}', {named_collection}, connection_string = '{masked_azure_connection_string}', container = 'test', blob_path = 'test')", ] if has_paimon: diff --git a/tests/queries/0_stateless/05458_azure_credentials_hidden_in_logged_queries.reference b/tests/queries/0_stateless/05458_azure_credentials_hidden_in_logged_queries.reference new file mode 100644 index 0000000000000..cf669808c66d3 --- /dev/null +++ b/tests/queries/0_stateless/05458_azure_credentials_hidden_in_logged_queries.reference @@ -0,0 +1,39 @@ +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', '[HIDDEN]', 'account', '[HIDDEN]') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', '[HIDDEN]') +SELECT * FROM azureBlobStorageCluster('cluster', 'https://account.blob.core.windows.net/container/blob.csv', '[HIDDEN]') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', '[HIDDEN]', SETTINGS input_format_csv_skip_first_lines = 0) +SELECT * FROM azureBlobStorageCluster('cluster', 'https://account.blob.core.windows.net/container/blob.csv', '[HIDDEN]', SETTINGS input_format_csv_skip_first_lines = 0) +SELECT * FROM icebergAzure('https://account.blob.core.windows.net/container/blob.csv', '[HIDDEN]', SETTINGS input_format_csv_skip_first_lines = 0) +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', '[HIDDEN]', partition_strategy = 'none') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', '[HIDDEN]') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', '[HIDDEN]', '[HIDDEN]') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', account_key = '[HIDDEN]') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', partition_strategy = 'none', '[HIDDEN]') +SELECT * FROM azureBlobStorage('[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]') +SELECT * FROM azureBlobStorageCluster('[HIDDEN]', 'https://account.blob.core.windows.net/container/blob.csv', '[HIDDEN]') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', '[HIDDEN]') +SELECT * FROM azureBlobStorage('AccountName=account;AccountKey=[HIDDEN];BlobEndpoint=https://account.blob.core.windows.net', 'container', 'blob', 'account', '[HIDDEN]') +SELECT * FROM azureBlobStorage('AccountName=account;AccountKey=[HIDDEN];SharedAccessSignature=[HIDDEN]', 'container', 'blob') +SELECT * FROM azureBlobStorage('AccountName=account;AccountKey=[HIDDEN];AccountKey=[HIDDEN]', 'container', 'blob') +SELECT * FROM azureBlobStorage('AccountName=account;BlobEndpoint=[HIDDEN]', 'container', 'blob') +SELECT * FROM azureBlobStorage('[HIDDEN]', 'container', 'blob') +SELECT * FROM azureBlobStorage('[HIDDEN]', 'container', 'blob') +SELECT * FROM azureBlobStorage(azure_conf, connection_string = 'AccountName=account;AccountKey=[HIDDEN]', account_key = '[HIDDEN]') +SELECT * FROM azureBlobStorage(azure_conf, '[HIDDEN]', blob_path = 'blob') +SELECT * FROM azureBlobStorage(azure_conf, container = '[HIDDEN]', blob_path = 'blob') +SELECT * FROM azureBlobStorage(azure_conf, connection_string = '[HIDDEN]', storage_account_url = '[HIDDEN]') +SELECT * FROM azureBlobStorageCluster('cluster', azure_conf, connection_string = '[HIDDEN]', connection_string = '[HIDDEN]') +CREATE TABLE test_azure (`key` UInt64) ENGINE = AzureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', '[HIDDEN]', partition_strategy = 'none') +CREATE TABLE test_azure (`key` UInt64) ENGINE = AzureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', partition_strategy = 'none', '[HIDDEN]') +CREATE TABLE test_azure (`key` UInt64) ENGINE = AzureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', 'CSV', '[HIDDEN]') +CREATE TABLE test_azure (`key` UInt64) ENGINE = AzureBlobStorage('[HIDDEN]', '[HIDDEN]') +CREATE TABLE test_azure (`key` UInt64) ENGINE = AzureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', '[HIDDEN]') +CREATE TABLE test_azure_queue (`key` UInt64) ENGINE = AzureQueue('https://account.blob.core.windows.net', 'container', '*', 'account', '[HIDDEN]') +CREATE TABLE test_azure_queue (`key` UInt64) ENGINE = AzureQueue('https://account.blob.core.windows.net', 'container', '*', 'CSV') SETTINGS after_processing_move_connection_string = '[HIDDEN]' +CREATE TABLE test_azure_queue (`key` UInt64) ENGINE = AzureQueue('https://account.blob.core.windows.net', 'container', '*', 'CSV') SETTINGS after_processing_move_connection_string = '[HIDDEN]' +CREATE TABLE test_azure_queue (`key` UInt64) ENGINE = AzureQueue('https://account.blob.core.windows.net', 'container', '*', 'CSV') SETTINGS after_processing_move_connection_string = 'AccountName=account;AccountKey=[HIDDEN];AccountKey=[HIDDEN]' +0 1 +0 1 +0 1 +0 1 +0 1 diff --git a/tests/queries/0_stateless/05458_azure_credentials_hidden_in_logged_queries.sh b/tests/queries/0_stateless/05458_azure_credentials_hidden_in_logged_queries.sh new file mode 100755 index 0000000000000..6246ea1d8cb32 --- /dev/null +++ b/tests/queries/0_stateless/05458_azure_credentials_hidden_in_logged_queries.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: the fast test build has no Azure table functions and engines, whose arguments it then hides whole + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `clickhouse-format` hides the credentials in the arguments of the Azure table functions and engines and in the +# `SETTINGS` clause of `AzureQueue`, wherever the parser reads them from. +while read -r query; do + $CLICKHOUSE_FORMAT --oneline --query "$query" +done <<'EOF' +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', extra_credentials(client_id = 'id', tenant_id = 'tid'), 'account', 'plain_account_key') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', 'sp=r&sig=plain_sas_signature') +SELECT * FROM azureBlobStorageCluster('cluster', 'https://account.blob.core.windows.net/container/blob.csv', 'sp=r&sig=plain_sas_signature') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', 'sp=r&sig=plain_sas_signature', SETTINGS input_format_csv_skip_first_lines = 0) +SELECT * FROM azureBlobStorageCluster('cluster', 'https://account.blob.core.windows.net/container/blob.csv', 'sp=r&sig=plain_sas_signature', SETTINGS input_format_csv_skip_first_lines = 0) +SELECT * FROM icebergAzure('https://account.blob.core.windows.net/container/blob.csv', 'sp=r&sig=plain_sas_signature', SETTINGS input_format_csv_skip_first_lines = 0) +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', 'sp=r&sig=plain_sas_signature', partition_strategy = 'none') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', 'sp=r&sig=plain_sas_signature' = 'x') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', 'sp=r&sig=plain_sas_signature', extra_credentials(client_id = 'id', client_secret = 'plain_client_secret', role_arn = 'plain_role_arn')) +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', account_key = 'plain_account_key') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', partition_strategy = 'none', 'plain_account_key') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', extra_credentials(client_id = 'id'), extra_credentials(tenant_id = 'tid'), 'plain_account_key', 'CSV') +SELECT * FROM azureBlobStorageCluster(extra_credentials(client_id = 'id'), 'https://account.blob.core.windows.net/container/blob.csv', 'sp=r&sig=plain_sas_signature') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', extra_credentials('plain_client_secret' = 'x', client_id = 'plain_client_id')) +SELECT * FROM azureBlobStorage('AccountName=account;AccountKey=plain_account_key;BlobEndpoint=https://account.blob.core.windows.net', 'container', 'blob', 'account', 'plain_account_key') +SELECT * FROM azureBlobStorage('AccountName=account;AccountKey=plain_account_key;SharedAccessSignature=sp=r&sig=plain_sas_signature', 'container', 'blob') +SELECT * FROM azureBlobStorage('AccountName=account;AccountKey=plain_account_key;AccountKey=plain_account_key_2', 'container', 'blob') +SELECT * FROM azureBlobStorage('AccountName=account;BlobEndpoint=https://account.blob.core.windows.net/?sig=plain_sas_signature', 'container', 'blob') +SELECT * FROM azureBlobStorage('account.blob.core.windows.net/?sig=plain_sas_signature', 'container', 'blob') +SELECT * FROM azureBlobStorage('HTTPS://user:plain_password=x@account.blob.core.windows.net/', 'container', 'blob') +SELECT * FROM azureBlobStorage(azure_conf, connection_string = 'AccountName=account;AccountKey=plain_account_key', account_key = 'plain_account_key') +SELECT * FROM azureBlobStorage(azure_conf, client_secret = 'plain_client_secret', blob_path = 'blob') +SELECT * FROM azureBlobStorage(azure_conf, container = headers('Authorization' = 'plain_header_secret'), blob_path = 'blob') +SELECT * FROM azureBlobStorage(azure_conf, connection_string = 'AccountName=account;AccountKey=plain_account_key', storage_account_url = 'AccountName=account;AccountKey=plain_account_key_2') +SELECT * FROM azureBlobStorageCluster('cluster', azure_conf, connection_string = concat('AccountName=account;AccountKey=', 'plain_account_key'), connection_string = 'AccountName=account;AccountKey=plain_account_key_2') +CREATE TABLE test_azure (key UInt64) ENGINE = AzureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', 'sp=r&sig=plain_sas_signature', partition_strategy = 'none') +CREATE TABLE test_azure (key UInt64) ENGINE = AzureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', partition_strategy = 'none', 'plain_account_key') +CREATE TABLE test_azure (key UInt64) ENGINE = AzureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', 'CSV', extra_credentials(client_id = 'id', client_secret = 'plain_client_secret', role_arn = 'plain_role_arn')) +CREATE TABLE test_azure (key UInt64) ENGINE = AzureBlobStorage(azure_conf, sas_token = 'plain_sas_signature') +CREATE TABLE test_azure (key UInt64) ENGINE = AzureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', 'sp=r&sig=plain_sas_signature' = 'x') +CREATE TABLE test_azure_queue (key UInt64) ENGINE = AzureQueue('https://account.blob.core.windows.net', 'container', '*', 'account', 'plain_account_key' = 'x') +CREATE TABLE test_azure_queue (key UInt64) ENGINE = AzureQueue('https://account.blob.core.windows.net', 'container', '*', 'CSV') SETTINGS after_processing_move_connection_string = 'HTTPS://account.blob.core.windows.net/?sig=plain_sas_signature' +CREATE TABLE test_azure_queue (key UInt64) ENGINE = AzureQueue('https://account.blob.core.windows.net', 'container', '*', 'CSV') SETTINGS after_processing_move_connection_string = 'https://user:plain_password=x@account.blob.core.windows.net/' +CREATE TABLE test_azure_queue (key UInt64) ENGINE = AzureQueue('https://account.blob.core.windows.net', 'container', '*', 'CSV') SETTINGS after_processing_move_connection_string = 'AccountName=account;AccountKey=plain_account_key;AccountKey=plain_account_key_2' +EOF + +# The query tree hides an argument masked whole, not only the right side of a comparison. +$CLICKHOUSE_CLIENT -q "SELECT countIf(explain LIKE '%plain_sas_signature%'), countIf(explain LIKE '%[HIDDEN]%') FROM (EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', 'sp=r&sig=plain_sas_signature' = 'x'))" +$CLICKHOUSE_CLIENT -q "SELECT countIf(explain LIKE '%plain_sas_signature%'), countIf(explain LIKE '%[HIDDEN]%') FROM (EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM azureBlobStorage(azure_conf, 'sp=r&sig=plain_sas_signature' = 'x'))" + +# Nothing inside an Azure `extra_credentials(...)` is dumped, keys included. +for explain in "AST" "QUERY TREE run_passes = 0"; do + $CLICKHOUSE_CLIENT -q "SELECT countIf(explain LIKE '%plain_client_%'), countIf(explain LIKE '%[HIDDEN]%') FROM (EXPLAIN $explain SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', extra_credentials('plain_client_secret' = 'x', client_id = 'plain_client_id')))" +done + +# The statement is logged with the credential hidden. +$CLICKHOUSE_CLIENT -q "EXPLAIN AST SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', 'sp=r&sig=plain_sas_logged')" > /dev/null +$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS query_log" +$CLICKHOUSE_CLIENT -q "SELECT countIf(query LIKE '%plain_sas_logged%'), countIf(query LIKE '%[HIDDEN]%') FROM system.query_log WHERE current_database = currentDatabase() AND query LIKE 'EXPLAIN AST%azureBlobStorage%' AND type = 'QueryFinish' AND is_initial_query" From d956466a5301f01f280ca8d7acf6ab18397e1e29 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 3 Nov 2026 18:55:11 +0000 Subject: [PATCH 078/219] Backport #128943 to 26.8: Reset the MsgPack parser between streamed inputs --- .../Formats/Impl/MsgPackRowInputFormat.cpp | 4 +-- .../Formats/Impl/MsgPackRowInputFormat.h | 3 +- ...async_insert_after_invalid_entry.reference | 6 ++++ ...sgpack_async_insert_after_invalid_entry.sh | 36 +++++++++++++++++++ 4 files changed, 46 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/05493_msgpack_async_insert_after_invalid_entry.reference create mode 100755 tests/queries/0_stateless/05493_msgpack_async_insert_after_invalid_entry.sh diff --git a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp index 8e630effe06df..6d1dd7232513e 100644 --- a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp @@ -61,7 +61,7 @@ MsgPackRowInputFormat::MsgPackRowInputFormat(SharedHeader header_, std::unique_p void MsgPackRowInputFormat::resetParser() { IRowInputFormat::resetParser(); - visitor.reset(); + parser.init(); } void MsgPackRowInputFormat::setReadBuffer(ReadBuffer & in_) @@ -85,7 +85,7 @@ void MsgPackVisitor::set_info(IColumn & column, DataTypePtr type, UInt8 & read) info_stack.push(Info{column, type, false, std::nullopt, &read}); } -void MsgPackVisitor::reset() +void MsgPackVisitor::init() { info_stack = {}; } diff --git a/src/Processors/Formats/Impl/MsgPackRowInputFormat.h b/src/Processors/Formats/Impl/MsgPackRowInputFormat.h index 386ced3098ecc..3ed9fd4c3e513 100644 --- a/src/Processors/Formats/Impl/MsgPackRowInputFormat.h +++ b/src/Processors/Formats/Impl/MsgPackRowInputFormat.h @@ -52,7 +52,8 @@ class MsgPackVisitor : public msgpack::null_visitor /// Update info_stack void set_info(IColumn & column, DataTypePtr type, UInt8 & read); - void reset(); + /// Called by msgpack's parse_helper::init. + void init(); private: /// Stack is needed to process arrays and maps diff --git a/tests/queries/0_stateless/05493_msgpack_async_insert_after_invalid_entry.reference b/tests/queries/0_stateless/05493_msgpack_async_insert_after_invalid_entry.reference new file mode 100644 index 0000000000000..2e0aea006e630 --- /dev/null +++ b/tests/queries/0_stateless/05493_msgpack_async_insert_after_invalid_entry.reference @@ -0,0 +1,6 @@ +1 +2 +3 +4 +1 [1,2,3] +1 [1,2,3] diff --git a/tests/queries/0_stateless/05493_msgpack_async_insert_after_invalid_entry.sh b/tests/queries/0_stateless/05493_msgpack_async_insert_after_invalid_entry.sh new file mode 100755 index 0000000000000..19aeaf05141fd --- /dev/null +++ b/tests/queries/0_stateless/05493_msgpack_async_insert_after_invalid_entry.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: MsgPack format is not available in the fasttest environment + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# One async insert batch reuses one MsgPack input format for all its entries. Each case queues an +# invalid entry followed by a valid one: the valid entry must be parsed from a clean state. + +url="${CLICKHOUSE_URL}&async_insert=1&wait_for_async_insert=0&async_insert_use_adaptive_busy_timeout=0&async_insert_busy_timeout_min_ms=100000&async_insert_busy_timeout_max_ms=100000" + +function insert_pair() +{ + local table=$1 bad=$2 good=$3 + printf '%b' "$bad" | ${CLICKHOUSE_CURL} -sS "${url}&query=INSERT+INTO+${table}+FORMAT+MsgPack" --data-binary @- + printf '%b' "$good" | ${CLICKHOUSE_CURL} -sS "${url}&query=INSERT+INTO+${table}+FORMAT+MsgPack" --data-binary @- + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH ASYNC INSERT QUEUE ${table}" + ${CLICKHOUSE_CLIENT} -q "SELECT * FROM ${table} ORDER BY ALL" +} + +${CLICKHOUSE_CLIENT} -q " + CREATE TABLE t_scalar (x UInt64) ENGINE = MergeTree ORDER BY x; + CREATE TABLE t_exception (id UInt64, arr Array(UInt8)) ENGINE = MergeTree ORDER BY id; + CREATE TABLE t_array (id UInt64, arr Array(UInt8)) ENGINE = MergeTree ORDER BY id; +" + +# A uint32 with 1 of its 4 bytes; the valid entry is four rows 1, 2, 3, 4. +insert_pair t_scalar '\xce\x00' '\x01\x02\x03\x04' +# A string inside Array(UInt8) is rejected in the middle of the array. +insert_pair t_exception '\x07\x92\x01\xa1x' '\x01\x93\x01\x02\x03' +# An array of 3 elements with only 1 present. +insert_pair t_array '\x07\x93\x01' '\x01\x93\x01\x02\x03' + +${CLICKHOUSE_CLIENT} -q "DROP TABLE t_scalar; DROP TABLE t_exception; DROP TABLE t_array" From 0183a8ae992d6041ba35881cfca2a23bdeb574e0 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 3 Nov 2026 19:05:22 +0000 Subject: [PATCH 079/219] Backport #128946 to 26.8: Fix flaky test `02302_defaults_in_columnar_formats` on release branches that randomize the Parquet reader --- .../0_stateless/02302_defaults_in_columnar_formats.sql | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/02302_defaults_in_columnar_formats.sql b/tests/queries/0_stateless/02302_defaults_in_columnar_formats.sql index eeea8bc121cea..31bf6c5e6524a 100644 --- a/tests/queries/0_stateless/02302_defaults_in_columnar_formats.sql +++ b/tests/queries/0_stateless/02302_defaults_in_columnar_formats.sql @@ -7,11 +7,12 @@ select * from file(currentDatabase() || '_data_02302.orc', auto, 'x UInt8, xx UI insert into function file(currentDatabase() || '_data_02302.arrow') select 1 as x, null::Nullable(UInt8) as xx settings engine_file_truncate_on_insert=1; select * from file(currentDatabase() || '_data_02302.arrow', auto, 'x UInt8, xx UInt8 default 10, y default 42, z default x + xx + y') settings input_format_arrow_allow_missing_columns=1; +-- The next two queries test the native Parquet reader, so they pin it (some release branches still randomize it off). -- A count-only read reads just the single smallest declared column, so `d` must stay strictly -- narrower than every other column for this to exercise a defaulted one. The count cache would -- answer before a reader exists, and the two count-only optimizations are randomized off in some runs, so all three are pinned. -- Missing columns are rejected so that a read which is not count-only fails here instead of -- synthesizing `d` into a correctly sized mask and returning the same count with no guard in place. -select count() from file(currentDatabase() || '_data_02302.parquet', auto, 'x UInt64, d UInt8 default 10') settings use_cache_for_count_from_files = 0, optimize_count_from_files = 1, optimize_trivial_count_query = 1, input_format_parquet_allow_missing_columns = 0; +select count() from file(currentDatabase() || '_data_02302.parquet', auto, 'x UInt64, d UInt8 default 10') settings use_cache_for_count_from_files = 0, optimize_count_from_files = 1, optimize_trivial_count_query = 1, input_format_parquet_allow_missing_columns = 0, input_format_parquet_use_native_reader_v3 = 1; -- With the setting off no `default` expression is applied, so the columns keep their type defaults. -select * from file(currentDatabase() || '_data_02302.parquet', auto, 'x UInt8, xx UInt8 default 10, y default 42, z default x + xx + y') settings input_format_parquet_allow_missing_columns=1, input_format_defaults_for_omitted_fields=0; +select * from file(currentDatabase() || '_data_02302.parquet', auto, 'x UInt8, xx UInt8 default 10, y default 42, z default x + xx + y') settings input_format_parquet_allow_missing_columns=1, input_format_defaults_for_omitted_fields=0, input_format_parquet_use_native_reader_v3=1; From 773ab95bb163c7e8eac4ed154fb0d294427cd3b0 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 3 Nov 2026 22:20:07 +0000 Subject: [PATCH 080/219] Backport #128567 to 26.8: Fix heap-buffer-overflow read in `SnappyFramedReadBuffer` --- src/IO/SnappyFramedReadBuffer.cpp | 4 +-- src/IO/SnappyFramedReadBuffer.h | 3 +- ...05480_snappy_framed_read_padding.reference | 4 +++ .../05480_snappy_framed_read_padding.sh | 32 +++++++++++++++++++ 4 files changed, 40 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/05480_snappy_framed_read_padding.reference create mode 100755 tests/queries/0_stateless/05480_snappy_framed_read_padding.sh diff --git a/src/IO/SnappyFramedReadBuffer.cpp b/src/IO/SnappyFramedReadBuffer.cpp index 0ae482223a2ee..238dc9f45a41c 100644 --- a/src/IO/SnappyFramedReadBuffer.cpp +++ b/src/IO/SnappyFramedReadBuffer.cpp @@ -46,6 +46,7 @@ uint32_t maskedCrc32c(const char * data, size_t size) SnappyFramedReadBuffer::SnappyFramedReadBuffer( std::unique_ptr in_, size_t buf_size, char * existing_memory, size_t alignment) : CompressedReadBufferWrapper(std::move(in_), buf_size, existing_memory, alignment) + , decompress_buffer(MAX_UNCOMPRESSED_CHUNK_SIZE) { } @@ -168,7 +169,6 @@ bool SnappyFramedReadBuffer::nextImpl() "Snappy chunk claims uncompressed length {} which exceeds the framing format limit of {}{}", uncompressed_length, MAX_UNCOMPRESSED_CHUNK_SIZE, getExceptionEntryWithFileName(*in)); - decompress_buffer.resize(uncompressed_length); if (!snappy::RawUncompress(compressed, payload_size, decompress_buffer.data())) throw Exception(ErrorCodes::SNAPPY_UNCOMPRESS_FAILED, "Snappy decompression failed{}", @@ -211,7 +211,7 @@ bool SnappyFramedReadBuffer::nextImpl() /// Snappy framing format encodes the masked CRC-32C as little-endian. uint32_t expected_crc = unalignedLoadLittleEndian(chunk_buf.data()); - decompress_buffer.assign(chunk_buf.data() + 4, payload_size); + memcpy(decompress_buffer.data(), chunk_buf.data() + 4, payload_size); uint32_t actual_crc = maskedCrc32c(decompress_buffer.data(), payload_size); if (actual_crc != expected_crc) diff --git a/src/IO/SnappyFramedReadBuffer.h b/src/IO/SnappyFramedReadBuffer.h index 500b6fa54d968..9f45423209ce1 100644 --- a/src/IO/SnappyFramedReadBuffer.h +++ b/src/IO/SnappyFramedReadBuffer.h @@ -33,7 +33,8 @@ class SnappyFramedReadBuffer : public CompressedReadBufferWrapper /// Read and validate the stream identifier chunk. bool readStreamIdentifier(); - String decompress_buffer; + /// Padded: working_buffer points here and the base reports isPadded(). + Memory<> decompress_buffer; bool identifier_read = false; }; diff --git a/tests/queries/0_stateless/05480_snappy_framed_read_padding.reference b/tests/queries/0_stateless/05480_snappy_framed_read_padding.reference new file mode 100644 index 0000000000000..1daaa760f7f48 --- /dev/null +++ b/tests/queries/0_stateless/05480_snappy_framed_read_padding.reference @@ -0,0 +1,4 @@ +01 +90 50 +00 +129 20 diff --git a/tests/queries/0_stateless/05480_snappy_framed_read_padding.sh b/tests/queries/0_stateless/05480_snappy_framed_read_padding.sh new file mode 100755 index 0000000000000..510f5a8accfff --- /dev/null +++ b/tests/queries/0_stateless/05480_snappy_framed_read_padding.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Reading a framed-snappy text file without parallel parsing must not read past the decompressed chunk. +# Each case prints the type of the first chunk (00 = compressed, 01 = uncompressed) and the read result. + +DIR="${USER_FILES_PATH}/${CLICKHOUSE_TEST_UNIQUE_NAME}" +mkdir -p "${DIR}" +chmod 777 "${DIR}" +trap 'rm -rf "${DIR}"' EXIT + +function check() +{ + local file="${DIR}/$1.snappy" + ${CLICKHOUSE_CLIENT} -q " + INSERT INTO TABLE FUNCTION file('${file}', 'LineAsString', 'line String', 'snappy') + $2 + SETTINGS snappy_mode = 'framed'; + " + od -A n -t x1 -j 10 -N 1 "${file}" | tr -d ' ' + ${CLICKHOUSE_CLIENT} -q " + SELECT sum(length(line)), count() FROM file('${file}', 'LineAsString', 'line String', 'snappy') + SETTINGS snappy_mode = 'framed', input_format_parallel_parsing = 0; + " +} + +check uncompressed "SELECT toString(number) FROM numbers(50)" +check compressed "SELECT repeat('a', number % 13) || toString(number) FROM numbers(20)" From b240f9a9f49e4f0729ff1272cfdb1d91921501d2 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 3 Nov 2026 22:16:14 +0000 Subject: [PATCH 081/219] Backport #128964 to 26.8: Include the storage in `s3` cache keys --- src/IO/ReadPipeline.h | 4 +-- .../StorageObjectStorageSource.cpp | 9 ++++- tests/integration/test_storage_s3/test.py | 35 +++++++++++++++++++ 3 files changed, 45 insertions(+), 3 deletions(-) diff --git a/src/IO/ReadPipeline.h b/src/IO/ReadPipeline.h index 58dc0d2210ae5..24696be3774bf 100644 --- a/src/IO/ReadPipeline.h +++ b/src/IO/ReadPipeline.h @@ -121,7 +121,7 @@ class ReadPipeline void needFilesystemCache(FileCachePtr cache, FilesystemCacheSettings cache_settings, std::shared_ptr cache_log = nullptr); /// Overload with a custom cache key and origin, bypassing the default `FileCacheKey::fromPath` derivation. - /// Used by `StorageObjectStorageSource` where the cache key is `SipHash(path + etag)`. + /// Used by `StorageObjectStorageSource` where the cache key is `SipHash( + path + etag)`. void needFilesystemCache( FileCachePtr cache, FileCacheKey cache_key, @@ -135,7 +135,7 @@ class ReadPipeline /// Overload with a fully custom page cache key (path + file_version), bypassing the default /// `cache_path_prefix + object.remote_path` derivation. - /// Used by `StorageObjectStorageSource` where the key is `"s3:" + path` with `"etag:" + etag`. + /// Used by `StorageObjectStorageSource` where the key is `"s3::" + path` with `"etag:" + etag`. void needMemoryCache( String custom_cache_path, String custom_file_version, diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 5fc14e1df78f5..11c2fe303af40 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -170,10 +170,16 @@ namespace return path.substr(0, position); } + String getObjectStorageHash(const IObjectStorage & object_storage) + { + const String description = object_storage.getDescription(); + return sipHash128String(description.substr(0, description.find('?')) + '\0' + object_storage.getObjectsNamespace()); + } + String getPageCachePathForObjectStorage(const RelativePathWithMetadata & object_info, const ObjectStoragePtr & object_storage) { if (object_storage->getType() != ObjectStorageType::Web) - return "s3:" + object_info.getPath(); + return fmt::format("s3:{}:{}", getObjectStorageHash(*object_storage), object_info.getPath()); if (!object_info.read_source_index) return "web:" + object_info.getPath(); @@ -1689,6 +1695,7 @@ std::unique_ptr createReadBuffer( else { SipHash hash; + hash.update(getObjectStorageHash(*object_storage)); hash.update(object_info.getPath()); hash.update(object_info.metadata->etag); diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 96e94b2a89e89..04a553a14f299 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -2938,6 +2938,41 @@ def test_page_cache(started_cluster): assert 0 < int(gets) +def test_cache_key_includes_storage(started_cluster): + # The same key with the same content, so the same ETag, in another bucket or behind another endpoint: + # the filesystem cache and the page cache must not serve it the entry filled by the first object. + instance = started_cluster.instances["dummy"] + table_name = f"test_cache_key_includes_storage-{uuid.uuid4()}" + host = f"{started_cluster.minio_host}:{started_cluster.minio_port}" + ip = f"{started_cluster.minio_ip}:{started_cluster.minio_port}" + + for bucket in [started_cluster.minio_bucket, started_cluster.minio_bucket_2]: + instance.query( + f"insert into function s3('http://{host}/{bucket}/{table_name}.tsv', auto, 'x UInt64') select number from numbers(100) SETTINGS s3_truncate_on_insert=1" + ) + + urls = [ + f"http://{host}/{started_cluster.minio_bucket}/{table_name}.tsv", + f"http://{ip}/{started_cluster.minio_bucket}/{table_name}.tsv", + f"http://{host}/{started_cluster.minio_bucket_2}/{table_name}.tsv", + ] + for settings in ["filesystem_cache_name = 'cache1', enable_filesystem_cache=1", "use_page_cache_for_object_storage=1"]: + for url in urls: + query_id = f"{table_name}-{uuid.uuid4()}" + assert "100\n" == instance.query( + f"select sum(ignore(*) + 1) from s3('{url}', 'TSV', 'x UInt64') SETTINGS {settings}", + query_id=query_id, + ) + + instance.query("SYSTEM FLUSH LOGS") + + assert 0 < int( + instance.query( + f"SELECT ProfileEvents['S3GetObject'] FROM system.query_log WHERE query_id = '{query_id}' AND type = 'QueryFinish'" + ) + ) + + def test_archive(started_cluster): uuid.uuid4() From a161e812041d669cc8fff5294f693a86c5aff827 Mon Sep 17 00:00:00 2001 From: Craig Ringer Date: Fri, 16 Oct 2026 14:39:28 +1300 Subject: [PATCH 082/219] Backport: accept all-zero IPv6 literals in `IPAddress::tryParse` and the string constructor Fix "Malformed initial_address received over the network" on IPv6 loopback. `IPv6AddressImpl::parse` signals failure by returning the all-zero address, which is also the correct result for the valid literal `::`. So tryParse and the string constructor could not tell the two apart and rejected `::` and `0:0:0:0:0:0:0:0`. To fix, when the result is all-zero, also check the string with `inet_pton`. Problem: https://github.com/ClickHouse/ClickHouse/pull/108410 (merge commit a115e2281161cc8955e2590afdd34cfd63084b1f, in 26.7 and 26.8) made `ClientInfo::read` parse `initial_address` of a `SECONDARY_QUERY` with `IPAddress::tryParse` and throw on failure. It exposed the bug above. Error: Code: 117. DB::Exception: Malformed initial_address received over the network: expected an IP literal with a numeric port. (INCORRECT_DATA) Master fix: https://github.com/ClickHouse/ClickHouse/pull/122785 (commit e1d921be0c0). This backport only includes the IPv6 address mis-parsing fix, which was somewhat incidental in the upstream fix. This commit DOES NOT fix the actual truncation of `::6` to `::` which is a separate bug (#126179) that still exists on master. Related: https://github.com/ClickHouse/ClickHouse/issues/63256 --- base/poco/Net/src/IPAddress.cpp | 18 ++++++++++++++++-- ...385_ipv6_all_zero_literal_host_ip.reference | 3 +++ .../05385_ipv6_all_zero_literal_host_ip.sql | 13 +++++++++++++ 3 files changed, 32 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/05385_ipv6_all_zero_literal_host_ip.reference create mode 100644 tests/queries/0_stateless/05385_ipv6_all_zero_literal_host_ip.sql diff --git a/base/poco/Net/src/IPAddress.cpp b/base/poco/Net/src/IPAddress.cpp index 5a7281e4be477..fdd86c30df0f1 100644 --- a/base/poco/Net/src/IPAddress.cpp +++ b/base/poco/Net/src/IPAddress.cpp @@ -38,6 +38,20 @@ using Poco::Net::Impl::IPv6AddressImpl; #endif +namespace +{ +#if defined(POCO_HAVE_IPv6) + /// `IPv6AddressImpl::parse` returns the all-zero address both on failure and for a valid unscoped all-zero + /// literal; a valid scoped result has a non-zero scope. macOS `inet_pton` accepts and ignores a `%scope` suffix. + bool isUnscopedIPv6Literal(const std::string& addr) + { + struct in6_addr ia; + return addr.find('%') == std::string::npos && inet_pton(AF_INET6, addr.c_str(), &ia) == 1; + } +#endif +} + + namespace Poco { namespace Net { @@ -104,7 +118,7 @@ IPAddress::IPAddress(const std::string& addr) } IPv6AddressImpl addr6(IPv6AddressImpl::parse(addr)); - if (addr6 != IPv6AddressImpl()) + if (addr6 != IPv6AddressImpl() || isUnscopedIPv6Literal(addr)) { newIPv6(addr6.addr(), addr6.scope()); return; @@ -518,7 +532,7 @@ bool IPAddress::tryParse(const std::string& addr, IPAddress& result) } #if defined(POCO_HAVE_IPv6) IPv6AddressImpl impl6(IPv6AddressImpl::parse(addr)); - if (impl6 != IPv6AddressImpl()) + if (impl6 != IPv6AddressImpl() || isUnscopedIPv6Literal(addr)) { result.newIPv6(impl6.addr(), impl6.scope()); return true; diff --git a/tests/queries/0_stateless/05385_ipv6_all_zero_literal_host_ip.reference b/tests/queries/0_stateless/05385_ipv6_all_zero_literal_host_ip.reference new file mode 100644 index 0000000000000..ffa35be221a6d --- /dev/null +++ b/tests/queries/0_stateless/05385_ipv6_all_zero_literal_host_ip.reference @@ -0,0 +1,3 @@ +CREATE USER u_05385_a IDENTIFIED WITH no_password HOST IP \'::\' +CREATE USER u_05385_b IDENTIFIED WITH no_password HOST IP \'::\' +CREATE USER u_05385_c IDENTIFIED WITH no_password HOST IP \'::\' diff --git a/tests/queries/0_stateless/05385_ipv6_all_zero_literal_host_ip.sql b/tests/queries/0_stateless/05385_ipv6_all_zero_literal_host_ip.sql new file mode 100644 index 0000000000000..48e9885892979 --- /dev/null +++ b/tests/queries/0_stateless/05385_ipv6_all_zero_literal_host_ip.sql @@ -0,0 +1,13 @@ +-- Every spelling of the IPv6 unspecified address is an address. The peer `::6` of a proxy is +-- serialized as `::`, and a secondary query is rejected if `::` is not parsed back. +DROP USER IF EXISTS u_05385_a, u_05385_b, u_05385_c; + +CREATE USER u_05385_a HOST IP '::'; +CREATE USER u_05385_b HOST IP '0:0:0:0:0:0:0:0'; +CREATE USER u_05385_c HOST IP '::0.0.0.0'; + +SHOW CREATE USER u_05385_a; +SHOW CREATE USER u_05385_b; +SHOW CREATE USER u_05385_c; + +DROP USER u_05385_a, u_05385_b, u_05385_c; From af185378379d64ef4ddc9653d55c45b5057b5112 Mon Sep 17 00:00:00 2001 From: Craig Ringer Date: Wed, 4 Nov 2026 11:38:17 +1300 Subject: [PATCH 083/219] Resolve PTR of IPv4-mapped addresses via the IPv4 record Now that `::` parses as a literal, a server listening on `::` without global IPv6 binds a dual-stack socket and IPv4 clients arrive as IPv4-mapped addresses. Their reverse lookup went to ip6.arpa, so `host_regexp` users were rejected. Co-Authored-By: Claude Sonnet 5.5 --- src/Common/DNSResolver.cpp | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/Common/DNSResolver.cpp b/src/Common/DNSResolver.cpp index 8774eec4c105f..d589842e624bd 100644 --- a/src/Common/DNSResolver.cpp +++ b/src/Common/DNSResolver.cpp @@ -161,6 +161,12 @@ std::unordered_set reverseResolveImpl(const Poco::Net::IPAddress & addre { return ptr_resolver->resolve(address.toString()); } + if (address.isIPv4Mapped()) + { + /// An IPv4-mapped address is an IPv4 peer of a dual-stack socket, so its PTR record is the IPv4 one. + Poco::Net::IPAddress ipv4(reinterpret_cast(address.addr()) + 12, 4); + return ptr_resolver->resolve(ipv4.toString()); + } return ptr_resolver->resolve_v6(address.toString()); } From 519a74628a9b6788bb1ba8c6edb40325b381eff3 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 3 Nov 2026 23:13:16 +0000 Subject: [PATCH 084/219] Backport #127603 to 26.8: Fix heap-buffer-overflow when parsing text formats read through the userspace page cache --- .../IO/AsynchronousBoundedReadBuffer.cpp | 1 + src/IO/CachedInMemoryReadBufferFromFile.cpp | 2 + src/IO/PipelineReadBuffer.cpp | 1 + ...mats_read_padding_object_storage.reference | 18 ++++++++ ...ext_formats_read_padding_object_storage.sh | 46 +++++++++++++++++++ 5 files changed, 68 insertions(+) create mode 100644 tests/queries/0_stateless/05448_text_formats_read_padding_object_storage.reference create mode 100755 tests/queries/0_stateless/05448_text_formats_read_padding_object_storage.sh diff --git a/src/Disks/IO/AsynchronousBoundedReadBuffer.cpp b/src/Disks/IO/AsynchronousBoundedReadBuffer.cpp index 0596c8a5fd6a3..593fc9c278578 100644 --- a/src/Disks/IO/AsynchronousBoundedReadBuffer.cpp +++ b/src/Disks/IO/AsynchronousBoundedReadBuffer.cpp @@ -86,6 +86,7 @@ AsynchronousBoundedReadBuffer::AsynchronousBoundedReadBuffer( { use_page_cache = true; buffer_size = page_cache_block_size; + padded = false; } LOG_TEST(log, "Using buffer size {} while reading {}", buffer_size, file_name); diff --git a/src/IO/CachedInMemoryReadBufferFromFile.cpp b/src/IO/CachedInMemoryReadBufferFromFile.cpp index db1f9c071aa3f..69e38bbda98bc 100644 --- a/src/IO/CachedInMemoryReadBufferFromFile.cpp +++ b/src/IO/CachedInMemoryReadBufferFromFile.cpp @@ -30,6 +30,8 @@ CachedInMemoryReadBufferFromFile::CachedInMemoryReadBufferFromFile( , in(std::move(in_)), read_until_position(file_size.value()) , inner_read_until_position(read_until_position) { + /// The working buffer is a page cache cell, an exact-size allocation: nothing past its end is readable. + padded = false; } bool CachedInMemoryReadBufferFromFile::innerSupportsReadAt() const diff --git a/src/IO/PipelineReadBuffer.cpp b/src/IO/PipelineReadBuffer.cpp index ee70b8746d9e4..d3eed4efb7ae7 100644 --- a/src/IO/PipelineReadBuffer.cpp +++ b/src/IO/PipelineReadBuffer.cpp @@ -17,6 +17,7 @@ PipelineReadBuffer::PipelineReadBuffer(std::unique_ptr executor_ , executor(std::move(executor_)) , read_position(executor->getPosition()) { + padded = false; LOG_DEBUG(log, "Created, total_size={}, read_position={}", executor->totalSize(), read_position); } diff --git a/tests/queries/0_stateless/05448_text_formats_read_padding_object_storage.reference b/tests/queries/0_stateless/05448_text_formats_read_padding_object_storage.reference new file mode 100644 index 0000000000000..f40a36898a891 --- /dev/null +++ b/tests/queries/0_stateless/05448_text_formats_read_padding_object_storage.reference @@ -0,0 +1,18 @@ +-- page cache, direct read +0 0 +1 10 +2 20 +1 +-- page cache, prefetching read +0 0 +1 10 +2 20 +1 +-- page cache, reader executor, read twice +0 0 +1 10 +2 20 +0 0 +1 10 +2 20 +1 diff --git a/tests/queries/0_stateless/05448_text_formats_read_padding_object_storage.sh b/tests/queries/0_stateless/05448_text_formats_read_padding_object_storage.sh new file mode 100755 index 0000000000000..6d3c2b25435db --- /dev/null +++ b/tests/queries/0_stateless/05448_text_formats_read_padding_object_storage.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: AzureBlobStorage (Azurite) is not available in the fast test. + +# Parsing a short CSV object through the userspace page cache must not read past the end of the page cache cell that +# holds it: with a direct read, a prefetching read and the reader executor (read twice, so that the second read is +# served from the cache). A sanitizer build aborts on such a read. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +AZURE_CONN="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;" +# Azure container names are limited to 63 characters and must be lowercase alphanumeric, so hash the unique name. +AZURE_CONT="cont$(echo "${CLICKHOUSE_TEST_UNIQUE_NAME}" | md5sum | cut -c1-24)" + +CONFIG_FILE="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}_page_cache.yaml" +echo "page_cache_max_size: 134217728" > "${CONFIG_FILE}" + +# A 14-byte object: "0,0\n1,10\n2,20\n". +${CLICKHOUSE_CLIENT} --query " + INSERT INTO FUNCTION azureBlobStorage('${AZURE_CONN}', '${AZURE_CONT}', 'data.csv', 'CSV', 'auto', 'a UInt32, b UInt32') + SELECT number, number * 10 FROM numbers(3) SETTINGS azure_truncate_on_insert = 1" + +SELECT_QUERY="SELECT * FROM azureBlobStorage('${AZURE_CONN}', '${AZURE_CONT}', 'data.csv', 'CSV', 'auto', 'a String, b UInt32') + ORDER BY b SETTINGS input_format_parallel_parsing = 0" +PAGE_CACHE_USED="SELECT value > 0 FROM system.metrics WHERE metric = 'PageCacheBytes'" +PAGE_CACHE_HIT="SELECT sum(value) > 0 FROM system.events WHERE event = 'PageCacheReadBytes'" + +echo "-- page cache, direct read" +${CLICKHOUSE_LOCAL} --config-file "${CONFIG_FILE}" --query "${SELECT_QUERY}, + use_page_cache_for_object_storage = 1, remote_filesystem_read_method = 'read', use_reader_executor = 0; + ${PAGE_CACHE_USED}" + +echo "-- page cache, prefetching read" +${CLICKHOUSE_LOCAL} --config-file "${CONFIG_FILE}" --query "${SELECT_QUERY}, + use_page_cache_for_object_storage = 1, remote_filesystem_read_method = 'threadpool', remote_filesystem_read_prefetch = 1, + use_reader_executor = 0; + ${PAGE_CACHE_USED}" + +echo "-- page cache, reader executor, read twice" +READER_EXECUTOR_QUERY="${SELECT_QUERY}, + use_page_cache_for_object_storage = 1, remote_filesystem_read_method = 'read', use_reader_executor = 1" +${CLICKHOUSE_LOCAL} --config-file "${CONFIG_FILE}" --query "${READER_EXECUTOR_QUERY}; ${READER_EXECUTOR_QUERY}; ${PAGE_CACHE_HIT}" + +rm "${CONFIG_FILE}" From 97e956efda7aa5bab4e48055d9e85342194c70f2 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 3 Nov 2026 23:41:48 +0000 Subject: [PATCH 085/219] Backport #127627 to 26.8: Check the source table `SELECT` grant in `parallel_distributed_insert_select` --- src/Core/Settings.cpp | 4 + src/Storages/StorageDistributed.cpp | 20 +++++ ...uted_insert_select_source_access.reference | 9 ++ ...distributed_insert_select_source_access.sh | 87 +++++++++++++++++++ 4 files changed, 120 insertions(+) create mode 100644 tests/queries/0_stateless/05443_parallel_distributed_insert_select_source_access.reference create mode 100755 tests/queries/0_stateless/05443_parallel_distributed_insert_select_source_access.sh diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index a80163cb14b0d..2ec93b883a1c8 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -1462,6 +1462,10 @@ Possible values: Since v25.4, `INSERT ... SELECT` from a `ReplicatedMergeTree` or `SharedMergeTree` source can also be parallelized across replicas. To enable it: - `parallel_distributed_insert_select = 2` - `enable_parallel_replicas = 1` + +:::note +The rewritten query, including every expression of the `SELECT` (subqueries, UDFs, and so on), is executed on the shards under the credentials of the cluster configuration — the same trust model as other shard-side processing, such as the subqueries of a non-`GLOBAL` `IN`. Configure an interserver secret (the `secret` element of the cluster configuration) if shard-side reads must be authorized as the calling user. +::: )", 0) \ DECLARE(UInt64, distributed_group_by_no_merge, 0, R"( Do not merge aggregation states from different servers for distributed query processing, you can use this in case it is for certain that there are different keys on different shards diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 645ab311957da..cd54bdce2cc3d 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -2,6 +2,9 @@ #include #include +#include +#include +#include #include @@ -183,6 +186,7 @@ namespace Setting extern const SettingsBool prefer_global_in_and_join; extern const SettingsBool skip_unavailable_shards; extern const SettingsBool enable_global_with_statement; + extern const SettingsMap additional_table_filters; } namespace DistributedSetting @@ -1517,6 +1521,7 @@ std::optional StorageDistributed::distributedWrite(const ASTInser auto & select = query.select->as(); StoragePtr src_storage; + bool src_is_table_function = false; /// Distributed write only works in the most trivial case INSERT ... SELECT /// without any unions or joins on the right side @@ -1533,6 +1538,7 @@ std::optional StorageDistributed::distributedWrite(const ASTInser if (joined_tables.tablesCount() == 1) { src_storage = joined_tables.getLeftTableStorage(); + src_is_table_function = joined_tables.isLeftTableFunction(); } } } @@ -1540,8 +1546,22 @@ std::optional StorageDistributed::distributedWrite(const ASTInser if (!src_storage) return {}; + /// This is the only place source SELECT access is enforced in this case + /// (table functions have been already checked in ITableFunction::execute) + const auto src_id = src_storage->getStorageID(); + if (!src_is_table_function + && !local_context->getAccess()->isGranted(AccessType::SELECT, src_id.database_name, src_id.table_name)) + return {}; + if (auto row_policy_filter = local_context->getRowPolicyFilter(src_id.database_name, src_id.table_name, RowPolicyFilterType::SELECT_FILTER); + row_policy_filter && !row_policy_filter->isAlwaysTrue()) + return {}; + /// It is not easy to match this table only, analyzer also do not do it + if (!settings[Setting::additional_table_filters].value.empty()) + return {}; + if (auto src_distributed = std::dynamic_pointer_cast(src_storage)) { + src_distributed->checkLocalShardAccess(AccessType::SELECT, local_context); return distributedWriteBetweenDistributedTables(*src_distributed, query, local_context); } if (auto src_storage_cluster = std::dynamic_pointer_cast(src_storage)) diff --git a/tests/queries/0_stateless/05443_parallel_distributed_insert_select_source_access.reference b/tests/queries/0_stateless/05443_parallel_distributed_insert_select_source_access.reference new file mode 100644 index 0000000000000..0d095d7059548 --- /dev/null +++ b/tests/queries/0_stateless/05443_parallel_distributed_insert_select_source_access.reference @@ -0,0 +1,9 @@ +parallel_distributed_insert_select=0 ACCESS_DENIED +parallel_distributed_insert_select=1 ACCESS_DENIED +parallel_distributed_insert_select=2 ACCESS_DENIED +0 +1 +row policy on source ILLEGAL_PREWHERE +additional_table_filters 1 +remote database proxy ACCESS_DENIED +2 diff --git a/tests/queries/0_stateless/05443_parallel_distributed_insert_select_source_access.sh b/tests/queries/0_stateless/05443_parallel_distributed_insert_select_source_access.sh new file mode 100755 index 0000000000000..fe40401030ee3 --- /dev/null +++ b/tests/queries/0_stateless/05443_parallel_distributed_insert_select_source_access.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +user="user_$CLICKHOUSE_DATABASE" + +$CLICKHOUSE_CLIENT -q " + CREATE TABLE secret_local (id UInt64, secret String) ENGINE = MergeTree ORDER BY id; + CREATE TABLE copy_local (id UInt64, secret String) ENGINE = MergeTree ORDER BY id; + INSERT INTO secret_local VALUES (1, 'secret'); + + CREATE TABLE src_dist (id UInt64, secret String) ENGINE = Distributed(test_shard_localhost, currentDatabase(), secret_local, id); + CREATE TABLE dst_dist (id UInt64, secret String) ENGINE = Distributed(test_shard_localhost, currentDatabase(), copy_local, id); + + DROP USER IF EXISTS $user; + CREATE USER $user IDENTIFIED WITH no_password; + GRANT SELECT, INSERT ON $CLICKHOUSE_DATABASE.dst_dist TO $user; +" + +# The cluster has no interserver secret, so the shard runs the forwarded query under the credentials +# from the cluster configuration rather than as the caller. +client=($CLICKHOUSE_CLIENT --user "$user" --prefer_localhost_replica 0) + +for setting in 0 1 2; do + echo -n "parallel_distributed_insert_select=$setting " + "${client[@]}" --parallel_distributed_insert_select "$setting" -q "INSERT INTO dst_dist SELECT * FROM src_dist" |& + grep -o -m1 'ACCESS_DENIED' || echo 'NOT DENIED' +done + +# Read the destination directly: resolving `dst_dist` also authorizes `copy_local` once the shard +# query is planned on the initiator, which depends on the distributed plan settings. +$CLICKHOUSE_CLIENT -q "SELECT count() FROM copy_local" + +# With the grant in place the optimized path must still work. +$CLICKHOUSE_CLIENT -q "GRANT SELECT ON $CLICKHOUSE_DATABASE.src_dist TO $user" +"${client[@]}" --parallel_distributed_insert_select 2 -q "INSERT INTO dst_dist SELECT * FROM src_dist" +$CLICKHOUSE_CLIENT -q "SELECT count() FROM copy_local" + +# A row policy on the source table cannot be applied by the optimized rewrite (it ships the query +# to the shards, where the policy on the Distributed table never applies), so it must fall back to +# the regular path, which rejects a row policy on a Distributed table. +# +# The two fallback cases below pin serialize_query_plan = 0: with plan serialization the initiator +# plans the read of the underlying table itself and requires the caller's own SELECT on it, which +# these cases do not grant. +$CLICKHOUSE_CLIENT -q "CREATE ROW POLICY policy_05443 ON src_dist USING id != 1 TO $user" +echo -n "row policy on source " +"${client[@]}" --parallel_distributed_insert_select 2 --enable_analyzer 1 --enable_parallel_replicas 0 \ + --serialize_query_plan 0 \ + -q "INSERT INTO dst_dist SELECT * FROM src_dist" |& grep -o -m1 'ILLEGAL_PREWHERE' || echo 'NOT REJECTED' +$CLICKHOUSE_CLIENT -q "DROP ROW POLICY policy_05443 ON src_dist" + +# additional_table_filters is keyed by the source table name, which the optimized rewrite would +# drop, so it must fall back to the regular path, which applies the filter: the only row has +# id = 1, nothing is copied and the count stays the same. +echo -n "additional_table_filters " +"${client[@]}" --parallel_distributed_insert_select 2 --enable_parallel_replicas 0 --serialize_query_plan 0 \ + --additional_table_filters "{'src_dist' : 'id != 1'}" -q "INSERT INTO dst_dist SELECT * FROM src_dist" +$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH DISTRIBUTED dst_dist; SELECT count() FROM copy_local" + +# A source from a `Remote` database whose shard points back to this server reads the underlying +# local table, so the caller's own SELECT on it is required (the stored credentials are only for +# genuinely remote servers), like the regular read path enforces. +remote_db="${CLICKHOUSE_DATABASE}_remote" +$CLICKHOUSE_CLIENT -q " + CREATE DATABASE $remote_db ENGINE = Remote('127.0.0.1', '$CLICKHOUSE_DATABASE', 'default', ''); + GRANT SHOW COLUMNS ON $CLICKHOUSE_DATABASE.* TO $user; + GRANT SELECT, INSERT ON $remote_db.* TO $user; + GRANT INSERT ON $CLICKHOUSE_DATABASE.copy_local TO $user; +" +echo -n "remote database proxy " +"${client[@]}" --parallel_distributed_insert_select 2 \ + -q "INSERT INTO $remote_db.copy_local SELECT * FROM $remote_db.secret_local" |& + grep -o -m1 'ACCESS_DENIED' || echo 'NOT DENIED' + +# ...and works once SELECT on the underlying table is granted. +$CLICKHOUSE_CLIENT -q "GRANT SELECT ON $CLICKHOUSE_DATABASE.secret_local TO $user" +"${client[@]}" --parallel_distributed_insert_select 2 \ + -q "INSERT INTO $remote_db.copy_local SELECT * FROM $remote_db.secret_local" +$CLICKHOUSE_CLIENT -q "SELECT count() FROM copy_local" + +$CLICKHOUSE_CLIENT -q " + DROP DATABASE $remote_db; + DROP USER $user; +" From ff4ca7aeff324331d291433066a0377f41b992f0 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 4 Nov 2026 01:52:08 +0000 Subject: [PATCH 086/219] Backport #128727 to 26.8: Fix malformed Bloom filter granule abort --- .../MergeTree/MergeTreeIndexBloomFilter.cpp | 19 ++++- .../gtest_merge_tree_index_bloom_filter.cpp | 74 +++++++++++++++++++ 2 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 src/Storages/MergeTree/tests/gtest_merge_tree_index_bloom_filter.cpp diff --git a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp index 19d5a3516a5b7..7d21b561a7575 100644 --- a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp @@ -29,6 +29,7 @@ #include #include #include +#include namespace DB @@ -42,6 +43,7 @@ namespace Setting namespace ErrorCodes { extern const int BAD_ARGUMENTS; + extern const int CORRUPTED_DATA; extern const int ILLEGAL_COLUMN; extern const int INCORRECT_QUERY; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; @@ -101,9 +103,22 @@ void MergeTreeIndexGranuleBloomFilter::deserializeBinary(ReadBuffer & istr, Merg throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown index version {}.", version); readVarUInt(total_rows, istr); + if (total_rows == 0) + throw Exception(ErrorCodes::CORRUPTED_DATA, "Bloom filter index granule has zero rows"); - static size_t atom_size = 8; - size_t bytes_size = (bits_per_row * total_rows + atom_size - 1) / atom_size; + static constexpr size_t atom_size = 8; + size_t bits_size = 0; + if (common::mulOverflow(bits_per_row, total_rows, bits_size) + || common::addOverflow(bits_size, atom_size - 1, bits_size)) + { + throw Exception( + ErrorCodes::CORRUPTED_DATA, + "Bloom filter index granule size overflows for {} rows and {} bits per row", + total_rows, + bits_per_row); + } + + size_t bytes_size = bits_size / atom_size; size_t read_size = bytes_size; for (auto & filter : bloom_filters) { diff --git a/src/Storages/MergeTree/tests/gtest_merge_tree_index_bloom_filter.cpp b/src/Storages/MergeTree/tests/gtest_merge_tree_index_bloom_filter.cpp new file mode 100644 index 0000000000000..007d155521ef5 --- /dev/null +++ b/src/Storages/MergeTree/tests/gtest_merge_tree_index_bloom_filter.cpp @@ -0,0 +1,74 @@ +#include + +#include +#include +#include +#include +#include + +#include + +using namespace DB; + +namespace DB::ErrorCodes +{ + extern const int CORRUPTED_DATA; +} + +namespace +{ + +String encodeRowCount(size_t row_count) +{ + WriteBufferFromOwnString out; + writeVarUInt(row_count, out); + return out.str(); +} + +void expectCorruptedRowCount(size_t bits_per_row, size_t row_count) +{ + MergeTreeIndexGranuleBloomFilter granule(bits_per_row, 3, 1); + String encoded = encodeRowCount(row_count); + ReadBufferFromString in(encoded); + + try + { + granule.deserializeBinary(in, 1); + FAIL() << "Expected a corrupted Bloom-filter granule to be rejected"; + } + catch (const Exception & exception) + { + EXPECT_EQ(exception.code(), ErrorCodes::CORRUPTED_DATA); + } +} + +} + +TEST(MergeTreeIndexBloomFilterTest, DeserializesValidGranule) +{ + String encoded = encodeRowCount(1); + encoded.append(sizeof(BloomFilter::UnderType), '\0'); + + MergeTreeIndexGranuleBloomFilter granule(12, 3, 1); + ReadBufferFromString in(encoded); + + EXPECT_NO_THROW(granule.deserializeBinary(in, 1)); + EXPECT_FALSE(granule.empty()); +} + +TEST(MergeTreeIndexBloomFilterTest, RejectsZeroRowGranule) +{ + expectCorruptedRowCount(12, 0); +} + +TEST(MergeTreeIndexBloomFilterTest, RejectsFilterSizeOverflow) +{ + constexpr size_t bits_per_row = 12; + constexpr size_t overflowing_rows = std::numeric_limits::max() / bits_per_row + 1; + expectCorruptedRowCount(bits_per_row, overflowing_rows); +} + +TEST(MergeTreeIndexBloomFilterTest, RejectsRoundedFilterSizeOverflow) +{ + expectCorruptedRowCount(1, std::numeric_limits::max()); +} From 7dbad3931e9d58171d1c4572aac1224c69c9a2e1 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 4 Nov 2026 01:56:06 +0000 Subject: [PATCH 087/219] Backport #126519 to 26.8: Fix `CANNOT_SCHEDULE_TASK` in partitioned Iceberg `INSERT` with many partitions --- .../DataLakes/Iceberg/MultipleFileWriter.cpp | 3 ++ .../DataLakes/Iceberg/Mutations.cpp | 4 ++ ...rite_many_partitions_thread_pool.reference | 3 ++ ...eberg_write_many_partitions_thread_pool.sh | 39 +++++++++++++++++++ 4 files changed, 49 insertions(+) create mode 100644 tests/queries/0_stateless/05406_iceberg_write_many_partitions_thread_pool.reference create mode 100755 tests/queries/0_stateless/05406_iceberg_write_many_partitions_thread_pool.sh diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/MultipleFileWriter.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/MultipleFileWriter.cpp index 1935421c3701f..a9045b1423847 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/MultipleFileWriter.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/MultipleFileWriter.cpp @@ -99,6 +99,9 @@ void MultipleFileWriter::finalize() completed_file_stats.push_back(std::move(current_file_stats)); data_file_byte_counts.push_back(file_bytes); data_file_row_counts.push_back(current_file_num_rows.value_or(0)); + + /// A finished format can still hold resources: a Parquet one keeps its idle encoder threads. + release(); } void MultipleFileWriter::release() diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp index 6ffc50d493b88..a77a367fa008e 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp @@ -277,6 +277,8 @@ static std::optional writeDataFiles( path_resolver.resolve(delete_data_result[partition_key].path), /*with_tags=*/ false).size_bytes; delete_data_result[partition_key].total_bytes = static_cast(delete_bytes); } + delete_data_writers[partition_key].reset(); + delete_data_write_buffers[partition_key].reset(); } } @@ -351,6 +353,8 @@ static std::optional writeDataFiles( path_resolver.resolve(update_data_result[partition_key].path), /*with_tags=*/ false).size_bytes; update_data_result[partition_key].total_bytes = static_cast(update_bytes); } + update_data_writers[partition_key].reset(); + update_data_write_buffers[partition_key].reset(); } } diff --git a/tests/queries/0_stateless/05406_iceberg_write_many_partitions_thread_pool.reference b/tests/queries/0_stateless/05406_iceberg_write_many_partitions_thread_pool.reference new file mode 100644 index 0000000000000..dfd940ddad714 --- /dev/null +++ b/tests/queries/0_stateless/05406_iceberg_write_many_partitions_thread_pool.reference @@ -0,0 +1,3 @@ +3000 30 +3000 4501500 +1000 499500 diff --git a/tests/queries/0_stateless/05406_iceberg_write_many_partitions_thread_pool.sh b/tests/queries/0_stateless/05406_iceberg_write_many_partitions_thread_pool.sh new file mode 100755 index 0000000000000..e213e850986de --- /dev/null +++ b/tests/queries/0_stateless/05406_iceberg_write_many_partitions_thread_pool.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# - no-fasttest: requires `IcebergLocal` (USE_AVRO build option) + +# Each finished Iceberg data file used to keep its Parquet encoder threads until the commit, so writing +# 30 partitions with `max_threads = 16` needed 480 threads of the 300-thread global pool. +# https://github.com/ClickHouse/ClickHouse/issues/126508 + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +TABLE_DIR="${CLICKHOUSE_USER_FILES}/${CLICKHOUSE_TEST_UNIQUE_NAME}" +rm -rf "${TABLE_DIR}" +mkdir -p "${TABLE_DIR}" +trap 'rm -rf "${TABLE_DIR}"' EXIT + +COLS=$(for i in $(seq 16); do printf ', c%d Int64' "$i"; done) +EXPRS=$(for i in $(seq 16); do printf ', number * %d' "$i"; done) +SETTINGS="SET allow_insert_into_iceberg = 1, max_insert_threads = 1, max_threads = 16, output_format_parquet_parallel_encoding = 1" + +# INSERT, then ALTER TABLE ... UPDATE, which writes one position delete file and one data file per partition. +${CLICKHOUSE_LOCAL} --query " + ${SETTINGS}; + CREATE TABLE t (p Int32 ${COLS}) ENGINE = IcebergLocal('${TABLE_DIR}/update/') PARTITION BY p; + INSERT INTO t SELECT number % 30 ${EXPRS} FROM numbers(3000); + SELECT count(), uniqExact(_path) FROM t; + ALTER TABLE t UPDATE c1 = c1 + 1 WHERE 1; + SELECT count(), sum(c1) FROM t; +" -- --max_thread_pool_size=300 --thread_pool_queue_size=300 + +# ALTER TABLE ... DELETE writes two-column position delete files, so it needs more partitions to fill the pool. +${CLICKHOUSE_LOCAL} --query " + ${SETTINGS}; + CREATE TABLE t (p Int32, x Int64) ENGINE = IcebergLocal('${TABLE_DIR}/delete/') PARTITION BY p; + INSERT INTO t SELECT number % 80, number FROM numbers(2000); + ALTER TABLE t DELETE WHERE x >= 1000; + SELECT count(), sum(x) FROM t; +" -- --max_thread_pool_size=300 --thread_pool_queue_size=300 From 62750f257a2363d988c7629357b434dd703d4ae6 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 4 Nov 2026 07:32:40 +0000 Subject: [PATCH 088/219] Backport #127656 to 26.8: Fix `SYSTEM RESTORE REPLICA` leaving a replica behind the table metadata version on concurrent restores --- src/Common/FailPoint.cpp | 1 + src/Storages/StorageReplicatedMergeTree.cpp | 41 +++++++++-- .../test.py | 73 +++++++++++++++++++ 3 files changed, 110 insertions(+), 5 deletions(-) diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index a32f91d56b7b9..08a3ecd04eda6 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -160,6 +160,7 @@ static struct InitFiu PAUSEABLE_ONCE(query_metric_log_pause_before_finish) \ PAUSEABLE_ONCE(replicated_table_remove_zk_before_get_children) \ PAUSEABLE_ONCE(replicated_table_remove_zk_before_final_multi) \ + PAUSEABLE_ONCE(rmt_restore_replica_pause_before_metadata_version_raise) \ PAUSEABLE_ONCE(kafka2_remove_zk_before_get_children) \ PAUSEABLE_ONCE(kafka2_remove_zk_before_final_multi) \ PAUSEABLE_ONCE(keeper_map_delete_pause_before_multi) \ diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 561a9eb4d24e6..bd1d2f6321ea9 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -269,6 +269,7 @@ namespace FailPoints extern const char rmt_delay_execute_drop_range[]; extern const char replicated_table_remove_zk_before_get_children[]; extern const char replicated_table_remove_zk_before_final_multi[]; + extern const char rmt_restore_replica_pause_before_metadata_version_raise[]; extern const char check_table_inject_retryable_zk_error[]; } @@ -7357,6 +7358,7 @@ void StorageReplicatedMergeTree::restoreMetadataInZooKeeper( createReplica(metadata_snapshot, zookeeper_retries_info); createNewZooKeeperNodes(zookeeper_retries_info); + FailPointInjection::pauseFailPoint(FailPoints::rmt_restore_replica_pause_before_metadata_version_raise); /// Bump the /metadata ZNode version to match the max parts metadata version. /// The table's in-memory metadata_version is derived from the /metadata ZNode stat.version @@ -7364,12 +7366,41 @@ void StorageReplicatedMergeTree::restoreMetadataInZooKeeper( if (max_parts_metadata_version > 0) { auto zookeeper = getZooKeeper(); - String metadata_str = zookeeper->get(zookeeper_path + "/metadata"); - for (int32_t i = 0; i < max_parts_metadata_version; ++i) - zookeeper->set(zookeeper_path + "/metadata", metadata_str); - zookeeper->set(replica_path + "/metadata_version", std::to_string(max_parts_metadata_version)); + /// Each batch logs an ALTER_METADATA entry of the version it reaches, so a replica that cloned an earlier one catches up. + Coordination::Stat metadata_stat; + Coordination::Stat columns_stat; + String metadata_str = zookeeper->get(zookeeper_path + "/metadata", &metadata_stat); + String columns_str = zookeeper->get(zookeeper_path + "/columns", &columns_stat); + while (metadata_stat.version < max_parts_metadata_version) + { + const auto target = static_cast(std::min( + max_parts_metadata_version, Int64{metadata_stat.version} + static_cast(zkutil::MULTI_BATCH_SIZE))); + + Coordination::Requests ops; + ops.emplace_back(zkutil::makeCheckRequest(zookeeper_path + "/columns", columns_stat.version)); + for (int32_t version = metadata_stat.version; version < target; ++version) + ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/metadata", version + 1 == target ? metadata_str : "", version)); + + ReplicatedMergeTreeLogEntryData entry; + entry.type = LogEntry::ALTER_METADATA; + entry.source_replica = replica_name; + entry.metadata_str = metadata_str; + entry.columns_str = columns_str; + entry.alter_version = target; + entry.create_time = time(nullptr); + entry.have_mutation = false; + ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); + + Coordination::Responses responses; + const Coordination::Error code = zookeeper->tryMulti(ops, responses); + if (code != Coordination::Error::ZOK && code != Coordination::Error::ZBADVERSION) + zkutil::KeeperMultiException::check(code, ops, responses); + + metadata_str = zookeeper->get(zookeeper_path + "/metadata", &metadata_stat); + columns_str = zookeeper->get(zookeeper_path + "/columns", &columns_stat); + } - LOG_INFO(log, "Bumped metadata version to {} to match parts", max_parts_metadata_version); + LOG_INFO(log, "Raised metadata version to {} to match parts", metadata_stat.version); } LOG_INFO(log, "Created ZK nodes for table"); diff --git a/tests/integration/test_restore_replica_metadata_version/test.py b/tests/integration/test_restore_replica_metadata_version/test.py index b35af4075c11c..4fc23ce116ae7 100644 --- a/tests/integration/test_restore_replica_metadata_version/test.py +++ b/tests/integration/test_restore_replica_metadata_version/test.py @@ -36,6 +36,79 @@ def start_cluster(): cluster.shutdown() +def test_alter_after_concurrent_restore(start_cluster): + zk = cluster.get_kazoo_client("zoo1") + table = "test_concurrent_restore" + zk_path = f"/clickhouse/tables/{table}" + failpoint = "rmt_restore_replica_pause_before_metadata_version_raise" + + for node in [node_1, node_2]: + node.query(f"DROP TABLE IF EXISTS {table} SYNC") + for node in [node_1, node_2]: + node.query( + f"CREATE TABLE {table} (n UInt32) " + f"ENGINE = ReplicatedMergeTree('{zk_path}/', '{node.name}') ORDER BY n" + ) + + # Two ALTERs bring the metadata version of the table, and of the part inserted after them, to 2 + for column in ["a", "b"]: + node_1.query( + f"ALTER TABLE {table} ADD COLUMN {column} UInt32", + settings={"alter_sync": 2}, + ) + node_1.query(f"INSERT INTO {table} VALUES (1, 1, 1)") + node_2.query(f"SYSTEM SYNC REPLICA {table}") + for node in [node_1, node_2]: + assert ( + node.query( + f"SELECT metadata_version FROM system.tables WHERE name = '{table}'" + ) + == "2\n" + ) + + zk_rmr_with_retries(zk, zk_path) + for node in [node_1, node_2]: + node.query(f"SYSTEM RESTART REPLICA {table}") + + # Restore both replicas at the same time, as ON CLUSTER does: node_1 stops after it has + # restored the table in Keeper, before it raises the metadata version to the one of its part, + # and node_2 restores in the meantime by copying node_1 + node_1.query(f"SYSTEM ENABLE FAILPOINT {failpoint}") + restore = node_1.get_query_request(f"SYSTEM RESTORE REPLICA {table}") + node_1.query(f"SYSTEM WAIT FAILPOINT {failpoint} PAUSE") + node_2.query(f"SYSTEM RESTORE REPLICA {table}") + assert ( + node_2.query(f"SELECT is_readonly FROM system.replicas WHERE table = '{table}'") + == "0\n" + ) + node_1.query(f"SYSTEM DISABLE FAILPOINT {failpoint}") + restore.get_answer() + + for node in [node_1, node_2]: + node.query(f"SYSTEM SYNC REPLICA {table}") + + _, stat = zk.get(f"{zk_path}/metadata") + assert stat.version == 2 + for node in [node_1, node_2]: + version, _ = zk.get(f"{zk_path}/replicas/{node.name}/metadata_version") + assert version == b"2" + assert ( + node.query( + f"SELECT metadata_version FROM system.tables WHERE name = '{table}'" + ) + == "2\n" + ) + + # A replica left below the metadata version of the table rejects every ALTER with CANNOT_ASSIGN_ALTER + node_2.query(f"ALTER TABLE {table} ADD COLUMN c UInt32", settings={"alter_sync": 2}) + node_1.query(f"ALTER TABLE {table} ADD COLUMN d UInt32", settings={"alter_sync": 2}) + for node in [node_1, node_2]: + assert node.query(f"SELECT n, a, b, c, d FROM {table}") == "1\t1\t1\t0\t0\n" + + for node in [node_1, node_2]: + node.query(f"DROP TABLE IF EXISTS {table} SYNC") + + def test_fix_metadata_version_on_attach_part_after_restore(start_cluster): zk = cluster.get_kazoo_client("zoo1") From c7cda082ca5ad403d76482507c99ccff99697dec Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 4 Nov 2026 17:36:23 +0000 Subject: [PATCH 089/219] Backport #127190 to 26.8: Do not exhaust the stack or use quadratic memory in column operations on a deeply nested Dynamic value --- src/Columns/ColumnArray.cpp | 24 ++++----- src/Columns/ColumnDynamic.cpp | 29 +++++++++++ src/Columns/ColumnDynamic.h | 35 ++++++++++++- ...mic_deep_value_column_operations.reference | 4 ++ ...07_dynamic_deep_value_column_operations.sh | 50 +++++++++++++++++++ 5 files changed, 128 insertions(+), 14 deletions(-) create mode 100644 tests/queries/0_stateless/05407_dynamic_deep_value_column_operations.reference create mode 100755 tests/queries/0_stateless/05407_dynamic_deep_value_column_operations.sh diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 983e8647d420e..17bf48ecdbd20 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -1015,17 +1015,17 @@ ColumnPtr ColumnArray::filterGeneric(const Filter & filt, ssize_t result_size_hi memset(&nested_filt[offsetAt(i)], 0, sizeAt(i)); } - auto res = ColumnArray::create(data->cloneEmpty()); - ssize_t nested_result_size_hint = 0; if (result_size_hint < 0) nested_result_size_hint = result_size_hint; else if (result_size_hint && result_size_hint < 1000000000 && data->size() < 1000000000) /// Avoid overflow. nested_result_size_hint = result_size_hint * data->size() / size; - res->data = data->filter(nested_filt, nested_result_size_hint); + /// See the comment in `indexImpl` about building the nested result first. + ColumnPtr res_data = data->filter(nested_filt, nested_result_size_hint); - Offsets & res_offsets = res->getOffsets(); + auto res_offsets_column = ColumnOffsets::create(); + Offsets & res_offsets = res_offsets_column->getData(); if (result_size_hint) res_offsets.reserve_exact(result_size_hint > 0 ? result_size_hint : size); @@ -1039,7 +1039,7 @@ ColumnPtr ColumnArray::filterGeneric(const Filter & filt, ssize_t result_size_hi } } - return res; + return ColumnArray::create(res_data, std::move(res_offsets_column)); } ColumnPtr ColumnArray::filterNullable(const Filter & filt, ssize_t result_size_hint) const @@ -1273,10 +1273,8 @@ ColumnPtr ColumnArray::indexImpl(const PaddedPODArray & indexes, size_t limit PaddedPODArray & nested_indexes = nested_indexes_column->getData(); nested_indexes.reserve_exact(getOffsets().back()); - auto res = ColumnArray::create(data->cloneEmpty()); - - Offsets & res_offsets = res->getOffsets(); - res_offsets.resize_exact(limit); + auto res_offsets_column = ColumnOffsets::create(limit); + Offsets & res_offsets = res_offsets_column->getData(); size_t current_offset = 0; for (size_t i = 0; i < limit; ++i) @@ -1287,10 +1285,10 @@ ColumnPtr ColumnArray::indexImpl(const PaddedPODArray & indexes, size_t limit res_offsets[i] = current_offset; } - if (current_offset != 0) - res->data = data->index(*nested_indexes_column, current_offset); - - return res; + /// Build the nested result first: an empty clone of `data` copies its whole nested structure, which for + /// `Dynamic` is as deep as the data, and every level of this recursion would hold one. + ColumnPtr res_data = current_offset != 0 ? data->index(*nested_indexes_column, current_offset) : data->cloneEmpty(); + return ColumnArray::create(res_data, std::move(res_offsets_column)); } INSTANTIATE_INDEX_IMPL(ColumnArray) diff --git a/src/Columns/ColumnDynamic.cpp b/src/Columns/ColumnDynamic.cpp index 073730faf6037..6937da418c568 100644 --- a/src/Columns/ColumnDynamic.cpp +++ b/src/Columns/ColumnDynamic.cpp @@ -338,6 +338,8 @@ void ColumnDynamic::get(size_t n, Field & res) const void ColumnDynamic::getValueNameImpl(WriteBufferFromOwnString & name_buf, size_t n, const Options & options) const { + checkStackSize(); + const auto & variant_col = getVariantColumn(); const auto discr = variant_col.globalDiscriminatorAt(n); if (discr == ColumnVariant::NULL_DISCRIMINATOR) @@ -374,6 +376,8 @@ void ColumnDynamic::insertFrom(const IColumn & src_, size_t n) void ColumnDynamic::doInsertFrom(const IColumn & src_, size_t n) #endif { + checkStackSize(); + const auto & dynamic_src = assert_cast(src_); /// Same variants: copy the Variant column directly. Skip this fast path when the source has values in @@ -453,6 +457,8 @@ void ColumnDynamic::insertRangeFrom(const IColumn & src_, size_t start, size_t l void ColumnDynamic::doInsertRangeFrom(const IColumn & src_, size_t start, size_t length) #endif { + checkStackSize(); + if (start + length > src_.size()) throw Exception(ErrorCodes::PARAMETER_OUT_OF_BOUND, "Parameter out of bound in ColumnDynamic::insertRangeFrom method. " "[start({}) + length({}) > src.size()({})]", start, length, src_.size()); @@ -685,6 +691,8 @@ void ColumnDynamic::insertManyFrom(const IColumn & src_, size_t position, size_t void ColumnDynamic::doInsertManyFrom(const IColumn & src_, size_t position, size_t length) #endif { + checkStackSize(); + const auto & dynamic_src = assert_cast(src_); auto & variant_col = getVariantColumn(); @@ -900,6 +908,8 @@ void ColumnDynamic::skipSerializedInArena(ReadBuffer & in) const void ColumnDynamic::updateHashWithValue(size_t n, SipHash & hash) const { + checkStackSize(); + const auto & variant_col = getVariantColumn(); auto discr = variant_col.globalDiscriminatorAt(n); if (discr == ColumnVariant::NULL_DISCRIMINATOR) @@ -930,6 +940,7 @@ void ColumnDynamic::updateHashWithValue(size_t n, SipHash & hash) const void ColumnDynamic::updateHashWithValueRange(size_t begin, size_t end, SipHash & hash) const { + checkStackSize(); variant_column_ptr->updateHashWithValueRange(begin, end, hash); } @@ -980,6 +991,8 @@ int ColumnDynamic::compareAt(size_t n, size_t m, const IColumn & rhs, int nan_di int ColumnDynamic::doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const #endif { + checkStackSize(); + const auto & left_variant = getVariantColumn(); const auto & right_dynamic = assert_cast(rhs); const auto & right_variant = right_dynamic.getVariantColumn(); @@ -1106,6 +1119,7 @@ void ColumnDynamic::updatePermutation(IColumn::PermutationSortDirection directio ColumnPtr ColumnDynamic::compress(bool force_compression) const { + checkStackSize(); ColumnPtr variant_compressed = variant_column_ptr->compress(force_compression); size_t byte_size = variant_compressed->byteSize(); return ColumnCompressed::create(size(), byte_size, @@ -1117,6 +1131,7 @@ ColumnPtr ColumnDynamic::compress(bool force_compression) const ColumnCheckpointPtr ColumnDynamic::getCheckpoint() const { + checkStackSize(); UnorderedMapWithMemoryTracking variants_checkpoints; for (const auto & [name, discr] : variant_info.variant_name_to_discriminator) variants_checkpoints[name] = variant_column_ptr->getVariantByGlobalDiscriminator(discr).getCheckpoint(); @@ -1125,6 +1140,7 @@ ColumnCheckpointPtr ColumnDynamic::getCheckpoint() const void ColumnDynamic::updateCheckpoint(ColumnCheckpoint & checkpoint) const { + checkStackSize(); auto & variants_checkpoints = assert_cast(checkpoint).variants_checkpoints; for (const auto & [name, discr] : variant_info.variant_name_to_discriminator) @@ -1142,6 +1158,7 @@ void ColumnDynamic::updateCheckpoint(ColumnCheckpoint & checkpoint) const void ColumnDynamic::rollback(const ColumnCheckpoint & checkpoint) { + checkStackSize(); const auto & variants_checkpoints = assert_cast(checkpoint).variants_checkpoints; /// Manually rollback internals of Variant column @@ -1214,6 +1231,7 @@ void ColumnDynamic::getAllTypeNamesInto(UnorderedSetWithMemoryTracking & void ColumnDynamic::prepareForSquashing(const VectorWithMemoryTracking & source_columns, size_t factor) { + checkStackSize(); if (source_columns.empty()) return; @@ -1237,6 +1255,7 @@ void ColumnDynamic::prepareForSquashing(const VectorWithMemoryTracking & source_columns, size_t factor) { + checkStackSize(); /// Internal variants of source dynamic columns may differ. /// We want to preallocate memory for all variants we will have after squashing. /// It may happen that the total number of variants in source columns will @@ -1341,6 +1360,8 @@ void ColumnDynamic::prepareVariantsForSquashing(const VectorWithMemoryTracking(&rhs)) return max_dynamic_types == rhs_concrete->max_dynamic_types && global_max_dynamic_types == rhs_concrete->global_max_dynamic_types && variant_info.variant_name == rhs_concrete->variant_info.variant_name @@ -1350,6 +1371,8 @@ bool ColumnDynamic::dynamicStructureEquals(const IColumn & rhs) const void ColumnDynamic::chooseDynamicStructureForMerge(const VectorWithMemoryTracking & source_columns, std::optional max_dynamic_subcolumns) { + checkStackSize(); + if (!empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "chooseDynamicStructureForMerge should be called only on empty Dynamic column"); @@ -1480,6 +1503,8 @@ void ColumnDynamic::chooseDynamicStructureForMerge(const VectorWithMemoryTrackin void ColumnDynamic::takeExactDynamicStructureFrom(const IColumn & source) { + checkStackSize(); + if (!empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "takeExactDynamicStructureFrom should be called only on empty Dynamic column"); @@ -1502,6 +1527,8 @@ void ColumnDynamic::takeExactDynamicStructureFrom(const IColumn & source) void ColumnDynamic::fixDynamicStructure() { + checkStackSize(); + /// Reduce max_dynamic_types to the number of selected variants, so there will be no possibility /// to extend selected variants on inserts into this column. /// -1 because we don't count shared variant in the limit. @@ -1536,6 +1563,8 @@ ColumnDynamic::StatisticsPtr ColumnDynamic::getOrCalculateStatistics() const void ColumnDynamic::takeOrCalculateStatisticsFrom(const VectorWithMemoryTracking & source_columns) { + checkStackSize(); + /// Assumes dynamic structure has already been set by `takeExactDynamicStructureFrom` or `chooseDynamicStructureForMerge`. Statistics new_statistics; /// Collect total sizes for variants that are not in our structure (candidates for shared variant statistics). diff --git a/src/Columns/ColumnDynamic.h b/src/Columns/ColumnDynamic.h index 66979244f0e63..118561cf921d0 100644 --- a/src/Columns/ColumnDynamic.h +++ b/src/Columns/ColumnDynamic.h @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB @@ -24,6 +25,10 @@ namespace DB * When the limit on number of dynamic types is exceeded, all values * with new types are inserted into special shared variant with type String * that contains values and their types in binary format. + * + * Values nest into each other through variants like Array(Dynamic) to a depth that comes + * from the data, not from the declared type, so methods that recurse into the Variant + * column call checkStackSize. */ class ColumnDynamic final : public COWHelper, ColumnDynamic> { @@ -116,12 +121,14 @@ class ColumnDynamic final : public COWHelper, Colum MutableColumnPtr cloneEmpty() const override { + checkStackSize(); /// Keep current dynamic structure return Base::create(variant_column->cloneEmpty(), variant_info, max_dynamic_types, global_max_dynamic_types, statistics); } MutableColumnPtr cloneResized(size_t size) const override { + checkStackSize(); return Base::create(variant_column->cloneResized(size), variant_info, max_dynamic_types, global_max_dynamic_types, statistics); } @@ -181,6 +188,7 @@ class ColumnDynamic final : public COWHelper, Colum void popBack(size_t n) override { + checkStackSize(); variant_column_ptr->popBack(n); } @@ -207,42 +215,50 @@ class ColumnDynamic final : public COWHelper, Colum void updateHashFast(SipHash & hash) const override { + checkStackSize(); variant_column_ptr->updateHashFast(hash); } ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override { + checkStackSize(); return create(variant_column_ptr->filter(filt, result_size_hint), variant_info, max_dynamic_types, global_max_dynamic_types, statistics); } void filter(const Filter & filt) override { + checkStackSize(); variant_column_ptr->filter(filt); statistics.reset(); } void expand(const Filter & mask, bool inverted) override { + checkStackSize(); variant_column_ptr->expand(mask, inverted); } ColumnPtr permute(const Permutation & perm, size_t limit) const override { + checkStackSize(); return create(variant_column_ptr->permute(perm, limit), variant_info, max_dynamic_types, global_max_dynamic_types, statistics); } ColumnPtr index(const IColumn & indexes, size_t limit) const override { + checkStackSize(); return create(variant_column_ptr->index(indexes, limit), variant_info, max_dynamic_types, global_max_dynamic_types, statistics); } ColumnPtr replicate(const Offsets & replicate_offsets) const override { + checkStackSize(); return create(variant_column_ptr->replicate(replicate_offsets), variant_info, max_dynamic_types, global_max_dynamic_types, statistics); } VectorWithMemoryTracking scatter(size_t num_columns, const Selector & selector) const override { + checkStackSize(); VectorWithMemoryTracking scattered_variant_columns = variant_column_ptr->scatter(num_columns, selector); VectorWithMemoryTracking scattered_columns; scattered_columns.reserve(num_columns); @@ -266,6 +282,7 @@ class ColumnDynamic final : public COWHelper, Colum bool hasEqualValues() const override { + checkStackSize(); return variant_column_ptr->hasEqualValues(); } @@ -296,31 +313,37 @@ class ColumnDynamic final : public COWHelper, Colum void shrinkToFit() override { + checkStackSize(); variant_column_ptr->shrinkToFit(); } void ensureOwnership() override { + checkStackSize(); variant_column_ptr->ensureOwnership(); } size_t byteSize() const override { + checkStackSize(); return variant_column_ptr->byteSize(); } size_t byteSizeAt(size_t n) const override { + checkStackSize(); return variant_column_ptr->byteSizeAt(n); } size_t allocatedBytes() const override { + checkStackSize(); return variant_column_ptr->allocatedBytes(); } void protect() override { + checkStackSize(); variant_column_ptr->protect(); } @@ -332,14 +355,21 @@ class ColumnDynamic final : public COWHelper, Colum void forEachMutableSubcolumn(MutableColumnCallback callback) override { + /// Callers like IColumn::mutate recurse into the subcolumns through this method. + checkStackSize(); callback(variant_column); variant_column_ptr = assert_cast(variant_column.get()); } - void forEachSubcolumn(ColumnCallback callback) const override { callback(variant_column); } + void forEachSubcolumn(ColumnCallback callback) const override + { + checkStackSize(); + callback(variant_column); + } void forEachMutableSubcolumnRecursively(RecursiveMutableColumnCallback callback) override { + checkStackSize(); callback(*variant_column); variant_column_ptr = assert_cast(variant_column.get()); variant_column->forEachMutableSubcolumnRecursively(callback); @@ -347,6 +377,7 @@ class ColumnDynamic final : public COWHelper, Colum void forEachSubcolumnRecursively(RecursiveColumnCallback callback) const override { + checkStackSize(); callback(*variant_column); variant_column->forEachSubcolumnRecursively(callback); } @@ -377,11 +408,13 @@ class ColumnDynamic final : public COWHelper, Colum void finalize() override { + checkStackSize(); variant_column_ptr->finalize(); } bool isFinalized() const override { + checkStackSize(); return variant_column_ptr->isFinalized(); } diff --git a/tests/queries/0_stateless/05407_dynamic_deep_value_column_operations.reference b/tests/queries/0_stateless/05407_dynamic_deep_value_column_operations.reference new file mode 100644 index 0000000000000..636bda45df5e6 --- /dev/null +++ b/tests/queries/0_stateless/05407_dynamic_deep_value_column_operations.reference @@ -0,0 +1,4 @@ +[[[[NULL]]],[[[NULL]]]] +[NULL,NULL] +4 2 +server is alive diff --git a/tests/queries/0_stateless/05407_dynamic_deep_value_column_operations.sh b/tests/queries/0_stateless/05407_dynamic_deep_value_column_operations.sh new file mode 100755 index 0000000000000..81d6382e19109 --- /dev/null +++ b/tests/queries/0_stateless/05407_dynamic_deep_value_column_operations.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# The fast-test build is unoptimized and re-runs a new test several times under a 60s cap, while +# materializing a deeply nested Dynamic value whose operation does not trip the recursion guard takes +# superlinear time. The optimized stateless configurations run it within budget. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Each repetition of 1E2B0801 is a value of type Array(Dynamic(max_types=8)) with one element, so the +# nesting depth of the Dynamic value comes from the data, not from the declared type. The second row is NULL. +nested_value() { echo "format(RowBinary, 'd Dynamic', unhex(concat(repeat('1E2B0801', $1), '0000')))"; } + +$CLICKHOUSE_CLIENT -q "SELECT arraySort([d, d]) FROM $(nested_value 3)" + +# Depending on the build and the stack size, a deep value is either processed or rejected with +# TOO_DEEP_RECURSION, but it must never crash the server or use memory quadratic in the depth. +# Parsing a value takes time quadratic in its depth (every level has a longer type name), so the deepest value, +# which exhausts the stack in the release build, is only used for the operations that recurse into the nested +# columns: `index`, `compareAt`, and `filter`. +check_query() { + out=$($CLICKHOUSE_CLIENT -q "$1 FORMAT Null" 2>&1) + [[ -z "$out" || "$out" == *TOO_DEEP_RECURSION* ]] || echo "$1: $out" +} +table=$(nested_value 1000) +for query in "SELECT [d] FROM $table" "SELECT arraySort([d, d]) FROM $table" "SELECT d, arrayJoin([1, 2]) FROM $table" "SELECT d FROM $table WHERE d IS NOT NULL"; do + check_query "$query" +done +table=$(nested_value 9000) +for query in "SELECT [d] FROM $table" "SELECT arraySort([d, d]) FROM $table" "SELECT d FROM $table WHERE d IS NOT NULL"; do + check_query "$query" +done + +# Writing a part and merging parts go through the dynamic structure helpers of the column. +# Writing a part takes time quadratic in the nesting depth (every level adds substreams with longer names), +# so a depth that exhausts the stack takes tens of seconds; use a small depth that checks the normal path +# and stays far from the stack limit in sanitizer builds, where depth 300 already hits it. +# The substream names of a deep value exceed the file name length limit of the filesystem in a wide part, +# so they must be replaced with hashes regardless of the randomized MergeTree settings. +$CLICKHOUSE_CLIENT -q "CREATE TABLE deep_dynamic (d Dynamic) ENGINE = MergeTree ORDER BY tuple() SETTINGS replace_long_file_name_to_hash = 1, max_file_name_length = 127" +table=$(nested_value 64) +for query in "INSERT INTO deep_dynamic SELECT d FROM $table" "INSERT INTO deep_dynamic SELECT d FROM $table" "OPTIMIZE TABLE deep_dynamic FINAL"; do + out=$($CLICKHOUSE_CLIENT -q "$query" 2>&1) + [[ -z "$out" ]] || echo "$query: $out" +done +$CLICKHOUSE_CLIENT -q "SELECT count(), countIf(d IS NULL) FROM deep_dynamic" +$CLICKHOUSE_CLIENT -q "DROP TABLE deep_dynamic" + +$CLICKHOUSE_CLIENT -q "SELECT 'server is alive'" From 6ca313f250c0fedc18268706b9558965c4f88d9b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 4 Nov 2026 10:48:11 +0000 Subject: [PATCH 090/219] Backport #128674 to 26.8: Do not pre-allocate memory for declared string length in Keeper --- src/Common/ZooKeeper/ZooKeeperConstants.h | 2 + src/Common/ZooKeeper/ZooKeeperIO.cpp | 23 +++++-- src/Coordination/tests/gtest_coordination.cpp | 63 +++++++++++++++++++ 3 files changed, 83 insertions(+), 5 deletions(-) diff --git a/src/Common/ZooKeeper/ZooKeeperConstants.h b/src/Common/ZooKeeper/ZooKeeperConstants.h index 68f9459ddd671..36de2427dbbef 100644 --- a/src/Common/ZooKeeper/ZooKeeperConstants.h +++ b/src/Common/ZooKeeper/ZooKeeperConstants.h @@ -95,6 +95,8 @@ static constexpr int32_t PASSWORD_LENGTH = 16; static constexpr int32_t MAX_STRING_OR_ARRAY_SIZE = 1 << 28; /// 256 MiB /// Upper bound on elements reserved up front when reading an array; the declared size is attacker-controlled static constexpr int32_t MAX_ARRAY_PREALLOCATION = 4096; +/// Bytes read per step when reading a string; the declared size is attacker-controlled, so memory grows with the bytes received +static constexpr int32_t MAX_STRING_PREALLOCATION = 1 << 16; /// 64 KiB /// Request length prefix is a signed int32, so this is a hard limit static constexpr uint64_t MAX_REQUEST_SIZE_HARD_LIMIT = std::numeric_limits::max(); /// Smallest server-advertised max_request_size we accept; anything below is treated as bogus and ignored diff --git a/src/Common/ZooKeeper/ZooKeeperIO.cpp b/src/Common/ZooKeeper/ZooKeeperIO.cpp index 1a2956e7fbe26..10c6d05591fc1 100644 --- a/src/Common/ZooKeeper/ZooKeeperIO.cpp +++ b/src/Common/ZooKeeper/ZooKeeperIO.cpp @@ -93,11 +93,24 @@ void read(std::string & s, ReadBuffer & in) if (size > MAX_STRING_OR_ARRAY_SIZE) throw Exception(Error::ZMARSHALLINGERROR,"Too large string size while reading from ZooKeeper {}", size); - s.resize(size); - size_t read_bytes = in.read(s.data(), size); - if (read_bytes != static_cast(size)) - throw Exception( - Error::ZMARSHALLINGERROR, "Buffer size read from Zookeeper is not big enough. Expected {}. Got {}", size, read_bytes); + /// Memory must track bytes actually read, not the declared size: + /// do not pre-allocate for an attacker-controlled length. + s.clear(); + size_t remaining = static_cast(size); + while (remaining > 0) + { + const size_t chunk = std::min(remaining, MAX_STRING_PREALLOCATION); + const size_t old_size = s.size(); + s.resize(old_size + chunk); + const size_t read_bytes = in.read(s.data() + old_size, chunk); + if (read_bytes != chunk) + throw Exception( + Error::ZMARSHALLINGERROR, + "Buffer size read from Zookeeper is not big enough. Expected {}. Got {}", + size, + old_size + read_bytes); + remaining -= chunk; + } } void read(ACL & acl, ReadBuffer & in) diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp index daf984bb6ac67..39d64147a3656 100644 --- a/src/Coordination/tests/gtest_coordination.cpp +++ b/src/Coordination/tests/gtest_coordination.cpp @@ -427,6 +427,69 @@ TEST_P(CoordinationTest, CreateRequestDoesNotPreallocateOversizedACLs) } } +TEST(CoordinationSerialization, StringReadDoesNotPreallocateDeclaredSize) +{ + /// Invariant: memory must track bytes actually read, not the declared string length. + const size_t chunk = Coordination::MAX_STRING_PREALLOCATION; + + auto expect_marshalling_error = [](DB::ReadBuffer & rbuf, std::string & s) + { + try + { + Coordination::read(s, rbuf); + FAIL() << "Expected ZMARSHALLINGERROR"; + } + catch (const Coordination::Exception & e) + { + EXPECT_EQ(e.code, Coordination::Error::ZMARSHALLINGERROR); + } + }; + + { + /// The largest allowed length is declared, but no payload bytes follow. + DB::WriteBufferFromNuraftBuffer wbuf; + Coordination::write(Coordination::MAX_STRING_OR_ARRAY_SIZE, wbuf); + + DB::ReadBufferFromNuraftBuffer rbuf(wbuf.getBuffer()); + std::string s; + expect_marshalling_error(rbuf, s); + EXPECT_LE(s.capacity(), 2 * chunk); + } + + { + /// The input ends after one complete chunk. + DB::WriteBufferFromNuraftBuffer wbuf; + Coordination::write(Coordination::MAX_STRING_OR_ARRAY_SIZE, wbuf); + const std::string first_chunk(chunk, 'x'); + wbuf.write(first_chunk.data(), first_chunk.size()); + + DB::ReadBufferFromNuraftBuffer rbuf(wbuf.getBuffer()); + std::string s; + expect_marshalling_error(rbuf, s); + EXPECT_LE(s.capacity(), 4 * chunk); + } + + { + /// A string that needs more than one chunk is read completely, and the next field is aligned. + std::string expected(chunk + 1, '\0'); + for (size_t i = 0; i < expected.size(); ++i) + expected[i] = static_cast(i % 251); + + DB::WriteBufferFromNuraftBuffer wbuf; + Coordination::write(expected, wbuf); + Coordination::write(static_cast(42), wbuf); + + DB::ReadBufferFromNuraftBuffer rbuf(wbuf.getBuffer()); + std::string s; + int32_t next = 0; + Coordination::read(s, rbuf); + Coordination::read(next, rbuf); + EXPECT_EQ(s, expected); + EXPECT_EQ(next, 42); + EXPECT_TRUE(rbuf.eof()); + } +} + template struct SimpliestRaftServer { From 9c680b0e8b2ba1755f1e8c847510a13bafb5c3f5 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 4 Nov 2026 12:07:14 +0000 Subject: [PATCH 091/219] Backport #129137 to 26.8: Stop the authentication quota from disclosing whether a user exists --- src/Access/AccessControl.cpp | 29 ++++++++++--------- .../02884_authentication_quota.reference | 12 ++++---- .../0_stateless/02884_authentication_quota.sh | 6 ++-- 3 files changed, 25 insertions(+), 22 deletions(-) diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index 5fe6573b5714f..bcca86281d53f 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -602,22 +602,25 @@ AuthResult AccessControl::authenticate(const Credentials & credentials, const Po // NOTE: In the case where the user has never been logged in using LDAP, // Then user_id is not generated, and the authentication quota will always be nullptr. auto authentication_quota = getAuthenticationQuota(credentials.getUserName(), address, client_info.getLastForwardedForHost()); - if (authentication_quota) - { - /// Reserve a single try from the quota to check whether we have another authentication try. - /// This is required for correct behavior in this situation: - /// User has 1 login failures quota. - /// * At the first login with an invalid password: Increase the quota counter. 1 (used) > 1 (max) is false. - /// Then try to authenticate the user and throw an AUTHENTICATION_FAILED error. - /// * In case of the second try: increase quota counter, 2 (used) > 1 (max), then throw QUOTA_EXCEED - /// and don't let the user authenticate. - /// - /// The authentication failures counter will be reset after successful authentication. - authentication_quota->used(QuotaType::FAILED_SEQUENTIAL_AUTHENTICATIONS, 1); - } try { + if (authentication_quota) + { + /// Reserve a single try from the quota to check whether we have another authentication try. + /// This is required for correct behavior in this situation: + /// User has 1 login failures quota. + /// * At the first login with an invalid password: Increase the quota counter. 1 (used) > 1 (max) is false. + /// Then try to authenticate the user and throw an AUTHENTICATION_FAILED error. + /// * In case of the second try: increase quota counter, 2 (used) > 1 (max), then throw QUOTA_EXCEED + /// and don't let the user authenticate. + /// + /// The authentication failures counter will be reset after successful authentication. + /// Inside the try block, so that an exceeded quota is reported like any other authentication failure + /// and doesn't reveal whether the user exists. + authentication_quota->used(QuotaType::FAILED_SEQUENTIAL_AUTHENTICATIONS, 1); + } + const auto auth_result = MultipleAccessStorage::authenticate(credentials, address, *external_authenticators, client_info, allow_no_password, allow_plaintext_password); if (authentication_quota) diff --git a/tests/queries/0_stateless/02884_authentication_quota.reference b/tests/queries/0_stateless/02884_authentication_quota.reference index d8ed6d97b5051..c412693811857 100644 --- a/tests/queries/0_stateless/02884_authentication_quota.reference +++ b/tests/queries/0_stateless/02884_authentication_quota.reference @@ -6,7 +6,7 @@ > Login to the user account using the wrong password. password is incorrect > Quota is exceeded 1 >= 1. Login with correct password should fail. -QUOTA_EXCEEDED +password is incorrect > Check the failed_sequential_authentications, max_failed_sequential_authentications fields. 2 1 > Alter the quota with MAX FAILED SEQUENTIAL AUTHENTICATIONS = 4 @@ -18,9 +18,9 @@ password is incorrect password is incorrect password is incorrect password is incorrect -QUOTA_EXCEEDED +password is incorrect > Also try to login with correct password. Quota should stay exceeded. -QUOTA_EXCEEDED +password is incorrect > Check the failed_sequential_authentications, max_failed_sequential_authentications fields. 6 4 > Reset the quota by increasing MAX FAILED SEQUENTIAL AUTHENTICATIONS and successful login @@ -32,7 +32,7 @@ QUOTA_EXCEEDED > Login to the user account using the wrong password. password is incorrect > Quota is exceeded 1 >= 1. Login with correct password should fail. -QUOTA_EXCEEDED +password is incorrect > Check the failed_sequential_authentications, max_failed_sequential_authentications fields. 2 1 > Alter the quota with MAX FAILED SEQUENTIAL AUTHENTICATIONS = 4 @@ -44,9 +44,9 @@ password is incorrect password is incorrect password is incorrect password is incorrect -QUOTA_EXCEEDED +password is incorrect > Also try to login with correct password. Quota should stay exceeded. -QUOTA_EXCEEDED +password is incorrect > Check the failed_sequential_authentications, max_failed_sequential_authentications fields. 6 4 > Reset the quota by increasing MAX FAILED SEQUENTIAL AUTHENTICATIONS and successful login diff --git a/tests/queries/0_stateless/02884_authentication_quota.sh b/tests/queries/0_stateless/02884_authentication_quota.sh index bca24bf6cc275..b092e0b35359a 100755 --- a/tests/queries/0_stateless/02884_authentication_quota.sh +++ b/tests/queries/0_stateless/02884_authentication_quota.sh @@ -18,7 +18,7 @@ function login_test() ${CLICKHOUSE_CLIENT} --user ${USER} --password "wrong_pass" --query "select 1 format Null" 2>&1 | grep -m1 -o 'password is incorrect' echo "> Quota is exceeded 1 >= 1. Login with correct password should fail." - ${CLICKHOUSE_CLIENT} --user ${USER} --password "pass" --query "select 1 format Null" 2>&1 | grep -m1 -o 'QUOTA_EXCEEDED' + ${CLICKHOUSE_CLIENT} --user ${USER} --password "pass" --query "select 1 format Null" 2>&1 | grep -m1 -o 'password is incorrect' echo "> Check the failed_sequential_authentications, max_failed_sequential_authentications fields." ${CLICKHOUSE_CLIENT} -q "SELECT failed_sequential_authentications, max_failed_sequential_authentications FROM system.quotas_usage WHERE quota_name = '${QUOTA}'" @@ -37,10 +37,10 @@ function login_test() ${CLICKHOUSE_CLIENT} --user ${USER} --password "wrong_pass" --query "select 1 format Null" 2>&1 | grep -m1 -o 'password is incorrect' ${CLICKHOUSE_CLIENT} --user ${USER} --password "wrong_pass" --query "select 1 format Null" 2>&1 | grep -m1 -o 'password is incorrect' ${CLICKHOUSE_CLIENT} --user ${USER} --password "wrong_pass" --query "select 1 format Null" 2>&1 | grep -m1 -o 'password is incorrect' - ${CLICKHOUSE_CLIENT} --user ${USER} --password "wrong_pass" --query "select 1 format Null" 2>&1 | grep -m1 -o 'QUOTA_EXCEEDED' + ${CLICKHOUSE_CLIENT} --user ${USER} --password "wrong_pass" --query "select 1 format Null" 2>&1 | grep -m1 -o 'password is incorrect' echo "> Also try to login with correct password. Quota should stay exceeded." - ${CLICKHOUSE_CLIENT} --user ${USER} --password "pass" --query "select 1 format Null" 2>&1 | grep -m1 -o 'QUOTA_EXCEEDED' + ${CLICKHOUSE_CLIENT} --user ${USER} --password "pass" --query "select 1 format Null" 2>&1 | grep -m1 -o 'password is incorrect' echo "> Check the failed_sequential_authentications, max_failed_sequential_authentications fields." ${CLICKHOUSE_CLIENT} -q "SELECT failed_sequential_authentications, max_failed_sequential_authentications FROM system.quotas_usage WHERE quota_name = '${QUOTA}'" From 794b287b872669cff5e65fbc5ce43f14ead3f543 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 4 Nov 2026 14:50:51 +0000 Subject: [PATCH 092/219] Backport #128248 to 26.8: Leave the row estimate of a filtered array join unknown in join reordering --- .../QueryPlan/Optimizations/optimizeJoin.cpp | 13 ++++++++-- ...06_join_swap_array_join_estimate.reference | 4 ++++ .../05406_join_swap_array_join_estimate.sql | 24 +++++++++++++++++++ 3 files changed, 39 insertions(+), 2 deletions(-) diff --git a/src/Processors/QueryPlan/Optimizations/optimizeJoin.cpp b/src/Processors/QueryPlan/Optimizations/optimizeJoin.cpp index 2fd31ceb62579..4b191debc0fa2 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeJoin.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeJoin.cpp @@ -527,8 +527,17 @@ RelationStats estimateReadRowsCount(QueryPlan::Node & node, const ActionsDAG::No { /// A filter above the array join can read its output, which the statistics of the input do not describe. auto stats = estimateReadRowsCount(*node.children.front(), nullptr); - if (stats.estimated_rows) - stats.estimated_rows = toUInt64Saturating(Float64(*stats.estimated_rows) * array_join_rows_multiplier); + if (stats.estimated_rows && *stats.estimated_rows != 0) + { + /// Such a filter leaves the row count unknown unless the input is empty. + if (filter) + { + stats.estimated_rows.reset(); + stats.imprecise_estimate = true; + } + else + stats.estimated_rows = toUInt64Saturating(Float64(*stats.estimated_rows) * array_join_rows_multiplier); + } if (expression_step) remapColumnStats(stats.column_stats, expression_step->getExpression()); else diff --git a/tests/queries/0_stateless/05406_join_swap_array_join_estimate.reference b/tests/queries/0_stateless/05406_join_swap_array_join_estimate.reference index a19d957fd1bd0..c826867912a98 100644 --- a/tests/queries/0_stateless/05406_join_swap_array_join_estimate.reference +++ b/tests/queries/0_stateless/05406_join_swap_array_join_estimate.reference @@ -7,3 +7,7 @@ ARRAY JOIN, 100 rows 0 1 0 10 2000 +ARRAY JOIN, element filter 1 0 1 +ARRAY JOIN, filter step 1 0 1 +arrayJoin, filter 1 0 1 +ARRAY JOIN, element filter, empty input 1 0 diff --git a/tests/queries/0_stateless/05406_join_swap_array_join_estimate.sql b/tests/queries/0_stateless/05406_join_swap_array_join_estimate.sql index 9be52cdbe674e..152d40621d340 100644 --- a/tests/queries/0_stateless/05406_join_swap_array_join_estimate.sql +++ b/tests/queries/0_stateless/05406_join_swap_array_join_estimate.sql @@ -55,6 +55,30 @@ SELECT count() FROM (SELECT arrayJoin(arr) AS k FROM aj_l WHERE id > 1000) AS s SELECT count() FROM (SELECT arrayJoin(arr) AS k FROM aj_s) AS s INNER JOIN aj_r ON aj_r.id = s.k; SELECT count() FROM (SELECT k FROM aj_l ARRAY JOIN arr AS k) AS s INNER JOIN aj_r ON aj_r.id = s.k; +-- A filter on the array-joined elements leaves 6 rows in `s`; the estimate of its input must not make `aj_r` the build side. +-- The third count is the plans that show no row estimate for `s`. +SELECT 'ARRAY JOIN, element filter', countIf(explain ILIKE '%Join: aj_r%'), countIf(explain ILIKE '%Join: s%'), countIf(explain ILIKE '%s[no_stats~?]%') +FROM (EXPLAIN actions = 1, keep_logical_steps = 1 + SELECT count() FROM aj_r INNER JOIN (SELECT id FROM aj_l ARRAY JOIN arr AS k WHERE k = 5) AS s ON aj_r.id = s.id) +WHERE explain ILIKE '%Join:%'; + +SELECT 'ARRAY JOIN, filter step', countIf(explain ILIKE '%Join: aj_r%'), countIf(explain ILIKE '%Join: s%'), countIf(explain ILIKE '%s[no_stats~?]%') +FROM (EXPLAIN actions = 1, keep_logical_steps = 1 + SELECT count() FROM aj_r INNER JOIN (SELECT id FROM aj_l ARRAY JOIN arr AS k WHERE k = 5) AS s ON aj_r.id = s.id) +WHERE explain ILIKE '%Join:%'; + +SELECT 'arrayJoin, filter', countIf(explain ILIKE '%Join: aj_r%'), countIf(explain ILIKE '%Join: s%'), countIf(explain ILIKE '%s[no_stats~?]%') +FROM (EXPLAIN actions = 1, keep_logical_steps = 1 + SELECT count() FROM aj_r INNER JOIN (SELECT id FROM (SELECT id, arrayJoin(arr) AS k FROM aj_l) WHERE k = 5) AS s ON aj_r.id = s.id + SETTINGS query_plan_merge_expressions = 0) +WHERE explain ILIKE '%Join:%'; + +-- A filter on the elements of an empty input keeps its estimate of 0 rows. +SELECT 'ARRAY JOIN, element filter, empty input', countIf(explain ILIKE '%Join: aj_r%'), countIf(explain ILIKE '%Join: s%') +FROM (EXPLAIN actions = 1, keep_logical_steps = 1 + SELECT count() FROM (SELECT id FROM aj_l ARRAY JOIN arr AS k WHERE id > 1000 AND k = 5) AS s INNER JOIN aj_r ON aj_r.id = s.id) +WHERE explain ILIKE '%Join:%'; + DROP TABLE aj_l; DROP TABLE aj_s; DROP TABLE aj_r; From a39f8968995a45a7007ea063eadc1c2cc0b429c7 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 4 Nov 2026 16:19:04 +0000 Subject: [PATCH 093/219] Backport #128607 to 26.8: Initialize parquet reader a bit earlier --- src/Processors/Formats/IInputFormat.h | 2 + .../Impl/ParquetV3BlockInputFormat.cpp | 32 +++++++++++++--- .../Formats/Impl/ParquetV3BlockInputFormat.h | 11 ++++-- .../StorageObjectStorageSource.cpp | 13 +++++-- .../StorageObjectStorageSource.h | 5 ++- ..._storage_parquet_footer_prefetch.reference | 4 ++ ..._object_storage_parquet_footer_prefetch.sh | 38 +++++++++++++++++++ 7 files changed, 91 insertions(+), 14 deletions(-) create mode 100644 tests/queries/0_stateless/05481_object_storage_parquet_footer_prefetch.reference create mode 100755 tests/queries/0_stateless/05481_object_storage_parquet_footer_prefetch.sh diff --git a/src/Processors/Formats/IInputFormat.h b/src/Processors/Formats/IInputFormat.h index 93dc66dd9b4c1..702784d97aeec 100644 --- a/src/Processors/Formats/IInputFormat.h +++ b/src/Processors/Formats/IInputFormat.h @@ -132,6 +132,8 @@ class IInputFormat : public ISource void needOnlyCount() { need_only_count = true; } + virtual void prefetch() {} + virtual std::optional, size_t>> getMatchedBuckets() const { return std::nullopt; } protected: diff --git a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp index 8d8cbabdd35f0..ad3bebffb872e 100644 --- a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp @@ -64,9 +64,16 @@ ParquetV3BlockInputFormat::ParquetV3BlockInputFormat( format_filter_info = std::make_shared(); } -void ParquetV3BlockInputFormat::initializeIfNeeded() +void ParquetV3BlockInputFormat::prefetch() { - if (!reader) + if (need_only_count) + return; + readFileFooterIfNeeded(); +} + +void ParquetV3BlockInputFormat::readFileFooterIfNeeded() +{ + if (!reader && !reader_with_footer) { format_filter_info->initKeyConditionOnce(getPort().getHeader()); parser_shared_resources->initOnce([&] @@ -95,11 +102,24 @@ void ParquetV3BlockInputFormat::initializeIfNeeded() parser_shared_resources->opaque = ext; }); + auto new_reader = std::make_unique(); + new_reader->reader.prefetcher.init(in, read_options, parser_shared_resources); + new_reader->reader.file_metadata = getFileMetadata(new_reader->reader.prefetcher); + + std::lock_guard lock(reader_mutex); + reader_with_footer = std::move(new_reader); + } +} + +void ParquetV3BlockInputFormat::initializeIfNeeded() +{ + if (!reader) + { + readFileFooterIfNeeded(); + { std::lock_guard lock(reader_mutex); - reader.emplace(); - reader->reader.prefetcher.init(in, read_options, parser_shared_resources); - reader->reader.file_metadata = getFileMetadata(reader->reader.prefetcher); + reader = std::move(reader_with_footer); reader->reader.init(read_options, getPort().getHeader(), format_filter_info); reader->init(parser_shared_resources, buckets_to_read ? std::optional(buckets_to_read->row_group_ids) : std::nullopt); } @@ -203,6 +223,7 @@ void ParquetV3BlockInputFormat::resetReadBuffer() std::lock_guard lock(reader_mutex); if (reader) reader->shutdownTasks(); + reader_with_footer.reset(); } IInputFormat::resetReadBuffer(); } @@ -212,6 +233,7 @@ void ParquetV3BlockInputFormat::resetParser() { std::lock_guard lock(reader_mutex); reader.reset(); + reader_with_footer.reset(); } previous_block_missing_values.clear(); IInputFormat::resetParser(); diff --git a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.h b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.h index 0ee6d2226a554..b0767cd75886c 100644 --- a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.h +++ b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.h @@ -68,6 +68,8 @@ class ParquetV3BlockInputFormat final : public IInputFormat private: Chunk read() override; + void prefetch() override; + void onCancel() noexcept override; const FormatSettings format_settings; @@ -77,17 +79,20 @@ class ParquetV3BlockInputFormat final : public IInputFormat ParquetMetadataCachePtr metadata_cache; const std::optional object_with_metadata; - /// (This mutex is not important. It protects `reader.emplace` in a weird case where onCancel() + /// (This mutex is not important. It protects setting `reader` in a weird case where onCancel() /// may be called in parallel with first read(). ReadManager itself is thread safe for that, - /// but initializing vs checking the std::optional would race without this mutex.) + /// but initializing vs checking the pointer would race without this mutex.) std::mutex reader_mutex; - std::optional reader; + std::unique_ptr reader; + /// Has the file footer, but is not initialized: initializing it starts reading and decoding the column chunks. + std::unique_ptr reader_with_footer; bool reported_count = false; // if need_only_count BlockMissingValues previous_block_missing_values; size_t previous_approx_bytes_read_for_chunk = 0; + void readFileFooterIfNeeded(); void initializeIfNeeded(); std::shared_ptr buckets_to_read; diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 11c2fe303af40..d96efabe196dd 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -935,7 +935,7 @@ void StorageObjectStorageSource::addNumRowsToCache(const ObjectInfo & object_inf schema_cache.addNumRows(cache_key, num_rows); } -StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReader() +StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReader(bool prefetch_input_format) { return createReader( 0, @@ -951,7 +951,8 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade max_block_size, parser_shared_resources, format_filter_info, - need_only_count); + need_only_count, + prefetch_input_format); } StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReader( @@ -968,7 +969,8 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade size_t max_block_size, FormatParserSharedResourcesPtr parser_shared_resources, FormatFilterInfoPtr format_filter_info, - bool need_only_count) + bool need_only_count, + bool prefetch_input_format) { ObjectInfoPtr object_info; auto query_settings = configuration->getQuerySettings(context_); @@ -1378,6 +1380,9 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade if (need_only_count) input_format->needOnlyCount(); + if (prefetch_input_format) + input_format->prefetch(); + builder.init(Pipe(input_format)); if (!identity_partition_columns.empty()) @@ -1541,7 +1546,7 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade std::future StorageObjectStorageSource::createReaderAsync() { - return create_reader_scheduler([=, this] { return createReader(); }, Priority{}); + return create_reader_scheduler([=, this] { return createReader(/* prefetch_input_format */ true); }, Priority{}); } std::unique_ptr createReadBuffer( diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.h b/src/Storages/ObjectStorage/StorageObjectStorageSource.h index ec9b155d08608..7d3654b5ee646 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.h +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.h @@ -154,9 +154,10 @@ class StorageObjectStorageSource final : public ISource size_t max_block_size, FormatParserSharedResourcesPtr parser_shared_resources, FormatFilterInfoPtr format_filter_info, - bool need_only_count); + bool need_only_count, + bool prefetch_input_format = false); - ReaderHolder createReader(); + ReaderHolder createReader(bool prefetch_input_format = false); std::future createReaderAsync(); diff --git a/tests/queries/0_stateless/05481_object_storage_parquet_footer_prefetch.reference b/tests/queries/0_stateless/05481_object_storage_parquet_footer_prefetch.reference new file mode 100644 index 0000000000000..8588a3ce6bb67 --- /dev/null +++ b/tests/queries/0_stateless/05481_object_storage_parquet_footer_prefetch.reference @@ -0,0 +1,4 @@ +400 79800 0 399 +58 11571 +INCORRECT_DATA +1 diff --git a/tests/queries/0_stateless/05481_object_storage_parquet_footer_prefetch.sh b/tests/queries/0_stateless/05481_object_storage_parquet_footer_prefetch.sh new file mode 100755 index 0000000000000..aae1c258a604e --- /dev/null +++ b/tests/queries/0_stateless/05481_object_storage_parquet_footer_prefetch.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: needs MinIO + +# A source of an object storage reads the Parquet footer of its next file in the background, while it reads the current one. +# One stream (`max_threads = 1`) reads all files, so every file but the first one is read through a prefetched footer. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +DIR="${CLICKHOUSE_TEST_UNIQUE_NAME}" +Q="${CLICKHOUSE_TEST_UNIQUE_NAME}_" + +for i in 0 1 2 3; do + ${CLICKHOUSE_CLIENT} --query "INSERT INTO FUNCTION s3(s3_conn, filename = '${DIR}/good/${i}.parquet', format = Parquet) SELECT number + ${i} * 100 AS x FROM numbers(100)" +done +${CLICKHOUSE_CLIENT} --query "INSERT INTO FUNCTION s3(s3_conn, filename = '${DIR}/bad/0.parquet', format = Parquet) SELECT number AS x FROM numbers(100)" +${CLICKHOUSE_CLIENT} --query "INSERT INTO FUNCTION s3(s3_conn, filename = '${DIR}/bad/1.parquet', format = RawBLOB) SELECT 'not a parquet file'" + +SETTINGS="--max_threads 1 --use_parquet_metadata_cache 1" +${CLICKHOUSE_CLIENT} $SETTINGS --query_id "${Q}1" --query "SELECT count(x), sum(x), min(x), max(x) FROM s3(s3_conn, filename = '${DIR}/good/*.parquet')" +${CLICKHOUSE_CLIENT} $SETTINGS --query_id "${Q}2" --query "SELECT count(x), sum(x) FROM s3(s3_conn, filename = '${DIR}/good/*.parquet') WHERE x % 7 = 0" + +# The file that is not Parquet fails the query as it did before the footer was read in advance. +${CLICKHOUSE_CLIENT} $SETTINGS --query "SELECT sum(x) FROM s3(s3_conn, filename = '${DIR}/bad/*.parquet')" 2>&1 | grep -o -m1 "INCORRECT_DATA" + +${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS query_log" + +# The second query finds all footers in the Parquet metadata cache. +# With parallel replicas, `s3` is rewritten to `s3Cluster` and the files are read by secondary queries, +# which run with `current_database = 'default'`, so the hits are summed over `initial_query_id`. +${CLICKHOUSE_CLIENT} --query " + SELECT sum(ProfileEvents['ParquetMetadataCacheHits']) >= 4 + FROM system.query_log + WHERE initial_query_id = '${Q}2' AND type = 'QueryFinish' + AND (current_database = currentDatabase() OR NOT is_initial_query) +" From 794d9f0b7205ce798f0ea0cf427c6e6e82e7904e Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 4 Nov 2026 17:56:33 +0000 Subject: [PATCH 094/219] Backport #122707 to 26.8: Fix loading `AzureBlobStorage` tables whose engine arguments end with `partition_strategy = 'none'` --- .../ObjectStorage/Azure/Configuration.cpp | 105 +++++++++++++++-- .../tests/gtest_azure_configuration.cpp | 34 ++++++ ...age_partition_strategy_key_value.reference | 11 ++ ...b_storage_partition_strategy_key_value.sql | 106 ++++++++++++++++++ 4 files changed, 244 insertions(+), 12 deletions(-) create mode 100644 tests/queries/0_stateless/05291_azure_blob_storage_partition_strategy_key_value.reference create mode 100644 tests/queries/0_stateless/05291_azure_blob_storage_partition_strategy_key_value.sql diff --git a/src/Storages/ObjectStorage/Azure/Configuration.cpp b/src/Storages/ObjectStorage/Azure/Configuration.cpp index e83b2eed55701..ff087488fa893 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.cpp +++ b/src/Storages/ObjectStorage/Azure/Configuration.cpp @@ -281,6 +281,21 @@ static ASTPtr extractExtraCredentials(ASTs & args) return nullptr; } +/// Returns the number of arguments before the trailing `identifier = value` arguments. +static size_t countPositionalArguments(const ASTs & args) +{ + size_t count = args.size(); + while (count > 0) + { + const auto * function = args[count - 1]->as(); + if (!function || function->name != "equals" || !function->arguments || function->arguments->children.size() != 2 + || !function->arguments->children[0]->as()) + break; + --count; + } + return count; +} + bool AzureStorageParsedArguments::collectCredentials( ASTPtr maybe_credentials, std::optional & client_id, std::optional & tenant_id, ContextPtr local_context) { @@ -374,7 +389,31 @@ void AzureStorageParsedArguments::fromAST(ASTs & engine_args, ContextPtr context { auto extra_credentials = extractExtraCredentials(engine_args); - if (engine_args.empty() || engine_args.size() > AzureStorageParsedArguments::getMaxNumberOfArguments(with_structure)) + const size_t count = countPositionalArguments(engine_args); + /// Like the positional arguments, the key-value arguments are persisted with evaluated values. + for (size_t i = count; i < engine_args.size(); ++i) + { + const auto & key_value = engine_args[i]->as().arguments->children; + engine_args[i] = makeASTOperator("equals", key_value[0], evaluateConstantExpressionOrIdentifierAsLiteral(key_value[1], context)); + } + const auto key_value_args = parseKeyValueArguments(ASTs(engine_args.begin() + count, engine_args.end()), context); + for (const auto & [key, _] : key_value_args) + { + if (key != "partition_strategy" && key != "partition_columns_in_data_file") + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Unexpected key-value argument `{}`: only `partition_strategy` and `partition_columns_in_data_file` " + "can be passed as key-value arguments", + key); + } + + if (!key_value_args.empty() && count < 3) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Key-value arguments require at least three positional arguments: " + "connection string or storage account URL, container name and blob path"); + + if (count == 0 || count > AzureStorageParsedArguments::getMaxNumberOfArguments(with_structure)) { throw Exception( ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, @@ -383,12 +422,12 @@ void AzureStorageParsedArguments::fromAST(ASTs & engine_args, ContextPtr context AzureStorageParsedArguments::getSignatures(with_structure)); } - for (auto & engine_arg : engine_args) - engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, context); + for (size_t i = 0; i < count; ++i) + engine_args[i] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[i], context); /// This is only for lightweight loading of tables, so does not contain credentials /// for listing tables of Unity Catalog - if (engine_args.size() == 1) + if (count == 1) { connection_params.endpoint.storage_account_url = checkAndGetLiteralArgument(engine_args[0], "connection_string/storage_account_url"); @@ -396,7 +435,7 @@ void AzureStorageParsedArguments::fromAST(ASTs & engine_args, ContextPtr context return; } - if (engine_args.size() == 2) + if (count == 2) { String connection_url = checkAndGetLiteralArgument(engine_args[0], "connection_string/storage_account_url"); String sas_token = checkAndGetLiteralArgument(engine_args[1], "sas_token"); @@ -425,7 +464,7 @@ void AzureStorageParsedArguments::fromAST(ASTs & engine_args, ContextPtr context return s == "auto" || FormatFactory::instance().getAllFormats().contains(Poco::toLower(s)); }; - if (engine_args.size() == 4) + if (count == 4) { auto fourth_arg = checkAndGetLiteralArgument(engine_args[3], "format/account_name"); if (is_format_arg(fourth_arg)) @@ -442,7 +481,7 @@ void AzureStorageParsedArguments::fromAST(ASTs & engine_args, ContextPtr context "Unknown format or account name specified without account key: {}", fourth_arg); } } - else if (engine_args.size() == 5) + else if (count == 5) { auto fourth_arg = checkAndGetLiteralArgument(engine_args[3], "format/account_name"); if (is_format_arg(fourth_arg)) @@ -456,7 +495,7 @@ void AzureStorageParsedArguments::fromAST(ASTs & engine_args, ContextPtr context account_key = checkAndGetLiteralArgument(engine_args[4], "account_key"); } } - else if (engine_args.size() == 6) + else if (count == 6) { auto fourth_arg = checkAndGetLiteralArgument(engine_args[3], "format/account_name"); if (is_format_arg(fourth_arg)) @@ -499,7 +538,7 @@ void AzureStorageParsedArguments::fromAST(ASTs & engine_args, ContextPtr context } } } - else if (engine_args.size() == 7) + else if (count == 7) { const auto fourth_arg = checkAndGetLiteralArgument(engine_args[3], "format/account_name"); @@ -552,7 +591,7 @@ void AzureStorageParsedArguments::fromAST(ASTs & engine_args, ContextPtr context compression_method = checkAndGetLiteralArgument(engine_args[6], "compression"); } } - else if (engine_args.size() == 8) + else if (count == 8) { auto fourth_arg = checkAndGetLiteralArgument(engine_args[3], "format/account_name"); @@ -608,7 +647,7 @@ void AzureStorageParsedArguments::fromAST(ASTs & engine_args, ContextPtr context } } } - else if (engine_args.size() == 9) + else if (count == 9) { auto fourth_arg = checkAndGetLiteralArgument(engine_args[3], "format/account_name"); account_name = fourth_arg; @@ -646,7 +685,7 @@ void AzureStorageParsedArguments::fromAST(ASTs & engine_args, ContextPtr context partition_columns_in_data_file_was_set = true; } } - else if (engine_args.size() == 10 && with_structure) + else if (count == 10 && with_structure) { auto fourth_arg = checkAndGetLiteralArgument(engine_args[3], "format/account_name"); account_name = fourth_arg; @@ -671,6 +710,36 @@ void AzureStorageParsedArguments::fromAST(ASTs & engine_args, ContextPtr context structure = checkAndGetLiteralArgument(engine_args[9], "structure"); } + if (auto it = key_value_args.find("partition_strategy"); it != key_value_args.end()) + { + if (partition_strategy_was_set) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, "`partition_strategy` is specified both as a positional and as a key-value argument"); + + const auto & partition_strategy_name = it->second.safeGet(); + const auto partition_strategy_type_opt = magic_enum::enum_cast(partition_strategy_name, magic_enum::case_insensitive); + if (!partition_strategy_type_opt) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown partition strategy {}", partition_strategy_name); + + partition_strategy_type = partition_strategy_type_opt.value(); + partition_strategy_was_set = true; + } + + if (auto it = key_value_args.find("partition_columns_in_data_file"); it != key_value_args.end()) + { + if (partition_columns_in_data_file_was_set) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`partition_columns_in_data_file` is specified both as a positional and as a key-value argument"); + + partition_columns_in_data_file = it->second.safeGet(); + partition_columns_in_data_file_was_set = true; + } + + /// A key-value strategy gets the same default as the named-collection form. + if (key_value_args.contains("partition_strategy") && !partition_columns_in_data_file_was_set) + partition_columns_in_data_file = partition_strategy_type != PartitionStrategyFactory::StrategyType::HIVE; + connection_params = getAzureConnectionParams(connection_url, container_name, account_name, account_key, client_id, tenant_id, context); } @@ -700,6 +769,14 @@ static void addStructureAndFormatToArgsIfNeededAzure( } else { + /// extra_credentials is not a constant expression; take it out while rewriting. + auto extra_credentials = extractExtraCredentials(args); + + /// The trailing key-value arguments are appended back after the positional ones. + const size_t count = countPositionalArguments(args); + ASTs key_value_args(args.begin() + count, args.end()); + args.resize(count); + if (args.size() < 3 || args.size() > AzureStorageParsedArguments::getMaxNumberOfArguments()) throw Exception( ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, @@ -843,6 +920,10 @@ static void addStructureAndFormatToArgsIfNeededAzure( if (with_structure && checkAndGetLiteralArgument(args[7], "structure") == "auto") args[7] = structure_literal; } + + args.insert(args.end(), key_value_args.begin(), key_value_args.end()); + if (extra_credentials) + args.push_back(extra_credentials); } } diff --git a/src/Storages/tests/gtest_azure_configuration.cpp b/src/Storages/tests/gtest_azure_configuration.cpp index 89ed7547bf72c..d793dc8ec9fc2 100644 --- a/src/Storages/tests/gtest_azure_configuration.cpp +++ b/src/Storages/tests/gtest_azure_configuration.cpp @@ -296,6 +296,40 @@ TEST(StorageAzureConfiguration, AddStructureAndFormatTooManyArgs) "Expected 3 to"); } +/// Distributed query preparation rewrites the args; extra_credentials must be skipped and kept. +TEST(StorageAzureConfiguration, AddStructureAndFormatKeepsExtraCredentials) +{ + std::string query = "DESCRIBE TABLE azureBlobStorage('https://azurite1:10000/devstoreaccount1', 'test_container', 'test_blob.csv', extra_credentials(client_id='test_client_id', tenant_id='test_tenant_id'))"; + + ASTs engine_args = getEngineArgs(query); + StorageAzureConfigurationFriend conf; + conf.addStructureAndFormatToArgsIfNeeded(engine_args, "a Int32", "Parquet", Context::getGlobalContextInstance(), /*with_structure=*/true); + + /// (url, container, blobpath, format, compression, structure, extra_credentials) + ASSERT_EQ(engine_args.size(), 7u); + EXPECT_EQ(engine_args[3]->as()->value.safeGet(), "Parquet"); + EXPECT_EQ(engine_args[5]->as()->value.safeGet(), "a Int32"); + const auto * credentials = engine_args.back()->as(); + ASSERT_TRUE(credentials); + EXPECT_EQ(credentials->name, "extra_credentials"); +} + +TEST(StorageAzureConfiguration, AddStructureAndFormatKeepsKeyValueArgumentsAndExtraCredentials) +{ + std::string query = "DESCRIBE TABLE azureBlobStorage('https://azurite1:10000/devstoreaccount1', 'test_container', 'test_blob.csv', extra_credentials(client_id='test_client_id', tenant_id='test_tenant_id'), partition_strategy = 'none')"; + + ASTs engine_args = getEngineArgs(query); + StorageAzureConfigurationFriend conf; + conf.addStructureAndFormatToArgsIfNeeded(engine_args, "a Int32", "Parquet", Context::getGlobalContextInstance(), /*with_structure=*/true); + + /// (url, container, blobpath, format, compression, structure, partition_strategy = 'none', extra_credentials) + ASSERT_EQ(engine_args.size(), 8u); + EXPECT_EQ(engine_args[5]->as()->value.safeGet(), "a Int32"); + EXPECT_EQ(engine_args[6]->formatWithSecretsOneLine(), "partition_strategy = 'none'"); + const auto * credentials = engine_args[7]->as(); + ASSERT_TRUE(credentials); + EXPECT_EQ(credentials->name, "extra_credentials"); +} } #endif diff --git a/tests/queries/0_stateless/05291_azure_blob_storage_partition_strategy_key_value.reference b/tests/queries/0_stateless/05291_azure_blob_storage_partition_strategy_key_value.reference new file mode 100644 index 0000000000000..cf17f1f3312f6 --- /dev/null +++ b/tests/queries/0_stateless/05291_azure_blob_storage_partition_strategy_key_value.reference @@ -0,0 +1,11 @@ +t_implicit 1 +t_implicit 1 +t_kv_hive id +t_kv_hive id +t_kv_expr id +1 +2 +t_kv_none 1 +t_kv_none 1 +id UInt64 +t_positional_equals id diff --git a/tests/queries/0_stateless/05291_azure_blob_storage_partition_strategy_key_value.sql b/tests/queries/0_stateless/05291_azure_blob_storage_partition_strategy_key_value.sql new file mode 100644 index 0000000000000..c2b93e5231bdb --- /dev/null +++ b/tests/queries/0_stateless/05291_azure_blob_storage_partition_strategy_key_value.sql @@ -0,0 +1,106 @@ +-- Tags: no-fasttest +-- Tag no-fasttest: needs the AzureBlobStorage engine and Azurite, which the fast test does not have + +-- AzureBlobStorage engine arguments may end with key-value `partition_strategy` and +-- `partition_columns_in_data_file` arguments. The server itself writes `partition_strategy = 'none'` +-- into the table definition when `PARTITION BY` resolves to no strategy, so such a table must attach again. +-- The write arms use Azurite; the other statements use a SAS token in the URL and stay off the network. + +DROP TABLE IF EXISTS t_implicit; +DROP TABLE IF EXISTS t_kv_hive; +DROP TABLE IF EXISTS t_kv_expr; +DROP TABLE IF EXISTS t_kv_hive_write; +DROP TABLE IF EXISTS t_kv_hive_write_pcdf; +DROP TABLE IF EXISTS t_kv_none; +DROP TABLE IF EXISTS t_positional_equals; + +SET file_like_engine_default_partition_strategy = 'wildcard'; + +CREATE TABLE t_implicit (id UInt64, v String) +ENGINE = AzureBlobStorage('http://localhost:11111/devstoreaccount1?sig=X', 'cont', 'data.csv', 'CSV') +PARTITION BY id; +SELECT name, empty(partition_key) FROM system.tables WHERE database = currentDatabase() AND name = 't_implicit'; +DETACH TABLE t_implicit; +ATTACH TABLE t_implicit; +SELECT name, empty(partition_key) FROM system.tables WHERE database = currentDatabase() AND name = 't_implicit'; + +CREATE TABLE t_kv_hive (id UInt64, v String) +ENGINE = AzureBlobStorage('http://localhost:11111/devstoreaccount1?sig=X', 'cont', 'data', 'CSV', partition_strategy = 'hive') +PARTITION BY id; +SELECT name, partition_key FROM system.tables WHERE database = currentDatabase() AND name = 't_kv_hive'; +DETACH TABLE t_kv_hive; +ATTACH TABLE t_kv_hive; +SELECT name, partition_key FROM system.tables WHERE database = currentDatabase() AND name = 't_kv_hive'; + +-- The stored definition keeps the value of a key-value argument, not its expression. +SET use_hive_partitioning = 1; +CREATE TABLE t_kv_expr (id UInt64, v String) +ENGINE = AzureBlobStorage('http://localhost:11111/devstoreaccount1?sig=X', 'cont', 'data', 'CSV', partition_strategy = if(getSetting('use_hive_partitioning'), 'hive', 'none')) +PARTITION BY id; +DETACH TABLE t_kv_expr; +SET use_hive_partitioning = 0; +ATTACH TABLE t_kv_expr; +SELECT name, partition_key FROM system.tables WHERE database = currentDatabase() AND name = 't_kv_expr'; + +-- `num_columns` of the written file: a key-value `hive` keeps the partition column out of it by default. +CREATE TABLE t_kv_hive_write (id UInt64, v String) +ENGINE = AzureBlobStorage('DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;', 'cont05291', concat(currentDatabase(), '_kv_hive'), 'Parquet', partition_strategy = 'hive') +PARTITION BY id; +INSERT INTO t_kv_hive_write VALUES (1, 'a'); +SELECT DISTINCT num_columns FROM azureBlobStorage('DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;', 'cont05291', concat(currentDatabase(), '_kv_hive/**.parquet'), 'ParquetMetadata') SETTINGS use_hive_partitioning = 0; + +CREATE TABLE t_kv_hive_write_pcdf (id UInt64, v String) +ENGINE = AzureBlobStorage('DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;', 'cont05291', concat(currentDatabase(), '_kv_hive_pcdf'), 'Parquet', partition_strategy = 'hive', partition_columns_in_data_file = 1) +PARTITION BY id; +INSERT INTO t_kv_hive_write_pcdf VALUES (1, 'a'); +SELECT DISTINCT num_columns FROM azureBlobStorage('DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;', 'cont05291', concat(currentDatabase(), '_kv_hive_pcdf/**.parquet'), 'ParquetMetadata') SETTINGS use_hive_partitioning = 0; + +SET file_like_engine_default_partition_strategy = 'hive'; + +CREATE TABLE t_kv_none (id UInt64, v String) +ENGINE = AzureBlobStorage('http://localhost:11111/devstoreaccount1?sig=X', 'cont', 'data.csv', 'CSV', partition_strategy = 'none') +PARTITION BY id; +SELECT name, empty(partition_key) FROM system.tables WHERE database = currentDatabase() AND name = 't_kv_none'; +DETACH TABLE t_kv_none; +ATTACH TABLE t_kv_none; +SELECT name, empty(partition_key) FROM system.tables WHERE database = currentDatabase() AND name = 't_kv_none'; + +DESCRIBE TABLE azureBlobStorage('http://localhost:11111/devstoreaccount1?sig=X', 'cont', 'data.csv', 'CSV', 'auto', 'id UInt64', partition_strategy = 'none'); + +-- Cluster queries accept `extra_credentials` together with the key-value arguments, in both argument orders and +-- when parallel replicas turn `azureBlobStorage` into a cluster query. Reading then fails: its token is not sent over plain http. +SET s3_allow_server_credentials_in_user_queries = 1; +SELECT * FROM azureBlobStorageCluster('test_shard_localhost', 'http://localhost:11111/devstoreaccount1?sig=X', 'cont', 'data.csv', 'CSV', 'auto', 'id UInt64', extra_credentials(client_id = 'x', tenant_id = 'y'), partition_strategy = 'none'); -- { serverError STD_EXCEPTION } +SELECT * FROM azureBlobStorageCluster('test_shard_localhost', 'http://localhost:11111/devstoreaccount1?sig=X', 'cont', 'data.csv', 'CSV', 'auto', 'id UInt64', partition_strategy = 'none', extra_credentials(client_id = 'x', tenant_id = 'y')); -- { serverError STD_EXCEPTION } +SELECT * FROM azureBlobStorage('http://localhost:11111/devstoreaccount1?sig=X', 'cont', 'data.csv', 'CSV', 'auto', 'id UInt64', extra_credentials(client_id = 'x', tenant_id = 'y'), partition_strategy = 'none') +SETTINGS enable_parallel_replicas = 1, automatic_parallel_replicas_mode = 0, max_parallel_replicas = 3, parallel_replicas_for_cluster_engines = 1, + cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost'; -- { serverError STD_EXCEPTION } + +-- Invalid key-value arguments. +CREATE TABLE t_bad (id UInt64, v String) +ENGINE = AzureBlobStorage('http://localhost:11111/devstoreaccount1?sig=X', 'cont', 'data.csv', 'CSV', format = 'CSV'); -- { serverError BAD_ARGUMENTS } +CREATE TABLE t_bad (id UInt64, v String) +ENGINE = AzureBlobStorage('http://localhost:11111/devstoreaccount1?sig=X', 'cont', 'data.csv', 'CSV', partition_strategy = 'unknown') +PARTITION BY id; -- { serverError BAD_ARGUMENTS } +CREATE TABLE t_bad (id UInt64, v String) +ENGINE = AzureBlobStorage('http://localhost:11111/devstoreaccount1?sig=X', 'cont', 'data', 'CSV', 'auto', 'hive', partition_strategy = 'hive') +PARTITION BY id; -- { serverError BAD_ARGUMENTS } +CREATE TABLE t_bad (id UInt64, v String) +ENGINE = AzureBlobStorage('http://localhost:11111/devstoreaccount1?sig=X', 'cont', 'data.csv', 'CSV', partition_strategy = 'none', partition_columns_in_data_file = 0) +PARTITION BY id; -- { serverError BAD_ARGUMENTS } +CREATE TABLE t_bad (id UInt64, v String) +ENGINE = AzureBlobStorage('http://localhost:11111/devstoreaccount1/cont/data.csv?sig=X', 'X', partition_strategy = 'none'); -- { serverError BAD_ARGUMENTS } + +-- A constant `1 = 1` stays a positional argument (`partition_columns_in_data_file`). +CREATE TABLE t_positional_equals (id UInt64, v String) +ENGINE = AzureBlobStorage('http://localhost:11111/devstoreaccount1?sig=X', 'cont', 'data', 'CSV', 'auto', 'hive', 1 = 1) +PARTITION BY id; +SELECT name, partition_key FROM system.tables WHERE database = currentDatabase() AND name = 't_positional_equals'; + +DROP TABLE t_implicit; +DROP TABLE t_kv_hive; +DROP TABLE t_kv_expr; +DROP TABLE t_kv_hive_write; +DROP TABLE t_kv_hive_write_pcdf; +DROP TABLE t_kv_none; +DROP TABLE t_positional_equals; From 9a470ba70ecefba80bad0b5268355643af1ec8e1 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 4 Nov 2026 20:07:48 +0000 Subject: [PATCH 095/219] Backport #129240 to 26.8: Validate the SSE-C key when creating an S3 client --- src/IO/S3/Client.cpp | 21 +++++++++++-- src/IO/S3/Client.h | 2 ++ ...498_s3_disk_sse_c_key_validation.reference | 1 + .../05498_s3_disk_sse_c_key_validation.sql | 31 +++++++++++++++++++ 4 files changed, 52 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/05498_s3_disk_sse_c_key_validation.reference create mode 100644 tests/queries/0_stateless/05498_s3_disk_sse_c_key_validation.sql diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index dc9acdbdaa3e4..b16f2316a3477 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -35,6 +35,7 @@ #include #include +#include #include #include #include @@ -82,6 +83,7 @@ namespace Setting namespace ErrorCodes { + extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; extern const int TOO_MANY_REDIRECTS; } @@ -1314,6 +1316,21 @@ std::unique_ptr ClientFactory::create( // NOLINT if (!server_side_encryption_customer_key_base64.empty()) { + /// `base64Decode` throws on characters outside the alphabet but skips whitespace and ignores padding bits, + /// so the round trip below accepts only the canonical base64 of an AES-256 key, the only key SSE-C takes. + String customer_key; + try + { + customer_key = base64Decode(server_side_encryption_customer_key_base64); + } + catch (const Poco::DataFormatException &) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "`server_side_encryption_customer_key_base64` must be a base64-encoded 256-bit key"); + } + + if (customer_key.size() != AES256_KEY_BYTES || base64Encode(customer_key) != server_side_encryption_customer_key_base64) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "`server_side_encryption_customer_key_base64` must be a base64-encoded 256-bit key"); + /// See Client::GeneratePresignedUrlWithSSEC(). headers.push_back({Aws::S3::SSEHeaders::SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, @@ -1322,10 +1339,8 @@ std::unique_ptr ClientFactory::create( // NOLINT headers.push_back({Aws::S3::SSEHeaders::SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, server_side_encryption_customer_key_base64}); - Aws::Utils::ByteBuffer buffer = Aws::Utils::HashingUtils::Base64Decode(server_side_encryption_customer_key_base64); - String str_buffer(reinterpret_cast(buffer.GetUnderlyingData()), buffer.GetLength()); headers.push_back({Aws::S3::SSEHeaders::SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, - Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::HashingUtils::CalculateMD5(str_buffer))}); + Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::HashingUtils::CalculateMD5(customer_key))}); } // These will be added after request signing diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h index ae5312812549a..1c728fccddeca 100644 --- a/src/IO/S3/Client.h +++ b/src/IO/S3/Client.h @@ -349,6 +349,8 @@ class Client : private Aws::S3::S3Client class ClientFactory { public: + static constexpr size_t AES256_KEY_BYTES = 32; + ~ClientFactory(); static ClientFactory & instance(); diff --git a/tests/queries/0_stateless/05498_s3_disk_sse_c_key_validation.reference b/tests/queries/0_stateless/05498_s3_disk_sse_c_key_validation.reference new file mode 100644 index 0000000000000..573541ac9702d --- /dev/null +++ b/tests/queries/0_stateless/05498_s3_disk_sse_c_key_validation.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/05498_s3_disk_sse_c_key_validation.sql b/tests/queries/0_stateless/05498_s3_disk_sse_c_key_validation.sql new file mode 100644 index 0000000000000..1247bc62fb1f4 --- /dev/null +++ b/tests/queries/0_stateless/05498_s3_disk_sse_c_key_validation.sql @@ -0,0 +1,31 @@ +-- Tags: no-fasttest +-- no-fasttest: needs S3 support in the build. + +-- An SSE-C key that is not the base64 of a 256-bit key is rejected when the S3 client is created, before any request. + +-- Non-ASCII bytes. +CREATE TABLE t_sse_c (x UInt8) ENGINE = MergeTree ORDER BY x +SETTINGS disk = disk(type = 's3', endpoint = 'http://localhost:11111/test/05498/', no_sign_request = 1, skip_access_check = 1, + server_side_encryption_customer_key_base64 = 'éAAA'); -- { serverError BAD_ARGUMENTS } + +-- Valid base64 of 3 bytes. +CREATE TABLE t_sse_c (x UInt8) ENGINE = MergeTree ORDER BY x +SETTINGS disk = disk(type = 's3', endpoint = 'http://localhost:11111/test/05498/', no_sign_request = 1, skip_access_check = 1, + server_side_encryption_customer_key_base64 = 'AAAA'); -- { serverError BAD_ARGUMENTS } + +-- An invalid character in a key of the right length. +CREATE TABLE t_sse_c (x UInt8) ENGINE = MergeTree ORDER BY x +SETTINGS disk = disk(type = 's3', endpoint = 'http://localhost:11111/test/05498/', no_sign_request = 1, skip_access_check = 1, + server_side_encryption_customer_key_base64 = 'MDEyMzQ1Njc4OTAx!jM0NTY3ODkwMTIzNDU2Nzg5MDE='); -- { serverError BAD_ARGUMENTS } + +-- A valid key followed by more characters. +CREATE TABLE t_sse_c (x UInt8) ENGINE = MergeTree ORDER BY x +SETTINGS disk = disk(type = 's3', endpoint = 'http://localhost:11111/test/05498/', no_sign_request = 1, skip_access_check = 1, + server_side_encryption_customer_key_base64 = 'MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDE=!'); -- { serverError BAD_ARGUMENTS } + +-- A valid key with surrounding whitespace. +CREATE TABLE t_sse_c (x UInt8) ENGINE = MergeTree ORDER BY x +SETTINGS disk = disk(type = 's3', endpoint = 'http://localhost:11111/test/05498/', no_sign_request = 1, skip_access_check = 1, + server_side_encryption_customer_key_base64 = ' MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDE= '); -- { serverError BAD_ARGUMENTS } + +SELECT count() FROM system.tables WHERE database = currentDatabase() AND name = 't_sse_c'; From 5286e5d222f68b3df9fc16a7131c3495444e1d5a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 3 Nov 2026 23:22:32 +0000 Subject: [PATCH 096/219] Backport #118822 to 26.8: Store the effective nested SETTINGS clause on the query node and validate it --- src/Access/SettingsConstraints.cpp | 39 +++-- src/Access/SettingsConstraints.h | 13 +- src/Analyzer/QueryTreeBuilder.cpp | 34 ++-- src/Interpreters/Context.cpp | 22 +++ src/Interpreters/Context.h | 2 + ...raints_nested_settings_effective.reference | 35 +++++ ...s_constraints_nested_settings_effective.sh | 146 ++++++++++++++++++ ...ttings_equal_value_marks_changed.reference | 18 +++ ...sted_settings_equal_value_marks_changed.sh | 48 ++++++ 9 files changed, 329 insertions(+), 28 deletions(-) create mode 100644 tests/queries/0_stateless/05141_settings_constraints_nested_settings_effective.reference create mode 100755 tests/queries/0_stateless/05141_settings_constraints_nested_settings_effective.sh create mode 100644 tests/queries/0_stateless/05175_nested_settings_equal_value_marks_changed.reference create mode 100755 tests/queries/0_stateless/05175_nested_settings_equal_value_marks_changed.sh diff --git a/src/Access/SettingsConstraints.cpp b/src/Access/SettingsConstraints.cpp index fcf380632efef..9bfcb7946495b 100644 --- a/src/Access/SettingsConstraints.cpp +++ b/src/Access/SettingsConstraints.cpp @@ -311,14 +311,22 @@ void SettingsConstraints::clamp(const Settings & current_settings, SettingsChang checkOrClamp(current_settings, changes, CLAMP_ON_VIOLATION, source); } +void SettingsConstraints::clampRejectingInvalidChanges(const Settings & current_settings, SettingsChanges & changes, SettingSource source) const +{ + checkOrClamp(current_settings, changes, CLAMP_ON_VIOLATION_THROW_ON_INVALID, source); +} + void SettingsConstraints::checkOrClamp(const Settings & current_settings, SettingsChanges & changes, ReactionOnViolation reaction, SettingSource source) const { /// If we filter out settings that match the current default here, `compatibility` will silently override them. /// So when `compatibility` is present, we keep unchanged settings so they are applied after `compatibility`. bool has_compatibility_setting = changes.tryGet("compatibility") != nullptr; + /// With `CLAMP_ON_VIOLATION_THROW_ON_INVALID` the surviving list is stored on the query node as the settings + /// set explicitly on that node, so a change equal to the current value must stay in the list. + bool ignore_unchanged_settings = has_compatibility_setting || reaction == CLAMP_ON_VIOLATION_THROW_ON_INVALID; std::erase_if(changes, [&](SettingChange & change) { - return !checkImpl(current_settings, change, reaction, source, /*ignore_unchanged_settings=*/has_compatibility_setting); + return !checkImpl(current_settings, change, reaction, source, ignore_unchanged_settings); }); } @@ -415,7 +423,10 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, if (setting_name == "profile") return true; - if (reaction == THROW_ON_VIOLATION) + /// Invalid changes (an unknown or disallowed name, an uncastable value) throw in every mode but `CLAMP_ON_VIOLATION`. + const bool throw_on_invalid_change = reaction != CLAMP_ON_VIOLATION; + + if (throw_on_invalid_change) { try { @@ -436,7 +447,8 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, else if (!access_control->isSettingNameAllowed(setting_name)) return false; - Field new_value = getNewValueToCheck(current_settings, change, ignore_unchanged_settings, reaction == THROW_ON_VIOLATION); + Field new_value = getNewValueToCheck( + current_settings, change, ignore_unchanged_settings, /*throw_on_failure=*/ throw_on_invalid_change); if (new_value.isNull()) return false; @@ -479,9 +491,12 @@ bool SettingsConstraints::Checker::check(SettingChange & change, ReactionOnViolation reaction, SettingSource source) const { + /// Every reaction other than `THROW_ON_VIOLATION` clamps or drops a violating change. + const bool throw_on_violation = reaction == THROW_ON_VIOLATION; + if (!explain.text.empty()) { - if (reaction == THROW_ON_VIOLATION) + if (throw_on_violation) throw Exception(explain, code); return false; } @@ -490,7 +505,7 @@ bool SettingsConstraints::Checker::check(SettingChange & change, auto less_or_cannot_compare = [=](const Field & left, const Field & right) { - if (reaction == THROW_ON_VIOLATION) + if (throw_on_violation) return accurateLess(left, right); try { @@ -504,7 +519,7 @@ bool SettingsConstraints::Checker::check(SettingChange & change, auto equals_or_cannot_compare = [=](const Field & left, const Field & right) { - if (reaction == THROW_ON_VIOLATION) + if (throw_on_violation) return accurateEquals(left, right); try { @@ -519,7 +534,7 @@ bool SettingsConstraints::Checker::check(SettingChange & change, if (constraint.writability == SettingConstraintWritability::CONST) { - if (reaction == THROW_ON_VIOLATION) + if (throw_on_violation) throw Exception(ErrorCodes::SETTING_CONSTRAINT_VIOLATION, "Setting {} should not be changed", setting_name); return false; } @@ -530,7 +545,7 @@ bool SettingsConstraints::Checker::check(SettingChange & change, if (!min_value.isNull() && !max_value.isNull() && less_or_cannot_compare(max_value, min_value)) { - if (reaction == THROW_ON_VIOLATION) + if (throw_on_violation) throw Exception( ErrorCodes::SETTING_CONSTRAINT_VIOLATION, "The maximum ({}) value is less than the minimum ({}) value for setting {}", @@ -547,7 +562,7 @@ bool SettingsConstraints::Checker::check(SettingChange & change, if (!min_value.isNull() && less_or_cannot_compare(effective_value, min_value)) { - if (reaction == THROW_ON_VIOLATION) + if (throw_on_violation) { throw Exception(ErrorCodes::SETTING_CONSTRAINT_VIOLATION, "Setting {} shouldn't be less than {}", setting_name, applyVisitor(FieldVisitorToString(), min_value)); @@ -558,7 +573,7 @@ bool SettingsConstraints::Checker::check(SettingChange & change, if (!max_value.isNull() && less_or_cannot_compare(max_value, effective_value)) { - if (reaction == THROW_ON_VIOLATION) + if (throw_on_violation) { throw Exception(ErrorCodes::SETTING_CONSTRAINT_VIOLATION, "Setting {} shouldn't be greater than {}", setting_name, applyVisitor(FieldVisitorToString(), max_value)); @@ -572,7 +587,7 @@ bool SettingsConstraints::Checker::check(SettingChange & change, bool equals = equals_or_cannot_compare(value, effective_value); if (equals) { - if (reaction == THROW_ON_VIOLATION) + if (throw_on_violation) throw Exception(ErrorCodes::SETTING_CONSTRAINT_VIOLATION, "Setting {} shouldn't be {}", setting_name, applyVisitor(FieldVisitorToString(), value)); /// On clamp paths there is no sensible value to clamp to — disallowed entries are a @@ -584,7 +599,7 @@ bool SettingsConstraints::Checker::check(SettingChange & change, if (!getSettingSourceRestrictions(setting_name).isSourceAllowed(source)) { - if (reaction == THROW_ON_VIOLATION) + if (throw_on_violation) throw Exception(ErrorCodes::READONLY, "Setting {} is not allowed to be set by {}", setting_name, toString(source)); return false; } diff --git a/src/Access/SettingsConstraints.h b/src/Access/SettingsConstraints.h index 1e7470566281f..d522b3971622c 100644 --- a/src/Access/SettingsConstraints.h +++ b/src/Access/SettingsConstraints.h @@ -80,6 +80,7 @@ class SettingsConstraints void merge(const SettingsConstraints & other); /// Checks whether `change` violates these constraints and throws an exception if so. + /// The non-const `SettingsChanges &` overload also drops a change equal to the current value from `changes` (unless `compatibility` is present). void check(const Settings & current_settings, const SettingChange & change, SettingSource source) const; void check(const Settings & current_settings, const SettingsChanges & changes, SettingSource source) const; void check(const Settings & current_settings, SettingsChanges & changes, SettingSource source) const; @@ -95,8 +96,12 @@ class SettingsConstraints void check(const MergeTreeSettings & current_settings, const SettingsChanges & changes) const; /// Checks whether `change` violates these and clamps the `change` if so. + /// A change equal to the current value is dropped from `changes` (unless `compatibility` is present). void clamp(const Settings & current_settings, SettingsChanges & changes, SettingSource source) const; + /// Same as `clamp`, but an unknown or disallowed setting name and an uncastable value throw, as in `check`, + /// and a change equal to the current value is kept in `changes`. + void clampRejectingInvalidChanges(const Settings & current_settings, SettingsChanges & changes, SettingSource source) const; friend bool operator ==(const SettingsConstraints & left, const SettingsConstraints & right); friend bool operator !=(const SettingsConstraints & left, const SettingsConstraints & right) { return !(left == right); } @@ -104,8 +109,12 @@ class SettingsConstraints private: enum ReactionOnViolation { + /// Any violation, an unknown name or an uncastable value throws. THROW_ON_VIOLATION, + /// Violations are clamped or dropped; an unknown name or an uncastable value is dropped. CLAMP_ON_VIOLATION, + /// Violations are clamped or dropped; an unknown name or an uncastable value throws. + CLAMP_ON_VIOLATION_THROW_ON_INVALID, }; struct Constraint @@ -162,8 +171,8 @@ class SettingsConstraints } }; - /// Common logic for `check(Settings, SettingsChanges&)` and `clamp`. Both filter out unchanged settings - /// (unless `compatibility` is present) and differ only in whether violations throw or get clamped to the nearest bound. + /// Common logic for `check`, `clamp` and `clampRejectingInvalidChanges`. The modes differ in whether violations throw or get clamped + /// to the nearest bound and whether an invalid change throws or is dropped. void checkOrClamp(const Settings & current_settings, SettingsChanges & changes, ReactionOnViolation reaction, SettingSource source) const; diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp index 0b5e861df1473..126a792506caf 100644 --- a/src/Analyzer/QueryTreeBuilder.cpp +++ b/src/Analyzer/QueryTreeBuilder.cpp @@ -311,22 +311,28 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectExpression( set_query.changes.removeSetting("limit"); set_query.changes.removeSetting("offset"); - /// A nested `SETTINGS` clause (a subquery, a CTE, a view's inner query) used to be applied to - /// the per-node context unchecked, letting a user override `readonly`, `CONST` and `MIN`/`MAX` - /// constraints - e.g. `additional_table_filters`, which the Planner reads from this context. - /// Clamp it instead of throwing, as done for other settings crossing execution contexts - /// (`getSQLSecurityOverriddenContext`, secondary queries, DDL replay): violating changes are - /// dropped, out-of-bounds values are clamped, so a view whose inner clause violates the - /// reader's constraints keeps working. A top-level clause still throws - /// (`applySettingsFromQuery`). Clamp a copy: the `QueryNode` keeps the clause as written, so - /// the tree's AST and hash are unchanged, and every node executing the subquery clamps it - /// against its own constraints. `SETTINGS name = DEFAULT` stays ignored here - see #115415. + /// A nested `SETTINGS` clause (a subquery, a CTE, a view's inner query) is clamped to the session's + /// settings constraints, as done for other settings crossing execution contexts + /// (`getSQLSecurityOverriddenContext`, secondary queries, DDL replay): violating changes are dropped, + /// out-of-bounds values are clamped, so a view whose inner clause violates the reader's constraints + /// keeps working. A top-level clause still throws (`applySettingsFromQuery`). An unknown name or an + /// uncastable value throws as for a top-level clause, on every server that executes the query. The + /// node records the effective `changes`: the ones that survive, with their clamped values and the + /// literals as written, so `toAST`, the tree hash, the `Planner` and the shards receiving the + /// subquery see what was applied. `SETTINGS name = DEFAULT` stays ignored here - see #115415. if (!set_query.changes.empty()) { - auto checked_changes = set_query.changes; - updated_context->clampToSettingsConstraints(checked_changes, SettingSource::QUERY); - updated_context->applySettingsChanges(checked_changes); - settings_changes = set_query.changes; + /// One change at a time, so that each is clamped against the context as the preceding changes + /// left it. Clamping the whole clause as a batch checks every change against the context as it + /// was before the clause - so `SETTINGS use_query_cache = 1, use_query_cache = 0` could resolve + /// differently from the last-wins reading the clause has everywhere else. + for (const auto & change : set_query.changes) + { + SettingsChanges single_change{change}; + updated_context->clampToSettingsConstraintsRejectingInvalidChanges(single_change, SettingSource::QUERY); + updated_context->applySettingsChanges(single_change); + settings_changes.insert(settings_changes.end(), single_change.begin(), single_change.end()); + } } } diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 26a03ab3a6dbb..ebd3e4a70ba46 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -3620,6 +3620,13 @@ void Context::clampToSettingsConstraintsWithLock(SettingsChanges & changes, Sett doSettingsSanityCheckClamp(*settings, getLogger("SettingsSanity")); } +void Context::clampToSettingsConstraintsRejectingInvalidChangesWithLock(SettingsChanges & changes, SettingSource source) +{ + getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.clampRejectingInvalidChanges(*settings, changes, source); + if (getApplicationType() == ApplicationType::LOCAL || getApplicationType() == ApplicationType::SERVER) + doSettingsSanityCheckClamp(*settings, getLogger("SettingsSanity")); +} + void Context::checkMergeTreeSettingsConstraintsWithLock(const MergeTreeSettings & merge_tree_settings, const SettingsChanges & changes) const { getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.check(merge_tree_settings, changes); @@ -3704,6 +3711,21 @@ void Context::clampToSettingsConstraints(SettingsChanges & changes, SettingSourc clampToSettingsConstraintsWithLock(changes, source); } +void Context::clampToSettingsConstraintsRejectingInvalidChanges(SettingsChanges & changes, SettingSource source) +{ + if (std::ranges::any_of(changes, isProfileChange)) + { + changes = enforceConstraintsAlongProfileChanges(shared_from_this(), changes, [source](Context & context, SettingsChanges & segment) + { + context.clampToSettingsConstraintsRejectingInvalidChanges(segment, source); + }); + return; + } + + SharedLockGuard lock(mutex); + clampToSettingsConstraintsRejectingInvalidChangesWithLock(changes, source); +} + void Context::checkMergeTreeSettingsConstraints(const MergeTreeSettings & merge_tree_settings, const SettingsChanges & changes) const { SharedLockGuard lock(mutex); diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index f6a7dc9dfe8e8..e74cbddab6fcc 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -1252,6 +1252,7 @@ class Context: public ContextData, public std::enable_shared_from_this /// For the resets of a statement that also changes `profile`: `changes_applied_first` decides their constraints. void checkSettingsConstraintsForSettingsReset(const std::vector & names, const SettingsChanges & changes_applied_first, SettingSource source); void clampToSettingsConstraints(SettingsChanges & changes, SettingSource source); + void clampToSettingsConstraintsRejectingInvalidChanges(SettingsChanges & changes, SettingSource source); void checkMergeTreeSettingsConstraints(const MergeTreeSettings & merge_tree_settings, const SettingsChanges & changes) const; /// Reset settings to default value @@ -2100,6 +2101,7 @@ class Context: public ContextData, public std::enable_shared_from_this void checkSettingsConstraintsWithLock(SettingsChanges & changes, SettingSource source); void clampToSettingsConstraintsWithLock(SettingsChanges & changes, SettingSource source); + void clampToSettingsConstraintsRejectingInvalidChangesWithLock(SettingsChanges & changes, SettingSource source); void checkSettingsConstraintsWithLock(const AlterSettingsProfileElements & profile_elements, SettingSource source) const; void clampToSettingsConstraintsWithLock(SettingsChanges & changes, SettingSource source) const; diff --git a/tests/queries/0_stateless/05141_settings_constraints_nested_settings_effective.reference b/tests/queries/0_stateless/05141_settings_constraints_nested_settings_effective.reference new file mode 100644 index 0000000000000..a5714250422e8 --- /dev/null +++ b/tests/queries/0_stateless/05141_settings_constraints_nested_settings_effective.reference @@ -0,0 +1,35 @@ +-- use_query_cache = 0 CONST: a nested use_query_cache = 1 runs but caches nothing +3 +0 0 +-- readonly = 1: a nested use_query_cache = 1 runs but caches nothing +3 +0 0 +-- unconstrained: a nested use_query_cache = 1 caches the subquery +3 +1 0 +-- unconstrained: a nested use_query_cache = 1 equal to the outer value still caches the subquery, next to the outer query +3 +1 1 +-- unconstrained: a nested use_query_cache = 'true' given as a string is applied +3 +1 0 +-- a nested unknown setting still throws +1 +-- a nested value the setting cannot take still throws +1 +-- a nested clause forwarded to the shards carries the clamped value +1 +-- the same query for an unconstrained user reads both shards +6 +-- a view with an invalid nested setting in its stored body fails on a direct read +1 +-- and on a remote shard through Distributed +1 +-- also when the shard inlines the view +1 +-- a parameterized view with an invalid nested setting fails on a direct read +1 +-- and on a remote shard +1 +-- the session settings are untouched +false diff --git a/tests/queries/0_stateless/05141_settings_constraints_nested_settings_effective.sh b/tests/queries/0_stateless/05141_settings_constraints_nested_settings_effective.sh new file mode 100755 index 0000000000000..8206835476c28 --- /dev/null +++ b/tests/queries/0_stateless/05141_settings_constraints_nested_settings_effective.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# Tags: no-old-analyzer +# (the clamp is analyzer-path behaviour; the legacy interpreter throws on such a nested clause) +# A nested `SETTINGS` clause is clamped to the session's settings constraints, and the clamped clause is +# what the query node keeps. So a nested `use_query_cache = 1` the constraints dropped must not enable +# the query result cache in the `Planner` (issue #117226), a shard receives the clamped clause, and an +# unknown name or an uncastable value in a nested clause throws, as it does in a top-level clause - also +# when the clause sits in a view's stored body and the view is read on a remote shard. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Users and settings profiles are server-global, so their names carry the test database to keep this +# test safe against a concurrent copy of itself - which is how the flaky check runs it. +SUFFIX="05141_${CLICKHOUSE_DATABASE}" +CONST_USER="user_const_${SUFFIX}" +CONST_PROFILE="profile_const_${SUFFIX}" +RO_USER="user_ro_${SUFFIX}" +RO_PROFILE="profile_ro_${SUFFIX}" +FREE_USER="user_free_${SUFFIX}" +DIST_USER="user_dist_${SUFFIX}" +DIST_PROFILE="profile_dist_${SUFFIX}" +TABLE="t_05141" +DIST_TABLE="dist_05141" +VIEW="v_05141" +DIST_VIEW="dist_v_05141" +PARAM_VIEW="pv_05141" +# `system.query_cache` lists the entries of every user and every test, and the cache is never dropped +# here, so each cached subquery carries a per-run marker in its text and is looked up by it. +MARKER="marker_${SUFFIX}" + +${CLICKHOUSE_CLIENT} --query " +DROP USER IF EXISTS ${CONST_USER}, ${RO_USER}, ${FREE_USER}, ${DIST_USER}; +DROP SETTINGS PROFILE IF EXISTS ${CONST_PROFILE}, ${RO_PROFILE}, ${DIST_PROFILE}; +DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.${DIST_TABLE}; +DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.${TABLE}; +DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.${DIST_VIEW}; +DROP VIEW IF EXISTS ${CLICKHOUSE_DATABASE}.${VIEW}; +DROP VIEW IF EXISTS ${CLICKHOUSE_DATABASE}.${PARAM_VIEW}; + +CREATE TABLE ${CLICKHOUSE_DATABASE}.${TABLE} (id UInt32, s String) ENGINE = MergeTree ORDER BY id; +INSERT INTO ${CLICKHOUSE_DATABASE}.${TABLE} VALUES (1, 'a'), (2, 'b'), (3, 'c'); +CREATE TABLE ${CLICKHOUSE_DATABASE}.${DIST_TABLE} AS ${CLICKHOUSE_DATABASE}.${TABLE} + ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), ${TABLE}); + +CREATE SETTINGS PROFILE ${CONST_PROFILE} SETTINGS use_query_cache = 0 CONST; +CREATE SETTINGS PROFILE ${RO_PROFILE} SETTINGS readonly = 1 CONST; +CREATE SETTINGS PROFILE ${DIST_PROFILE} SETTINGS max_rows_to_read MAX 2; +CREATE USER ${CONST_USER} IDENTIFIED WITH no_password SETTINGS PROFILE ${CONST_PROFILE}; +CREATE USER ${RO_USER} IDENTIFIED WITH no_password SETTINGS PROFILE ${RO_PROFILE}; +CREATE USER ${FREE_USER} IDENTIFIED WITH no_password; +CREATE USER ${DIST_USER} IDENTIFIED WITH no_password SETTINGS PROFILE ${DIST_PROFILE}; +GRANT SELECT ON ${CLICKHOUSE_DATABASE}.* TO ${CONST_USER}, ${RO_USER}, ${FREE_USER}, ${DIST_USER}; +" + +# `CREATE` does not analyze the body of a view with an explicit column list nor of a parameterized view, so +# such definitions can carry an invalid nested setting. +${CLICKHOUSE_CLIENT} --query " +CREATE VIEW ${CLICKHOUSE_DATABASE}.${VIEW} (x UInt8) AS SELECT * FROM (SELECT 1 AS x SETTINGS max_threds = 1); +CREATE TABLE ${CLICKHOUSE_DATABASE}.${DIST_VIEW} (x UInt8) ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), ${VIEW}); +CREATE VIEW ${CLICKHOUSE_DATABASE}.${PARAM_VIEW} AS SELECT * FROM (SELECT 1 AS x SETTINGS max_threds = 1) WHERE x = {p:UInt8}; +" + +# The bare binary, not ${CLICKHOUSE_CLIENT}: the harness's randomized settings would be rejected for the readonly user. +RESTRICTED_CLIENT="${CLICKHOUSE_CLIENT_BINARY} --host=${CLICKHOUSE_HOST} --port=${CLICKHOUSE_PORT_TCP} --database=${CLICKHOUSE_DATABASE}" +CONST="${RESTRICTED_CLIENT} --user=${CONST_USER}" +RO="${RESTRICTED_CLIENT} --user=${RO_USER}" +FREE="${RESTRICTED_CLIENT} --user=${FREE_USER}" +DIST="${RESTRICTED_CLIENT} --user=${DIST_USER}" + +# Cache entries whose text carries the marker: subquery entries, then top-level entries. +function cache_entries() +{ + ${CLICKHOUSE_CLIENT} --query "SELECT countIf(is_subquery), countIf(NOT is_subquery) FROM system.query_cache WHERE query LIKE '%$1%'" +} + +# The marker is compared against a real column so it survives constant folding and stays in the +# subquery text the cache entry is keyed and listed by. +echo "-- use_query_cache = 0 CONST: a nested use_query_cache = 1 runs but caches nothing" +${CONST} --query "SELECT count() FROM (SELECT id FROM ${TABLE} WHERE s != '${MARKER}_const' SETTINGS use_query_cache = 1)" +cache_entries "${MARKER}_const" + +echo "-- readonly = 1: a nested use_query_cache = 1 runs but caches nothing" +${RO} --query "SELECT count() FROM (SELECT id FROM ${TABLE} WHERE s != '${MARKER}_ro' SETTINGS use_query_cache = 1)" +cache_entries "${MARKER}_ro" + +echo "-- unconstrained: a nested use_query_cache = 1 caches the subquery" +${FREE} --query "SELECT count() FROM (SELECT id FROM ${TABLE} WHERE s != '${MARKER}_free' SETTINGS use_query_cache = 1)" +cache_entries "${MARKER}_free" + +echo "-- unconstrained: a nested use_query_cache = 1 equal to the outer value still caches the subquery, next to the outer query" +${FREE} --query "SELECT count() FROM (SELECT id FROM ${TABLE} WHERE s != '${MARKER}_both' SETTINGS use_query_cache = 1) SETTINGS use_query_cache = 1" +cache_entries "${MARKER}_both" + +echo "-- unconstrained: a nested use_query_cache = 'true' given as a string is applied" +${FREE} --query "SELECT count() FROM (SELECT id FROM ${TABLE} WHERE s != '${MARKER}_string' SETTINGS use_query_cache = 'true')" +cache_entries "${MARKER}_string" + +echo "-- a nested unknown setting still throws" +${CONST} --query "SELECT count() FROM (SELECT id FROM ${TABLE} SETTINGS max_threds = 1)" 2>&1 | grep -c -F "UNKNOWN_SETTING" + +echo "-- a nested value the setting cannot take still throws" +${CONST} --query "SELECT count() FROM (SELECT id FROM ${TABLE} SETTINGS max_threads = 'abc')" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED" + +# The IN subquery runs on the shards, whose user has no constraints. It must arrive with the value the +# initiator clamped (2, so reading 3 rows fails there), not with the value as written. +echo "-- a nested clause forwarded to the shards carries the clamped value" +${DIST} --query "SELECT count() FROM ${DIST_TABLE} WHERE id IN (SELECT id FROM ${TABLE} SETTINGS max_rows_to_read = 100)" 2>&1 | grep -c -F "TOO_MANY_ROWS" + +echo "-- the same query for an unconstrained user reads both shards" +${FREE} --query "SELECT count() FROM ${DIST_TABLE} WHERE id IN (SELECT id FROM ${TABLE} SETTINGS max_rows_to_read = 100)" + +# The bare binary again: `${CLICKHOUSE_CLIENT}` adds `--send_logs_level`, so the server's error log line would +# be printed too and `grep -c` would count two lines. +echo "-- a view with an invalid nested setting in its stored body fails on a direct read" +${RESTRICTED_CLIENT} --query "SELECT * FROM ${VIEW}" 2>&1 | grep -c -F "UNKNOWN_SETTING" + +# `prefer_localhost_replica = 0` is explicit: the local shard is analyzed under the initiator's query kind +# and already throws, so only the remote path exercises the fix, and the harness randomizes this setting. +echo "-- and on a remote shard through Distributed" +${RESTRICTED_CLIENT} --query "SELECT * FROM ${DIST_VIEW} SETTINGS prefer_localhost_replica = 0" 2>&1 | grep -c -F "UNKNOWN_SETTING" + +echo "-- also when the shard inlines the view" +${RESTRICTED_CLIENT} --query "SELECT * FROM ${DIST_VIEW} SETTINGS prefer_localhost_replica = 0, analyzer_inline_views = 1" 2>&1 | grep -c -F "UNKNOWN_SETTING" + +echo "-- a parameterized view with an invalid nested setting fails on a direct read" +${RESTRICTED_CLIENT} --query "SELECT * FROM ${PARAM_VIEW}(p = 1)" 2>&1 | grep -c -F "UNKNOWN_SETTING" + +# 127.0.0.2 is the non-local shard of `test_cluster_two_shards`; the port is explicit so the test runs on +# a non-default port; the database is qualified because the remote session does not inherit the test database. +echo "-- and on a remote shard" +${RESTRICTED_CLIENT} --query "SELECT * FROM remote('127.0.0.2:${CLICKHOUSE_PORT_TCP}', view(SELECT * FROM ${CLICKHOUSE_DATABASE}.${PARAM_VIEW}(p = 1)))" 2>&1 | grep -c -F "UNKNOWN_SETTING" + +echo "-- the session settings are untouched" +${CONST} --query "SELECT getSetting('use_query_cache')" + +${CLICKHOUSE_CLIENT} --query " +DROP TABLE ${CLICKHOUSE_DATABASE}.${DIST_TABLE}; +DROP TABLE ${CLICKHOUSE_DATABASE}.${TABLE}; +DROP TABLE ${CLICKHOUSE_DATABASE}.${DIST_VIEW}; +DROP VIEW ${CLICKHOUSE_DATABASE}.${VIEW}; +DROP VIEW ${CLICKHOUSE_DATABASE}.${PARAM_VIEW}; +DROP USER ${CONST_USER}, ${RO_USER}, ${FREE_USER}, ${DIST_USER}; +DROP SETTINGS PROFILE ${CONST_PROFILE}, ${RO_PROFILE}, ${DIST_PROFILE}; +" diff --git a/tests/queries/0_stateless/05175_nested_settings_equal_value_marks_changed.reference b/tests/queries/0_stateless/05175_nested_settings_equal_value_marks_changed.reference new file mode 100644 index 0000000000000..95ca9df0fcece --- /dev/null +++ b/tests/queries/0_stateless/05175_nested_settings_equal_value_marks_changed.reference @@ -0,0 +1,18 @@ +-- control: the setting is not changed in the session +0 1 +-- control: compatibility = '26.7' in a nested clause moves the setting to its old default +1 0 +-- an equal nested assignment is applied and marked changed in the subquery's context, as at the top level +1 1 +-- and stays on the query node +1 +-- same clause: an equal assignment before compatibility keeps the current value +1 1 +-- same clause: an equal assignment before a profile that carries compatibility keeps the current value +1 1 +-- child scope: an equal assignment in the outer subquery protects the setting from compatibility in the inner one +1 +-- child scope: the same with a profile that carries compatibility in the inner one +1 +-- top level over HTTP: an equal assignment protects the setting from compatibility in a subquery +1 diff --git a/tests/queries/0_stateless/05175_nested_settings_equal_value_marks_changed.sh b/tests/queries/0_stateless/05175_nested_settings_equal_value_marks_changed.sh new file mode 100755 index 0000000000000..4f9000f02f8c0 --- /dev/null +++ b/tests/queries/0_stateless/05175_nested_settings_equal_value_marks_changed.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Tags: no-old-analyzer + +# A nested `SETTINGS` change equal to the current value is applied and marks the setting as changed +# in the subquery's context, as at the top level. That flag is what `compatibility` respects +# ("changed manually" settings are left alone), also when `compatibility` sits in a profile or in +# a nested query, so the flag must not be dropped as a "no-op". + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Settings profiles are server-global, so the name carries the test database. +PROFILE="p_compat_05175_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} --query "DROP SETTINGS PROFILE IF EXISTS ${PROFILE}" +${CLICKHOUSE_CLIENT} --query "CREATE SETTINGS PROFILE ${PROFILE} SETTINGS compatibility = '26.7'" + +Q="SELECT changed, value FROM system.settings WHERE name = 'enable_group_by_top_k_optimization'" + +echo "-- control: the setting is not changed in the session" +${CLICKHOUSE_CLIENT} --query "${Q}" + +echo "-- control: compatibility = '26.7' in a nested clause moves the setting to its old default" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM (${Q} SETTINGS compatibility = '26.7')" + +echo "-- an equal nested assignment is applied and marked changed in the subquery's context, as at the top level" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM (${Q} SETTINGS enable_group_by_top_k_optimization = 1)" + +echo "-- and stays on the query node" +${CLICKHOUSE_CLIENT} --query "EXPLAIN QUERY TREE SELECT * FROM (${Q} SETTINGS enable_group_by_top_k_optimization = 1)" | grep -c "SETTINGS enable_group_by_top_k_optimization=" + +echo "-- same clause: an equal assignment before compatibility keeps the current value" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM (${Q} SETTINGS enable_group_by_top_k_optimization = 1, compatibility = '26.7')" + +echo "-- same clause: an equal assignment before a profile that carries compatibility keeps the current value" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM (${Q} SETTINGS enable_group_by_top_k_optimization = 1, profile = '${PROFILE}')" + +echo "-- child scope: an equal assignment in the outer subquery protects the setting from compatibility in the inner one" +${CLICKHOUSE_CLIENT} --query "SELECT value FROM (SELECT * FROM (${Q} SETTINGS compatibility = '26.7') SETTINGS enable_group_by_top_k_optimization = 1)" + +echo "-- child scope: the same with a profile that carries compatibility in the inner one" +${CLICKHOUSE_CLIENT} --query "SELECT value FROM (SELECT * FROM (${Q} SETTINGS profile = '${PROFILE}') SETTINGS enable_group_by_top_k_optimization = 1)" + +echo "-- top level over HTTP: an equal assignment protects the setting from compatibility in a subquery" +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "SELECT value FROM (${Q} SETTINGS compatibility = '26.7') SETTINGS enable_group_by_top_k_optimization = 1" + +${CLICKHOUSE_CLIENT} --query "DROP SETTINGS PROFILE ${PROFILE}" From fcf64f19fa59c6120ef019b3deedc071062fb062 Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Wed, 4 Nov 2026 14:39:00 +0000 Subject: [PATCH 097/219] Read `use_query_cache` of a subquery from its context in the Planner `shouldUseQueryCacheForSubquery` read the nested `use_query_cache` with `safeGet` on the literal, so a value written as a string such as `'true'` raised `BAD_GET` and failed `05141_settings_constraints_nested_settings_effective`. The clause is now clamped and applied to the node's context, so read the value from there, as master does. Co-Authored-By: Claude Opus 5.5 --- src/Planner/Planner.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 0d438f4f8581b..b312f5c22898f 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -113,6 +113,7 @@ namespace DB namespace Setting { extern const SettingsMap additional_table_filters; + extern const SettingsBool use_query_cache; extern const SettingsUInt64 aggregation_in_order_max_block_bytes; extern const SettingsUInt64 aggregation_memory_efficient_merge_threads; extern const SettingsUInt64 allow_experimental_parallel_reading_from_replicas; @@ -2474,7 +2475,8 @@ static bool shouldUseQueryCacheForSubquery( for (const auto & change : query_node.getSettingsChanges()) { if (change.name == "use_query_cache") - return change.value.safeGet(); + /// The node's context holds the clause as clamped and applied; the literal may be a string such as `'true'`. + return query_node.getContext()->getSettingsRef()[Setting::use_query_cache]; } } From c592f9a98f3dfab3f6fdc93429d80f9449be7a22 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 4 Nov 2026 14:11:16 +0000 Subject: [PATCH 098/219] Backport #128604 to 26.8: Don't do s3 requests to get metadata in delta tables --- .../DataLakes/DeltaLake/TableSnapshot.cpp | 36 +++++++++++++------ 1 file changed, 26 insertions(+), 10 deletions(-) diff --git a/src/Storages/ObjectStorage/DataLakes/DeltaLake/TableSnapshot.cpp b/src/Storages/ObjectStorage/DataLakes/DeltaLake/TableSnapshot.cpp index 5c14fcf47aec9..302c600c34f9e 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeltaLake/TableSnapshot.cpp +++ b/src/Storages/ObjectStorage/DataLakes/DeltaLake/TableSnapshot.cpp @@ -66,6 +66,30 @@ namespace DB::FailPoints namespace DeltaLake { +namespace +{ + +/// A data file with the size its `add` action records. +struct DataObjectInfo : public DB::ObjectInfo +{ + DataObjectInfo(std::string path_, int64_t size_) + : DB::ObjectInfo(DB::RelativePathWithMetadata(std::move(path_))) + , size(size_) + { + } + + std::optional getFileSizeHint() const override + { + if (size > 0) + return static_cast(size); + return std::nullopt; + } + + const int64_t size; +}; + +} + class TableSnapshot::Iterator final : public DB::IObjectIterator, private DB::WithContext { private: @@ -96,7 +120,6 @@ class TableSnapshot::Iterator final : public DB::IObjectIterator, private DB::Wi const TableSchema & table_schema_, const DB::NameToNameMap & physical_names_map_, const DB::Names & partition_columns_, - DB::ObjectStoragePtr object_storage_, const DB::ActionsDAG * filter_, DB::IDataLakeMetadata::FileProgressCallback callback_, size_t list_batch_size_, @@ -113,7 +136,6 @@ class TableSnapshot::Iterator final : public DB::IObjectIterator, private DB::Wi , read_schema(read_schema_) , expression_schema(table_schema_) , partition_columns(partition_columns_) - , object_storage(object_storage_) , callback(callback_) , list_batch_size(list_batch_size_) , log(log_) @@ -415,13 +437,9 @@ class TableSnapshot::Iterator final : public DB::IObjectIterator, private DB::Wi } parseDVHandle(*scan_item, object); - object->setObjectMetadata(object_storage->getObjectMetadata(object->getPath(), /*with_tags=*/ false)); if (callback) - { - chassert(object->getObjectMetadata()); - callback(DB::FileProgress(0, object->getObjectMetadata()->size_bytes)); - } + callback(DB::FileProgress(0, object->getFileSizeHint().value_or(0))); return object; } } @@ -595,7 +613,7 @@ class TableSnapshot::Iterator final : public DB::IObjectIterator, private DB::Wi std::string full_path = DB::resolvePathInsideTable( context->getDataPath(), DB::unescapeForFileName(KernelUtils::fromDeltaString(path))); - auto object = std::make_shared(DB::RelativePathWithMetadata(std::move(full_path))); + auto object = std::make_shared(std::move(full_path), size); object->data_lake_metadata.emplace(); LOG_TEST( @@ -639,7 +657,6 @@ class TableSnapshot::Iterator final : public DB::IObjectIterator, private DB::Wi DB::NamesAndTypesList read_schema; DB::NamesAndTypesList expression_schema; DB::Names partition_columns; - const DB::ObjectStoragePtr object_storage; const DB::IDataLakeMetadata::FileProgressCallback callback; const size_t list_batch_size; const LoggerPtr log; @@ -994,7 +1011,6 @@ DB::ObjectIterator TableSnapshot::iterate( schema->table_schema, schema->physical_names_map, schema->partition_columns, - object_storage, filter_dag, callback, list_batch_size, From 6485d61ef2322aac2ad6cbdee1480ecd1671adb4 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 5 Nov 2026 12:47:13 +0000 Subject: [PATCH 099/219] Backport #127297 to 26.8: Mark `02372_analyzer_join` as `no-msan` --- tests/queries/0_stateless/02372_analyzer_join.sql.j2 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/02372_analyzer_join.sql.j2 b/tests/queries/0_stateless/02372_analyzer_join.sql.j2 index c9dc41d157090..201e69b64f00d 100644 --- a/tests/queries/0_stateless/02372_analyzer_join.sql.j2 +++ b/tests/queries/0_stateless/02372_analyzer_join.sql.j2 @@ -1,4 +1,4 @@ --- Tags: long +-- Tags: long, no-msan SET enable_analyzer = 1; SET single_join_prefer_left_table = 0; From aa3b0718266a4c32bcf9d1951a4404f8b0e3427a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 3 Nov 2026 11:50:29 +0000 Subject: [PATCH 100/219] Backport #122717 to 26.8: Hide secrets in system mutations --- src/Storages/MergeTree/MergeTreeData.cpp | 2 +- .../MergeTree/MutateFromLogEntryTask.cpp | 2 +- .../System/StorageSystemMutations.cpp | 116 ++++++++++++++---- src/Storages/System/StorageSystemMutations.h | 3 +- ...91_system_mutations_hide_secrets.reference | 5 + .../05291_system_mutations_hide_secrets.sql | 47 +++++++ 6 files changed, 151 insertions(+), 24 deletions(-) create mode 100644 tests/queries/0_stateless/05291_system_mutations_hide_secrets.reference create mode 100644 tests/queries/0_stateless/05291_system_mutations_hide_secrets.sql diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index d56e8d105b718..8c039d07acbec 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -12263,7 +12263,7 @@ MergeTreeData::LightweightUpdateResult MergeTreeData::updateLightweightImpl(cons if (it != commands.end()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Got unexpected command with type {} in lightweight update", it->type); - LOG_DEBUG(log, "Executing lightweight update with commands: {}", commands.toString(false)); + LOG_DEBUG(log, "Executing lightweight update with commands: {}", commands.ast(false)->formatForLogging()); NamesAndTypesList system_columns; MutationCommands commands_to_run; diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp index 10c3dfaf0ca49..fd361e332e5b5 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp @@ -180,7 +180,7 @@ ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare() Strings mutation_ids; commands = std::make_shared(storage.queue.getMutationCommands(source_part, new_part_info.mutation, mutation_ids)); LOG_TRACE(log, "Mutating part {} with mutation commands from {} mutations ({}): {}", - entry.new_part_name, commands->size(), fmt::join(mutation_ids, ", "), commands->toString(true)); + entry.new_part_name, commands->size(), fmt::join(mutation_ids, ", "), commands->ast(true)->formatForLogging()); /// mutation_ids can be empty here. mutation_ids_for_log = mutation_ids; diff --git a/src/Storages/System/StorageSystemMutations.cpp b/src/Storages/System/StorageSystemMutations.cpp index 095673f0f43c0..5b6a75756d609 100644 --- a/src/Storages/System/StorageSystemMutations.cpp +++ b/src/Storages/System/StorageSystemMutations.cpp @@ -11,14 +11,68 @@ #include #include #include +#include +#include #include #include #include +#include +#include +#include namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_parser_depth; + extern const SettingsUInt64 max_parser_backtracks; +} + +namespace ErrorCodes +{ + extern const int SYNTAX_ERROR; + extern const int TOO_DEEP_RECURSION; + extern const int TOO_SLOW_PARSING; +} + +/// The mutation entry keeps the real text of the command, because the mutation is executed from it +/// (on every replica, for replicated tables). Hide the secrets on the way out instead, the way +/// `system.distributed_ddl_queue` does. The command is formatted the same way as the stored text, +/// so only the secrets differ from it. +static String hideSecretsInCommand(const ContextPtr & context, const String & command) +{ + /// Nothing to parse, see `MutationCommand::ast`. + if (command.empty()) + return command; + + const auto & settings = context->getSettingsRef(); + ParserAlterCommandList parser; + ASTPtr ast; + try + { + ast = parseQuery(parser, command, 0, settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks]); + } + catch (const Exception & e) + { + /// The text may be unparseable on another server version, so hide the whole command instead of failing the table. + if (e.code() != ErrorCodes::SYNTAX_ERROR + && e.code() != ErrorCodes::TOO_DEEP_RECURSION + && e.code() != ErrorCodes::TOO_SLOW_PARSING) + throw; + return String(HIDDEN_SECRET); + } + + return ast->formatWithPossiblyHidingSensitiveData( + /*max_length=*/0, + /*one_line=*/true, + /*show_secrets=*/false, + /*print_pretty_type_names=*/false, + /*identifier_quoting_rule=*/IdentifierQuotingRule::WhenNecessary, + /*identifier_quoting_style=*/IdentifierQuotingStyle::Backticks); +} + ColumnsDescription StorageSystemMutations::getColumnsDescription() { @@ -72,10 +126,11 @@ Block StorageSystemMutations::getFilterSampleBlock() const }; } -void StorageSystemMutations::fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node * predicate, std::vector) const +void StorageSystemMutations::fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node * predicate, std::vector columns_mask) const { const auto access = context->getAccess(); const bool check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES); + const bool show_secrets = canDisplaySecrets(context); /// Collect a set of *MergeTree tables. std::map> merge_tree_tables; @@ -176,26 +231,45 @@ void StorageSystemMutations::fillData(MutableColumns & res_columns, ContextPtr c parts_postpone_reasons_map.emplace_back(std::move(key_value)); } - size_t col_num = 0; - res_columns[col_num++]->insert(database); - res_columns[col_num++]->insert(table); - - res_columns[col_num++]->insert(status.id); - res_columns[col_num++]->insert(status.command); - res_columns[col_num++]->insert(UInt64(status.create_time)); - res_columns[col_num++]->insert(UInt64(status.finish_time)); - res_columns[col_num++]->insert(block_partition_ids); - res_columns[col_num++]->insert(block_numbers); - res_columns[col_num++]->insert(parts_in_progress_names); - res_columns[col_num++]->insert(parts_to_do_names); - res_columns[col_num++]->insert(parts_to_do_names.size()); - res_columns[col_num++]->insert(parts_postpone_reasons_map); - res_columns[col_num++]->insert(status.is_done); - res_columns[col_num++]->insert(status.is_killed); - res_columns[col_num++]->insert(status.latest_failed_part); - res_columns[col_num++]->insert(UInt64(status.latest_fail_time)); - res_columns[col_num++]->insert(status.latest_fail_reason); - res_columns[col_num++]->insert(status.latest_fail_error_code_name); + size_t src_index = 0; + size_t res_index = 0; + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(database); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(table); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(status.id); + /// Parsing the command costs much more than everything else here, so it is done only when the column is queried. + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(show_secrets ? status.command : hideSecretsInCommand(context, status.command)); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(UInt64(status.create_time)); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(UInt64(status.finish_time)); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(block_partition_ids); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(block_numbers); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(parts_in_progress_names); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(parts_to_do_names); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(parts_to_do_names.size()); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(parts_postpone_reasons_map); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(status.is_done); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(status.is_killed); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(status.latest_failed_part); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(UInt64(status.latest_fail_time)); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(status.latest_fail_reason); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(status.latest_fail_error_code_name); } } } diff --git a/src/Storages/System/StorageSystemMutations.h b/src/Storages/System/StorageSystemMutations.h index 5341838a65e3c..c3f897491fc76 100644 --- a/src/Storages/System/StorageSystemMutations.h +++ b/src/Storages/System/StorageSystemMutations.h @@ -21,8 +21,9 @@ class StorageSystemMutations final : public IStorageSystemOneBlock protected: using IStorageSystemOneBlock::IStorageSystemOneBlock; - void fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node * predicate, std::vector) const override; + void fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node * predicate, std::vector columns_mask) const override; Block getFilterSampleBlock() const override; + bool supportsColumnsMask() const override { return true; } }; } diff --git a/tests/queries/0_stateless/05291_system_mutations_hide_secrets.reference b/tests/queries/0_stateless/05291_system_mutations_hide_secrets.reference new file mode 100644 index 0000000000000..c9a233584ace8 --- /dev/null +++ b/tests/queries/0_stateless/05291_system_mutations_hide_secrets.reference @@ -0,0 +1,5 @@ +t_05291 (DELETE WHERE id IN (SELECT CAST(c1, \'UInt64\') FROM s3(\'http://127.0.0.1:1/nope.csv\', \'AKIAFAKEKEYID\', \'[HIDDEN]\', \'CSV\', \'c1 String\'))) +t_05291 (UPDATE v = 0 WHERE id IN (SELECT CAST(c1, \'UInt64\') FROM s3(\'http://127.0.0.1:1/nope.csv\', \'AKIAFAKEKEYID\', \'[HIDDEN]\', \'CSV\', \'c1 String\'))) +t_05291 (UPDATE _row_exists = 0 WHERE id IN (SELECT CAST(c1, \'UInt64\') FROM s3(\'http://127.0.0.1:1/nope.csv\', \'AKIAFAKEKEYID\', \'[HIDDEN]\', \'CSV\', \'c1 String\'))) +t_05291_replicated (DELETE WHERE id IN (SELECT CAST(c1, \'UInt64\') FROM s3(\'http://127.0.0.1:1/nope.csv\', \'AKIAFAKEKEYID\', \'[HIDDEN]\', \'CSV\', \'c1 String\'))) +0 diff --git a/tests/queries/0_stateless/05291_system_mutations_hide_secrets.sql b/tests/queries/0_stateless/05291_system_mutations_hide_secrets.sql new file mode 100644 index 0000000000000..58aa5021b160c --- /dev/null +++ b/tests/queries/0_stateless/05291_system_mutations_hide_secrets.sql @@ -0,0 +1,47 @@ +-- Tags: no-fasttest +-- no-fasttest: the `s3` table function is not in the fast test build. + +-- A mutation entry keeps the real text of its commands, because the mutation is executed from it (on every +-- replica, for replicated tables). `system.mutations.command` must still hide the credentials of the table +-- functions in the mutation subqueries, the way the query log does. + +SET mutations_sync = 0; +SET lightweight_deletes_sync = 0; +SET allow_nondeterministic_mutations = 1; + +DROP TABLE IF EXISTS t_05291; +DROP TABLE IF EXISTS t_05291_replicated; + +CREATE TABLE t_05291 (id UInt64, v UInt64) ENGINE = MergeTree ORDER BY id; +CREATE TABLE t_05291_replicated (id UInt64, v UInt64) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/t_05291_replicated', 'r1') ORDER BY id; + +INSERT INTO t_05291 SELECT number, number FROM numbers(10); +INSERT INTO t_05291_replicated SELECT number, number FROM numbers(10); + +-- Keep the mutations pending, so that they stay in `system.mutations`. +SYSTEM STOP MERGES t_05291; +SYSTEM STOP MERGES t_05291_replicated; + +ALTER TABLE t_05291 DELETE WHERE id IN + (SELECT c1::UInt64 FROM s3('http://127.0.0.1:1/nope.csv', 'AKIAFAKEKEYID', 'SECRET05291DELETE', 'CSV', 'c1 String')); +ALTER TABLE t_05291 UPDATE v = 0 WHERE id IN + (SELECT c1::UInt64 FROM s3('http://127.0.0.1:1/nope.csv', 'AKIAFAKEKEYID', 'SECRET05291UPDATE', 'CSV', 'c1 String')); +DELETE FROM t_05291 WHERE id IN + (SELECT c1::UInt64 FROM s3('http://127.0.0.1:1/nope.csv', 'AKIAFAKEKEYID', 'SECRET05291LIGHTWEIGHT', 'CSV', 'c1 String')); +ALTER TABLE t_05291_replicated DELETE WHERE id IN + (SELECT c1::UInt64 FROM s3('http://127.0.0.1:1/nope.csv', 'AKIAFAKEKEYID', 'SECRET05291REPLICATED', 'CSV', 'c1 String')); + +-- The replica loads the mutations from ZooKeeper asynchronously; `PULL` fetches them synchronously. +SYSTEM SYNC REPLICA t_05291_replicated PULL; + +SELECT table, command FROM system.mutations WHERE database = currentDatabase() ORDER BY table, mutation_id; + +-- A secret cannot be probed with a filter on the column either. +SELECT count() FROM system.mutations WHERE database = currentDatabase() AND position(command, 'SECRET05291') > 0; + +-- `KILL MUTATION` parses the command that it reads from `system.mutations` to check the access rights. +KILL MUTATION WHERE database = currentDatabase() AND table = 't_05291' FORMAT Null; + +DROP TABLE t_05291; +DROP TABLE t_05291_replicated; From 89ebc45ccadb3b6d60ecb2a7fe98b3a6b30e066c Mon Sep 17 00:00:00 2001 From: alesapin Date: Thu, 5 Nov 2026 12:12:33 +0100 Subject: [PATCH 101/219] Fix build: `Common/HiddenSecret.h` does not exist in 26.8 The header was added to master in 23f1076dba51 and was not backported, so use the `[HIDDEN]` literal directly, as the rest of 26.8 does. https://github.com/ClickHouse/ClickHouse/pull/128949 Co-Authored-By: Claude Opus 5.5 --- src/Storages/System/StorageSystemMutations.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/Storages/System/StorageSystemMutations.cpp b/src/Storages/System/StorageSystemMutations.cpp index 5b6a75756d609..8e31750dce96d 100644 --- a/src/Storages/System/StorageSystemMutations.cpp +++ b/src/Storages/System/StorageSystemMutations.cpp @@ -11,7 +11,6 @@ #include #include #include -#include #include #include #include @@ -61,7 +60,7 @@ static String hideSecretsInCommand(const ContextPtr & context, const String & co && e.code() != ErrorCodes::TOO_DEEP_RECURSION && e.code() != ErrorCodes::TOO_SLOW_PARSING) throw; - return String(HIDDEN_SECRET); + return "[HIDDEN]"; } return ast->formatWithPossiblyHidingSensitiveData( From 793e6080791158fddaf8a7fa1ce4b4ce5ec4df9d Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 5 Nov 2026 12:49:31 +0000 Subject: [PATCH 102/219] Backport #126984 to 26.8: Reject malformed URLs in the Azure `(url, sas_token)` form with `BAD_ARGUMENTS` --- src/Storages/ObjectStorage/Azure/Configuration.cpp | 11 +++++------ .../0_stateless/05427_azure_sas_url_parsing.reference | 0 .../0_stateless/05427_azure_sas_url_parsing.sql | 9 +++++++++ 3 files changed, 14 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/05427_azure_sas_url_parsing.reference create mode 100644 tests/queries/0_stateless/05427_azure_sas_url_parsing.sql diff --git a/src/Storages/ObjectStorage/Azure/Configuration.cpp b/src/Storages/ObjectStorage/Azure/Configuration.cpp index ff087488fa893..ac661969cb7af 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.cpp +++ b/src/Storages/ObjectStorage/Azure/Configuration.cpp @@ -44,7 +44,6 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int LOGICAL_ERROR; } const std::unordered_set required_configuration_keys = { @@ -166,9 +165,9 @@ void AzureStorageParsedArguments::fillBlobsFromURLCommon(String & connection_url if (pos_container != std::string::npos) { - String container_blob_path = connection_url.substr(pos_container+5); - connection_url = connection_url.substr(0,pos_container+4); - container_name = connection_url.substr(pos_container+4); + size_t host_end = pos_container + suffix.size(); + String container_blob_path = host_end < connection_url.size() ? connection_url.substr(host_end + 1) : ""; + connection_url.resize(host_end); auto pos_blob_path = container_blob_path.find('/'); if (pos_blob_path != std::string::npos) @@ -188,9 +187,9 @@ void AzureStorageParsedArguments::fillBlobsFromURLCommon(String & connection_url auto pos_net = connection_url.find(suffix); if (pos_slash == std::string::npos || pos_at == std::string::npos|| pos_dot == std::string::npos || pos_net == std::string::npos - || pos_at-pos_slash-3 <= 0 || pos_dot-pos_at-1 <= 0) + || pos_at <= pos_slash + 3 || pos_dot <= pos_at + 1) { - throw Exception(ErrorCodes::LOGICAL_ERROR, "Incorrect url format for a abfss url {}", connection_url); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Incorrect url format for a abfss url {}", connection_url); } auto container_name_abfss = connection_url.substr(pos_slash+3, pos_at-pos_slash-3); auto name = connection_url.substr(pos_at+1, pos_dot-pos_at-1); diff --git a/tests/queries/0_stateless/05427_azure_sas_url_parsing.reference b/tests/queries/0_stateless/05427_azure_sas_url_parsing.reference new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/queries/0_stateless/05427_azure_sas_url_parsing.sql b/tests/queries/0_stateless/05427_azure_sas_url_parsing.sql new file mode 100644 index 0000000000000..329dfc62bcdae --- /dev/null +++ b/tests/queries/0_stateless/05427_azure_sas_url_parsing.sql @@ -0,0 +1,9 @@ +-- Tags: no-fasttest +-- no-fasttest: needs Azure Blob Storage support, which is not built in the fast test. + +-- The (url, sas_token) form of the Azure table functions parses a URL that ends right after the host like the same URL +-- with a trailing slash (both then fail at format detection), and rejects a malformed abfss URL with BAD_ARGUMENTS. +SELECT * FROM azureBlobStorage('https://acc.blob.core.windows.net', 'sv=2022'); -- { serverError CANNOT_DETECT_FORMAT } +SELECT * FROM azureBlobStorage('abfss://foo.net', 'sv=2022'); -- { serverError BAD_ARGUMENTS } +SELECT * FROM azureBlobStorage('abfss://foo.net/c/x.csv', 'sv=2022'); -- { serverError BAD_ARGUMENTS } +SELECT * FROM azureBlobStorage('abfss://c.x@acc.dfs.core.windows.net/p/x.csv', 'sv=2022'); -- { serverError BAD_ARGUMENTS } From 621969c1a6655506d2da7ce371852a195c839810 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 5 Nov 2026 13:17:01 +0000 Subject: [PATCH 103/219] Backport #121470 to 26.8: Fix the chronically flaky tests of the last month --- .../integration/dotnet_client/Dockerfile | 11 ++++- tests/integration/conftest.py | 30 ++++++++++++ .../helpers/keeper_snapshot_utils.py | 33 +++++++++++-- tests/integration/test_crash_log/test.py | 5 ++ tests/integration/test_dotnet_client/test.py | 9 ++-- tests/integration/test_git_import/test.py | 29 +++++++++++- .../test_storage_nats/test_nats_core.py | 38 +++++++++++++-- ...01514_distributed_cancel_query_on_error.sh | 47 ++++++++++++++----- .../01721_join_implicit_cast_long.sql.j2 | 6 ++- ...0_commit_order_projection_merge_levels.sql | 15 +++++- 10 files changed, 197 insertions(+), 26 deletions(-) diff --git a/ci/docker/integration/dotnet_client/Dockerfile b/ci/docker/integration/dotnet_client/Dockerfile index 3fe1ba7dafc50..82bf1d6867186 100644 --- a/ci/docker/integration/dotnet_client/Dockerfile +++ b/ci/docker/integration/dotnet_client/Dockerfile @@ -1,5 +1,5 @@ # docker build . -# docker run -it --rm --network=host 14f23e59669c dotnet run --host localhost --port 8123 --user default --database default +# docker run -it --rm --network=host 14f23e59669c dotnet /client/out/clickhouse.test.dll --host localhost --port 8123 --user default --database default # Rebuild to enable SBOM/provenance attestations (see #97903, #98511) FROM mcr.microsoft.com/dotnet/sdk:8.0 @@ -9,3 +9,12 @@ COPY *.cs *.csproj /client/ ARG VERSION=4.1.0 RUN dotnet add package ClickHouse.Client -v ${VERSION} + +# Build here, not in the test. `dotnet run` restores, builds and runs, which starts an MSBuild node +# and a Roslyn compiler server alongside the application: three .NET processes where the test needs +# one, and the two extra ones are not what this test covers. They are already a known source of +# noise in this suite - the analyzer crash that +# https://github.com/ClickHouse/ClickHouse/pull/107375 had to work around came from the in-test +# build. Publishing here leaves the test with a single `dotnet ` process. +ENV DOTNET_CLI_TELEMETRY_OPTOUT=1 DOTNET_NOLOGO=1 MSBUILDDISABLENODEREUSE=1 +RUN dotnet publish -c Release -o /client/out && rm -rf /client/obj /client/bin diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 52d86d70fa6e2..b5c6beca7e631 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -68,8 +68,38 @@ def tune_local_port_range(): ) +def is_xdist_replacement_worker(): + """True when this worker was spawned to replace one that died mid-run. + + `pytest-xdist` numbers a replacement past the size of the original pool, so `gw8` in a pool of + eight (`gw0`..`gw7`) is a replacement. A replacement starts a fresh pytest *session*, which + re-runs every session-scoped fixture - and `cleanup_environment` kills every container on the + host, including the ones the workers that are still running own. One worker dying, from an OOM + kill for instance, would otherwise take the containers of the whole job with it. + """ + worker = os.environ.get("PYTEST_XDIST_WORKER", "") + count = os.environ.get("PYTEST_XDIST_WORKER_COUNT", "") + if not worker.startswith("gw") or not count.isdigit(): + return False + + index = worker.removeprefix("gw") + if not index.isdigit(): + return False + + return int(index) >= int(count) + + @pytest.fixture(autouse=True, scope="session") def cleanup_environment(): + if is_xdist_replacement_worker(): + logging.info( + "Worker %s replaces a worker that died, so the containers of the workers which are " + "still running are not ours to clean up. Skipping the environment cleanup.", + os.environ["PYTEST_XDIST_WORKER"], + ) + yield + return + try: result = run_and_check(["docker ps | wc -l"], shell=True) if int(result) > 1: diff --git a/tests/integration/helpers/keeper_snapshot_utils.py b/tests/integration/helpers/keeper_snapshot_utils.py index 30610a82d9568..2231a44823308 100644 --- a/tests/integration/helpers/keeper_snapshot_utils.py +++ b/tests/integration/helpers/keeper_snapshot_utils.py @@ -108,11 +108,38 @@ def stop_zk(zk): pass +# A Keeper these tests have partitioned with `iptables ... DROP` can lose a response, and kazoo's +# own pings keep the socket readable, so `KazooClient.get` and `exists` wait on an `AsyncResult` +# that never completes. Unbounded, one lost response burns the whole 900s `pytest-timeout` budget +# and takes the rest of the module with it, reported as a bare `Timeout (>900.0s) from +# pytest-timeout` that names neither the request nor the node. Bound every wait instead. +ZK_REQUEST_TIMEOUT_SEC = 60 + + +def zk_get(zk, path, timeout=ZK_REQUEST_TIMEOUT_SEC): + """`zk.get(path)`, giving up after `timeout` instead of waiting forever.""" + return zk.retry(lambda: zk.get_async(path).get(timeout=timeout)) + + +def zk_exists(zk, path, timeout=ZK_REQUEST_TIMEOUT_SEC): + """`zk.exists(path)`, giving up after `timeout` instead of waiting forever.""" + return zk.retry(lambda: zk.exists_async(path).get(timeout=timeout)) + + def fill_test_tree(zk, base, count=300): import os as _os + from kazoo.exceptions import NodeExistsError zk.ensure_path(base) for i in range(count): - zk.create(f"{base}/{i}", _os.urandom(1024)) # random to resist ZSTD compression + data = _os.urandom(1024) # random to resist ZSTD compression + try: + zk.create(f"{base}/{i}", data) + except NodeExistsError: + # `KazooClientWithImplicitRetries.create` retries, and `create` is not idempotent: a + # create whose response was lost has already taken effect, so the retry fails on a node + # it wrote itself. Write the data the caller asked for and carry on - the postcondition + # is that the node exists with that data, however many attempts it took. + zk.set(f"{base}/{i}", data) for i in range(0, count, 10): zk.delete(f"{base}/{i}") @@ -135,9 +162,9 @@ def verify_test_tree(leader_zk, lagging_zk, base, count=300): lagging_zk.sync(base) for i in range(count): if i % 10 != 0: - assert lagging_zk.get(f"{base}/{i}")[0] == leader_zk.get(f"{base}/{i}")[0] + assert zk_get(lagging_zk, f"{base}/{i}")[0] == zk_get(leader_zk, f"{base}/{i}")[0] else: - assert lagging_zk.exists(f"{base}/{i}") is None + assert zk_exists(lagging_zk, f"{base}/{i}") is None def get_kill_timestamp(node): diff --git a/tests/integration/test_crash_log/test.py b/tests/integration/test_crash_log/test.py index 8d88e7afb79fe..fbe94ccc39932 100644 --- a/tests/integration/test_crash_log/test.py +++ b/tests/integration/test_crash_log/test.py @@ -69,9 +69,14 @@ def test_crash_log_synchronous(started_node): wait_for_clickhouse_stop(started_node) started_node.restart_clickhouse() crashes_count += 1 + # Report the rows that are there. Without them a lost row leaves only `'1' == '2'` behind, + # which says nothing about which of the two crashes failed to record itself. assert ( started_node.query("SELECT COUNT(*) FROM system.crash_log") == f"{crashes_count}\n" + ), started_node.query( + "SELECT event_time_microseconds, signal, signal_description, query " + "FROM system.crash_log ORDER BY event_time_microseconds FORMAT Vertical" ) diff --git a/tests/integration/test_dotnet_client/test.py b/tests/integration/test_dotnet_client/test.py index 4d23f3296f047..149e57eefa5c3 100644 --- a/tests/integration/test_dotnet_client/test.py +++ b/tests/integration/test_dotnet_client/test.py @@ -49,11 +49,14 @@ def test_dotnet_client(started_cluster, dotnet_container): reference = fp.read() code, (stdout, stderr) = dotnet_container.exec_run( - "dotnet run --host {host} --port {port} --user default --password 123 --database default".format( + "dotnet /client/out/clickhouse.test.dll --host {host} --port {port} --user default --password 123 --database default".format( host=started_cluster.get_instance_ip("node"), port=8123 ), demux=True, ) - assert code == 0 - assert stdout == reference + # These two streams exist nowhere else: the client runs in its own container and nothing in the + # job collects its output. Reporting them here is what makes a client that died on a signal + # diagnosable at all, instead of leaving a bare exit code behind. + assert code == 0, f"stdout:\n{stdout!r}\nstderr:\n{stderr!r}" + assert stdout == reference, f"stderr:\n{stderr!r}" diff --git a/tests/integration/test_git_import/test.py b/tests/integration/test_git_import/test.py index 51558ea4ee377..dc7add2acfc8c 100644 --- a/tests/integration/test_git_import/test.py +++ b/tests/integration/test_git_import/test.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import logging +import re import time import pytest @@ -40,8 +41,24 @@ def clone_git_repository(repo, dir, commit=None): time.sleep(1) +# `ClickHouseCluster` gives the whole container the running server's profraw merge pool +# (`LLVM_PROFILE_FILE=/debug/it-%c%4m.profraw`), which every `docker exec`-ed process inherits. +# A second writer cannot merge into a continuous-mode profile - it records its own writer's counter +# bias - so in a coverage build the profiling runtime writes +# `LLVM Profile Warning: Unable to merge profile data: source profile file is not compatible.` and +# two `File exists` errors to stderr. Give this one-shot process a file of its own: its coverage is +# still collected and it no longer touches the server's pool. +GIT_IMPORT_PROFILE_FILE = "/debug/git-import-%p.profraw" + +# One progress line per processed commit, which `git-import` writes to stderr. +COMMIT_PROGRESS_RE = re.compile(r"^\d+% \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [0-9a-f]{40} ") + + def run_git_import(dir): - command = f"cd {dir} && /usr/bin/clickhouse git-import 2>&1" + command = ( + f"cd {dir} && LLVM_PROFILE_FILE={GIT_IMPORT_PROFILE_FILE} " + "/usr/bin/clickhouse git-import 2>&1" + ) return node.exec_in_container(["bash", "-c", command]) @@ -176,7 +193,15 @@ def test_git_import(): create_tables() insert_into_tables(dir) - assert output.count("\n") == 26 + # `git-import` prints the `git log` command it runs and the commit count to stdout, and one + # progress line per commit to stderr, which the `2>&1` above merges into the same stream. Match + # those shapes instead of counting every newline, so that a line written by anything other than + # `git-import` - a profiling or sanitizer runtime, a loader warning - fails the thing it + # actually broke rather than this count. + lines = output.splitlines() + assert "git log --reverse --no-merges --pretty=%H" in lines, output + assert "Total 24 commits to process." in lines, output + assert sum(1 for line in lines if COMMIT_PROGRESS_RE.match(line)) == 24, output assert node.query("SELECT count() FROM commits") == "24\n" assert node.query("SELECT count() FROM file_changes") == "35\n" assert node.query("SELECT count(), round(avg(indent), 1) FROM line_changes") == TSV( diff --git a/tests/integration/test_storage_nats/test_nats_core.py b/tests/integration/test_storage_nats/test_nats_core.py index 5edf9dd292854..905c12cf4103a 100644 --- a/tests/integration/test_storage_nats/test_nats_core.py +++ b/tests/integration/test_storage_nats/test_nats_core.py @@ -1030,10 +1030,42 @@ def test_nats_restore_failed_connection_without_losses_on_write(nats_cluster): time.sleep(4) nats_helpers.revive_nats(nats_cluster) - result = instance.query_with_retry("SELECT count(DISTINCT key) FROM test.view", check_callback = lambda num_rows: int(num_rows) == messages_num) + # Core NATS is at-most-once: the broker keeps no message it has not already handed to a + # subscriber, so whatever was still in flight when it was stopped is gone for good and the + # count of the first batch cannot be asserted. The no-loss guarantee belongs to JetStream and + # is asserted by the test of the same name in `test_nats_jet_stream.py`. What this test is + # named after is the connection, so what is asserted here is that publishing works again. + # + # A core subscription comes back only when the client reconnects and re-sends `SUB` - + # `NATSCoreConsumer` does not override `needsResubscribe`, so nothing on the ClickHouse side + # resubscribes it and nothing is logged when it happens - and a message published before that + # lands on a subject with no subscriber and is dropped. Waiting longer after publishing cannot + # recover it, so the same batch is published until it arrives: its keys are fixed, so + # `DISTINCT` collapses the duplicates. + probe_num = 1000 + probe_values = ",".join( + "({i}, {i})".format(i=i) for i in range(messages_num, messages_num + probe_num) + ) + probe_count = "SELECT count(DISTINCT key) FROM test.view WHERE key >= {}".format(messages_num) + + received = 0 + deadline = time.monotonic() + 120 + while received != probe_num and time.monotonic() < deadline: + instance.query_with_retry( + "INSERT INTO test.producer_reconnect VALUES {}".format(probe_values) + ) + received = int( + instance.query_with_retry( + probe_count, + retry_count=10, + sleep_time=0.5, + check_callback=lambda num_rows: int(num_rows) == probe_num, + ) + ) - assert int(result) == messages_num, "ClickHouse lost some messages: {}".format( - result + assert received == probe_num, ( + "ClickHouse did not restore the connection: {} of the {} messages republished after the " + "broker came back arrived within 120s".format(received, probe_num) ) diff --git a/tests/queries/0_stateless/01514_distributed_cancel_query_on_error.sh b/tests/queries/0_stateless/01514_distributed_cancel_query_on_error.sh index 8e3659d2d07cd..960f2f03174fe 100755 --- a/tests/queries/0_stateless/01514_distributed_cancel_query_on_error.sh +++ b/tests/queries/0_stateless/01514_distributed_cancel_query_on_error.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash # Tags: distributed, no-llvm-coverage -# no-llvm-coverage: the test needs the shard allocating `groupArray(repeat('a', 100000))` -# per row to hit `max_memory_usage=1G` and emit `Query memory limit exceeded` before the -# other shard cancels. LLVM source-based coverage instrumentation perturbs this -# memory-tracking cadence, so the expected exception sometimes does not surface on `127.3` -# before cancellation, leaving an empty `grep` match and a flaky FAIL. +# no-llvm-coverage: the test needs the shard accumulating `repeat('a', 100000)` per row to hit +# `max_memory_usage=1G` and emit `Query memory limit exceeded` before the other shard cancels. +# LLVM source-based coverage instrumentation perturbs this memory-tracking cadence, so the +# expected exception sometimes does not surface on `127.3` before cancellation, leaving an empty +# `grep` match and a flaky FAIL. CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -14,7 +14,11 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # 1 on 127.2 # 2 on 127.3 # max_block_size to fail faster -# max_memory_usage/_shard_num/repeat() will allow failure on the first shard earlier. +# `groupArrayIf` on `_shard_num` makes `127.3` the only shard that can run out of memory: on +# `127.2` the condition is false for every row, so its aggregate states never grow and it just +# streams `system.numbers` until the initiator cancels it. Before this, `127.2` accumulated +# one-byte strings and reached the same `max_memory_usage` only about 20% later than `127.3`, so +# which shard failed first was a race. opts=( "--max_memory_usage=1G" "--max_block_size=50" @@ -22,9 +26,30 @@ opts=( "--max_distributed_connections=2" "--max_bytes_before_external_group_by=0" "--max_bytes_ratio_before_external_group_by=0" + # The query reads `system.numbers` until a shard runs out of memory, so `max_memory_usage` + # has to be the only limit that can stop it. The test profile of the CI configuration + # (tests/config/users.d/limits.yaml) sets `max_rows_to_read = 20000000`, which `127.2` and + # the initiator reach at about the same time as `127.3` reaches `max_memory_usage`; the query + # then fails with `TOO_MANY_ROWS` instead of `MEMORY_LIMIT_EXCEEDED` and the `grep` below + # finds nothing. + "--max_rows_to_read=0" + # `127.2` now never fails on its own, so if the cancellation this test is about ever breaks, + # the query would stream forever and the test would be killed by the runner's timeout with no + # output. Fail it with `TIMEOUT_EXCEEDED` instead, far above the ~1s the expected exception + # needs. + "--max_execution_time=100" ) -${CLICKHOUSE_CLIENT} "${opts[@]}" -q "SELECT groupArray(repeat('a', if(_shard_num == 2, 100000, 1))), number%100000 k from remote('127.{2,3}', system.numbers) GROUP BY k LIMIT 10e6" |& { - # the query should fail earlier on 127.3 and 127.2 should not even go to the memory limit exceeded error. - grep -F -q "DB::Exception: Received from 127.3:${CLICKHOUSE_PORT_TCP}. DB::Exception: Query memory limit exceeded:" - # while if this will not correctly then it will got the exception from the 127.2:9000 and fail -} +LOG="$CLICKHOUSE_TMP/err-$CLICKHOUSE_DATABASE" +trap 'rm -f "$LOG"' EXIT + +${CLICKHOUSE_CLIENT} "${opts[@]}" -q "SELECT groupArrayIf(repeat('a', 100000), _shard_num == 2), number%100000 k from remote('127.{2,3}', system.numbers) GROUP BY k LIMIT 10e6" > "$LOG" 2>&1 +CODE=$? + +# the query should fail earlier on 127.3 and 127.2 should not even go to the memory limit exceeded error. +# while if this will not correctly then it will got the exception from the 127.2:9000 and fail +if ! grep -F -q "DB::Exception: Received from 127.3:${CLICKHOUSE_PORT_TCP}. DB::Exception: Query memory limit exceeded:" "$LOG"; then + # stderr only on the failing path: a test that writes there and exits 0 is a FAIL for the runner. + echo "Fail, Code: $CODE, client output:" >&2 + cat "$LOG" >&2 + exit 1 +fi diff --git a/tests/queries/0_stateless/01721_join_implicit_cast_long.sql.j2 b/tests/queries/0_stateless/01721_join_implicit_cast_long.sql.j2 index 8d73d1ce9e651..ec0e7ced42581 100644 --- a/tests/queries/0_stateless/01721_join_implicit_cast_long.sql.j2 +++ b/tests/queries/0_stateless/01721_join_implicit_cast_long.sql.j2 @@ -1,4 +1,8 @@ --- Tags: long +-- Tags: long, no-msan +-- no-msan: 5 join algorithms x ~78 statements, several of them deliberately spilling, average +-- ~183s under MSan against ~4s in a release build; the p99 is 73% of the 600s per-test budget, so +-- a run that gets killed on the timeout is an ordinary tail event. Join implicit-cast semantics +-- get no MSan-specific coverage, and the test still runs under ASan, UBSan and TSan. SET enable_analyzer = 1; DROP TABLE IF EXISTS t1; diff --git a/tests/queries/0_stateless/04040_commit_order_projection_merge_levels.sql b/tests/queries/0_stateless/04040_commit_order_projection_merge_levels.sql index 594092c7f3e61..77021b6f561a9 100644 --- a/tests/queries/0_stateless/04040_commit_order_projection_merge_levels.sql +++ b/tests/queries/0_stateless/04040_commit_order_projection_merge_levels.sql @@ -2,6 +2,11 @@ set enable_analyzer = 1; +-- Bound the wait, so that a scheduled merge that never runs fails this test with +-- `TIMEOUT_EXCEEDED` instead of polling until the harness kills it: `SYSTEM SYNC MERGES` treats +-- `max_execution_time = 0`, the default of a stateless test, as "wait forever". +set max_execution_time = 120; + drop table if exists mt_merge_levels sync; CREATE TABLE mt_merge_levels( @@ -29,6 +34,14 @@ SYSTEM SYNC MERGES mt_merge_levels; SYSTEM SCHEDULE MERGE mt_merge_levels PARTS 'all_1_2_1', 'all_3_4_1'; SYSTEM SYNC MERGES mt_merge_levels; +-- `SYSTEM SYNC MERGES` returns as soon as the merged part is among the active parts, which +-- `MergePlainMergeTreeTask::finish` makes true in `transaction.commit()` - before it pushes the +-- `MergeParts` row of that merge with `write_part_log`. Flushing `part_log` right after +-- `SYSTEM SYNC MERGES` can therefore miss the row of the last merge. Dropping the table first +-- closes that window: the drop waits for the merge task to run to completion, and `part_log` +-- keeps the database and table names as strings, so the queries below still find the rows. +drop table mt_merge_levels sync; + -- Check part_log ProfileEvents to verify rebuild vs merge behavior system flush logs part_log; @@ -43,5 +56,3 @@ select part_name, ProfileEvents['RebuiltProjections'] as rebuilt, ProfileEvents[ from system.part_log where database = currentDatabase() and table = 'mt_merge_levels' and event_type = 'MergeParts' and part_name = 'all_1_4_2' order by part_name; - -drop table mt_merge_levels sync; From 40212c0a4af0dee8663ffb13d6316e1fab4080fc Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 5 Nov 2026 14:36:29 +0000 Subject: [PATCH 104/219] Backport #129291 to 26.8: Fix Keeper heap corruption from bad log pack --- src/Coordination/Changelog.cpp | 2 + .../tests/gtest_coordination_changelog.cpp | 40 +++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index a5d204ea5bce5..88e2e7ebc3b89 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -4580,6 +4580,8 @@ void Changelog::applyEntriesFromBuffer(uint64_t index, nuraft::buffer & buffer) { uint64_t cur_index = index + i; int buf_size = buffer.get_int(); + if (buf_size < 0 || static_cast(buf_size) > buffer.size() - buffer.pos()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "Log entry {} in a log pack has invalid size {}", cur_index, buf_size); nuraft::ptr buf_local = nuraft::buffer::alloc(buf_size); buffer.get(buf_local); diff --git a/src/Coordination/tests/gtest_coordination_changelog.cpp b/src/Coordination/tests/gtest_coordination_changelog.cpp index ef2a02920b17f..cf601aebd88df 100644 --- a/src/Coordination/tests/gtest_coordination_changelog.cpp +++ b/src/Coordination/tests/gtest_coordination_changelog.cpp @@ -772,6 +772,46 @@ TEST_P(CoordinationTestWithCompression, ChangelogTestBatchOperationsEmpty) changelog_reader.init(5, 0); } +TEST_P(CoordinationTestWithCompression, ChangelogTestBatchOperationsInvalidSize) +{ + ChangelogDirTest test("./logs"); + this->setLogDirectory("./logs"); + + DB::KeeperLogStore changelog( + DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 100}, + DB::FlushSettings(), + DB::ReadAheadSettings{}, + this->keeper_context); + changelog.init(0, 0); + auto entry = getLogEntry("hello_world", 10); + changelog.append(entry); + changelog.end_of_append_batch(0, 0); + waitDurableLogs(changelog); + + /// A pack of one entry holds the entry count, the entry size and the entry itself. A peer controls the size, + /// so only the size is rewritten here. + auto entries = changelog.pack(1, 1); + const auto entry_size = static_cast(entries->size() - 2 * sizeof(int32_t)); + auto apply_with_size = [&](int32_t size) + { + entries->pos(sizeof(int32_t)); + entries->put(size); + changelog.apply_pack(2, *entries); + }; + + /// On master a size past the entry over-reads the pack in `nuraft::buffer::get` and a negative size + /// wraps the allocation size in `nuraft::buffer::alloc`. + EXPECT_THROW(apply_with_size(entry_size + 1), DB::Exception); + EXPECT_THROW(apply_with_size(-1), DB::Exception); + EXPECT_EQ(changelog.next_slot(), 2); + + apply_with_size(entry_size); + changelog.apply_pack(3, *changelog.pack(1, 0)); + changelog.end_of_append_batch(0, 0); + EXPECT_EQ(changelog.next_slot(), 3); + EXPECT_EQ(changelog.entry_at(2)->get_term(), 10); +} + TEST_P(CoordinationTestWithCompression, ChangelogTestWriteAtPreviousFile) { From 039117eb7e9625f862d43081b1c7faa71c7eb9ef Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 4 Nov 2026 15:11:04 +0000 Subject: [PATCH 105/219] Backport #128955 to 26.8: Hide secrets in `system.s3_queue_settings` and `system.azure_queue_settings` --- .../ObjectStorageQueueSettings.cpp | 17 ++++++++++++-- .../ObjectStorageQueueSettings.h | 3 ++- ...torageSystemObjectStorageQueueSettings.cpp | 4 +++- ...ttings_system_table_hide_secrets.reference | 4 ++++ ...eue_settings_system_table_hide_secrets.sql | 22 +++++++++++++++++++ 5 files changed, 46 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/05493_s3queue_settings_system_table_hide_secrets.reference create mode 100644 tests/queries/0_stateless/05493_s3queue_settings_system_table_hide_secrets.sql diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp index ff1148a959da0..4bc066f9acc0b 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp @@ -5,10 +5,12 @@ #include #include #include +#include #include #include #include #include +#include #include namespace DB @@ -92,7 +94,8 @@ void ObjectStorageQueueSettings::dumpToSystemEngineSettingsColumns( MutableColumnsAndConstraints & params, const std::string & table_name, const std::string & database_name, - const StorageObjectStorageQueue & storage) const + const StorageObjectStorageQueue & storage, + bool display_secrets) const { MutableColumns & res_columns = params.res_columns; auto metadata_snapshot = storage.getInMemoryMetadataPtr(CurrentThread::tryGetQueryContext(), false); @@ -113,6 +116,16 @@ void ObjectStorageQueueSettings::dumpToSystemEngineSettingsColumns( { return StorageObjectStorageQueue::isSettingChangeable(setting_name, (*this)[ObjectStorageQueueSetting::mode]); }; + /// Hides what `SHOW CREATE TABLE` hides. A secret is hidden whole, as its masker renders SQL. + auto render_value = [&](const std::string & setting_name, const Field & value) -> String + { + if (!display_secrets) + { + if (auto masked = getSecretArgumentsFinder().renderSecretSetting(setting_name, value); masked && masked->contains(HIDDEN_SECRET)) + return String(HIDDEN_SECRET); + } + return convertFieldToString(value); + }; for (const auto & change : impl->all()) { @@ -121,7 +134,7 @@ void ObjectStorageQueueSettings::dumpToSystemEngineSettingsColumns( res_columns[i++]->insert(database_name); res_columns[i++]->insert(table_name); res_columns[i++]->insert(name); - res_columns[i++]->insert(convertFieldToString(change.getValue())); + res_columns[i++]->insert(render_value(name, change.getValue())); res_columns[i++]->insert(change.getTypeName()); res_columns[i++]->insert(is_changed(name)); res_columns[i++]->insert(change.getDescription()); diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.h index 4455fadff3bf2..6dbd51326fbb6 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.h @@ -63,7 +63,8 @@ struct ObjectStorageQueueSettings MutableColumnsAndConstraints & params, const std::string & table_name, const std::string & database_name, - const StorageObjectStorageQueue & storage) const; + const StorageObjectStorageQueue & storage, + bool display_secrets) const; void loadFromQuery(ASTStorage & storage_def, bool is_attach, const StorageID & storage_id); diff --git a/src/Storages/System/StorageSystemObjectStorageQueueSettings.cpp b/src/Storages/System/StorageSystemObjectStorageQueueSettings.cpp index ec3f099ba390c..d0208fe4c1300 100644 --- a/src/Storages/System/StorageSystemObjectStorageQueueSettings.cpp +++ b/src/Storages/System/StorageSystemObjectStorageQueueSettings.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -45,6 +46,7 @@ void StorageSystemObjectStorageQueueSettings::fillData( const ActionsDAG::Node *, std::vector) const { + const bool display_secrets = canDisplaySecrets(context); auto add_table = [&]( const DatabaseTablesIteratorPtr & it, StorageObjectStorageQueue & storage) { @@ -54,7 +56,7 @@ void StorageSystemObjectStorageQueueSettings::fillData( auto constraints_and_current_profiles = context->getSettingsConstraintsAndCurrentProfiles(); const auto & constraints = constraints_and_current_profiles->constraints; MutableColumnsAndConstraints params(res_columns, constraints); - storage.getSettings().dumpToSystemEngineSettingsColumns(params, it->name(), it->databaseName(), storage); + storage.getSettings().dumpToSystemEngineSettingsColumns(params, it->name(), it->databaseName(), storage, display_secrets); }; const auto access = context->getAccess(); diff --git a/tests/queries/0_stateless/05493_s3queue_settings_system_table_hide_secrets.reference b/tests/queries/0_stateless/05493_s3queue_settings_system_table_hide_secrets.reference new file mode 100644 index 0000000000000..6ccddfe23c735 --- /dev/null +++ b/tests/queries/0_stateless/05493_s3queue_settings_system_table_hide_secrets.reference @@ -0,0 +1,4 @@ +after_processing_move_access_key_id key_id_05493 +after_processing_move_connection_string [HIDDEN] +after_processing_move_secret_access_key [HIDDEN] +mode unordered diff --git a/tests/queries/0_stateless/05493_s3queue_settings_system_table_hide_secrets.sql b/tests/queries/0_stateless/05493_s3queue_settings_system_table_hide_secrets.sql new file mode 100644 index 0000000000000..63571197469ef --- /dev/null +++ b/tests/queries/0_stateless/05493_s3queue_settings_system_table_hide_secrets.sql @@ -0,0 +1,22 @@ +-- Tags: no-fasttest, zookeeper +-- no-fasttest: the `S3Queue` engine is not built in the fast test. + +-- `system.s3_queue_settings` hides the secrets that `SHOW CREATE TABLE` hides. + +DROP TABLE IF EXISTS t_secrets; + +CREATE TABLE t_secrets (a String) +ENGINE = S3Queue('http://whatever-we-dont-care:9001/root/t_secrets/*', NOSIGN, 'CSV') +SETTINGS mode = 'unordered', + after_processing_move_access_key_id = 'key_id_05493', + after_processing_move_secret_access_key = 'SEKRIT_05493_1', + after_processing_move_connection_string = 'DefaultEndpointsProtocol=https;AccountName=a;AccountKey=SEKRIT_05493_2;'; + +SELECT name, value +FROM system.s3_queue_settings +WHERE database = currentDatabase() AND table = 't_secrets' + AND name IN ('after_processing_move_access_key_id', 'after_processing_move_secret_access_key', + 'after_processing_move_connection_string', 'mode') +ORDER BY name; + +DROP TABLE t_secrets; From 8c83cd8ac09e2668706e11a1e3f02f785ea1ad8c Mon Sep 17 00:00:00 2001 From: Antonio Andelic Date: Thu, 5 Nov 2026 14:57:36 +0100 Subject: [PATCH 106/219] Fix build of the backport to 26.8: use the queue engines' `SETTINGS_TO_HIDE` `Parsers/SecretArguments.h` and `Common/HiddenSecret.h` do not exist on 26.8: they came with later refactors on master. On 26.8 the secret engine settings are declared in the per-engine `SETTINGS_TO_HIDE` maps, which `ASTSetQuery` also uses for `SHOW CREATE TABLE`. Look up the setting in the maps of `S3Queue` and `AzureQueue` instead of `getSecretArgumentsFinder`. CI report: https://s3.amazonaws.com/clickhouse-test-reports/json.html?PR=129304&sha=f92b82ef385e8fefb391cba5c3c942616c83545c&name_0=BackportPR&name_1=Build%20%28amd_debug%29 PR: https://github.com/ClickHouse/ClickHouse/pull/129304 Co-Authored-By: Claude Opus 5.5 --- .../ObjectStorageQueueSettings.cpp | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp index 4bc066f9acc0b..c54b74cb82c01 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp @@ -5,12 +5,12 @@ #include #include #include -#include +#include #include +#include #include #include #include -#include #include namespace DB @@ -119,10 +119,18 @@ void ObjectStorageQueueSettings::dumpToSystemEngineSettingsColumns( /// Hides what `SHOW CREATE TABLE` hides. A secret is hidden whole, as its masker renders SQL. auto render_value = [&](const std::string & setting_name, const Field & value) -> String { + static constexpr std::string_view hidden_secret = "[HIDDEN]"; if (!display_secrets) { - if (auto masked = getSecretArgumentsFinder().renderSecretSetting(setting_name, value); masked && masked->contains(HIDDEN_SECRET)) - return String(HIDDEN_SECRET); + /// Both queue engines take the same settings, so the secrets of both apply. + for (const auto * settings_to_hide : {&S3Queue::SETTINGS_TO_HIDE, &AzureQueue::SETTINGS_TO_HIDE}) + { + auto it = settings_to_hide->find(setting_name); + if (it == settings_to_hide->end()) + continue; + if (auto masked = it->second(value); masked && masked->contains(hidden_secret)) + return String(hidden_secret); + } } return convertFieldToString(value); }; From d9d2f638181a7cedcb640b6d7290c33a91a77ba5 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 5 Nov 2026 16:37:12 +0000 Subject: [PATCH 107/219] Backport #129201 to 26.8: Fix double counting in the network charts on the dashboard.html --- src/Storages/System/StorageSystemDashboards.cpp | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/src/Storages/System/StorageSystemDashboards.cpp b/src/Storages/System/StorageSystemDashboards.cpp index c6ae173c713bf..db530c6c31c76 100644 --- a/src/Storages/System/StorageSystemDashboards.cpp +++ b/src/Storages/System/StorageSystemDashboards.cpp @@ -1262,6 +1262,9 @@ GROUP BY t ORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1 )EOQ") } }, + /// With `asynchronous_metrics_key_values_mode = 'both'` every network interface is published twice: + /// as `NetworkReceiveBytes` with a key and as the legacy `NetworkReceiveBytes_eth0`, and summing the + /// LIKE pattern counts every byte twice. Prefer the key-value form, fall back to the legacy names. { { "dashboard", "Cloud overview" }, { "title", "Network receive bytes/sec" }, @@ -1270,7 +1273,7 @@ WITH toDateTimeOrDefault({from:String}, '', now() - {seconds:UInt32}) AS from, toDateTimeOrDefault({to:String}, '', now()) AS to SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(value) FROM ( - SELECT event_time, sum(value) AS value + SELECT event_time, if(countIf(metric = 'NetworkReceiveBytes') != 0, sumIf(value, metric = 'NetworkReceiveBytes'), sum(value)) AS value FROM clusterAllReplicas(default, merge('system', '^asynchronous_metric_log')) WHERE event_date BETWEEN toDate(from) AND toDate(to) AND event_time BETWEEN from AND to AND metric LIKE 'NetworkReceiveBytes%' @@ -1288,7 +1291,7 @@ WITH toDateTimeOrDefault({from:String}, '', now() - {seconds:UInt32}) AS from, toDateTimeOrDefault({to:String}, '', now()) AS to SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(value) FROM ( - SELECT event_time, sum(value) AS value + SELECT event_time, if(countIf(metric = 'NetworkSendBytes') != 0, sumIf(value, metric = 'NetworkSendBytes'), sum(value)) AS value FROM clusterAllReplicas(default, merge('system', '^asynchronous_metric_log')) WHERE event_date BETWEEN toDate(from) AND toDate(to) AND event_time BETWEEN from AND to AND metric LIKE 'NetworkSendBytes%' @@ -2054,7 +2057,7 @@ WITH toDateTimeOrDefault({from:String}, '', now() - {seconds:UInt32}) AS from, toDateTimeOrDefault({to:String}, '', now()) AS to SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, hostname, avg(value) FROM ( - SELECT event_time, hostname, sum(value) AS value + SELECT event_time, hostname, if(countIf(metric = 'NetworkReceiveBytes') != 0, sumIf(value, metric = 'NetworkReceiveBytes'), sum(value)) AS value FROM clusterAllReplicas(default, merge('system', '^asynchronous_metric_log')) WHERE event_date BETWEEN toDate(from) AND toDate(to) AND event_time BETWEEN from AND to AND metric LIKE 'NetworkReceiveBytes%' @@ -2072,7 +2075,7 @@ WITH toDateTimeOrDefault({from:String}, '', now() - {seconds:UInt32}) AS from, toDateTimeOrDefault({to:String}, '', now()) AS to SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, hostname, avg(value) FROM ( - SELECT event_time, hostname, sum(value) AS value + SELECT event_time, hostname, if(countIf(metric = 'NetworkSendBytes') != 0, sumIf(value, metric = 'NetworkSendBytes'), sum(value)) AS value FROM clusterAllReplicas(default, merge('system', '^asynchronous_metric_log')) WHERE event_date BETWEEN toDate(from) AND toDate(to) AND event_time BETWEEN from AND to AND metric LIKE 'NetworkSendBytes%' From 4088b83baf40a92ade4b638d239aad8d103b743e Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 5 Nov 2026 18:40:28 +0000 Subject: [PATCH 108/219] Backport #122029 to 26.8: Visit only matching typed paths when resolving a `JSON` subcolumn --- src/DataTypes/IDataType.cpp | 1 + src/DataTypes/Serializations/ISerialization.h | 3 + .../Serializations/SerializationObject.cpp | 34 ++- .../tests/gtest_DataTypeObject_create.cpp | 129 ++++++++++ .../json_typed_path_subcolumn_lookup.xml | 223 ++++++++++++++++++ 5 files changed, 388 insertions(+), 2 deletions(-) create mode 100644 tests/performance/json_typed_path_subcolumn_lookup.xml diff --git a/src/DataTypes/IDataType.cpp b/src/DataTypes/IDataType.cpp index ccc8e24295bef..c573881c50837 100644 --- a/src/DataTypes/IDataType.cpp +++ b/src/DataTypes/IDataType.cpp @@ -213,6 +213,7 @@ std::unique_ptr IDataType::getSubcolumnData( settings.enumerate_dynamic_streams = false; settings.enumerate_virtual_streams = true; settings.array_level = initial_array_level; + settings.subcolumn_name = subcolumn_name; data.serialization->enumerateStreams(settings, callback_with_data, data); if (!res && data.type->hasDynamicSubcolumnsData()) diff --git a/src/DataTypes/Serializations/ISerialization.h b/src/DataTypes/Serializations/ISerialization.h index b94270507e95b..b6e854b799128 100644 --- a/src/DataTypes/Serializations/ISerialization.h +++ b/src/DataTypes/Serializations/ISerialization.h @@ -19,6 +19,7 @@ #include #include #include +#include #include namespace DB @@ -362,6 +363,8 @@ class ISerialization : private boost::noncopyable, public std::enable_shared_fro struct EnumerateStreamsSettings { SubstreamPath path; + /// When resolving a subcolumn, serializers may omit streams that cannot contribute to its lookup. + std::optional subcolumn_name; bool position_independent_encoding = true; /// If set to false, don't enumerate dynamic subcolumns /// (such as dynamic types in Dynamic column or dynamic paths in JSON column). diff --git a/src/DataTypes/Serializations/SerializationObject.cpp b/src/DataTypes/Serializations/SerializationObject.cpp index 35ad4488a2880..934a510c66221 100644 --- a/src/DataTypes/Serializations/SerializationObject.cpp +++ b/src/DataTypes/Serializations/SerializationObject.cpp @@ -206,8 +206,7 @@ void SerializationObject::enumerateStreams(EnumerateStreamsSettings & settings, settings.path.push_back(Substream::ObjectData); - /// First, iterate over typed paths in sorted order, we will always serialize them. - for (const auto & path : sorted_typed_paths) + auto enumerate_typed_path = [&](const String & path) { settings.path.back().creator = std::make_shared(path); settings.path.push_back(Substream::ObjectTypedPath); @@ -222,6 +221,37 @@ void SerializationObject::enumerateStreams(EnumerateStreamsSettings & settings, serialization->enumerateStreams(settings, callback, path_data); settings.path.pop_back(); settings.path.back().creator.reset(); + }; + + if (settings.subcolumn_name) + { + /// Include the separator using the canonical stream naming, including empty path elements. + settings.path.push_back(Substream::ObjectTypedPath); + const auto prefix = getSubcolumnNameForStream(settings.path); + settings.path.pop_back(); + auto name = *settings.subcolumn_name; + if (name.starts_with(prefix)) + { + name.remove_prefix(prefix.size()); + /// Only an exact typed path or its descendants can match the requested name. + /// Prefixes are already in the same order as `sorted_typed_paths`, preserving collisions. + size_t end = 0; + while (true) + { + end = name.find('.', end); + if (auto it = typed_paths_serializations.find(String(name.substr(0, end))); it != typed_paths_serializations.end()) + enumerate_typed_path(it->first); + if (end == std::string_view::npos) + break; + ++end; + } + } + } + else + { + /// Full enumeration retains every typed path in serialization order. + for (const auto & path : sorted_typed_paths) + enumerate_typed_path(path); } /// If column or deserialization state was provided, iterate over dynamic paths, diff --git a/src/DataTypes/tests/gtest_DataTypeObject_create.cpp b/src/DataTypes/tests/gtest_DataTypeObject_create.cpp index 8563b3d8343d0..eb8548a3c9322 100644 --- a/src/DataTypes/tests/gtest_DataTypeObject_create.cpp +++ b/src/DataTypes/tests/gtest_DataTypeObject_create.cpp @@ -1,4 +1,7 @@ +#include +#include #include +#include #include #include #include @@ -10,6 +13,29 @@ using namespace DB; +namespace +{ + +void expectSubcolumnType(const IDataType & type, std::string_view subcolumn, std::string_view expected_name) +{ + SCOPED_TRACE(String(subcolumn)); + auto result = type.tryGetSubcolumnType(subcolumn); + EXPECT_EQ(type.hasSubcolumn(subcolumn), !expected_name.empty()); + + if (expected_name.empty()) + { + EXPECT_EQ(result, nullptr); + EXPECT_THROW(type.getSubcolumnType(subcolumn), Exception); + return; + } + + ASSERT_NE(result, nullptr); + EXPECT_EQ(result->getName(), expected_name); + EXPECT_TRUE(result->equals(*type.getSubcolumnType(subcolumn))); +} + +} + /// Regression tests for STID 1477-2777 (BuzzHouse serverfuzz, amd_msan, 2026-04-25): /// `DataTypeObject::createObject` accessed `object_type_argument->parameter`, /// `path_with_type`, and `function->arguments->children[0]` / `[1]` without @@ -195,3 +221,106 @@ TEST(DataTypeObject, CreateJSONWithValidAST) auto type_default = factory.get("JSON"); ASSERT_NE(type_default, nullptr); } + +TEST(DataTypeObject, SubcolumnLookup) +{ + auto type = DataTypeFactory::instance().get( + "JSON(" + "max_dynamic_types=5, max_dynamic_paths=3, " + "typed_string String, typed_array Array(UInt64), " + "nested Array(JSON(inner UInt32)), nullable Nullable(String), " + "a Array(JSON), a.b Int64, `escaped.dot` UInt8, `spaced path` UInt8, obj.x UInt16)"); + const auto & object = assert_cast(*type); + + const std::vector> subcolumns = { + {"typed_string", "String"}, + {"typed_array", "Array(UInt64)"}, + {"typed_array.size0", "UInt64"}, + {"arbitrary", "Dynamic(max_types=5)"}, + {"arbitrary.:`Int64`", "Nullable(Int64)"}, + {"arbitrary.:`Array(String)`.size0", "UInt64"}, + {"nested.inner", "Array(UInt32)"}, + {"nested.inner.:`UInt32`", "Array(UInt32)"}, + {"a.x", "Array(Dynamic)"}, + {"a.b", "Int64"}, + {"a.b.c", "Array(Dynamic)"}, + {"a.:`Array(JSON)`.x", "Array(Dynamic)"}, + {"a.:`String`", "Array(Dynamic)"}, + {"nullable", "Nullable(String)"}, + {"escaped.dot", "UInt8"}, + {"spaced path", "UInt8"}, + {"^`obj`", "JSON(max_dynamic_types=5, max_dynamic_paths=3, x UInt16)"}, + {"@`obj`.x", "UInt16"}, + {"@`missing`", "Dynamic(max_types=5)"}, + {DataTypeObject::SPECIAL_SUBCOLUMN_NAME_FOR_DISTINCT_PATHS_CALCULATION, "Array(String)"}, + {":`Int64`", ""}, + }; + + for (const auto & [subcolumn, expected_name] : subcolumns) + expectSubcolumnType(object, subcolumn, expected_name); +} + +TEST(DataTypeObject, SubcolumnLookupPreservesSerializationPrecedence) +{ + auto & factory = DataTypeFactory::instance(); + + auto array_collision = factory.get("JSON(a Array(UInt64), a.size0 String)"); + expectSubcolumnType(*array_collision, "a.size0", "UInt64"); + + auto tuple_collision = factory.get("JSON(a Tuple(size0 String), a.size0 UInt64)"); + expectSubcolumnType(*tuple_collision, "a.size0", "String"); + + auto multiple_prefixes = factory.get("JSON(a Array(JSON), a.b Int64)"); + expectSubcolumnType(*multiple_prefixes, "a.b", "Int64"); + expectSubcolumnType(*multiple_prefixes, "a.b.c", "Array(Dynamic)"); + + auto special_name_collision = factory.get(fmt::format( + "JSON({} UInt8)", DataTypeObject::SPECIAL_SUBCOLUMN_NAME_FOR_DISTINCT_PATHS_CALCULATION)); + expectSubcolumnType( + *special_name_collision, DataTypeObject::SPECIAL_SUBCOLUMN_NAME_FOR_DISTINCT_PATHS_CALCULATION, "UInt8"); + + auto prefixed_name_collision = std::make_shared( + DataTypeObject::SchemaFormat::JSON, + std::unordered_map{{"^`obj`", factory.get("JSON(x UInt8)")}}); + expectSubcolumnType(*prefixed_name_collision, "^`obj`.x", "UInt8"); +} + +TEST(DataTypeObject, SubcolumnLookupUsesCanonicalPathNames) +{ + auto nested_type = DataTypeFactory::instance().get("JSON(x UInt8)"); + auto object = std::make_shared( + DataTypeObject::SchemaFormat::JSON, + std::unordered_map{ + {"escaped.dot", nested_type}, + {R"(back\slash)", nested_type}, + {"back`tick", nested_type}, + {"spaced path", nested_type}, + {"", nested_type}, + {".x", DataTypeFactory::instance().get("UInt16")}, + }); + + expectSubcolumnType(*object, "escaped.dot.x", "UInt8"); + expectSubcolumnType(*object, R"(back\slash.x)", "UInt8"); + expectSubcolumnType(*object, "back`tick.x", "UInt8"); + expectSubcolumnType(*object, "spaced path.x", "UInt8"); + expectSubcolumnType(*object, ".x", "UInt8"); +} + +TEST(DataTypeObject, SubcolumnLookupDefersToCustomSerialization) +{ + auto object = std::make_shared( + DataTypeObject::SchemaFormat::JSON, + std::unordered_map{{"a", DataTypeFactory::instance().get("UInt8")}}); + auto serialization = object->getDefaultSerialization(); + object->setCustomization(std::make_unique(DataTypeCustomNamePtr{}, std::move(serialization))); + + expectSubcolumnType(*object, "a", "UInt8"); +} + +TEST(DataTypeObject, SubcolumnsThroughContainers) +{ + auto & factory = DataTypeFactory::instance(); + expectSubcolumnType(*factory.get("Array(JSON(x UInt8))"), "x", "Array(UInt8)"); + expectSubcolumnType(*factory.get("Tuple(j JSON(x UInt8))"), "j.x", "UInt8"); + expectSubcolumnType(*factory.get("Dynamic"), "JSON.x", "Dynamic"); +} diff --git a/tests/performance/json_typed_path_subcolumn_lookup.xml b/tests/performance/json_typed_path_subcolumn_lookup.xml new file mode 100644 index 0000000000000..2c12273924bb2 --- /dev/null +++ b/tests/performance/json_typed_path_subcolumn_lookup.xml @@ -0,0 +1,223 @@ + + + + 1 + + + + CREATE TABLE t_json_subcolumn_lookup_compact (id UInt64, json JSON(t0 String, t1 String, t2 String, t3 String, t4 String, t5 String, t6 String, t7 String, t8 String, t9 String, t10 String, t11 String, t12 String, t13 String, t14 String, t15 String, t16 String, t17 String, t18 String, t19 String, t20 String, t21 String, t22 String, t23 String, t24 String, t25 String, t26 String, t27 String, t28 String, t29 String, t30 String, t31 String, t32 String, t33 String, t34 String, t35 String, t36 String, t37 String, t38 String, t39 String, t40 String, t41 String, t42 String, t43 String, t44 String, t45 String, t46 String, t47 String, t48 String, t49 String, t50 String, t51 String, t52 String, t53 String, t54 String, t55 String, t56 String, t57 String, t58 String, t59 String, t60 String, t61 String, t62 String, t63 String, t64 String, t65 String, t66 String, t67 String, t68 String, t69 String, t70 String, t71 String, t72 String, t73 String, t74 String, t75 String, t76 String, t77 String, t78 String, t79 String, t80 String, t81 String, t82 String, t83 String, t84 String, t85 String, t86 String, t87 String, t88 String, t89 String, t90 String, t91 String, t92 String, t93 String, t94 String, t95 String, t96 String, t97 String, t98 String, t99 String, t100 String, t101 String, t102 String, t103 String, t104 String, t105 String, t106 String, t107 String, t108 String, t109 String, t110 String, t111 String, t112 String, t113 String, t114 String, t115 String, t116 String, t117 String, t118 String, t119 String, t120 String, t121 String, t122 String, t123 String, t124 String, t125 String, t126 String, t127 String, t128 String, t129 String, t130 String, t131 String, t132 String, t133 String, t134 String, t135 String, t136 String, t137 String, t138 String, t139 String, t140 String, t141 String, t142 String, t143 String, t144 String, t145 String, t146 String, t147 String, t148 String, t149 String, t150 String, t151 String, t152 String, t153 String, t154 String, t155 String, t156 String, t157 String, t158 String, t159 String, t160 String, t161 String, t162 String, t163 String, t164 String, t165 String, t166 String, t167 String, t168 String, t169 String, t170 String, t171 String, t172 String, t173 String, t174 String, t175 String, t176 String, t177 String, t178 String, t179 String, t180 String, t181 String, t182 String, t183 String, t184 String, t185 String, t186 String, t187 String, t188 String, t189 String, t190 String, t191 String, t192 String, t193 String, t194 String, t195 String, t196 String, t197 String, t198 String, t199 String, t200 String, t201 String, t202 String, t203 String, t204 String, t205 String, t206 String, t207 String, t208 String, t209 String, t210 String, t211 String, t212 String, t213 String, t214 String, t215 String, t216 String, t217 String, t218 String, t219 String, t220 String, t221 String, t222 String, t223 String, t224 String, t225 String, t226 String, t227 String, t228 String, t229 String, t230 String, t231 String, t232 String, t233 String, t234 String, t235 String, t236 String, t237 String, t238 String, t239 String, t240 String, t241 String, t242 String, t243 String, t244 String, t245 String, t246 String, t247 String, t248 String, t249 String, t250 String, t251 String, t252 String, t253 String, t254 String, t255 String, t256 String, t257 String, t258 String, t259 String, t260 String, t261 String, t262 String, t263 String, t264 String, t265 String, t266 String, t267 String, t268 String, t269 String, t270 String, t271 String, t272 String, t273 String, t274 String, t275 String, t276 String, t277 String, t278 String, t279 String, t280 String, t281 String, t282 String, t283 String, t284 String, t285 String, t286 String, t287 String, t288 String, t289 String, t290 String, t291 String, t292 String, t293 String, t294 String, t295 String, t296 String, t297 String, t298 String, t299 String, t300 String, t301 String, t302 String, t303 String, t304 String, t305 String, t306 String, t307 String, t308 String, t309 String, t310 String, t311 String, t312 String, t313 String, t314 String, t315 String, t316 String, t317 String, t318 String, t319 String, t320 String, t321 String, t322 String, t323 String, t324 String, t325 String, t326 String, t327 String, t328 String, t329 String, t330 String, t331 String, t332 String, t333 String, t334 String, t335 String, t336 String, t337 String, t338 String, t339 String, t340 String, t341 String, t342 String, t343 String, t344 String, t345 String, t346 String, t347 String, t348 String, t349 String, t350 String, t351 String, t352 String, t353 String, t354 String, t355 String, t356 String, t357 String, t358 String, t359 String, t360 String, t361 String, t362 String, t363 String, t364 String, t365 String, t366 String, t367 String, t368 String, t369 String, t370 String, t371 String, t372 String, t373 String, t374 String, t375 String, t376 String, t377 String, t378 String, t379 String, t380 String, t381 String, t382 String, t383 String, t384 String, t385 String, t386 String, t387 String, t388 String, t389 String, t390 String, t391 String, t392 String, t393 String, t394 String, t395 String, t396 String, t397 String, t398 String, t399 String, t400 String, t401 String, t402 String, t403 String, t404 String, t405 String, t406 String, t407 String, t408 String, t409 String, t410 String, t411 String, t412 String, t413 String, t414 String, t415 String, t416 String, t417 String, t418 String, t419 String, t420 String, t421 String, t422 String, t423 String, t424 String, t425 String, t426 String, t427 String, t428 String, t429 String, t430 String, t431 String, t432 String, t433 String, t434 String, t435 String, t436 String, t437 String, t438 String, t439 String, t440 String, t441 String, t442 String, t443 String, t444 String, t445 String, t446 String, t447 String, t448 String, t449 String, t450 String, t451 String, t452 String, t453 String, t454 String, t455 String, t456 String, t457 String, t458 String, t459 String, t460 String, t461 String, t462 String, t463 String, t464 String, t465 String, t466 String, t467 String, t468 String, t469 String, t470 String, t471 String, t472 String, t473 String, t474 String, t475 String, t476 String, t477 String, t478 String, t479 String, t480 String, t481 String, t482 String, t483 String, t484 String, t485 String, t486 String, t487 String, t488 String, t489 String, t490 String, t491 String, t492 String, t493 String, t494 String, t495 String, t496 String, t497 String, t498 String, t499 String)) + ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = '100G', min_rows_for_wide_part = 1000000000 + + + CREATE TABLE t_json_subcolumn_lookup_wide (id UInt64, json JSON(t0 String, t1 String, t2 String, t3 String, t4 String, t5 String, t6 String, t7 String, t8 String, t9 String, t10 String, t11 String, t12 String, t13 String, t14 String, t15 String, t16 String, t17 String, t18 String, t19 String, t20 String, t21 String, t22 String, t23 String, t24 String, t25 String, t26 String, t27 String, t28 String, t29 String, t30 String, t31 String, t32 String, t33 String, t34 String, t35 String, t36 String, t37 String, t38 String, t39 String, t40 String, t41 String, t42 String, t43 String, t44 String, t45 String, t46 String, t47 String, t48 String, t49 String, t50 String, t51 String, t52 String, t53 String, t54 String, t55 String, t56 String, t57 String, t58 String, t59 String, t60 String, t61 String, t62 String, t63 String, t64 String, t65 String, t66 String, t67 String, t68 String, t69 String, t70 String, t71 String, t72 String, t73 String, t74 String, t75 String, t76 String, t77 String, t78 String, t79 String, t80 String, t81 String, t82 String, t83 String, t84 String, t85 String, t86 String, t87 String, t88 String, t89 String, t90 String, t91 String, t92 String, t93 String, t94 String, t95 String, t96 String, t97 String, t98 String, t99 String, t100 String, t101 String, t102 String, t103 String, t104 String, t105 String, t106 String, t107 String, t108 String, t109 String, t110 String, t111 String, t112 String, t113 String, t114 String, t115 String, t116 String, t117 String, t118 String, t119 String, t120 String, t121 String, t122 String, t123 String, t124 String, t125 String, t126 String, t127 String, t128 String, t129 String, t130 String, t131 String, t132 String, t133 String, t134 String, t135 String, t136 String, t137 String, t138 String, t139 String, t140 String, t141 String, t142 String, t143 String, t144 String, t145 String, t146 String, t147 String, t148 String, t149 String, t150 String, t151 String, t152 String, t153 String, t154 String, t155 String, t156 String, t157 String, t158 String, t159 String, t160 String, t161 String, t162 String, t163 String, t164 String, t165 String, t166 String, t167 String, t168 String, t169 String, t170 String, t171 String, t172 String, t173 String, t174 String, t175 String, t176 String, t177 String, t178 String, t179 String, t180 String, t181 String, t182 String, t183 String, t184 String, t185 String, t186 String, t187 String, t188 String, t189 String, t190 String, t191 String, t192 String, t193 String, t194 String, t195 String, t196 String, t197 String, t198 String, t199 String, t200 String, t201 String, t202 String, t203 String, t204 String, t205 String, t206 String, t207 String, t208 String, t209 String, t210 String, t211 String, t212 String, t213 String, t214 String, t215 String, t216 String, t217 String, t218 String, t219 String, t220 String, t221 String, t222 String, t223 String, t224 String, t225 String, t226 String, t227 String, t228 String, t229 String, t230 String, t231 String, t232 String, t233 String, t234 String, t235 String, t236 String, t237 String, t238 String, t239 String, t240 String, t241 String, t242 String, t243 String, t244 String, t245 String, t246 String, t247 String, t248 String, t249 String, t250 String, t251 String, t252 String, t253 String, t254 String, t255 String, t256 String, t257 String, t258 String, t259 String, t260 String, t261 String, t262 String, t263 String, t264 String, t265 String, t266 String, t267 String, t268 String, t269 String, t270 String, t271 String, t272 String, t273 String, t274 String, t275 String, t276 String, t277 String, t278 String, t279 String, t280 String, t281 String, t282 String, t283 String, t284 String, t285 String, t286 String, t287 String, t288 String, t289 String, t290 String, t291 String, t292 String, t293 String, t294 String, t295 String, t296 String, t297 String, t298 String, t299 String, t300 String, t301 String, t302 String, t303 String, t304 String, t305 String, t306 String, t307 String, t308 String, t309 String, t310 String, t311 String, t312 String, t313 String, t314 String, t315 String, t316 String, t317 String, t318 String, t319 String, t320 String, t321 String, t322 String, t323 String, t324 String, t325 String, t326 String, t327 String, t328 String, t329 String, t330 String, t331 String, t332 String, t333 String, t334 String, t335 String, t336 String, t337 String, t338 String, t339 String, t340 String, t341 String, t342 String, t343 String, t344 String, t345 String, t346 String, t347 String, t348 String, t349 String, t350 String, t351 String, t352 String, t353 String, t354 String, t355 String, t356 String, t357 String, t358 String, t359 String, t360 String, t361 String, t362 String, t363 String, t364 String, t365 String, t366 String, t367 String, t368 String, t369 String, t370 String, t371 String, t372 String, t373 String, t374 String, t375 String, t376 String, t377 String, t378 String, t379 String, t380 String, t381 String, t382 String, t383 String, t384 String, t385 String, t386 String, t387 String, t388 String, t389 String, t390 String, t391 String, t392 String, t393 String, t394 String, t395 String, t396 String, t397 String, t398 String, t399 String, t400 String, t401 String, t402 String, t403 String, t404 String, t405 String, t406 String, t407 String, t408 String, t409 String, t410 String, t411 String, t412 String, t413 String, t414 String, t415 String, t416 String, t417 String, t418 String, t419 String, t420 String, t421 String, t422 String, t423 String, t424 String, t425 String, t426 String, t427 String, t428 String, t429 String, t430 String, t431 String, t432 String, t433 String, t434 String, t435 String, t436 String, t437 String, t438 String, t439 String, t440 String, t441 String, t442 String, t443 String, t444 String, t445 String, t446 String, t447 String, t448 String, t449 String, t450 String, t451 String, t452 String, t453 String, t454 String, t455 String, t456 String, t457 String, t458 String, t459 String, t460 String, t461 String, t462 String, t463 String, t464 String, t465 String, t466 String, t467 String, t468 String, t469 String, t470 String, t471 String, t472 String, t473 String, t474 String, t475 String, t476 String, t477 String, t478 String, t479 String, t480 String, t481 String, t482 String, t483 String, t484 String, t485 String, t486 String, t487 String, t488 String, t489 String, t490 String, t491 String, t492 String, t493 String, t494 String, t495 String, t496 String, t497 String, t498 String, t499 String)) + ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0 + + + SYSTEM STOP MERGES t_json_subcolumn_lookup_compact + SYSTEM STOP MERGES t_json_subcolumn_lookup_wide + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 0, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 1000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 2000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 3000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 4000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 5000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 6000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 7000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 8000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 9000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 10000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 11000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 12000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 13000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 14000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 15000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 16000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 17000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 18000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 19000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 20000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 21000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 22000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 23000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 24000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 25000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 26000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 27000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 28000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 29000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 30000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 31000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 32000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 33000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 34000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 35000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 36000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 37000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 38000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 39000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 40000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 41000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 42000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 43000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 44000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 45000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 46000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 47000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 48000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 49000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 50000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 51000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 52000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 53000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 54000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 55000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 56000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 57000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 58000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 59000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 60000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 61000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 62000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 63000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 64000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 65000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 66000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 67000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 68000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 69000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 70000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 71000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 72000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 73000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 74000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 75000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 76000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 77000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 78000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 79000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 80000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 81000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 82000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 83000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 84000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 85000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 86000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 87000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 88000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 89000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 90000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 91000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 92000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 93000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 94000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 95000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 0, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 1000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 2000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 3000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 4000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 5000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 6000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 7000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 8000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 9000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 10000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 11000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 12000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 13000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 14000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 15000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 16000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 17000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 18000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 19000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 20000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 21000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 22000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 23000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 24000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 25000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 26000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 27000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 28000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 29000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 30000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 31000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 32000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 33000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 34000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 35000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 36000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 37000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 38000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 39000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 40000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 41000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 42000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 43000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 44000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 45000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 46000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 47000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 48000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 49000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 50000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 51000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 52000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 53000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 54000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 55000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 56000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 57000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 58000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 59000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 60000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 61000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 62000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 63000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 64000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 65000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 66000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 67000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 68000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 69000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 70000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 71000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 72000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 73000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 74000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 75000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 76000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 77000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 78000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 79000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 80000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 81000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 82000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 83000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 84000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 85000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 86000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 87000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 88000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 89000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 90000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 91000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 92000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 93000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 94000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 95000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + + SELECT json.t250 FROM t_json_subcolumn_lookup_compact FORMAT Null + SELECT json.d25 FROM t_json_subcolumn_lookup_compact FORMAT Null + SELECT json.t1, json.t50, json.t100, json.t150, json.t200, json.t250, json.t300, json.t350, json.t400, json.t450 FROM t_json_subcolumn_lookup_compact FORMAT Null + SELECT json.t250 FROM t_json_subcolumn_lookup_compact WHERE id = 50000 FORMAT Null + SELECT json.t250 FROM t_json_subcolumn_lookup_wide FORMAT Null + SELECT json.d25 FROM t_json_subcolumn_lookup_wide FORMAT Null + SELECT json.t1, json.t50, json.t100, json.t150, json.t200, json.t250, json.t300, json.t350, json.t400, json.t450 FROM t_json_subcolumn_lookup_wide FORMAT Null + SELECT json.t250 FROM t_json_subcolumn_lookup_wide WHERE id = 50000 FORMAT Null + + DROP TABLE IF EXISTS t_json_subcolumn_lookup_compact + DROP TABLE IF EXISTS t_json_subcolumn_lookup_wide + From 72c3dae18938c3056e38327d4b02cf824957977d Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 5 Nov 2026 16:06:54 +0000 Subject: [PATCH 109/219] Backport #129338 to 26.8: Fix the crash of `merge`, `loop` and `executable` reading a materialized view's source table on INSERT --- src/Processors/QueryPlan/ReadFromLoopStep.cpp | 4 +- src/Storages/StorageExecutable.cpp | 6 +- src/Storages/StorageMerge.cpp | 11 +- ...urce_merge_loop_executable_crash.reference | 11 ++ ...7_mv_source_merge_loop_executable_crash.sh | 148 ++++++++++++++++++ 5 files changed, 177 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/05507_mv_source_merge_loop_executable_crash.reference create mode 100755 tests/queries/0_stateless/05507_mv_source_merge_loop_executable_crash.sh diff --git a/src/Processors/QueryPlan/ReadFromLoopStep.cpp b/src/Processors/QueryPlan/ReadFromLoopStep.cpp index 60e00d420ec99..4735796e0b744 100644 --- a/src/Processors/QueryPlan/ReadFromLoopStep.cpp +++ b/src/Processors/QueryPlan/ReadFromLoopStep.cpp @@ -72,7 +72,9 @@ namespace if (context->getSettingsRef()[Setting::allow_experimental_analyzer]) { - InterpreterSelectQueryAnalyzer interpreter(select_ast, context, options, column_names); + /// The analyzer substitutes the inserted block of a materialized view for its source table only when given it. + InterpreterSelectQueryAnalyzer interpreter( + select_ast, context, options, context->isViewInnerQuery() ? nullptr : context->getViewSource(), column_names); if (query_info.storage_limits) interpreter.addStorageLimits(*query_info.storage_limits); plan = std::move(interpreter).extractQueryPlan(); diff --git a/src/Storages/StorageExecutable.cpp b/src/Storages/StorageExecutable.cpp index cd6eaf90454b2..c8b438f4430b0 100644 --- a/src/Storages/StorageExecutable.cpp +++ b/src/Storages/StorageExecutable.cpp @@ -202,7 +202,11 @@ void StorageExecutable::readImpl( { QueryPipelineBuilder builder; if (context->getSettingsRef()[Setting::allow_experimental_analyzer]) - builder = InterpreterSelectQueryAnalyzer(input_query, context, {}).buildQueryPipeline(); + { + /// The analyzer substitutes the inserted block of a materialized view for its source table only when given it. + builder = InterpreterSelectQueryAnalyzer( + input_query, context, {}, context->isViewInnerQuery() ? nullptr : context->getViewSource()).buildQueryPipeline(); + } else builder = InterpreterSelectWithUnionQuery(input_query, context, {}).buildQueryPipeline(); inputs.emplace_back(QueryPipelineBuilder::getPipe(std::move(builder), resources)); diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index b46619e7d2450..cf7aed6565382 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -1641,9 +1641,18 @@ ReadFromMerge::ChildPlan ReadFromMerge::createPlanForTable( /// Converting query to AST because types might be different in the source table. /// Need to resolve types again. auto ast = modified_query_info.query_tree->toAST(); + /// The analyzer substitutes the inserted block of a materialized view for its source table only when given it. + /// A child that is the source table itself reads the table, as it does at `FetchColumns`. + auto view_source = modified_context->isViewInnerQuery() ? nullptr : modified_context->getViewSource(); + if (view_source && view_source->getStorageID().getFullNameNotQuoted() == storage->getStorageID().getFullNameNotQuoted()) + { + modified_context->clearViewSource(); + view_source = nullptr; + } InterpreterSelectQueryAnalyzer interpreter(ast, modified_context, - child_select_query_options); + child_select_query_options, + view_source); auto & planner = interpreter.getPlanner(); planner.buildQueryPlanIfNeeded(); diff --git a/tests/queries/0_stateless/05507_mv_source_merge_loop_executable_crash.reference b/tests/queries/0_stateless/05507_mv_source_merge_loop_executable_crash.reference new file mode 100644 index 0000000000000..9e4184c192eb5 --- /dev/null +++ b/tests/queries/0_stateless/05507_mv_source_merge_loop_executable_crash.reference @@ -0,0 +1,11 @@ +insert loop 3 3 +insert merge 3 1051 +insert merge_policy 3 1 +append_refresh loop 3 6 +append_refresh merge 3 1066 +append_refresh merge_policy 3 4 +refreshable loop 16 109 +refreshable merge 16 1066 +insert executable 3 6 +append_refresh executable 3 15 +refreshable executable 16 1066 diff --git a/tests/queries/0_stateless/05507_mv_source_merge_loop_executable_crash.sh b/tests/queries/0_stateless/05507_mv_source_merge_loop_executable_crash.sh new file mode 100755 index 0000000000000..161eb3541b0a6 --- /dev/null +++ b/tests/queries/0_stateless/05507_mv_source_merge_loop_executable_crash.sh @@ -0,0 +1,148 @@ +#!/usr/bin/env bash +# Tags: no-old-analyzer +# no-old-analyzer: the fix is in the analyzer path; the old interpreter reads the view source in its own way. + +# Inside a materialized view query, the source table read by the nested interpreter of `loop` or `executable` +# is the inserted block, like the rest of the view query, while a `merge` child reads the table whatever its +# siblings are. The database name is spelled out: `currentDatabase()` in the view query does not resolve to the +# test database during the push. +# A refreshable view appending to the source table behaves like an INSERT; a refreshable view reading +# the source table reads the table. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -m -q " +DROP DATABASE IF EXISTS ${CLICKHOUSE_DATABASE_1}; +DROP TABLE IF EXISTS mv_merge; +DROP TABLE IF EXISTS mv_merge_policy; +DROP ROW POLICY IF EXISTS policy_05507 ON child; +DROP TABLE IF EXISTS child; +DROP TABLE IF EXISTS mv_loop; +DROP TABLE IF EXISTS dst; +DROP TABLE IF EXISTS dist; +DROP TABLE IF EXISTS dist_target; +DROP TABLE IF EXISTS src; + +CREATE TABLE src (x UInt64) ENGINE = MergeTree ORDER BY x; +INSERT INTO src SELECT number + 100 FROM numbers(10); + +CREATE TABLE dist_target (x UInt64) ENGINE = MergeTree ORDER BY x; +CREATE TABLE dist AS dist_target ENGINE = Distributed(test_shard_localhost, ${CLICKHOUSE_DATABASE}, dist_target); + +CREATE TABLE dst (name String, block_rows UInt64, value UInt64) ENGINE = MergeTree ORDER BY name; + +-- 'dist' raises the merge read above FetchColumns, so 'src' is planned by a nested interpreter. +CREATE MATERIALIZED VIEW mv_merge TO dst AS + SELECT 'merge' AS name, count() AS block_rows, + (SELECT sum(x) FROM merge(${CLICKHOUSE_DATABASE}, '^(src|dist)\$') WHERE x > 0) AS value + FROM src; + +-- Another child planned by the nested interpreter keeps reading the inserted block in its row policy, as at FetchColumns. +CREATE TABLE child (x UInt64) ENGINE = MergeTree ORDER BY x; +INSERT INTO child VALUES (1), (4), (100); +CREATE ROW POLICY OR REPLACE policy_05507 ON child USING x IN (SELECT x FROM ${CLICKHOUSE_DATABASE}.src) TO ALL; +CREATE MATERIALIZED VIEW mv_merge_policy TO dst AS + SELECT 'merge_policy' AS name, count() AS block_rows, + (SELECT sum(x) FROM merge(${CLICKHOUSE_DATABASE}, '^(child|dist)\$')) AS value + FROM src; + +CREATE MATERIALIZED VIEW mv_loop TO dst AS + SELECT 'loop' AS name, count() AS block_rows, + (SELECT max(x) FROM (SELECT x FROM loop(${CLICKHOUSE_DATABASE}, src) WHERE x > 0 LIMIT 25)) AS value + FROM src; + +INSERT INTO src VALUES (1), (2), (3); + +SELECT 'insert', * FROM dst ORDER BY name; + +-- A refreshable view appending to the source table pushes its block to these views like an INSERT. It lives in an Atomic +-- database: in a Replicated one an APPEND refresh runs on one replica only. +TRUNCATE TABLE dst; +CREATE DATABASE ${CLICKHOUSE_DATABASE_1} ENGINE = Atomic; +CREATE MATERIALIZED VIEW ${CLICKHOUSE_DATABASE_1}.rmv_append REFRESH EVERY 1 YEAR APPEND TO ${CLICKHOUSE_DATABASE}.src EMPTY AS + SELECT number + 4 AS x FROM numbers(3); +SYSTEM REFRESH VIEW ${CLICKHOUSE_DATABASE_1}.rmv_append; +SYSTEM WAIT VIEW ${CLICKHOUSE_DATABASE_1}.rmv_append; + +SELECT 'append_refresh', * FROM dst ORDER BY name; + +-- A refreshable view reading the source table has no inserted block, so it reads the table. +TRUNCATE TABLE dst; +CREATE MATERIALIZED VIEW ${CLICKHOUSE_DATABASE_1}.rmv_read REFRESH EVERY 1 YEAR APPEND TO ${CLICKHOUSE_DATABASE}.dst EMPTY AS + SELECT 'merge' AS name, count() AS block_rows, + (SELECT sum(x) FROM merge(${CLICKHOUSE_DATABASE}, '^(src|dist)\$') WHERE x > 0) AS value + FROM ${CLICKHOUSE_DATABASE}.src + UNION ALL + SELECT 'loop', count(), (SELECT max(x) FROM (SELECT x FROM loop(${CLICKHOUSE_DATABASE}, src) WHERE x > 0 LIMIT 25)) + FROM ${CLICKHOUSE_DATABASE}.src; +SYSTEM REFRESH VIEW ${CLICKHOUSE_DATABASE_1}.rmv_read; +SYSTEM WAIT VIEW ${CLICKHOUSE_DATABASE_1}.rmv_read; + +SELECT 'refreshable', * FROM dst ORDER BY name; + +DROP DATABASE ${CLICKHOUSE_DATABASE_1}; +DROP TABLE mv_merge; +DROP TABLE mv_merge_policy; +DROP ROW POLICY policy_05507 ON child; +DROP TABLE child; +DROP TABLE mv_loop; +DROP TABLE dst; +DROP TABLE dist; +DROP TABLE dist_target; +DROP TABLE src; +" + +# The input query of `executable` needs `user_scripts_path`, which is a server config option. +WORK_DIR=$(mktemp -d "${CLICKHOUSE_TMP}/05507_XXXXXX") +trap 'rm -rf "${WORK_DIR}"' EXIT +mkdir -p "${WORK_DIR}/user_scripts" "${WORK_DIR}/data" + +cat > "${WORK_DIR}/user_scripts/echo_rows.sh" <<'SCRIPT' +#!/usr/bin/env bash +while IFS= read -r x; do + printf '%s\n' "${x}" +done +SCRIPT +chmod +x "${WORK_DIR}/user_scripts/echo_rows.sh" + +cat > "${WORK_DIR}/config.xml" < + ${WORK_DIR}/user_scripts/ + ${WORK_DIR}/data/ + +EOF + +$CLICKHOUSE_LOCAL --config-file="${WORK_DIR}/config.xml" -m --query " +CREATE DATABASE db_05507; +CREATE TABLE db_05507.src (x UInt64) ENGINE = MergeTree ORDER BY x; +INSERT INTO db_05507.src SELECT number + 100 FROM numbers(10); +CREATE TABLE db_05507.dst (name String, block_rows UInt64, value UInt64) ENGINE = MergeTree ORDER BY name; + +CREATE MATERIALIZED VIEW db_05507.mv_executable TO db_05507.dst AS + SELECT 'executable' AS name, count() AS block_rows, + (SELECT sum(x) FROM executable('echo_rows.sh', 'TSV', 'x UInt64', (SELECT x FROM db_05507.src WHERE x > 0))) AS value + FROM db_05507.src; + +INSERT INTO db_05507.src VALUES (1), (2), (3); + +SELECT 'insert', * FROM db_05507.dst ORDER BY name; + +TRUNCATE TABLE db_05507.dst; +CREATE MATERIALIZED VIEW db_05507.rmv_append REFRESH EVERY 1 YEAR APPEND TO db_05507.src EMPTY AS SELECT number + 4 AS x FROM numbers(3); +SYSTEM REFRESH VIEW db_05507.rmv_append; +SYSTEM WAIT VIEW db_05507.rmv_append; + +SELECT 'append_refresh', * FROM db_05507.dst ORDER BY name; + +TRUNCATE TABLE db_05507.dst; +CREATE MATERIALIZED VIEW db_05507.rmv_read REFRESH EVERY 1 YEAR APPEND TO db_05507.dst EMPTY AS + SELECT 'executable' AS name, count() AS block_rows, + (SELECT sum(x) FROM executable('echo_rows.sh', 'TSV', 'x UInt64', (SELECT x FROM db_05507.src WHERE x > 0))) AS value + FROM db_05507.src; +SYSTEM REFRESH VIEW db_05507.rmv_read; +SYSTEM WAIT VIEW db_05507.rmv_read; + +SELECT 'refreshable', * FROM db_05507.dst ORDER BY name; +" From e7da66abe24831fce7df8811579277af5bb6412f Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Thu, 5 Nov 2026 20:23:11 +0000 Subject: [PATCH 110/219] Drop the `merge` row-policy case from 05507: it needs #128013, which is not on this branch On this branch a `merge` child's row policy is applied above the child's read, so with a `Distributed` sibling raising the stage it fails with NOT_FOUND_COLUMN_IN_BLOCK even without a view. Co-Authored-By: Claude Opus 5.5 --- ...v_source_merge_loop_executable_crash.reference | 2 -- ...05507_mv_source_merge_loop_executable_crash.sh | 15 --------------- 2 files changed, 17 deletions(-) diff --git a/tests/queries/0_stateless/05507_mv_source_merge_loop_executable_crash.reference b/tests/queries/0_stateless/05507_mv_source_merge_loop_executable_crash.reference index 9e4184c192eb5..e1ac9521d29dd 100644 --- a/tests/queries/0_stateless/05507_mv_source_merge_loop_executable_crash.reference +++ b/tests/queries/0_stateless/05507_mv_source_merge_loop_executable_crash.reference @@ -1,9 +1,7 @@ insert loop 3 3 insert merge 3 1051 -insert merge_policy 3 1 append_refresh loop 3 6 append_refresh merge 3 1066 -append_refresh merge_policy 3 4 refreshable loop 16 109 refreshable merge 16 1066 insert executable 3 6 diff --git a/tests/queries/0_stateless/05507_mv_source_merge_loop_executable_crash.sh b/tests/queries/0_stateless/05507_mv_source_merge_loop_executable_crash.sh index 161eb3541b0a6..9ca9db387b390 100755 --- a/tests/queries/0_stateless/05507_mv_source_merge_loop_executable_crash.sh +++ b/tests/queries/0_stateless/05507_mv_source_merge_loop_executable_crash.sh @@ -16,9 +16,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) $CLICKHOUSE_CLIENT -m -q " DROP DATABASE IF EXISTS ${CLICKHOUSE_DATABASE_1}; DROP TABLE IF EXISTS mv_merge; -DROP TABLE IF EXISTS mv_merge_policy; -DROP ROW POLICY IF EXISTS policy_05507 ON child; -DROP TABLE IF EXISTS child; DROP TABLE IF EXISTS mv_loop; DROP TABLE IF EXISTS dst; DROP TABLE IF EXISTS dist; @@ -39,15 +36,6 @@ CREATE MATERIALIZED VIEW mv_merge TO dst AS (SELECT sum(x) FROM merge(${CLICKHOUSE_DATABASE}, '^(src|dist)\$') WHERE x > 0) AS value FROM src; --- Another child planned by the nested interpreter keeps reading the inserted block in its row policy, as at FetchColumns. -CREATE TABLE child (x UInt64) ENGINE = MergeTree ORDER BY x; -INSERT INTO child VALUES (1), (4), (100); -CREATE ROW POLICY OR REPLACE policy_05507 ON child USING x IN (SELECT x FROM ${CLICKHOUSE_DATABASE}.src) TO ALL; -CREATE MATERIALIZED VIEW mv_merge_policy TO dst AS - SELECT 'merge_policy' AS name, count() AS block_rows, - (SELECT sum(x) FROM merge(${CLICKHOUSE_DATABASE}, '^(child|dist)\$')) AS value - FROM src; - CREATE MATERIALIZED VIEW mv_loop TO dst AS SELECT 'loop' AS name, count() AS block_rows, (SELECT max(x) FROM (SELECT x FROM loop(${CLICKHOUSE_DATABASE}, src) WHERE x > 0 LIMIT 25)) AS value @@ -84,9 +72,6 @@ SELECT 'refreshable', * FROM dst ORDER BY name; DROP DATABASE ${CLICKHOUSE_DATABASE_1}; DROP TABLE mv_merge; -DROP TABLE mv_merge_policy; -DROP ROW POLICY policy_05507 ON child; -DROP TABLE child; DROP TABLE mv_loop; DROP TABLE dst; DROP TABLE dist; From 915b8b0d3061c07ba0a6a95107bae281a8a86f95 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 5 Nov 2026 18:36:54 +0000 Subject: [PATCH 111/219] Backport #128662 to 26.8: Fix reading a view whose query has more than one `ARRAY JOIN` --- src/Storages/StorageView.cpp | 3 +- .../05481_view_with_two_array_joins.reference | 40 +++++++++++++++ .../05481_view_with_two_array_joins.sql | 49 +++++++++++++++++++ 3 files changed, 91 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05481_view_with_two_array_joins.reference create mode 100644 tests/queries/0_stateless/05481_view_with_two_array_joins.sql diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp index 1d27a13b7d6e3..1f0fb18f96c1f 100644 --- a/src/Storages/StorageView.cpp +++ b/src/Storages/StorageView.cpp @@ -250,7 +250,7 @@ StoragePtr tryGetTrivialViewUnderlyingStorage(const ASTPtr & inner_query, Contex || select->limitLength() || select->limitOffset() || select->limitBy() || select->limit_by_all || select->limitByLength() || select->limitByOffset() - || select->distinct || select->arrayJoinExpressionList().first + || select->distinct || select->settings()) { return nullptr; @@ -284,6 +284,7 @@ StoragePtr tryGetTrivialViewUnderlyingStorage(const ASTPtr & inner_query, Contex } const auto * tables = select->tables().get(); + /// Every JOIN and ARRAY JOIN clause is a separate element of this list. if (!tables || tables->children.size() != 1) { return nullptr; diff --git a/tests/queries/0_stateless/05481_view_with_two_array_joins.reference b/tests/queries/0_stateless/05481_view_with_two_array_joins.reference new file mode 100644 index 0000000000000..4782026402557 --- /dev/null +++ b/tests/queries/0_stateless/05481_view_with_two_array_joins.reference @@ -0,0 +1,40 @@ +v 1 1 +v 1 2 +v 1 3 +v_definer 1 1 +v_definer 1 2 +v_definer 1 3 +v_definer, no pushdown 1 1 +v_definer, no pushdown 1 2 +v_definer, no pushdown 1 3 +v_dist 1 1 +v_dist 1 2 +v_dist 1 3 +v_full 1 1 1 +v_full 1 2 1 +v_full 1 3 1 +v_paste 1 1 0 +v_paste 1 2 0 +v_paste 1 3 1 +v_asof 1 1 1 +v_asof 1 2 1 +v_asof 1 3 1 +v_cross 1 1 1 +v_cross 1 1 2 +v_cross 1 2 1 +v_cross 1 2 2 +v_cross 1 3 1 +v_cross 1 3 2 +v_left_semi 1 1 1 +v_left_semi 1 2 1 +v_left_semi 1 3 1 +v_right_semi 1 1 1 +v_right_semi 1 2 1 +v_right_semi 1 3 1 +v_left 1 0 +v_left 1 3 +mv 1 1 +mv 1 2 +mv 1 3 +v_dist_plain pushdown 1 +v_dist no pushdown 1 diff --git a/tests/queries/0_stateless/05481_view_with_two_array_joins.sql b/tests/queries/0_stateless/05481_view_with_two_array_joins.sql new file mode 100644 index 0000000000000..63a9363f97936 --- /dev/null +++ b/tests/queries/0_stateless/05481_view_with_two_array_joins.sql @@ -0,0 +1,49 @@ +-- Tags: distributed +-- A view whose query has more than one ARRAY JOIN must be readable (issue #128410). +CREATE TABLE t (id UInt32, a Array(Array(UInt32))) ENGINE = MergeTree ORDER BY id; +INSERT INTO t VALUES (1, [[1, 2], [3]]); +CREATE TABLE t_dist AS t ENGINE = Distributed(test_shard_localhost, currentDatabase(), t); + +CREATE VIEW v AS SELECT id, y FROM t ARRAY JOIN a AS x ARRAY JOIN x AS y; +CREATE VIEW v_definer SQL SECURITY DEFINER AS SELECT id, y FROM t ARRAY JOIN a AS x ARRAY JOIN x AS y; +CREATE VIEW v_dist AS SELECT id, y FROM t_dist ARRAY JOIN a AS x ARRAY JOIN x AS y; + +SELECT 'v', * FROM v ORDER BY y SETTINGS optimize_trivial_view_pushdown_to_distributed = 1; +SELECT 'v_definer', * FROM v_definer ORDER BY y SETTINGS optimize_trivial_view_pushdown_to_distributed = 1; +SELECT 'v_definer, no pushdown', * FROM v_definer ORDER BY y SETTINGS optimize_trivial_view_pushdown_to_distributed = 0; +SELECT 'v_dist', * FROM v_dist ORDER BY y SETTINGS optimize_trivial_view_pushdown_to_distributed = 1; + +-- Other JOIN kinds between or around the ARRAY JOINs, and LEFT ARRAY JOIN. +CREATE TABLE r (id UInt32, k UInt32) ENGINE = MergeTree ORDER BY id; +INSERT INTO r VALUES (1, 1), (2, 2); + +CREATE VIEW v_full AS SELECT id, y, k FROM t ARRAY JOIN a AS x FULL OUTER JOIN r ON t.id = r.id ARRAY JOIN x AS y; +CREATE VIEW v_paste AS SELECT id, y, n FROM t ARRAY JOIN a AS x PASTE JOIN (SELECT number AS n FROM numbers(2)) AS p ARRAY JOIN x AS y; +CREATE VIEW v_asof AS SELECT id, y, k FROM t ARRAY JOIN a AS x ASOF JOIN r ON t.id = r.id AND length(x) >= r.k ARRAY JOIN x AS y; +CREATE VIEW v_cross AS SELECT id, y, k FROM t ARRAY JOIN a AS x CROSS JOIN r ARRAY JOIN x AS y; +CREATE VIEW v_left_semi AS SELECT id, y, k FROM t ARRAY JOIN a AS x LEFT SEMI JOIN r ON t.id = r.id ARRAY JOIN x AS y; +CREATE VIEW v_right_semi AS SELECT id, y, k FROM t RIGHT SEMI JOIN r ON t.id = r.id ARRAY JOIN a AS x ARRAY JOIN x AS y; +CREATE VIEW v_left AS SELECT id, y FROM t LEFT ARRAY JOIN a AS x LEFT ARRAY JOIN arrayFilter(e -> e > 2, x) AS y; + +SELECT 'v_full', * FROM v_full ORDER BY ALL SETTINGS optimize_trivial_view_pushdown_to_distributed = 1; +SELECT 'v_paste', * FROM v_paste ORDER BY ALL SETTINGS optimize_trivial_view_pushdown_to_distributed = 1; +SELECT 'v_asof', * FROM v_asof ORDER BY ALL SETTINGS optimize_trivial_view_pushdown_to_distributed = 1; +SELECT 'v_cross', * FROM v_cross ORDER BY ALL SETTINGS optimize_trivial_view_pushdown_to_distributed = 1; +SELECT 'v_left_semi', * FROM v_left_semi ORDER BY ALL SETTINGS optimize_trivial_view_pushdown_to_distributed = 1; +SELECT 'v_right_semi', * FROM v_right_semi ORDER BY ALL SETTINGS optimize_trivial_view_pushdown_to_distributed = 1; +SELECT 'v_left', * FROM v_left ORDER BY ALL SETTINGS optimize_trivial_view_pushdown_to_distributed = 1; + +-- A refreshable materialized view reading the view. +CREATE TABLE dst (id UInt32, y UInt32) ENGINE = MergeTree ORDER BY (id, y); +CREATE MATERIALIZED VIEW mv REFRESH AFTER 1 YEAR SETTINGS all_replicas = 1 APPEND TO dst EMPTY AS SELECT id, y FROM v; +SYSTEM REFRESH VIEW mv; +SYSTEM WAIT VIEW mv; +SELECT 'mv', * FROM dst ORDER BY y; + +-- The view over `Distributed` is not trivial, so the pushdown does not fire for it; a plain view is the positive control. +-- The default EXPLAIN format does not print the `VIEW subquery` step. +SET explain_query_plan_default = 'legacy'; +SET prefer_localhost_replica = 0; +CREATE VIEW v_dist_plain AS SELECT id FROM t_dist; +SELECT 'v_dist_plain pushdown', countIf(explain LIKE '%VIEW subquery%') = 0 FROM (EXPLAIN SELECT id FROM v_dist_plain SETTINGS optimize_trivial_view_pushdown_to_distributed = 1); +SELECT 'v_dist no pushdown', countIf(explain LIKE '%VIEW subquery%') > 0 FROM (EXPLAIN SELECT id, y FROM v_dist SETTINGS optimize_trivial_view_pushdown_to_distributed = 1); From 402c77e6c3006631004d4898e51a29688cb1f01c Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Thu, 5 Nov 2026 20:58:38 +0000 Subject: [PATCH 112/219] Drop the lambda `LEFT ARRAY JOIN` case from 05481 On this branch ARRAY JOIN cutting a lambda column fails a debug-build assertion in `ColumnConst::doInsertRangeFrom`, unrelated to the view fix; master avoids it since #118788. Co-Authored-By: Claude Opus 5.5 --- .../0_stateless/05481_view_with_two_array_joins.reference | 2 -- tests/queries/0_stateless/05481_view_with_two_array_joins.sql | 4 +--- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/tests/queries/0_stateless/05481_view_with_two_array_joins.reference b/tests/queries/0_stateless/05481_view_with_two_array_joins.reference index 4782026402557..aa8e3286a4af2 100644 --- a/tests/queries/0_stateless/05481_view_with_two_array_joins.reference +++ b/tests/queries/0_stateless/05481_view_with_two_array_joins.reference @@ -31,8 +31,6 @@ v_left_semi 1 3 1 v_right_semi 1 1 1 v_right_semi 1 2 1 v_right_semi 1 3 1 -v_left 1 0 -v_left 1 3 mv 1 1 mv 1 2 mv 1 3 diff --git a/tests/queries/0_stateless/05481_view_with_two_array_joins.sql b/tests/queries/0_stateless/05481_view_with_two_array_joins.sql index 63a9363f97936..5b7100c3f32aa 100644 --- a/tests/queries/0_stateless/05481_view_with_two_array_joins.sql +++ b/tests/queries/0_stateless/05481_view_with_two_array_joins.sql @@ -13,7 +13,7 @@ SELECT 'v_definer', * FROM v_definer ORDER BY y SETTINGS optimize_trivial_view_p SELECT 'v_definer, no pushdown', * FROM v_definer ORDER BY y SETTINGS optimize_trivial_view_pushdown_to_distributed = 0; SELECT 'v_dist', * FROM v_dist ORDER BY y SETTINGS optimize_trivial_view_pushdown_to_distributed = 1; --- Other JOIN kinds between or around the ARRAY JOINs, and LEFT ARRAY JOIN. +-- Other JOIN kinds between or around the ARRAY JOINs. CREATE TABLE r (id UInt32, k UInt32) ENGINE = MergeTree ORDER BY id; INSERT INTO r VALUES (1, 1), (2, 2); @@ -23,7 +23,6 @@ CREATE VIEW v_asof AS SELECT id, y, k FROM t ARRAY JOIN a AS x ASOF JOIN r ON t. CREATE VIEW v_cross AS SELECT id, y, k FROM t ARRAY JOIN a AS x CROSS JOIN r ARRAY JOIN x AS y; CREATE VIEW v_left_semi AS SELECT id, y, k FROM t ARRAY JOIN a AS x LEFT SEMI JOIN r ON t.id = r.id ARRAY JOIN x AS y; CREATE VIEW v_right_semi AS SELECT id, y, k FROM t RIGHT SEMI JOIN r ON t.id = r.id ARRAY JOIN a AS x ARRAY JOIN x AS y; -CREATE VIEW v_left AS SELECT id, y FROM t LEFT ARRAY JOIN a AS x LEFT ARRAY JOIN arrayFilter(e -> e > 2, x) AS y; SELECT 'v_full', * FROM v_full ORDER BY ALL SETTINGS optimize_trivial_view_pushdown_to_distributed = 1; SELECT 'v_paste', * FROM v_paste ORDER BY ALL SETTINGS optimize_trivial_view_pushdown_to_distributed = 1; @@ -31,7 +30,6 @@ SELECT 'v_asof', * FROM v_asof ORDER BY ALL SETTINGS optimize_trivial_view_pushd SELECT 'v_cross', * FROM v_cross ORDER BY ALL SETTINGS optimize_trivial_view_pushdown_to_distributed = 1; SELECT 'v_left_semi', * FROM v_left_semi ORDER BY ALL SETTINGS optimize_trivial_view_pushdown_to_distributed = 1; SELECT 'v_right_semi', * FROM v_right_semi ORDER BY ALL SETTINGS optimize_trivial_view_pushdown_to_distributed = 1; -SELECT 'v_left', * FROM v_left ORDER BY ALL SETTINGS optimize_trivial_view_pushdown_to_distributed = 1; -- A refreshable materialized view reading the view. CREATE TABLE dst (id UInt32, y UInt32) ENGINE = MergeTree ORDER BY (id, y); From 8d70f2f1d5286e5cd4841ede6de55eae3123b1b1 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 5 Nov 2026 21:27:31 +0000 Subject: [PATCH 113/219] Backport #128731 to 26.8: Fix Hive partition with empty partition key --- src/Storages/Hive/StorageHive.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/Storages/Hive/StorageHive.cpp b/src/Storages/Hive/StorageHive.cpp index acaba8fe71a25..7c71181264681 100644 --- a/src/Storages/Hive/StorageHive.cpp +++ b/src/Storages/Hive/StorageHive.cpp @@ -609,6 +609,10 @@ HiveFilesWithSkipSplits StorageHive::collectHiveFilesFromPartition( throw Exception(ErrorCodes::INVALID_PARTITION_VALUE, "Partition value size not match, expect {}, but got {}", partition_names.size(), partition.values.size()); + if (!partition_key_expr) + throw Exception(ErrorCodes::INVALID_PARTITION_VALUE, + "Hive metastore returned a partition for table {}.{}, but the table has no partition key", hive_database, hive_table); + /// Join partition values in CSV format WriteBufferFromOwnString wb; for (size_t i = 0; i < partition.values.size(); ++i) From 9e732e8b6adf1a5b0bdd313b3dce10c1c9141edc Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 6 Nov 2026 08:44:57 +0000 Subject: [PATCH 114/219] Backport #129308 to 26.8: Allow jemalloc percpu arenas even if CPU counts are non deterministic --- contrib/jemalloc | 2 +- programs/main.cpp | 2 - .../test_jemalloc_percpu_arena/test.py | 67 ++++++++++++++----- .../01502_jemalloc_percpu_arena.sh | 10 +-- .../01505_pipeline_executor_UAF.sh | 3 - 5 files changed, 58 insertions(+), 26 deletions(-) diff --git a/contrib/jemalloc b/contrib/jemalloc index 5f864d8bd4c4f..9e5a679b18b9c 160000 --- a/contrib/jemalloc +++ b/contrib/jemalloc @@ -1 +1 @@ -Subproject commit 5f864d8bd4c4fa7a99cb47c374d5c6465a160578 +Subproject commit 9e5a679b18b9c8d6b7f6fc319cb63808e88bc5ba diff --git a/programs/main.cpp b/programs/main.cpp index 74e1b893ba321..18ea7ad034189 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -296,8 +296,6 @@ static __attribute__((constructor(0))) void init_je_malloc_message() std::string_view message_view{str}; if (message_view == ": background thread creation failed (11)\n") return; - if (message_view == ": Number of CPUs detected is not deterministic. Per-CPU arena disabled.\n") - return; # if defined(SYS_write) syscall(SYS_write, STDERR_FILENO, message_view.data(), message_view.size()); diff --git a/tests/integration/test_jemalloc_percpu_arena/test.py b/tests/integration/test_jemalloc_percpu_arena/test.py index 761d5357349cc..7829fddc9f8c6 100755 --- a/tests/integration/test_jemalloc_percpu_arena/test.py +++ b/tests/integration/test_jemalloc_percpu_arena/test.py @@ -52,6 +52,19 @@ def run_with_cpu_limit(cmd, *args): return run_command_in_container(cmd, *args) +def run_with_cpu_possible_limit(cmd, possible, *args): + with NamedTemporaryFile() as possible_cpu: + possible_cpu.write(f"{possible}".encode()) + possible_cpu.flush() + + args += ( + "--volume", + f"{possible_cpu.name}:/sys/devices/system/cpu/possible:ro", + ) + + return run_command_in_container(cmd, *args) + + def skip_if_jemalloc_disabled(): output = run_command_in_container( """clickhouse local -q " @@ -81,28 +94,33 @@ def test_jemalloc_percpu_arena(): all_cpus = int(run_with_cpu_limit("getconf _NPROCESSORS_CONF")) assert all_cpus == multiprocessing.cpu_count(), all_cpus - # implicitly disable percpu arena + # smoke under the online override result = run_with_cpu_limit( 'clickhouse local -q "select 1"', - # NOTE: explicitly disable, since it is enabled by default in debug build - # (and even though debug builds are not in CI let's state this). "--env", "MALLOC_CONF=abort_conf:false", ) assert int(result) == int(1), result - # jemalloc compares sysconf(_SC_NPROCESSORS_ONLN) and sysconf(_SC_NPROCESSORS_CONF) to determine the CPU topology info - # Apparently sometimes _SC_NPROCESSORS_ONLN SOMETIMES ignores the override of /sys/devices/system/cpu/online and it should be equal to - # _SC_NPROCESSORS_CONF - # Latter it also checks sched_getaffinity() + CPU_COUNT(&set), so, we rely also on --cpuset-cpus (taskset) in order to trigger the - # second condition - with pytest.raises(subprocess.CalledProcessError): - # should fail because of abort_conf:true but: - run_with_cpu_limit( - 'clickhouse local -q "select 1"', - "--cpuset-cpus", f"{CPU_ID}", - "--env", "MALLOC_CONF=abort_conf:true", - ) + # Restricted affinity (cpuset) on top of the online override: percpu arenas + # are sized by the possible CPU count (nr_cpu_ids), so this must not abort + # even with abort_conf:true. + # NOTE: abort:false to make it compatible with debug build + result = run_with_cpu_limit( + 'clickhouse local -q "select 1"', + "--cpuset-cpus", f"{CPU_ID}", + "--env", "MALLOC_CONF=abort_conf:true,abort:false", + ) + assert int(result) == int(1), result + + # ... and percpu arena must stay enabled (the old behavior was to disable + # it with "Number of CPUs detected is not deterministic"). + stats = run_with_cpu_limit( + '''clickhouse local -q "select 1" 2>&1 | grep -F "opt.percpu_arena"''', + "--cpuset-cpus", f"{CPU_ID}", + "--env", "MALLOC_CONF=abort_conf:true,abort:false,stats_print:true", + ) + assert b'"percpu"' in stats, stats # should not fail even with abort_conf:true, due to explicit narenas # NOTE: abort:false to make it compatible with debug build @@ -112,6 +130,25 @@ def test_jemalloc_percpu_arena(): ) +# If /sys/devices/system/cpu/possible is masked and does not cover the CPUs in +# the affinity mask, it is lying and percpu arena cannot be sized safely: +# jemalloc must fail close (disable percpu arena, abort with abort_conf:true) +# instead of indexing arenas out of range. +def test_jemalloc_percpu_arena_inconsistent_possible(): + skip_if_jemalloc_disabled() + + assert multiprocessing.cpu_count() > CPU_ID + + with pytest.raises(subprocess.CalledProcessError): + run_with_cpu_possible_limit( + 'clickhouse local -q "select 1"', + "0", + "--cpuset-cpus", f"{CPU_ID}", + "--env", "MALLOC_CONF=abort_conf:true", + ) + + # For manual run. if __name__ == "__main__": test_jemalloc_percpu_arena() + test_jemalloc_percpu_arena_inconsistent_possible() diff --git a/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh b/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh index c1bd1e0e1fa14..f35ad99c1a859 100755 --- a/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh +++ b/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh @@ -9,8 +9,10 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ncpus="$(getconf _NPROCESSORS_ONLN)" -# Disable them explicitly, to avoid failing on "Number of CPUs is not deterministic". -export MALLOC_CONF=abort_conf:false,abort:false +# percpu arenas are sized by the possible CPU count (nr_cpu_ids), so a +# restricted affinity mask must not disable them or abort, even with abort_conf:true. +# NOTE: abort:false for compatibility with debug builds. +export MALLOC_CONF=abort_conf:true,abort:false # Regression for: # @@ -22,6 +24,4 @@ taskset --cpu-list $((ncpus-1)) ${CLICKHOUSE_LOCAL} -q 'select 1' 2>&1 # just in case something more complicated taskset --cpu-list $((ncpus-1)) ${CLICKHOUSE_LOCAL} -q 'select count() from numbers_mt(100000000) settings max_threads=100' 2>&1 -# this command should not fail because we specify narenas explicitly -# (even with abort_conf:true) -MALLOC_CONF=abort_conf:true,abort:false,narenas:$((ncpus)) taskset --cpu-list $((ncpus-1)) ${CLICKHOUSE_LOCAL} -q 'select 1' 2>&1 +MALLOC_CONF=$MALLOC_CONF,narenas:$((ncpus)) taskset --cpu-list $((ncpus-1)) ${CLICKHOUSE_LOCAL} -q 'select 1' 2>&1 diff --git a/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh b/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh index 7fa0bb6f47eb5..2a124fe4c3dab 100755 --- a/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh +++ b/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh @@ -5,9 +5,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -# Avoid failures due to "Number of CPUs is not deterministic" error from jemalloc. -export MALLOC_CONF=abort_conf:false,abort:false - # Regression for UAF in ThreadPool. # (Triggered under TSAN) for _ in {1..10}; do From dd48baa77bf2024527b5a3eec8d513d1e7811c2f Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 6 Nov 2026 15:56:30 +0000 Subject: [PATCH 115/219] Backport #119422 to 26.8: Skip single replica kafka tests on SharedCatalog --- tests/queries/0_stateless/03923_kafka2_keeper_offsets.sh | 2 +- tests/queries/0_stateless/04648_kafka2_race_creation_removal.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/03923_kafka2_keeper_offsets.sh b/tests/queries/0_stateless/03923_kafka2_keeper_offsets.sh index 8621731b831b2..fb6179f956a80 100755 --- a/tests/queries/0_stateless/03923_kafka2_keeper_offsets.sh +++ b/tests/queries/0_stateless/03923_kafka2_keeper_offsets.sh @@ -116,7 +116,7 @@ $CLICKHOUSE_CLIENT --allow_experimental_kafka_offsets_storage_in_keeper 1 -q " kafka_format = 'JSONEachRow', kafka_max_block_size = 100, kafka_keeper_path = '$KEEPER_PATH', - kafka_replica_name = 'r1'; + kafka_replica_name = '{replica}'; " # Create destination table diff --git a/tests/queries/0_stateless/04648_kafka2_race_creation_removal.sh b/tests/queries/0_stateless/04648_kafka2_race_creation_removal.sh index e56b48c954b4a..bc55b6ce957da 100755 --- a/tests/queries/0_stateless/04648_kafka2_race_creation_removal.sh +++ b/tests/queries/0_stateless/04648_kafka2_race_creation_removal.sh @@ -55,7 +55,7 @@ function create_table() kafka_group_name = '${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}', kafka_format = 'RawBLOB', kafka_keeper_path = '${ZK_PATH}', - kafka_replica_name = 'r1' + kafka_replica_name = '{replica}' " } From dad0d70a6f79bbec5569aea0acaa75f9d7a388d6 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 6 Nov 2026 15:34:06 +0000 Subject: [PATCH 116/219] Backport #129139 to 26.8: Reject MySQL hosts that the client library would reinterpret, and check `remote_url_allow_hosts` in the `MySQL` database engine --- src/Common/RemoteHostFilter.cpp | 4 + src/Common/mysqlxx/Connection.cpp | 8 ++ src/Common/mysqlxx/Pool.cpp | 53 +++++++++++++ src/Databases/MySQL/DatabaseMySQL.cpp | 11 ++- src/Dictionaries/MySQLDictionarySource.cpp | 6 +- src/Storages/StorageMySQL.cpp | 2 +- src/TableFunctions/TableFunctionMySQL.cpp | 2 +- .../test_allowed_url_from_config/test.py | 28 +++++++ ...5493_mysql_host_connector_syntax.reference | 13 ++++ .../05493_mysql_host_connector_syntax.sh | 77 +++++++++++++++++++ 10 files changed, 200 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/05493_mysql_host_connector_syntax.reference create mode 100755 tests/queries/0_stateless/05493_mysql_host_connector_syntax.sh diff --git a/src/Common/RemoteHostFilter.cpp b/src/Common/RemoteHostFilter.cpp index 6f953f48d48b0..5b5da5ec7c6b9 100644 --- a/src/Common/RemoteHostFilter.cpp +++ b/src/Common/RemoteHostFilter.cpp @@ -110,6 +110,10 @@ bool RemoteHostFilter::checkForDirectEntry(const std::string & str) const /// Allow everything by default. return true; + /// Connectors take the host as a C string and stop at the first NUL, so the checked string must not contain one. + if (str.contains('\0')) + return false; + std::lock_guard guard(hosts_mutex); if (primary_hosts.contains(str)) diff --git a/src/Common/mysqlxx/Connection.cpp b/src/Common/mysqlxx/Connection.cpp index 3f9accfb97eb0..281f8f33c9be2 100644 --- a/src/Common/mysqlxx/Connection.cpp +++ b/src/Common/mysqlxx/Connection.cpp @@ -120,6 +120,14 @@ void Connection::connect(const char* db, if (mysql_ssl_set(driver.get(), ifNotEmpty(ssl_key), ifNotEmpty(ssl_cert), ifNotEmpty(ssl_ca), nullptr, nullptr)) throw ConnectionFailed(errorMessage(driver.get()), mysql_errno(driver.get())); + /// Without an explicit socket, use TCP to the given port: otherwise `localhost` goes to the library's default socket. + if (!ifNotEmpty(socket)) + { + unsigned protocol = MYSQL_PROTOCOL_TCP; + if (mysql_options(driver.get(), MYSQL_OPT_PROTOCOL, &protocol)) + throw ConnectionFailed(errorMessage(driver.get()), mysql_errno(driver.get())); + } + if (!mysql_real_connect(driver.get(), server, user, password, db, port, ifNotEmpty(socket), driver->client_flag)) throw ConnectionFailed(errorMessage(driver.get()), mysql_errno(driver.get())); diff --git a/src/Common/mysqlxx/Pool.cpp b/src/Common/mysqlxx/Pool.cpp index 4f7c4ac57d171..112f0c370be8e 100644 --- a/src/Common/mysqlxx/Pool.cpp +++ b/src/Common/mysqlxx/Pool.cpp @@ -9,11 +9,61 @@ #include #include #include +#include #include #include #include +namespace DB::ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + +namespace +{ + +/// `mysql_real_connect` reinterprets these hosts and port 0; reject them so it dials exactly what `remote_url_allow_hosts` checked. +void checkConnectionTarget(const std::string & server, unsigned port, const std::string & socket) +{ + std::string_view meaning; + if (server.contains('\0')) + meaning = "the end of the host"; + else if (server.find_first_of(";=") != std::string::npos) + meaning = "a connection string"; + else if (server.contains("://")) + meaning = "a connection plugin URL"; + else if (server.contains(',')) + meaning = "a list of hosts"; + + if (!meaning.empty()) + { + std::string printable; + for (const char c : server) + printable += c == '\0' ? std::string_view("\\0") : std::string_view(&c, 1); + + throw DB::Exception( + DB::ErrorCodes::BAD_ARGUMENTS, + "MySQL host '{}' contains syntax which the MySQL client library would read as {}. " + "Specify a plain host and port, and use the `socket` and `ssl_*` settings instead", + printable, meaning); + } + + if (socket.contains('\0')) + throw DB::Exception( + DB::ErrorCodes::BAD_ARGUMENTS, "MySQL socket for host '{}' contains a NUL byte, where the MySQL client library would end it", server); + + /// The client library uses the socket only for an empty host or `localhost`, otherwise it connects over TCP. + const bool uses_socket = !socket.empty() && (server.empty() || server == "localhost"); + if (port == 0 && !uses_socket) + throw DB::Exception( + DB::ErrorCodes::BAD_ARGUMENTS, + "MySQL port 0 for host '{}' would be replaced by the default port of the MySQL client library. Specify the port explicitly", + server); +} + +} + namespace mysqlxx { @@ -380,6 +430,9 @@ void Pool::initialize() Pool::Connection * Pool::allocConnection(bool dont_throw_if_failed_first_time) { + /// All connections of the pool are created here. + checkConnectionTarget(server, port, socket); + auto conn_ptr = std::make_unique(); try diff --git a/src/Databases/MySQL/DatabaseMySQL.cpp b/src/Databases/MySQL/DatabaseMySQL.cpp index 8c93a25e5800c..5302b8e74c0fb 100644 --- a/src/Databases/MySQL/DatabaseMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMySQL.cpp @@ -39,6 +39,7 @@ # include # include # include +# include # include #if CLICKHOUSE_CLOUD @@ -682,6 +683,14 @@ void registerDatabaseMySQL(DatabaseFactory & factory) configuration.username = safeGetLiteralValue(arguments[2], engine_name); configuration.password = safeGetLiteralValue(arguments[3], engine_name); } + + /// Same policy as the `PostgreSQL` database engine: skip only the server's own metadata replay. + if (!(args.is_metadata_replay && args.mode >= LoadingStrictnessLevel::ATTACH)) + { + for (const auto & address : configuration.addresses) + args.context->getRemoteHostFilter().checkHostAndPort(address.first, toString(address.second)); + } + mysql_settings->loadFromQueryContext(args.context, *engine_define); if (engine_define->settings) mysql_settings->loadFromQuery(*engine_define); @@ -740,7 +749,7 @@ ENGINE = MySQL('host:port', ['database' | database], 'user', 'password') **Engine Parameters** -- `host:port` — MySQL server address. +- `host:port` — MySQL server address. The host must be a plain host name or IP address and is reached over TCP at the given port, also for `localhost`: Unix sockets are not supported. Other host syntax of the MySQL client library, such as connection strings or comma-separated host lists, is rejected. - `database` — Remote database name. - `user` — MySQL user. - `password` — User password. diff --git a/src/Dictionaries/MySQLDictionarySource.cpp b/src/Dictionaries/MySQLDictionarySource.cpp index e18fdb935deef..391d4de233007 100644 --- a/src/Dictionaries/MySQLDictionarySource.cpp +++ b/src/Dictionaries/MySQLDictionarySource.cpp @@ -344,7 +344,11 @@ The `table` or `where` fields cannot be used together with the `query` field. An There is no explicit parameter `secure`. When establishing an SSL-connection security is mandatory. -MySQL can be connected to on a local host via sockets. To do this, set `host` and `socket`. +MySQL can be connected to on a local host via sockets. To do this, set `host` to `localhost` and `socket`. For any other `host`, `socket` is ignored and the connection goes over TCP. Without `socket`, the connection goes over TCP to `port`, also for `localhost`. + + +`remote_url_allow_hosts` checks only `host` and `port`: the path in `socket` is not checked. + Example of settings: diff --git a/src/Storages/StorageMySQL.cpp b/src/Storages/StorageMySQL.cpp index dfe38fcda5f94..46f9e5c462d14 100644 --- a/src/Storages/StorageMySQL.cpp +++ b/src/Storages/StorageMySQL.cpp @@ -682,7 +682,7 @@ The table structure can differ from the original MySQL table structure: **Engine Parameters** -- `host:port` — MySQL server address. +- `host:port` — MySQL server address. The host must be a plain host name or IP address and is reached over TCP at the given port, also for `localhost`: Unix sockets are not supported. Other host syntax of the MySQL client library, such as connection strings or comma-separated host lists, is rejected. - `database` — Remote database name. - `table` — Remote table name, or a query passed to MySQL as is (see [Passing a query instead of a table name](#passing-a-query)). - `user` — MySQL user. diff --git a/src/TableFunctions/TableFunctionMySQL.cpp b/src/TableFunctions/TableFunctionMySQL.cpp index d454a9b0c4f11..681e541f01edf 100644 --- a/src/TableFunctions/TableFunctionMySQL.cpp +++ b/src/TableFunctions/TableFunctionMySQL.cpp @@ -187,7 +187,7 @@ mysql({host:port, database, table, user, password[, replace_query, on_duplicate_ | Argument | Description | |---------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `host:port` | MySQL server address. | +| `host:port` | MySQL server address. The host must be a plain host name or IP address and is reached over TCP at the given port, also for `localhost`: Unix sockets are not supported. Other host syntax of the MySQL client library, such as connection strings or comma-separated host lists, is rejected. | | `database` | Remote database name. | | `table` | Remote table name, or a query passed to MySQL as is (see [Passing a query instead of a table name](#passing-a-query)). | | `user` | MySQL user. | diff --git a/tests/integration/test_allowed_url_from_config/test.py b/tests/integration/test_allowed_url_from_config/test.py index d30410845a40f..b1786b7aa8011 100644 --- a/tests/integration/test_allowed_url_from_config/test.py +++ b/tests/integration/test_allowed_url_from_config/test.py @@ -263,6 +263,34 @@ def test_mysql_dictionary_host_filter(start_cluster): node5.query("DROP DICTIONARY IF EXISTS default.test_mysql_host_filter") +def test_mysql_database_host_filter(start_cluster): + # node5 denies all hosts; the check runs before any connection attempt. + assert "not allowed" in node5.query_and_get_error( + "CREATE DATABASE test_mysql_db_filter ENGINE = MySQL('10.0.0.1:3306', 'db', 'u', 'p')" + ) + node5.query( + "CREATE NAMED COLLECTION IF NOT EXISTS test_mysql_db_filter_nc AS " + "host = '10.0.0.1', port = 3306, database = 'db', user = 'u', password = 'p'" + ) + assert "not allowed" in node5.query_and_get_error( + "CREATE DATABASE test_mysql_db_filter ENGINE = MySQL(test_mysql_db_filter_nc)" + ) + node5.query("DROP NAMED COLLECTION test_mysql_db_filter_nc") + + # node1's suffix rule would match the whole string, but the connector stops at the NUL; reject it. + assert "not allowed" in node1.query_and_get_error( + "CREATE DATABASE test_mysql_db_filter ENGINE = MySQL('127.0.0.1\\0.allowed.example:1', 'db', 'u', 'p')" + ) + node1.query( + "CREATE NAMED COLLECTION IF NOT EXISTS test_mysql_db_filter_nul_nc AS " + "host = '127.0.0.1\\0.allowed.example', port = 1, database = 'db', user = 'u', password = 'p'" + ) + assert "not allowed" in node1.query_and_get_error( + "CREATE DATABASE test_mysql_db_filter ENGINE = MySQL(test_mysql_db_filter_nul_nc)" + ) + node1.query("DROP NAMED COLLECTION test_mysql_db_filter_nul_nc") + + def test_mysql_dictionary_replica_host_filter(start_cluster): """MySQL dictionary source with REPLICA(...) syntax should respect RemoteHostFilter.""" # Covers the replica branch: if (config.has(settings_config_prefix + ".replica")) diff --git a/tests/queries/0_stateless/05493_mysql_host_connector_syntax.reference b/tests/queries/0_stateless/05493_mysql_host_connector_syntax.reference new file mode 100644 index 0000000000000..60ef3fadf28cc --- /dev/null +++ b/tests/queries/0_stateless/05493_mysql_host_connector_syntax.reference @@ -0,0 +1,13 @@ +which the MySQL client library would read as a connection string +which the MySQL client library would read as a connection string +which the MySQL client library would read as a list of hosts +which the MySQL client library would read as a connection plugin URL +which the MySQL client library would read as the end of the host +connection failed +port 0: rejected +dictionary 127.0.0.1 port 0: rejected +dictionary localhost port 0: unix socket +dictionary localhost port 0: rejected +localhost: tcp +which the MySQL client library would read as a connection string +which the MySQL client library would read as a connection string diff --git a/tests/queries/0_stateless/05493_mysql_host_connector_syntax.sh b/tests/queries/0_stateless/05493_mysql_host_connector_syntax.sh new file mode 100755 index 0000000000000..bc24f80c96227 --- /dev/null +++ b/tests/queries/0_stateless/05493_mysql_host_connector_syntax.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: the MySQL client library is not built in the fast test + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The MySQL client library reinterprets some host strings: a connection string (which can set a socket, a plugin +# directory, TLS files), a connection plugin URL, a list of hosts, or a host cut at a NUL. They must be rejected +# before any connection attempt. + +# Prints how the host was rejected, or "connection failed" for an ordinary connection failure. +function check() +{ + local output + output=$($CLICKHOUSE_CLIENT --send_logs_level=fatal -q "$1" 2>&1 | tr -d '\000') + if grep -q -F "which the MySQL client library would read as" <<< "$output"; then + grep -o -m1 "which the MySQL client library would read as [a-zA-Z ]*" <<< "$output" + elif grep -q -F "ALL_CONNECTION_TRIES_FAILED" <<< "$output"; then + echo "connection failed" + else + echo "unexpected output: $output" + fi +} + +check "SELECT * FROM mysql('socket=/nonexistent/mysqld.sock', 'db', 't', 'u', 'p')" +check "SELECT * FROM mysql('host=localhost;socket=/nonexistent/mysqld.sock', 'db', 't', 'u', 'p')" +check "SELECT * FROM mysql('127.0.0.1,localhost:1', 'db', 't', 'u', 'p')" +check "SELECT * FROM mysql('[replication://127.0.0.1]:1', 'db', 't', 'u', 'p')" +check "SELECT * FROM mysql('127.0.0.1\\0.example.com:1', 'db', 't', 'u', 'p')" + +# A plain host is passed through and fails to connect as usual. +check "SELECT * FROM mysql('127.0.0.1:1', 'db', 't', 'u', 'p')" + +# Port 0 would be replaced by the client library's default port. +$CLICKHOUSE_CLIENT --send_logs_level=fatal -q "SELECT * FROM mysql('127.0.0.1:0', 'db', 't', 'u', 'p')" 2>&1 \ + | grep -q -F "MySQL port 0" && echo "port 0: rejected" || echo "port 0: connection failed" + +# A dictionary socket is used only for `localhost`; for another host it is ignored and port 0 is rejected. +function check_dictionary() +{ + $CLICKHOUSE_CLIENT --send_logs_level=fatal -q " + CREATE DICTIONARY ${CLICKHOUSE_DATABASE}.d_mysql (x UInt64, y String) PRIMARY KEY x + SOURCE(MYSQL(HOST '$1' PORT 0 SOCKET '$2' USER 'u' PASSWORD 'p' DB 'db' TABLE 't')) + LAYOUT(FLAT()) LIFETIME(0)" + local output + output=$($CLICKHOUSE_CLIENT --send_logs_level=fatal -q "SELECT * FROM ${CLICKHOUSE_DATABASE}.d_mysql" 2>&1) + if grep -q -E "MySQL port 0|MySQL socket" <<< "$output"; then + echo "dictionary $1 port 0: rejected" + elif grep -q -F "through socket" <<< "$output"; then + echo "dictionary $1 port 0: unix socket" + else + echo "unexpected output: $output" + fi + $CLICKHOUSE_CLIENT -q "DROP DICTIONARY IF EXISTS ${CLICKHOUSE_DATABASE}.d_mysql" +} +check_dictionary "127.0.0.1" "/nonexistent/mysqld.sock" +check_dictionary "localhost" "/nonexistent/mysqld.sock" +# The client library would read a socket starting with NUL as no socket and connect over TCP. +check_dictionary "localhost" "\\0" + +# `localhost` uses TCP to the given port, not the client library's default Unix socket. +output=$($CLICKHOUSE_CLIENT --send_logs_level=fatal -q "SELECT * FROM mysql('localhost:1', 'db', 't', 'u', 'p')" 2>&1) +if grep -q -F "Can't connect to server on 'localhost'" <<< "$output"; then + echo "localhost: tcp" +elif grep -q -F "through socket" <<< "$output"; then + echo "localhost: unix socket" +else + echo "unexpected output: $output" +fi + +# The `MySQL` database engine and table engine use the same connection pool. +check "CREATE DATABASE ${CLICKHOUSE_DATABASE}_mysql ENGINE = MySQL('socket=/nonexistent/mysqld.sock', 'db', 'u', 'p')" +check "CREATE TABLE ${CLICKHOUSE_DATABASE}.t_mysql (x UInt32) ENGINE = MySQL('socket=/nonexistent/mysqld.sock', 'db', 't', 'u', 'p'); SELECT * FROM ${CLICKHOUSE_DATABASE}.t_mysql" + +$CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS ${CLICKHOUSE_DATABASE}_mysql; DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.t_mysql; DROP DICTIONARY IF EXISTS ${CLICKHOUSE_DATABASE}.d_mysql" From d082a80969f78a09b81ea14ded738248fb75835a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 6 Nov 2026 17:26:07 +0000 Subject: [PATCH 117/219] Backport #128951 to 26.8: Ignore skip index archives with a newer format version instead of throwing --- .../MergeTree/DataPartStorageOnDiskBase.cpp | 31 ++++++++++- .../MergeTree/DataPartStorageOnDiskBase.h | 10 ++++ .../MergeTree/DataPartStorageOnDiskPacked.cpp | 8 ++- src/Storages/MergeTree/MutateTask.cpp | 4 +- ...dex_archive_newer_format_version.reference | 7 +++ ...skip_index_archive_newer_format_version.sh | 53 +++++++++++++++++++ 6 files changed, 109 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/05493_skip_index_archive_newer_format_version.reference create mode 100755 tests/queries/0_stateless/05493_skip_index_archive_newer_format_version.sh diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp index 37a9ad04aa93f..0d0896b98830e 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp @@ -252,6 +252,7 @@ void DataPartStorageOnDiskBase::setRelativePath(const std::string & path) std::lock_guard lock(skip_indices_packed_mutex); skip_indices_packed_probed = false; skip_indices_packed_reader.reset(); + skip_indices_packed_unsupported = false; } } @@ -1218,7 +1219,10 @@ std::shared_ptr DataPartStorageOnDiskBase::getSkipIndic Expect404ResponseScope scope; try { - skip_indices_packed_reader = std::make_shared(disk, packed_path, ReadSettings{}); + auto buf = disk->readFile(packed_path, ReadSettings{}.adjustBufferSize(4096)); + auto index = readSkipIndicesPackedIndex(*buf, packed_path); + skip_indices_packed_unsupported = !index; + skip_indices_packed_reader = std::make_shared(index ? std::move(*index) : PackedFilesIO::Index{}); } catch (const Exception &) { @@ -1232,6 +1236,23 @@ std::shared_ptr DataPartStorageOnDiskBase::getSkipIndic return skip_indices_packed_reader; } +std::optional DataPartStorageOnDiskBase::readSkipIndicesPackedIndex(ReadBuffer & in, const String & archive_path) +{ + /// An empty buffer is left to readIndex, which reports the truncated archive. + if (!in.eof()) + { + const auto version = static_cast(*in.position()); + if (version > PackedFilesIO::MAX_VERSION) + { + LOG_DEBUG(getLogger("DataPartStorageOnDiskBase"), + "Skip indices archive {} has format version {}, newer than the supported {}. Skip indices stored in it will not be used", + archive_path, static_cast(version), static_cast(PackedFilesIO::MAX_VERSION)); + return std::nullopt; + } + } + return PackedFilesReader::readIndex(in); +} + void DataPartStorageOnDiskBase::seedSkipIndicesPackedReader(const PackedFilesIO::Index & index) const { std::lock_guard lock(skip_indices_packed_mutex); @@ -1265,6 +1286,14 @@ bool DataPartStorageOnDiskBase::hasSkipIndicesPackedArchive() const return getSkipIndicesPackedReader() != nullptr; } +bool DataPartStorageOnDiskBase::hasUnsupportedSkipIndicesPackedArchive() const +{ + if (!getSkipIndicesPackedReader()) + return false; + std::lock_guard lock(skip_indices_packed_mutex); + return skip_indices_packed_unsupported; +} + void DataPartStorageOnDiskBase::copyArchiveEntryTo( const PackedFilesReader & source_archive, const String & file_name, diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskBase.h b/src/Storages/MergeTree/DataPartStorageOnDiskBase.h index bceb343c342db..627292c787fb4 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDiskBase.h +++ b/src/Storages/MergeTree/DataPartStorageOnDiskBase.h @@ -79,6 +79,10 @@ class DataPartStorageOnDiskBase : public IDataPartStorage /// to know whether the archive must be rebuilt/excluded from hardlinks. bool hasSkipIndicesPackedArchive() const; + /// True iff skp_idx.packed has a newer format version than this server reads. Its skip indices + /// are not used and cannot be kept up to date, so a mutated part must not inherit the archive. + bool hasUnsupportedSkipIndicesPackedArchive() const; + /// Copy the named virtual files from this storage's skp_idx.packed archive into @target so /// the writer can ship a complete archive after also writing fresh recalc'd entries. Used by /// mutations to preserve surviving in-archive indices that aren't being recomputed when the @@ -197,6 +201,10 @@ class DataPartStorageOnDiskBase : public IDataPartStorage /// so the disk-probe default always misses. virtual std::shared_ptr getSkipIndicesPackedReader() const; + /// Reads the index of a skp_idx.packed archive. Returns nullopt for an archive written in a newer + /// format version: its skip indices are not used, while the part itself stays readable. + static std::optional readSkipIndicesPackedIndex(ReadBuffer & in, const String & archive_path); + /// Cheap pre-filtered lookup for the file-read overlay: returns the archive reader only when /// @name is a "skp_idx_..." substream that the archive actually contains, else nullptr. The /// prefix gate keeps unrelated files (checksums.txt, count.txt, columns.txt, ...) from loading @@ -245,6 +253,8 @@ class DataPartStorageOnDiskBase : public IDataPartStorage mutable std::mutex skip_indices_packed_mutex; mutable bool skip_indices_packed_probed TSA_GUARDED_BY(skip_indices_packed_mutex) = false; mutable std::shared_ptr skip_indices_packed_reader TSA_GUARDED_BY(skip_indices_packed_mutex); + /// The archive exists but has an unsupported format version; the reader then has an empty index. + mutable bool skip_indices_packed_unsupported TSA_GUARDED_BY(skip_indices_packed_mutex) = false; template void executeWriteOperation(Op && op) diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskPacked.cpp b/src/Storages/MergeTree/DataPartStorageOnDiskPacked.cpp index d6f8debe5bf2d..d5a182f2c6d0f 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDiskPacked.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDiskPacked.cpp @@ -696,8 +696,12 @@ std::shared_ptr DataPartStorageOnDiskPacked::getSkipInd { auto inner_archive_buf = reader->readFile( volume->getDisk(), data_path, String(SKIP_INDICES_PACKED_FILENAME), getReadSettings(), std::nullopt); - auto inner_index = PackedFilesReader::readIndex(*inner_archive_buf); - seedSkipIndicesPackedReader(inner_index); + auto index = readSkipIndicesPackedIndex(*inner_archive_buf, data_path + "/" + String(SKIP_INDICES_PACKED_FILENAME)); + { + std::lock_guard lock(skip_indices_packed_mutex); + skip_indices_packed_unsupported = !index; + } + seedSkipIndicesPackedReader(index ? *index : PackedFilesIO::Index{}); } catch (const Exception &) { diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index cead60e122d59..80ed02c63cdf0 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -4014,8 +4014,10 @@ void updateIndicesToRecalculateAndDrop(std::shared_ptr & ctx) (*ctx->data->getSettings())[MergeTreeSetting::packed_skip_index_max_bytes] > 0 && !ctx->indices_to_recalc.empty(); + /// An unsupported archive cannot be updated, so it would keep granules of the old data. bool archive_dirty = !ctx->dropped_skip_index_archive_file_names.empty() - || (source_has_archive && writer_can_open_archive); + || (source_has_archive && writer_can_open_archive) + || source_disk_storage->hasUnsupportedSkipIndicesPackedArchive(); if (!archive_dirty) for (const auto & idx : ctx->indices_to_recalc) if (source_part->isSkipIndexInPackedArchive(*idx)) { archive_dirty = true; break; } diff --git a/tests/queries/0_stateless/05493_skip_index_archive_newer_format_version.reference b/tests/queries/0_stateless/05493_skip_index_archive_newer_format_version.reference new file mode 100644 index 0000000000000..8186bcc258c48 --- /dev/null +++ b/tests/queries/0_stateless/05493_skip_index_archive_newer_format_version.reference @@ -0,0 +1,7 @@ +read and merge +1000 +500 +1 +mutation, then upgrade +1 +1 diff --git a/tests/queries/0_stateless/05493_skip_index_archive_newer_format_version.sh b/tests/queries/0_stateless/05493_skip_index_archive_newer_format_version.sh new file mode 100755 index 0000000000000..c8f25f150af50 --- /dev/null +++ b/tests/queries/0_stateless/05493_skip_index_archive_newer_format_version.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +# A part whose skip index archive skp_idx.packed was written in a newer, unknown format version +# (as after a rollback to an older server) stays readable, mergeable and mutable; only its skip indices are not used. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +data_path="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_newer_archive" +rm -rf "$data_path" +trap 'rm -rf "$data_path"' EXIT + +# Creates a table with one part that has skp_idx.packed and replaces the archive's format version (its first byte) with 0xff. +function create_part_with_newer_archive() +{ + rm -rf "$data_path" + ${CLICKHOUSE_LOCAL} --path "$data_path" --query " + CREATE DATABASE d ENGINE = Atomic; + CREATE TABLE d.t (id UInt64, v UInt64, INDEX i v TYPE minmax GRANULARITY 1) + ENGINE = MergeTree ORDER BY id + SETTINGS packed_skip_index_max_bytes = '1M', index_granularity = 100, $1 + min_bytes_for_full_part_storage = 0, min_rows_for_full_part_storage = 0, + min_level_for_full_part_storage = 0; + INSERT INTO d.t SELECT number, number FROM numbers(1000); + " + archives=("$data_path"/store/*/*/all_1_1_0/skp_idx.packed) + test -f "${archives[0]}" || { echo "no skp_idx.packed"; exit 1; } + original_version=$(head -c 1 "${archives[0]}" | od -An -tx1 | tr -d ' \n') + printf '\xff' | dd of="${archives[0]}" bs=1 count=1 conv=notrunc 2>/dev/null +} + +echo "read and merge" +create_part_with_newer_archive "" +${CLICKHOUSE_LOCAL} --path "$data_path" --query " + SELECT count() FROM d.t; + SELECT id FROM d.t WHERE v = 500; + INSERT INTO d.t SELECT number, number FROM numbers(1000, 10); + OPTIMIZE TABLE d.t FINAL; + SELECT count() FROM d.t WHERE v = 500 SETTINGS force_data_skipping_indices = 'i'; +" + +# The mutated part must not keep the old archive: after an upgrade it would be readable again but describe the old values. +echo "mutation, then upgrade" +create_part_with_newer_archive "min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0," +${CLICKHOUSE_LOCAL} --path "$data_path" --query "ALTER TABLE d.t UPDATE v = 1000000 WHERE id = 500 SETTINGS mutations_sync = 2" +for archive in "$data_path"/store/*/*/all_*/skp_idx.packed; do + test -f "$archive" && printf "\\x${original_version}" | dd of="$archive" bs=1 count=1 conv=notrunc 2>/dev/null +done +${CLICKHOUSE_LOCAL} --path "$data_path" --query " + SELECT count() FROM d.t WHERE v = 1000000; + SELECT count() FROM d.t WHERE v = 1000000 SETTINGS use_skip_indexes = 0; +" From 10d0bdd94b82736d0fe48f273dfb0e080427342e Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 6 Nov 2026 17:37:07 +0000 Subject: [PATCH 118/219] Backport #129529 to 26.8: Fix unbounded JIT memory growth with distinct literals --- src/Interpreters/InterpreterSystemQuery.cpp | 3 +- src/Interpreters/JIT/CHJIT.cpp | 9 ++++-- src/Interpreters/JIT/CHJIT.h | 6 ++-- ..._not_grow_with_distinct_literals.reference | 1 + ...xt_does_not_grow_with_distinct_literals.sh | 31 +++++++++++++++++++ 5 files changed, 42 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/05509_jit_llvm_context_does_not_grow_with_distinct_literals.reference create mode 100755 tests/queries/0_stateless/05509_jit_llvm_context_does_not_grow_with_distinct_literals.sh diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index a8b7f75ad33da..a152a634c05d3 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -574,8 +574,7 @@ BlockIO InterpreterSystemQuery::execute() getContext()->checkAccess(AccessType::SYSTEM_DROP_COMPILED_EXPRESSION_CACHE); if (auto * cache = CompiledExpressionCacheFactory::instance().tryGetCache()) cache->clear(); - /// Drop the static CHJIT slots so persistent LLVM state (`TargetMachine`, `Subtarget`, - /// `LLVMContext`-interned types/constants accumulated across compiles) is reclaimed too. + /// Drop the static CHJIT slots so persistent LLVM state (`TargetMachine`, `Subtarget`) is reclaimed too. /// Each in-flight compile and each cache holder retains its own `shared_ptr`, so /// concurrent operations are not affected: the actual CHJIT survives until every user /// has released its handle. After the cache->clear() above, the only remaining diff --git a/src/Interpreters/JIT/CHJIT.cpp b/src/Interpreters/JIT/CHJIT.cpp index 1f26d8e61d4b1..ace15933eb553 100644 --- a/src/Interpreters/JIT/CHJIT.cpp +++ b/src/Interpreters/JIT/CHJIT.cpp @@ -481,6 +481,11 @@ CHJIT::CompiledModule CHJIT::compileModule(std::function { std::lock_guard lock(jit_lock); + /// A fresh context per compile: `LLVMContext` never frees the constants it interns, and the optimizer + /// creates new ones for every distinct literal, so a long-lived context grows without bound. + /// Declared before `module`, which must be destroyed first. + auto context = JITAllocationScope::run([] { return std::make_unique(); }); + /// LLVM-touching work is wrapped in `JITAllocationScope` so its allocations land in the JIT /// arena. Caller-side ClickHouse code that runs between LLVM calls (error handling further /// down, symbol-name strings, result-map population) is intentionally left outside the scopes @@ -491,7 +496,7 @@ CHJIT::CompiledModule CHJIT::compileModule(std::function /// belong in the JIT arena alongside the module itself. auto module = JITAllocationScope::run([&] { - auto new_module = createModuleForCompilation(); + auto new_module = createModuleForCompilation(*context); compile_function(*new_module); return new_module; }); @@ -502,7 +507,7 @@ CHJIT::CompiledModule CHJIT::compileModule(std::function return module_info; } -std::unique_ptr CHJIT::createModuleForCompilation() +std::unique_ptr CHJIT::createModuleForCompilation(llvm::LLVMContext & context) { std::unique_ptr module = std::make_unique("jit" + std::to_string(current_module_key), context); module->setDataLayout(layout); diff --git a/src/Interpreters/JIT/CHJIT.h b/src/Interpreters/JIT/CHJIT.h index 3646451f6abf1..9296b100bb97c 100644 --- a/src/Interpreters/JIT/CHJIT.h +++ b/src/Interpreters/JIT/CHJIT.h @@ -89,7 +89,7 @@ class CHJIT private: - std::unique_ptr createModuleForCompilation(); + std::unique_ptr createModuleForCompilation(llvm::LLVMContext & context); CompiledModule compileModule(std::unique_ptr module); @@ -99,7 +99,6 @@ class CHJIT static std::unique_ptr getTargetMachine(); - llvm::LLVMContext context; std::unique_ptr machine; llvm::DataLayout layout; std::unique_ptr compiler; @@ -114,8 +113,7 @@ class CHJIT /// Drop the static CHJIT instances used for compiled expressions and compiled aggregate functions. /// On the next compile request, fresh instances are constructed (paying the one-time -/// `TargetMachine`/`Subtarget`/`LLVMContext` initialization cost again, ~few MiB), reclaiming any -/// LLVMContext-retained type/constant uniquing accumulated by previous compilations. +/// `TargetMachine`/`Subtarget` initialization cost again, ~few MiB). /// /// Concurrency: each in-flight compile and each cache holder retains its own `shared_ptr`, /// so the actual instance survives a reset until every user has released its handle. The reset diff --git a/tests/queries/0_stateless/05509_jit_llvm_context_does_not_grow_with_distinct_literals.reference b/tests/queries/0_stateless/05509_jit_llvm_context_does_not_grow_with_distinct_literals.reference new file mode 100644 index 0000000000000..9766475a4185a --- /dev/null +++ b/tests/queries/0_stateless/05509_jit_llvm_context_does_not_grow_with_distinct_literals.reference @@ -0,0 +1 @@ +ok diff --git a/tests/queries/0_stateless/05509_jit_llvm_context_does_not_grow_with_distinct_literals.sh b/tests/queries/0_stateless/05509_jit_llvm_context_does_not_grow_with_distinct_literals.sh new file mode 100755 index 0000000000000..91766cd82d837 --- /dev/null +++ b/tests/queries/0_stateless/05509_jit_llvm_context_does_not_grow_with_distinct_literals.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, use_jemalloc +# no-fasttest: requires USE_EMBEDDED_COMPILER, which the fast-test image disables. +# use_jemalloc: measures the dedicated jemalloc arena for JIT allocations. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Every query compiles a filter with different literals. The compiled expression cache holds one entry, +# so the memory used by JIT compilation must stay flat instead of growing with each compiled query. + +JIT_ARENA_BYTES="SELECT sum(curregs * size) FROM system.jemalloc_arena_bins WHERE purpose = 'jit' FORMAT TSV;" + +{ + echo "CREATE TABLE t (key UInt64, value UInt64) ENGINE = Memory;" + echo "INSERT INTO t SELECT number, number FROM numbers(1000);" + for i in $(seq 1 4); do + printf "SELECT count() FROM t WHERE " + for j in $(seq 1 150); do + [ "$j" -gt 1 ] && printf " OR " + printf "(key = %d AND value > %d)" $((i * 100000 + j)) $((i * 100000 + j * 7)) + done + echo " SETTINGS compile_expressions = 1, min_count_to_compile_expression = 0 FORMAT Null;" + [ "$i" -eq 1 ] && echo "$JIT_ARENA_BYTES" + done + echo "$JIT_ARENA_BYTES" +} > "${CLICKHOUSE_TMP}/queries.sql" + +$CLICKHOUSE_LOCAL --queries-file "${CLICKHOUSE_TMP}/queries.sql" -- --compiled_expression_cache_elements_size=1 < /dev/null \ + | tr '\n' ' ' | awk '{ growth = $2 - $1; print (growth < 2000000 ? "ok" : "JIT arena grew by " growth " bytes") }' From c3dbb8d976f7199a26cb284ea73ae37f011ac483 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 6 Nov 2026 17:32:24 +0000 Subject: [PATCH 119/219] Backport #120582 to 26.8: Require `READ ON FILE` for `ATTACH TABLE ... FROM` a directory in `user_files` --- src/Interpreters/InterpreterCreateQuery.cpp | 7 ++ ...ble_from_path_read_on_file_grant.reference | 7 ++ ...tach_table_from_path_read_on_file_grant.sh | 67 +++++++++++++++++++ 3 files changed, 81 insertions(+) create mode 100644 tests/queries/0_stateless/05218_attach_table_from_path_read_on_file_grant.reference create mode 100755 tests/queries/0_stateless/05218_attach_table_from_path_read_on_file_grant.sh diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 10d1aec703852..afc133008a0d2 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -3672,6 +3672,13 @@ AccessRightsElements InterpreterCreateQuery::getRequiredAccess() const /// Settings that send the table's data elsewhere, e.g. the move destination of a queue, need the grants of that place. required_access.append_range(StorageFactory::instance().getExportAccess(create)); + /// `ATTACH TABLE ... FROM ''` reads a directory in `user_files` as the table data and then moves it to the + /// data path of the new table. Reading it needs `READ` on the FILE source like the `file` function does, and + /// taking it away from `user_files` needs `WRITE`, like renaming files after processing does. Without this, + /// `CREATE TABLE` on a table of their own would let a user read or consume any data staged in `user_files`. + if (create.has_attach_from_path) + required_access.emplace_back(AccessType::READ | AccessType::WRITE, toStringSource(AccessTypeObjects::Source::FILE)); + return required_access; } diff --git a/tests/queries/0_stateless/05218_attach_table_from_path_read_on_file_grant.reference b/tests/queries/0_stateless/05218_attach_table_from_path_read_on_file_grant.reference new file mode 100644 index 0000000000000..06604339ef491 --- /dev/null +++ b/tests/queries/0_stateless/05218_attach_table_from_path_read_on_file_grant.reference @@ -0,0 +1,7 @@ +ACCESS_DENIED: necessary to have the grant READ, WRITE ON FILE +tables named stolen: 0 +staged directory is still in place +ACCESS_DENIED: Missing permissions: WRITE ON FILE +tables named stolen: 0 +staged directory is still in place +424242 diff --git a/tests/queries/0_stateless/05218_attach_table_from_path_read_on_file_grant.sh b/tests/queries/0_stateless/05218_attach_table_from_path_read_on_file_grant.sh new file mode 100755 index 0000000000000..1fa58284b9aa2 --- /dev/null +++ b/tests/queries/0_stateless/05218_attach_table_from_path_read_on_file_grant.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Tags: no-object-storage, no-replicated-database +# no-object-storage: the query attaches a directory on the local disk, and there the table data lives on +# object storage. +# no-replicated-database: a Replicated database enqueues the query before the path is resolved against +# `user_files`, and the DDL worker then rejects the path as outside of `user_files`. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `ATTACH TABLE ... FROM ''` reads a directory in `user_files` as the data of the new table and moves it to +# the data path of the table, so it needs `READ ON FILE` like the `file` function and `WRITE ON FILE` like +# renaming files after processing. Without that check `CREATE TABLE` on a table of their own is enough for a +# user to read and consume data that somebody else staged in `user_files`. + +user="user_${CLICKHOUSE_TEST_UNIQUE_NAME}" +staged_root="${CLICKHOUSE_TEST_UNIQUE_NAME}_staged" +staged_table_dir="${staged_root}/data/staging/protected" + +# Stage a MergeTree table directory in `user_files` with clickhouse-local, as an admin or a migration +# workflow would do. An Ordinary database keeps the table directory at a known place. +${CLICKHOUSE_LOCAL} --path "${USER_FILES_PATH}/${staged_root}" --allow_deprecated_database_ordinary=1 --query " + CREATE DATABASE staging ENGINE = Ordinary; + CREATE TABLE staging.protected (id UInt64, secret UInt64) ENGINE = MergeTree ORDER BY id; + INSERT INTO staging.protected VALUES (1, 424242); +" + +${CLICKHOUSE_CLIENT} <&1 | grep -qF "$1"; then + echo "ACCESS_DENIED: $1" + else + echo "UNEXPECTED: the attach was not denied with: $1" + fi + ${CLICKHOUSE_CLIENT} --query "SELECT 'tables named stolen:', count() FROM system.tables WHERE database = currentDatabase() AND name = 'stolen'" + if [ -d "${USER_FILES_PATH}/${staged_table_dir}" ]; then + echo "staged directory is still in place" + else + echo "UNEXPECTED: the staged directory is gone" + fi +} +check_denied_and_untouched "necessary to have the grant READ, WRITE ON FILE" +${CLICKHOUSE_CLIENT} --query "GRANT READ ON FILE TO $user" +check_denied_and_untouched "Missing permissions: WRITE ON FILE" + +# With both grants the same query attaches the data. +${CLICKHOUSE_CLIENT} --query "GRANT WRITE ON FILE TO $user" +${CLICKHOUSE_CLIENT} --user "$user" --query "$attach_query" +${CLICKHOUSE_CLIENT} --user "$user" --query "SELECT secret FROM stolen" + +${CLICKHOUSE_CLIENT} < Date: Fri, 6 Nov 2026 22:21:50 +0000 Subject: [PATCH 120/219] Backport #129336 to 26.8: Fix LOGICAL_ERROR on invalidate_query of a remote ClickHouse dictionary source --- .../ClickHouseDictionarySource.cpp | 9 ++-- src/Dictionaries/readInvalidateQuery.cpp | 33 ++++++++++-- src/Dictionaries/readInvalidateQuery.h | 5 ++ ...mote_clickhouse_invalidate_query.reference | 4 ++ ...nary_remote_clickhouse_invalidate_query.sh | 52 +++++++++++++++++++ 5 files changed, 94 insertions(+), 9 deletions(-) create mode 100644 tests/queries/0_stateless/05457_dictionary_remote_clickhouse_invalidate_query.reference create mode 100755 tests/queries/0_stateless/05457_dictionary_remote_clickhouse_invalidate_query.sh diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index 250a3a504f3de..12a897ba958da 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -241,11 +241,10 @@ std::string ClickHouseDictionarySource::doInvalidateQuery(const std::string & re return result; } - /// We pass empty block to RemoteQueryExecutor, because we don't know the structure of the result. - auto invalidate_sample_block = std::make_shared(Block{}); - QueryPipeline pipeline(std::make_shared( - std::make_shared(pool, request, invalidate_sample_block, context_copy), false, false, false)); - return readInvalidateQuery(pipeline); + /// The structure of the result is not known in advance, so the executor gets an empty header + /// and is read directly, not through a `RemoteSource`. + RemoteQueryExecutor executor(pool, request, std::make_shared(Block{}), context_copy); + return readInvalidateQuery(executor); } void registerDictionarySourceClickHouse(DictionarySourceFactory & factory); diff --git a/src/Dictionaries/readInvalidateQuery.cpp b/src/Dictionaries/readInvalidateQuery.cpp index a4aaa63cc84fd..00a4107733719 100644 --- a/src/Dictionaries/readInvalidateQuery.cpp +++ b/src/Dictionaries/readInvalidateQuery.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -19,12 +20,15 @@ namespace ErrorCodes extern const int RECEIVED_EMPTY_DATA; } -std::string readInvalidateQuery(QueryPipeline & pipeline) +namespace { - PullingPipelineExecutor executor(pipeline); +/// `pull` fills the next block of the result and returns false when there are no more blocks. +template +std::string readInvalidateQueryImpl(Pull && pull) +{ Block block; - while (executor.pull(block)) + while (pull(block)) if (!block.empty()) break; @@ -45,7 +49,7 @@ std::string readInvalidateQuery(QueryPipeline & pipeline) auto & column_type = block.getByPosition(0); column_type.type->getDefaultSerialization()->serializeTextQuoted(*column_type.column->convertToFullColumnIfConst(), 0, out, FormatSettings()); - while (executor.pull(block)) + while (pull(block)) if (block.rows() > 0) throw Exception(ErrorCodes::TOO_MANY_ROWS, "Expected single row in resultset, got at least {}", rows + 1); @@ -53,3 +57,24 @@ std::string readInvalidateQuery(QueryPipeline & pipeline) } } + +std::string readInvalidateQuery(QueryPipeline & pipeline) +{ + PullingPipelineExecutor executor(pipeline); + return readInvalidateQueryImpl([&](Block & block) { return executor.pull(block); }); +} + +std::string readInvalidateQuery(RemoteQueryExecutor & executor) +{ + executor.sendQuery(); + auto result = readInvalidateQueryImpl([&](Block & block) + { + /// An empty block marks the end of the stream. + block = executor.readBlock(); + return !block.empty(); + }); + executor.finish(); + return result; +} + +} diff --git a/src/Dictionaries/readInvalidateQuery.h b/src/Dictionaries/readInvalidateQuery.h index 99e20ad54e4b6..8ad34982f28e5 100644 --- a/src/Dictionaries/readInvalidateQuery.h +++ b/src/Dictionaries/readInvalidateQuery.h @@ -6,8 +6,13 @@ namespace DB { class QueryPipeline; +class RemoteQueryExecutor; /// Using in MySQLDictionarySource and XDBCDictionarySource after processing invalidate_query. std::string readInvalidateQuery(QueryPipeline & pipeline); +/// For a remote query whose result structure is not known in advance, so it cannot be a pipeline source: +/// a port with an empty header does not accept the received columns. +std::string readInvalidateQuery(RemoteQueryExecutor & executor); + } diff --git a/tests/queries/0_stateless/05457_dictionary_remote_clickhouse_invalidate_query.reference b/tests/queries/0_stateless/05457_dictionary_remote_clickhouse_invalidate_query.reference new file mode 100644 index 0000000000000..e443bdf2664a0 --- /dev/null +++ b/tests/queries/0_stateless/05457_dictionary_remote_clickhouse_invalidate_query.reference @@ -0,0 +1,4 @@ +a +0 +b c + diff --git a/tests/queries/0_stateless/05457_dictionary_remote_clickhouse_invalidate_query.sh b/tests/queries/0_stateless/05457_dictionary_remote_clickhouse_invalidate_query.sh new file mode 100755 index 0000000000000..59efc91c52fff --- /dev/null +++ b/tests/queries/0_stateless/05457_dictionary_remote_clickhouse_invalidate_query.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# 127.0.0.2 is not a local address for the dictionary source, so the invalidate query is sent through +# RemoteQueryExecutor, like for a source on another server. Its result structure is unknown in advance. + +$CLICKHOUSE_CLIENT --query " +CREATE TABLE dict_src (id UInt64, val String, upd DateTime) ENGINE = Memory; +INSERT INTO dict_src VALUES (1, 'a', '2020-01-01 00:00:00'); + +CREATE DICTIONARY dict_remote (id UInt64, val String) +PRIMARY KEY id +SOURCE(CLICKHOUSE(HOST '127.0.0.2' PORT tcpPort() USER 'default' DB currentDatabase() TABLE 'dict_src' INVALIDATE_QUERY 'SELECT max(upd) FROM dict_src')) +LAYOUT(FLAT()) +LIFETIME(1); + +SELECT dictGet('dict_remote', 'val', toUInt64(1)); +" + +function wait_until() +{ + local deadline=$((SECONDS + 60)) + while [ "$($CLICKHOUSE_CLIENT --query "$1")" != "1" ] && [ $SECONDS -lt $deadline ] + do + sleep 1 + done +} + +DICT_INFO="FROM system.dictionaries WHERE database = currentDatabase() AND name = 'dict_remote'" +# Invalidate queries seen by the "remote" server. +CHECKS="FROM system.query_log WHERE current_database = currentDatabase() AND type != 'QueryStart' AND query LIKE 'SELECT max(upd)%'" + +# The first periodic check always reloads: there is no previous invalidate query response to compare with. +t0=$($CLICKHOUSE_CLIENT --query "SELECT toUnixTimestamp(last_successful_update_time) $DICT_INFO") +wait_until "SELECT toUnixTimestamp(last_successful_update_time) > $t0 $DICT_INFO" + +# A new row that does not change the invalidate query result. +$CLICKHOUSE_CLIENT --query "INSERT INTO dict_src VALUES (2, 'b', '2019-01-01 00:00:00')" +c0=$($CLICKHOUSE_CLIENT --query "SYSTEM FLUSH LOGS query_log; SELECT count() $CHECKS") +wait_until "SYSTEM FLUSH LOGS query_log; SELECT count() >= $c0 + 2 $CHECKS" +# Two checks have passed since the insert, the dictionary must not have been reloaded. +$CLICKHOUSE_CLIENT --query "SELECT dictHas('dict_remote', toUInt64(2))" + +# A new row that changes the invalidate query result. +$CLICKHOUSE_CLIENT --query "INSERT INTO dict_src VALUES (3, 'c', '2021-01-01 00:00:00')" +wait_until "SELECT dictHas('dict_remote', toUInt64(3))" +$CLICKHOUSE_CLIENT --query "SELECT dictGet('dict_remote', 'val', toUInt64(2)), dictGet('dict_remote', 'val', toUInt64(3))" + +$CLICKHOUSE_CLIENT --query "SELECT last_exception $DICT_INFO" From 16d373fde55ab02418fc4b478efe4d997359137c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 4 Nov 2026 11:32:36 +0000 Subject: [PATCH 121/219] Backport #117754 to 26.8: Split an oversized chunk across several Arrow record batches --- src/Core/FormatFactorySettings.h | 7 + src/Core/SettingsChangesHistory.cpp | 1 + src/Formats/FormatFactory.cpp | 1 + src/Formats/FormatSettings.h | 8 + .../Formats/Impl/ArrowBatchRowLimit.cpp | 319 ++++++++++++++++++ .../Formats/Impl/ArrowBatchRowLimit.h | 47 +++ .../ArrowIPC/ArrowIPCBlockOutputFormat.cpp | 40 ++- .../Impl/ArrowIPC/ArrowIPCBlockOutputFormat.h | 4 + .../Impl/ArrowIPC/RecordBatchEncoder.cpp | 285 ++++++++++------ .../Impl/ArrowIPC/RecordBatchEncoder.h | 18 +- .../Formats/Impl/CHColumnToArrowColumn.cpp | 125 +++++-- .../Formats/Impl/CHColumnToArrowColumn.h | 3 + ...umn_to_arrow_column_record_batch_split.cpp | 127 +++++++ .../DataLakes/DeltaLake/WriteTransaction.cpp | 78 +++-- .../test_arrowflight_interface/test.py | 37 ++ ...5_arrow_split_large_record_batch.reference | 10 + .../05055_arrow_split_large_record_batch.sh | 56 +++ ...ow_record_batch_split_boundaries.reference | 4 + ...056_arrow_record_batch_split_boundaries.sh | 52 +++ ..._record_batch_split_many_batches.reference | 2 + ...3_arrow_record_batch_split_many_batches.sh | 41 +++ 21 files changed, 1088 insertions(+), 177 deletions(-) create mode 100644 src/Processors/Formats/Impl/ArrowBatchRowLimit.cpp create mode 100644 src/Processors/Formats/Impl/ArrowBatchRowLimit.h create mode 100644 src/Processors/tests/gtest_ch_column_to_arrow_column_record_batch_split.cpp create mode 100644 tests/queries/0_stateless/05055_arrow_split_large_record_batch.reference create mode 100755 tests/queries/0_stateless/05055_arrow_split_large_record_batch.sh create mode 100644 tests/queries/0_stateless/05056_arrow_record_batch_split_boundaries.reference create mode 100755 tests/queries/0_stateless/05056_arrow_record_batch_split_boundaries.sh create mode 100644 tests/queries/0_stateless/05483_arrow_record_batch_split_many_batches.reference create mode 100755 tests/queries/0_stateless/05483_arrow_record_batch_split_many_batches.sh diff --git a/src/Core/FormatFactorySettings.h b/src/Core/FormatFactorySettings.h index f4bb179a550f3..9f610dedf5878 100644 --- a/src/Core/FormatFactorySettings.h +++ b/src/Core/FormatFactorySettings.h @@ -1519,6 +1519,13 @@ Note that `LowCardinality` columns can produce Arrow batches much larger or smal Buffering blocks can increase memory use and delay the first record batch until the query finishes. Leave both targets at `0` to write record batches as blocks arrive. `0` (the default) disables the byte target. Try `1048576` (1 MiB) as a starting value. +)", 0) \ + DECLARE(UInt64, output_format_arrow_max_record_batch_size_bytes, 2147483646, R"( +Upper bound on the `Utf8`, `Binary` and `List` buffers of a single record batch in the `Arrow` and `ArrowStream` output formats. A block whose strings, binary values or list elements exceed this is written as several record batches. + +Unlike [output_format_arrow_record_batch_size_bytes](#output_format_arrow_record_batch_size_bytes), which is a target that combines small blocks, this is a limit that splits large ones. The default is the largest value Arrow can address, because it offsets those buffers with 32-bit integers; a larger value has no effect. + +Only the buffers named above are bounded: a column of a fixed-width type has no offsets to overflow and stays in one batch. A single row never splits either, so a row above the limit still produces a batch that exceeds it. )", 0) \ \ DECLARE(Bool, output_format_orc_string_as_string, true, R"( diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index 4a0d5fe98e44c..194abc3ffd6c2 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -173,6 +173,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"output_format_arrow_record_batch_size_bytes", 0, 0, "New setting to combine small blocks in `Arrow` and `ArrowStream` output using a target size in bytes of accumulated data. The default `0` preserves one record batch per block."}, {"query_plan_filter_push_down_below_limit_by", false, true, "New setting to control pushing a filter on the `LIMIT BY` key columns below the `LIMIT BY` step. Set it to false to keep the filter above the `LIMIT BY`."}, {"query_plan_read_in_order_through_join", true, true, "Added the value `force`: keep reading in order through a join even when the parts announce no virtual rows."}, + {"output_format_arrow_max_record_batch_size_bytes", 2147483646, 2147483646, "New setting bounding the size of one record batch in `Arrow` and `ArrowStream` output. The default is Arrow's own addressing limit, so it does not change behavior."}, }); addSettingsChanges(settings_changes_history, "26.7", { diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index e3ed598c84a9a..4db92386e44d7 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -350,6 +350,7 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se format_settings.arrow.output_unsupported_types_as_binary = settings[Setting::output_format_arrow_unsupported_types_as_binary]; format_settings.arrow.output_record_batch_rows = settings[Setting::output_format_arrow_record_batch_size]; format_settings.arrow.output_record_batch_bytes = settings[Setting::output_format_arrow_record_batch_size_bytes]; + format_settings.arrow.output_max_record_batch_bytes = settings[Setting::output_format_arrow_max_record_batch_size_bytes]; format_settings.orc.allow_missing_columns = settings[Setting::input_format_orc_allow_missing_columns]; format_settings.orc.row_batch_size = settings[Setting::input_format_orc_row_batch_size]; format_settings.orc.skip_columns_with_unsupported_types_in_schema_inference = settings[Setting::input_format_orc_skip_columns_with_unsupported_types_in_schema_inference]; diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 99385f454d624..a4d4115b77129 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -5,9 +5,15 @@ #include #include +#include + namespace DB { +/// Arrow offsets its `Utf8`, `Binary` and `List` buffers with 32-bit integers, and the Apache Arrow +/// builders additionally reject the last representable value. +constexpr UInt64 MAX_ARROW_BUFFER_SIZE = std::numeric_limits::max() - 1; + /** * Various tweaks for input/output formats. Text serialization/deserialization * of data types also depend on some of these settings. It is different from @@ -203,6 +209,8 @@ struct FormatSettings bool output_unsupported_types_as_binary = true; UInt64 output_record_batch_rows = 0; UInt64 output_record_batch_bytes = 0; + /// Upper bound on one record batch, as opposed to the targets above, which combine small blocks. + UInt64 output_max_record_batch_bytes = MAX_ARROW_BUFFER_SIZE; } arrow{}; struct AvroSchemaRegistryTimeouts diff --git a/src/Processors/Formats/Impl/ArrowBatchRowLimit.cpp b/src/Processors/Formats/Impl/ArrowBatchRowLimit.cpp new file mode 100644 index 0000000000000..8f76772b9ef62 --- /dev/null +++ b/src/Processors/Formats/Impl/ArrowBatchRowLimit.cpp @@ -0,0 +1,319 @@ +#include + +#if USE_ARROW || USE_PARQUET + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace DB +{ + +namespace +{ + +/// The largest `n <= end - begin` for which the non-decreasing `offsets` grow by at most `limit` over +/// rows [begin, begin + n). +size_t maxRowsByOffsets(const IColumn::Offsets & offsets, size_t begin, size_t end, UInt64 limit) +{ + const UInt64 base = offsets[static_cast(begin) - 1]; + const auto * first = offsets.data() + begin; + const auto * it = std::upper_bound(first, offsets.data() + end, base + limit); + return static_cast(it - first); +} + +/// A writer either materializes the column or dictionary-encodes it; measuring it as materialized bounds +/// both, since a batch's dictionary holds at most the distinct values of its rows. `canBeInsideLowCardinality` +/// allows only fixed-width types, `String` and `FixedString`, and of those only the two string types reach +/// a 32-bit-offset buffer. +size_t maxRowsForLowCardinality( + const IColumn & column, + const DataTypePtr & type, + size_t begin, + size_t end, + bool fixed_string_as_fixed_byte_array, + UInt64 max_batch_bytes) +{ + const size_t num_rows = end - begin; + const DataTypePtr values_type = removeNullable(removeLowCardinality(type)); + const auto & low_cardinality = assert_cast(column); + const IColumn & values_column = *low_cardinality.getDictionary().getNestedNotNullableColumn(); + + if (isFixedString(values_type)) + { + if (fixed_string_as_fixed_byte_array) + return num_rows; + const size_t n = assert_cast(values_column).getN(); + return n == 0 ? num_rows : std::min(num_rows, max_batch_bytes / n); + } + + if (!isString(values_type)) + return num_rows; + + const auto & values = assert_cast(values_column); + const auto & value_offsets = values.getOffsets(); + + /// Scanning the indexes is only needed when the whole dictionary repeated on every row could overflow: + /// no single value is larger than all of them together. This is O(1), unlike finding the largest value, + /// so it does not add work proportional to the dictionary for every batch of a split chunk. + const UInt64 dictionary_bytes = values.empty() ? 0 : value_offsets.back(); + if (dictionary_bytes == 0 || num_rows <= max_batch_bytes / dictionary_bytes) + return num_rows; + + const auto & indexes = low_cardinality.getIndexes(); + UInt64 total = 0; + for (size_t row = begin; row != end; ++row) + { + const auto index = static_cast(indexes.getUInt(row)); + total += value_offsets[index] - value_offsets[index - 1]; + if (total > max_batch_bytes) + return row - begin; + } + return num_rows; +} + +/// A `Variant` becomes an Arrow dense union, whose own buffers are bounded by the row count; only its +/// children, holding one alternative's rows each, can overflow. +size_t maxRowsForVariant( + const IColumn & column, + const DataTypePtr & type, + size_t begin, + size_t end, + bool fixed_string_as_fixed_byte_array, + UInt64 max_batch_bytes) +{ + const size_t num_rows = end - begin; + const auto & variant_column = assert_cast(column); + const auto & variant_type = assert_cast(*type); + const size_t num_variants = variant_column.getNumVariants(); + const auto & local_discriminators = variant_column.getLocalDiscriminators(); + const auto & variant_offsets = variant_column.getOffsets(); + + /// Within a contiguous row range the offsets of each alternative are contiguous too (see + /// `ColumnVariant::updateHashWithValueRange`), so [begin, end) maps to the range + /// [nested_begin, nested_begin + nested_rows) of every alternative. + std::vector nested_begin(num_variants, 0); + std::vector nested_rows(num_variants, 0); + for (size_t row = begin; row != end; ++row) + { + const auto local_discr = local_discriminators[row]; + if (local_discr == ColumnVariant::NULL_DISCRIMINATOR) + continue; + if (nested_rows[local_discr] == 0) + nested_begin[local_discr] = variant_offsets[row]; + ++nested_rows[local_discr]; + } + + std::vector nested_rows_that_fit(num_variants); + bool all_fit = true; + for (size_t local_discr = 0; local_discr != num_variants; ++local_discr) + { + const auto global_discr + = variant_column.globalDiscriminatorByLocal(static_cast(local_discr)); + nested_rows_that_fit[local_discr] = maxRowsFittingOneArrowBatch( + variant_column.getVariantByLocalDiscriminator(local_discr), + variant_type.getVariant(global_discr), + nested_begin[local_discr], + nested_begin[local_discr] + nested_rows[local_discr], + fixed_string_as_fixed_byte_array, max_batch_bytes); + all_fit &= nested_rows_that_fit[local_discr] == nested_rows[local_discr]; + } + if (all_fit) + return num_rows; + + /// Some alternative cannot take all of its rows: cut at the row where it runs out. + std::vector nested_rows_taken(num_variants, 0); + for (size_t row = begin; row != end; ++row) + { + const auto local_discr = local_discriminators[row]; + if (local_discr == ColumnVariant::NULL_DISCRIMINATOR) + continue; + if (nested_rows_taken[local_discr] == nested_rows_that_fit[local_discr]) + return row - begin; + ++nested_rows_taken[local_discr]; + } + return num_rows; +} + +} + +size_t maxRowsFittingOneArrowBatch( + const IColumn & column, + const DataTypePtr & type, + size_t begin, + size_t end, + bool fixed_string_as_fixed_byte_array, + UInt64 max_batch_bytes) +{ + const size_t num_rows = end - begin; + if (num_rows == 0) + return 0; + + /// Arrow cannot address more than this however the caller was configured. + max_batch_bytes = std::min(max_batch_bytes, MAX_ARROW_BUFFER_SIZE); + + if (type->lowCardinality()) + return maxRowsForLowCardinality(column, type, begin, end, fixed_string_as_fixed_byte_array, max_batch_bytes); + + if (isVariant(type)) + return maxRowsForVariant(column, type, begin, end, fixed_string_as_fixed_byte_array, max_batch_bytes); + + /// A null row is written as a zero-length slot, so measuring the nested column only overestimates. + if (type->isNullable()) + { + return maxRowsFittingOneArrowBatch( + assert_cast(column).getNestedColumn(), + removeNullable(type), + begin, + end, + fixed_string_as_fixed_byte_array, max_batch_bytes); + } + + switch (WhichDataType(type).idx) + { + case TypeIndex::String: + { + return maxRowsByOffsets(assert_cast(column).getOffsets(), begin, end, max_batch_bytes); + } + case TypeIndex::FixedString: + { + if (fixed_string_as_fixed_byte_array) + return num_rows; + const size_t n = assert_cast(column).getN(); + return n == 0 ? num_rows : std::min(num_rows, max_batch_bytes / n); + } + case TypeIndex::Array: + { + const auto & array = assert_cast(column); + const auto & offsets = array.getOffsets(); + const size_t nested_begin = offsets[static_cast(begin) - 1]; + const size_t nested_rows = maxRowsFittingOneArrowBatch( + array.getData(), + assert_cast(*type).getNestedType(), + nested_begin, + offsets[end - 1], + fixed_string_as_fixed_byte_array, max_batch_bytes); + return maxRowsByOffsets(offsets, begin, end, std::min(nested_rows, max_batch_bytes)); + } + case TypeIndex::Map: + { + /// Both writers encode a Map exactly as its nested `Array(Tuple(key, value))`. + return maxRowsFittingOneArrowBatch( + assert_cast(column).getNestedColumn(), + assert_cast(*type).getNestedType(), + begin, + end, + fixed_string_as_fixed_byte_array, max_batch_bytes); + } + case TypeIndex::Tuple: + { + const auto & tuple = assert_cast(column); + const auto & elements = assert_cast(*type).getElements(); + size_t rows = num_rows; + for (size_t i = 0; i != elements.size(); ++i) + { + rows = std::min( + rows, + maxRowsFittingOneArrowBatch( + tuple.getColumn(i), elements[i], begin, begin + rows, fixed_string_as_fixed_byte_array, max_batch_bytes)); + } + return rows; + } + /// Fixed-width Arrow types: a single values buffer, no offsets, nothing to exceed. + case TypeIndex::UInt8: + case TypeIndex::UInt16: + case TypeIndex::UInt32: + case TypeIndex::UInt64: + case TypeIndex::UInt128: + case TypeIndex::UInt256: + case TypeIndex::Int8: + case TypeIndex::Int16: + case TypeIndex::Int32: + case TypeIndex::Int64: + case TypeIndex::Int128: + case TypeIndex::Int256: + case TypeIndex::Float32: + case TypeIndex::Float64: + case TypeIndex::Enum8: + case TypeIndex::Enum16: + case TypeIndex::Date: + case TypeIndex::Date32: + case TypeIndex::DateTime: + case TypeIndex::DateTime64: + case TypeIndex::Time: + case TypeIndex::Time64: + case TypeIndex::Decimal32: + case TypeIndex::Decimal64: + case TypeIndex::Decimal128: + case TypeIndex::Decimal256: + case TypeIndex::IPv4: + case TypeIndex::IPv6: + case TypeIndex::UUID: + case TypeIndex::Interval: + { + return num_rows; + } + default: + { + /// A type with no Arrow equivalent is written by serializing each value, so bounding its + /// encoded size would mean serializing twice. The encoder rejects a batch that overflows. + return num_rows; + } + } +} + +size_t maxRowsFittingOneArrowBatch( + const Columns & columns, + const DataTypes & types, + size_t begin, + size_t end, + bool fixed_string_as_fixed_byte_array, + UInt64 max_batch_bytes) +{ + chassert(columns.size() == types.size()); + const size_t num_rows = end - begin; + if (num_rows == 0) + return 0; + + /// Some limits (`Variant`, `LowCardinality`) take time linear in the rows they are given, so measuring + /// the whole remainder of a chunk for each of its batches would be quadratic. The window grows instead: + /// the limit of a prefix is the limit of the whole range capped at the prefix's length, so the first + /// window that does not fit entirely gives the answer, and the work stays proportional to the rows + /// emitted, plus the initial window. + static constexpr size_t initial_window = 8192; + size_t window = std::min(num_rows, initial_window); + while (true) + { + /// Two columns never share an Arrow buffer, so the smallest per-column limit applies. + size_t rows = window; + for (size_t i = 0; i < columns.size() && rows != 0; ++i) + rows = std::min( + rows, + maxRowsFittingOneArrowBatch( + *columns[i], types[i], begin, begin + rows, fixed_string_as_fixed_byte_array, max_batch_bytes)); + + if (rows < window || window == num_rows) + return rows; + window = std::min(num_rows, window * 2); + } +} + +} + +#endif diff --git a/src/Processors/Formats/Impl/ArrowBatchRowLimit.h b/src/Processors/Formats/Impl/ArrowBatchRowLimit.h new file mode 100644 index 0000000000000..c532138a8da0d --- /dev/null +++ b/src/Processors/Formats/Impl/ArrowBatchRowLimit.h @@ -0,0 +1,47 @@ +#pragma once + +#include "config.h" + +#if USE_ARROW || USE_PARQUET + +#include +#include +#include + +namespace DB +{ + +/// The largest `n` such that rows [begin, begin + n) of `column` stay within `max_batch_bytes`; writers +/// split an oversized chunk accordingly. Returns 0 only when the first row does not fit on its own, which +/// the writer then lets Arrow reject. `ColumnConst` and the sparse/replicated representations are not +/// handled; callers materialize first. +/// +/// `LowCardinality` is measured as materialized, which only over-splits when it is written as an Arrow +/// dictionary. A dictionary larger than one buffer is not covered at all: its bytes do not depend on the +/// row count, because `ColumnLowCardinality::insertRangeFrom` keeps a shared source dictionary whole, so +/// every slice re-emits it. Arrow rejects such a chunk. +/// +/// A type with no Arrow equivalent is not covered either, since only serializing its values reveals their +/// encoded size. Both uncovered shapes are rejected by the writer instead of being split. +size_t maxRowsFittingOneArrowBatch( + const IColumn & column, + const DataTypePtr & type, + size_t begin, + size_t end, + bool fixed_string_as_fixed_byte_array, + UInt64 max_batch_bytes = MAX_ARROW_BUFFER_SIZE); + +/// The largest `n` such that rows [begin, begin + n) of every column stay within `max_batch_bytes`: the +/// number of rows of the next record batch of a chunk. The work is proportional to the returned number of +/// rows rather than to `end - begin`, so splitting a chunk into many batches stays linear. +size_t maxRowsFittingOneArrowBatch( + const Columns & columns, + const DataTypes & types, + size_t begin, + size_t end, + bool fixed_string_as_fixed_byte_array, + UInt64 max_batch_bytes); + +} + +#endif diff --git a/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.cpp index 6253eb086fd8d..39b8406f86c31 100644 --- a/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.cpp @@ -3,6 +3,7 @@ #if USE_ARROW #include +#include #include #include #include @@ -24,6 +25,7 @@ #include #include +#include #include namespace DB @@ -255,7 +257,7 @@ std::pair ArrowIPCBlockOutputFormat::encodeDictionaryCol if (!state.emitted || delta_size > 0) { ColumnPtr delta = state.values->cut(delta_start, delta_size); - auto dict_batch = encoder->encode({delta}, {value_type}, delta_size); + auto dict_batch = encoder->encode({delta}, {value_type}, 0, delta_size); auto written = writeBatchMessage(dict_batch, dict.id, /*is_delta=*/state.emitted); if (!stream) dictionary_blocks.push_back({written.offset, written.metadata_length, written.body_length}); @@ -351,6 +353,15 @@ std::pair ArrowIPCBlockOutputFormat::substituteDictionar return {column, type}; } +void ArrowIPCBlockOutputFormat::writeRecordBatch( + const Columns & columns, const DataTypes & types, size_t begin, size_t end) +{ + auto batch = encoder->encode(columns, types, begin, end); + auto written = writeBatchMessage(batch); + if (!stream) + record_blocks.push_back({written.offset, written.metadata_length, written.body_length}); +} + void ArrowIPCBlockOutputFormat::consume(Chunk chunk) { /// Not in `writeChunk`: a reader gets the schema on the first chunk even when batches are combined. @@ -400,10 +411,10 @@ void ArrowIPCBlockOutputFormat::writeChunk(Chunk chunk) const Columns & columns = chunk.getColumns(); /// Dictionary-encoded columns are replaced in the record batch by their integer index column; their - /// values go into separate DictionaryBatch messages (extended across batches via deltas). + /// values go into separate DictionaryBatch messages, written here so they precede every record batch + /// of this chunk (extended across chunks via deltas). Columns record_columns(columns.begin(), columns.end()); DataTypes record_types = column_types; - for (size_t i = 0; i < columns.size(); ++i) { auto [substituted_column, substituted_type] = substituteDictionaries(columns[i], column_types[i], column_dict_plans[i]); @@ -411,10 +422,25 @@ void ArrowIPCBlockOutputFormat::writeChunk(Chunk chunk) record_types[i] = std::move(substituted_type); } - auto batch = encoder->encode(record_columns, record_types, num_rows); - auto written = writeBatchMessage(batch); - if (!stream) - record_blocks.push_back({written.offset, written.metadata_length, written.body_length}); + /// One record batch per chunk, unless the chunk exceeds what a record batch can address; then it is + /// written as several batches over successive row ranges. The limit is measured on the chunk's own + /// columns, since a dictionary-encoded one is already reduced to fixed-width indexes by now. + size_t offset = 0; + do + { + size_t rows = maxRowsFittingOneArrowBatch( + columns, + column_types, + offset, + num_rows, + format_settings.arrow.output_fixed_string_as_fixed_byte_array, + format_settings.arrow.output_max_record_batch_bytes); + /// A row that does not fit on its own cannot be represented at all; let the encoder reject it. + rows = std::min(std::max(rows, 1), num_rows - offset); + + writeRecordBatch(record_columns, record_types, offset, offset + rows); + offset += rows; + } while (offset < num_rows); } void ArrowIPCBlockOutputFormat::finalizeImpl() diff --git a/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.h b/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.h index 2be27e359c1f9..965cd893deea8 100644 --- a/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.h @@ -31,6 +31,8 @@ namespace DB /// metadata without the Apache Arrow C++ library. /// `LowCardinality` is written as its full column, or — when `output_format_arrow_low_cardinality_as_dictionary` /// is on — as an Arrow dictionary-encoded column (a single dictionary per id, extended across batches via deltas). +/// Each chunk becomes one record batch, except when it holds more data than the 32-bit offsets of the Arrow +/// IPC buffers can address, in which case it is split across several record batches. class ArrowIPCBlockOutputFormat final : public IOutputFormat { public: @@ -47,6 +49,8 @@ class ArrowIPCBlockOutputFormat final : public IOutputFormat void writeChunk(Chunk chunk); void writeSchemaIfNeeded(); + /// Writes rows [begin, end) of the already dictionary-substituted `columns` as one record batch. + void writeRecordBatch(const Columns & columns, const DataTypes & types, size_t begin, size_t end); /// Writes one encapsulated message for an encoded batch (a record batch, or a dictionary batch /// when `dictionary_id` is set), returning its location for recording an Arrow file `Block`. ArrowIPC::MessageWriter::WrittenMessage writeBatchMessage( diff --git a/src/Processors/Formats/Impl/ArrowIPC/RecordBatchEncoder.cpp b/src/Processors/Formats/Impl/ArrowIPC/RecordBatchEncoder.cpp index ff53303c7921c..1b9ac495fad2a 100644 --- a/src/Processors/Formats/Impl/ArrowIPC/RecordBatchEncoder.cpp +++ b/src/Processors/Formats/Impl/ArrowIPC/RecordBatchEncoder.cpp @@ -2,6 +2,7 @@ #if USE_ARROW +#include #include #include #include @@ -47,6 +48,19 @@ namespace DB::ArrowIPC namespace { +/// The writer splits an oversized chunk across batches, so normally only a single row too large on its own +/// gets here; it is also the backstop for a shape the row limit does not cover, such as a `LowCardinality` +/// dictionary exceeding one buffer. +[[noreturn]] void throwValueDoesNotFitArrowBuffer(Int64 bytes) +{ + throw Exception( + ErrorCodes::TOO_LARGE_ARRAY_SIZE, + "Cannot write {} bytes of string data to Arrow IPC: `Utf8`/`Binary` offsets are 32-bit, so one record " + "batch cannot hold more than {} bytes", + bytes, MAX_ARROW_BUFFER_SIZE); +} + + /// Pack one bit per input byte into `out` (size (num_rows + 7) / 8, zero-initialised by the caller): /// bit = 1 where the byte is non-zero, optionally inverted. Arrow validity uses invert = true (1 = not-null), /// Bool uses invert = false. Processes 64 bytes per step via `bytes64MaskToBits64Mask`, scalar for the tail. @@ -92,7 +106,7 @@ void RecordBatchEncoder::appendEmptyBuffer() buffers.emplace_back(static_cast(body.size()), 0); } -Int64 RecordBatchEncoder::appendValidity(const IColumn * null_map_column, size_t num_rows) +Int64 RecordBatchEncoder::appendValidity(const IColumn * null_map_column, size_t begin, size_t end) { if (!null_map_column) { @@ -101,22 +115,32 @@ Int64 RecordBatchEncoder::appendValidity(const IColumn * null_map_column, size_t return 0; } + const size_t num_rows = end - begin; const auto & null_map = assert_cast(*null_map_column).getData(); PODArray bitmap((num_rows + 7) / 8, 0); - packBytesToBits(null_map.data(), num_rows, bitmap.data(), /*invert=*/true); /// Arrow validity bit: 1 = valid. + packBytesToBits(null_map.data() + begin, num_rows, bitmap.data(), /*invert=*/true); /// Arrow validity bit: 1 = valid. appendBuffer(bitmap.data(), bitmap.size()); - return countBytesInFilter(null_map.data(), 0, num_rows); + return countBytesInFilter(null_map.data(), begin, end); } -void RecordBatchEncoder::appendOffsets(const IColumn::Offsets & ch_offsets, size_t num_rows) +void RecordBatchEncoder::appendOffsets(const IColumn::Offsets & ch_offsets, size_t begin, size_t end) { + const size_t num_rows = end - begin; + const UInt64 base = ch_offsets[static_cast(begin) - 1]; PODArray arrow_offsets(num_rows + 1); arrow_offsets[0] = 0; for (size_t i = 0; i < num_rows; ++i) { - if (ch_offsets[i] > static_cast(std::numeric_limits::max())) - throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Arrow IPC offset {} exceeds 32 bits", ch_offsets[i]); - arrow_offsets[i + 1] = static_cast(ch_offsets[i]); + const UInt64 offset = ch_offsets[begin + i] - base; + /// The row limit bounds the element count of `Array`/`Map` exactly, so only a row too large on its + /// own gets here. + if (offset > MAX_ARROW_BUFFER_SIZE) + throw Exception( + ErrorCodes::TOO_LARGE_ARRAY_SIZE, + "Cannot write a row of {} elements to Arrow IPC: `List` offsets are 32-bit, so a single row " + "cannot hold more than {} elements", + offset, MAX_ARROW_BUFFER_SIZE); + arrow_offsets[i + 1] = static_cast(offset); } appendBuffer(arrow_offsets.data(), (num_rows + 1) * sizeof(Int32)); } @@ -126,11 +150,11 @@ namespace /// Appends a fixed-width column's contiguous data straight into the body (a single copy, no temporary). template -void appendFixedWidth(RecordBatchEncoder & self, const IColumn & column, size_t num_rows) +void appendFixedWidth(RecordBatchEncoder & self, const IColumn & column, size_t begin, size_t end) { using V = typename Col::ValueType; const auto & data = assert_cast(column).getData(); - self.appendBuffer(data.data(), num_rows * sizeof(V)); + self.appendBuffer(data.data() + begin, (end - begin) * sizeof(V)); } int arrowTimeUnitForScale(UInt32 scale) @@ -167,49 +191,50 @@ Int64 timeUnitsPerDay(UInt32 scale) } void RecordBatchEncoder::encodeValues( - const IColumn & column, const DataTypePtr & type, size_t num_rows, const IColumn * null_map_column) + const IColumn & column, const DataTypePtr & type, size_t begin, size_t end, const IColumn * null_map_column) { const WhichDataType which(type); + const size_t num_rows = end - begin; if (isBool(type)) { /// Pack the 0/1 UInt8 column into an Arrow bit-packed boolean buffer. const auto & data = assert_cast(column).getData(); PODArray bitmap((num_rows + 7) / 8, 0); - packBytesToBits(data.data(), num_rows, bitmap.data(), /*invert=*/false); + packBytesToBits(data.data() + begin, num_rows, bitmap.data(), /*invert=*/false); appendBuffer(bitmap.data(), bitmap.size()); return; } switch (which.idx) { - case TypeIndex::UInt8: appendFixedWidth(*this, column, num_rows); return; - case TypeIndex::UInt16: appendFixedWidth(*this, column, num_rows); return; - case TypeIndex::UInt32: appendFixedWidth(*this, column, num_rows); return; - case TypeIndex::UInt64: appendFixedWidth(*this, column, num_rows); return; - case TypeIndex::Int8: appendFixedWidth(*this, column, num_rows); return; - case TypeIndex::Enum8: appendFixedWidth(*this, column, num_rows); return; - case TypeIndex::Int16: appendFixedWidth(*this, column, num_rows); return; - case TypeIndex::Enum16: appendFixedWidth(*this, column, num_rows); return; - case TypeIndex::Int32: appendFixedWidth(*this, column, num_rows); return; - case TypeIndex::Int64: appendFixedWidth(*this, column, num_rows); return; - case TypeIndex::Float32: appendFixedWidth(*this, column, num_rows); return; - case TypeIndex::Float64: appendFixedWidth(*this, column, num_rows); return; - case TypeIndex::Date32: appendFixedWidth(*this, column, num_rows); return; - case TypeIndex::DateTime: appendFixedWidth(*this, column, num_rows); return; + case TypeIndex::UInt8: appendFixedWidth(*this, column, begin, end); return; + case TypeIndex::UInt16: appendFixedWidth(*this, column, begin, end); return; + case TypeIndex::UInt32: appendFixedWidth(*this, column, begin, end); return; + case TypeIndex::UInt64: appendFixedWidth(*this, column, begin, end); return; + case TypeIndex::Int8: appendFixedWidth(*this, column, begin, end); return; + case TypeIndex::Enum8: appendFixedWidth(*this, column, begin, end); return; + case TypeIndex::Int16: appendFixedWidth(*this, column, begin, end); return; + case TypeIndex::Enum16: appendFixedWidth(*this, column, begin, end); return; + case TypeIndex::Int32: appendFixedWidth(*this, column, begin, end); return; + case TypeIndex::Int64: appendFixedWidth(*this, column, begin, end); return; + case TypeIndex::Float32: appendFixedWidth(*this, column, begin, end); return; + case TypeIndex::Float64: appendFixedWidth(*this, column, begin, end); return; + case TypeIndex::Date32: appendFixedWidth(*this, column, begin, end); return; + case TypeIndex::DateTime: appendFixedWidth(*this, column, begin, end); return; case TypeIndex::Date: { if (settings.arrow.output_date_as_uint16) { /// Backwards-compatible mode: write Date as a plain uint16. - appendFixedWidth(*this, column, num_rows); + appendFixedWidth(*this, column, begin, end); return; } /// Date is UInt16 days; widen to Int32 to match Arrow date32. const auto & data = assert_cast(column).getData(); PODArray widened(num_rows); for (size_t i = 0; i < num_rows; ++i) - widened[i] = static_cast(data[i]); + widened[i] = static_cast(data[begin + i]); appendBuffer(widened.data(), num_rows * sizeof(Int32)); return; } @@ -229,14 +254,14 @@ void RecordBatchEncoder::encodeValues( /// A null row carries an arbitrary nested value that the validity bitmap already masks as /// NULL; do not rescale it (matching the Apache Arrow writer), so junk in a NULL row cannot /// raise DECIMAL_OVERFLOW for a value that is emitted as NULL anyway. - if (null_map && (*null_map)[i]) + if (null_map && (*null_map)[begin + i]) { values[i] = 0; continue; } /// Widening to a coarser Arrow unit multiplies the value; guard against silent wraparound, /// matching the Apache Arrow writer which raises `DECIMAL_OVERFLOW` for the same case. - if (common::mulOverflow(data[i].value, factor, values[i])) + if (common::mulOverflow(data[begin + i].value, factor, values[i])) throw Exception( ErrorCodes::DECIMAL_OVERFLOW, "Decimal value is too large to convert to the Arrow timestamp scale"); } @@ -254,12 +279,12 @@ void RecordBatchEncoder::encodeValues( = null_map_column ? &assert_cast(*null_map_column).getData() : nullptr; for (size_t i = 0; i < num_rows; ++i) { - if (null_map && (*null_map)[i]) + if (null_map && (*null_map)[begin + i]) continue; - if (data[i] < 0 || data[i] >= 86400) - throwArrowTimeOutOfRange(data[i], 86400); + if (data[begin + i] < 0 || data[begin + i] >= 86400) + throwArrowTimeOutOfRange(data[begin + i], 86400); } - appendFixedWidth(*this, column, num_rows); + appendFixedWidth(*this, column, begin, end); return; } case TypeIndex::Time64: @@ -287,12 +312,12 @@ void RecordBatchEncoder::encodeValues( { /// A null row's arbitrary value is masked by the validity bitmap; do not rescale or /// range-check it (matching the library writer's `AppendNull`). - if (null_map && (*null_map)[i]) + if (null_map && (*null_map)[begin + i]) { values[i] = 0; continue; } - Int64 value = data[i].value; + Int64 value = data[begin + i].value; if (value < 0 || value >= source_units_per_day) throwArrowTimeOutOfRange(value, source_units_per_day); if (common::mulOverflow(value, factor, value) @@ -307,14 +332,14 @@ void RecordBatchEncoder::encodeValues( PODArray values(num_rows); for (size_t i = 0; i < num_rows; ++i) { - if (null_map && (*null_map)[i]) + if (null_map && (*null_map)[begin + i]) { values[i] = 0; continue; } - if (data[i].value < 0 || data[i].value >= source_units_per_day) - throwArrowTimeOutOfRange(data[i].value, source_units_per_day); - if (common::mulOverflow(data[i].value, factor, values[i])) + if (data[begin + i].value < 0 || data[begin + i].value >= source_units_per_day) + throwArrowTimeOutOfRange(data[begin + i].value, source_units_per_day); + if (common::mulOverflow(data[begin + i].value, factor, values[i])) throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "Decimal math overflow"); } appendBuffer(values.data(), num_rows * sizeof(Int64)); @@ -333,7 +358,7 @@ void RecordBatchEncoder::encodeValues( const auto & data = col.getData(); for (size_t i = 0; i < num_rows; ++i) { - const char * src = reinterpret_cast(&data[i]); + const char * src = reinterpret_cast(&data[begin + i]); char * dst = out.data() + i * arrow_width; memcpy(dst, src, ch_width); /// Sign-extend to the wider Arrow representation (little-endian two's complement). @@ -358,22 +383,40 @@ void RecordBatchEncoder::encodeValues( const auto & offs = cs.getOffsets(); const NullMap * null_map = null_map_column ? &assert_cast(*null_map_column).getData() : nullptr; + /// ClickHouse ColumnString stores no trailing '\0', so the offsets rebased against the row + /// before the range already are the Arrow ones. + const UInt64 base = offs[static_cast(begin) - 1]; + const UInt64 total_bytes = offs[static_cast(end) - 1] - base; PODArray arrow_offsets(num_rows + 1); - PODArray data; arrow_offsets[0] = 0; + + if (!null_map) + { + /// The bytes of the whole range are contiguous, so no full-size temporary is needed. + if (total_bytes > MAX_ARROW_BUFFER_SIZE) + throwValueDoesNotFitArrowBuffer(static_cast(total_bytes)); + for (size_t i = 0; i < num_rows; ++i) + arrow_offsets[i + 1] = static_cast(offs[begin + i] - base); + appendBuffer(arrow_offsets.data(), (num_rows + 1) * sizeof(Int32)); + appendBuffer(chars.data() + base, total_bytes); + return; + } + + PODArray data; + data.reserve(total_bytes); /// An upper bound: null rows contribute nothing. Int64 cur = 0; for (size_t i = 0; i < num_rows; ++i) { /// A null row carries an arbitrary nested value that the validity bitmap already masks as /// NULL; emit a zero-length slot for it (matching the Apache Arrow writer's `AppendNull`) /// so the bytes of a logically-NULL row never reach the IPC body. - if (!(null_map && (*null_map)[i])) + if (!(*null_map)[begin + i]) { - const size_t start = i == 0 ? 0 : offs[i - 1]; - const size_t len = offs[i] - start; /// ClickHouse ColumnString stores no trailing '\0' + const size_t start = offs[static_cast(begin + i) - 1]; + const size_t len = offs[begin + i] - start; cur += static_cast(len); - if (cur > std::numeric_limits::max()) - throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Arrow IPC string offset exceeds 32 bits"); + if (static_cast(cur) > MAX_ARROW_BUFFER_SIZE) + throwValueDoesNotFitArrowBuffer(cur); const size_t old = data.size(); data.resize(old + len); if (len) @@ -390,7 +433,7 @@ void RecordBatchEncoder::encodeValues( const auto & cfs = assert_cast(column); if (settings.arrow.output_fixed_string_as_fixed_byte_array) { - appendBuffer(cfs.getChars().data(), num_rows * cfs.getN()); + appendBuffer(cfs.getChars().data() + begin * cfs.getN(), num_rows * cfs.getN()); return; } /// `output_fixed_string_as_fixed_byte_array = 0`: the schema advertises a variable-width @@ -408,13 +451,13 @@ void RecordBatchEncoder::encodeValues( /// nested buffer can be copied in one shot. for (size_t i = 0; i < num_rows; ++i) { - const UInt64 end = static_cast(i + 1) * n; - if (end > static_cast(std::numeric_limits::max())) - throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Arrow IPC string offset exceeds 32 bits"); - arrow_offsets[i + 1] = static_cast(end); + const UInt64 slot_end = static_cast(i + 1) * n; + if (slot_end > MAX_ARROW_BUFFER_SIZE) + throwValueDoesNotFitArrowBuffer(static_cast(slot_end)); + arrow_offsets[i + 1] = static_cast(slot_end); } appendBuffer(arrow_offsets.data(), (num_rows + 1) * sizeof(Int32)); - appendBuffer(chars.data(), num_rows * n); + appendBuffer(chars.data() + begin * n, num_rows * n); return; } /// A null row carries an arbitrary nested value that the validity bitmap already masks as NULL; @@ -424,15 +467,15 @@ void RecordBatchEncoder::encodeValues( Int64 cur = 0; for (size_t i = 0; i < num_rows; ++i) { - if (!(*null_map)[i]) + if (!(*null_map)[begin + i]) { cur += static_cast(n); - if (cur > std::numeric_limits::max()) - throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Arrow IPC string offset exceeds 32 bits"); + if (static_cast(cur) > MAX_ARROW_BUFFER_SIZE) + throwValueDoesNotFitArrowBuffer(cur); const size_t old = data.size(); data.resize(old + n); if (n) - memcpy(data.data() + old, &chars[i * n], n); + memcpy(data.data() + old, &chars[(begin + i) * n], n); } arrow_offsets[i + 1] = static_cast(cur); } @@ -440,13 +483,13 @@ void RecordBatchEncoder::encodeValues( appendBuffer(data.data(), data.size()); return; } - case TypeIndex::IPv4: appendFixedWidth>(*this, column, num_rows); return; - case TypeIndex::Int128: appendFixedWidth>(*this, column, num_rows); return; - case TypeIndex::UInt128: appendFixedWidth>(*this, column, num_rows); return; - case TypeIndex::Int256: appendFixedWidth>(*this, column, num_rows); return; - case TypeIndex::UInt256: appendFixedWidth>(*this, column, num_rows); return; - case TypeIndex::IPv6: appendFixedWidth>(*this, column, num_rows); return; - case TypeIndex::Interval: appendFixedWidth>(*this, column, num_rows); return; + case TypeIndex::IPv4: appendFixedWidth>(*this, column, begin, end); return; + case TypeIndex::Int128: appendFixedWidth>(*this, column, begin, end); return; + case TypeIndex::UInt128: appendFixedWidth>(*this, column, begin, end); return; + case TypeIndex::Int256: appendFixedWidth>(*this, column, begin, end); return; + case TypeIndex::UInt256: appendFixedWidth>(*this, column, begin, end); return; + case TypeIndex::IPv6: appendFixedWidth>(*this, column, begin, end); return; + case TypeIndex::Interval: appendFixedWidth>(*this, column, begin, end); return; case TypeIndex::UUID: { /// fixed_size_binary(16) with the two 64-bit halves byte-reversed, matching the library writer. @@ -454,7 +497,7 @@ void RecordBatchEncoder::encodeValues( PODArray out(num_rows * 16); for (size_t i = 0; i < num_rows; ++i) { - UUID value = data[i]; + UUID value = data[begin + i]; auto * bytes = reinterpret_cast(&value); std::reverse(bytes, bytes + 8); std::reverse(bytes + 8, bytes + 16); @@ -466,9 +509,11 @@ void RecordBatchEncoder::encodeValues( case TypeIndex::Array: { const auto & ca = assert_cast(column); - appendOffsets(ca.getOffsets(), num_rows); + const auto & array_offsets = ca.getOffsets(); + appendOffsets(array_offsets, begin, end); const DataTypePtr & element_type = assert_cast(*type).getNestedType(); - encodeField(ca.getData(), element_type, ca.getData().size()); + encodeField( + ca.getData(), element_type, array_offsets[static_cast(begin) - 1], array_offsets[end - 1]); return; } case TypeIndex::Tuple: @@ -476,23 +521,25 @@ void RecordBatchEncoder::encodeValues( const auto & ct = assert_cast(column); const auto & elem_types = assert_cast(*type).getElements(); for (size_t i = 0; i < elem_types.size(); ++i) - encodeField(ct.getColumn(i), elem_types[i], num_rows); + encodeField(ct.getColumn(i), elem_types[i], begin, end); return; } case TypeIndex::Map: { const auto & cm = assert_cast(column); const auto & arr = cm.getNestedColumn(); /// ColumnArray(ColumnTuple(key, value)) - appendOffsets(arr.getOffsets(), num_rows); + const auto & array_offsets = arr.getOffsets(); + appendOffsets(array_offsets, begin, end); const auto & entries = assert_cast(arr.getData()); - const size_t entries_rows = entries.size(); + const size_t entries_begin = array_offsets[static_cast(begin) - 1]; + const size_t entries_end = array_offsets[end - 1]; /// The entries struct node (non-nullable), then its key and value children. - nodes.emplace_back(static_cast(entries_rows), 0); + nodes.emplace_back(static_cast(entries_end - entries_begin), 0); appendEmptyBuffer(); const auto & map_type = assert_cast(*type); - encodeField(entries.getColumn(0), map_type.getKeyType(), entries_rows); - encodeField(entries.getColumn(1), map_type.getValueType(), entries_rows); + encodeField(entries.getColumn(0), map_type.getKeyType(), entries_begin, entries_end); + encodeField(entries.getColumn(1), map_type.getValueType(), entries_begin, entries_end); return; } default: @@ -501,7 +548,7 @@ void RecordBatchEncoder::encodeValues( /// Arrow `Binary` column (read back as `String`); otherwise reject it. if (settings.arrow.output_unsupported_types_as_binary) { - encodeAsBinary(column, num_rows, null_map_column); + encodeAsBinary(column, begin, end, null_map_column); return; } throw Exception( @@ -512,8 +559,9 @@ void RecordBatchEncoder::encodeValues( } } -void RecordBatchEncoder::encodeAsBinary(const IColumn & column, size_t num_rows, const IColumn * null_map_column) +void RecordBatchEncoder::encodeAsBinary(const IColumn & column, size_t begin, size_t end, const IColumn * null_map_column) { + const size_t num_rows = end - begin; const NullMap * null_map = null_map_column ? &assert_cast(*null_map_column).getData() : nullptr; PODArray arrow_offsets(num_rows + 1); @@ -524,12 +572,12 @@ void RecordBatchEncoder::encodeAsBinary(const IColumn & column, size_t num_rows, { /// Skip the payload of a logically-NULL row (matching the Apache Arrow writer's `AppendNull`): /// emit a zero-length slot instead of the arbitrary bytes the null row may carry. - if (!(null_map && (*null_map)[i])) + if (!(null_map && (*null_map)[begin + i])) { - const std::string_view value = column.getDataAt(i); + const std::string_view value = column.getDataAt(begin + i); total += value.size(); - if (total > static_cast(std::numeric_limits::max())) - throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Arrow IPC binary offset exceeds 32 bits"); + if (total > MAX_ARROW_BUFFER_SIZE) + throwValueDoesNotFitArrowBuffer(static_cast(total)); data.insert(data.end(), value.data(), value.data() + value.size()); } arrow_offsets[i + 1] = static_cast(total); @@ -538,12 +586,14 @@ void RecordBatchEncoder::encodeAsBinary(const IColumn & column, size_t num_rows, appendBuffer(data.data(), data.size()); } -void RecordBatchEncoder::encodeField(const IColumn & column, const DataTypePtr & type, size_t num_rows) +void RecordBatchEncoder::encodeField(const IColumn & column, const DataTypePtr & type, size_t begin, size_t end) { + /// Materializing only the range keeps a split batch from expanding the whole chunk. `cut` is cheap for + /// these representations: it slices indexes or resizes a constant, it does not copy values. if (isColumnConst(column)) { - auto full = column.convertToFullColumnIfConst(); - encodeField(*full, type, num_rows); + auto full = column.cut(begin, end - begin)->convertToFullColumnIfConst(); + encodeField(*full, type, 0, end - begin); return; } @@ -554,23 +604,23 @@ void RecordBatchEncoder::encodeField(const IColumn & column, const DataTypePtr & /// library rather than a fix for an observed failure. if (column.isReplicated()) { - auto full = column.convertToFullColumnIfReplicated(); - encodeField(*full, type, num_rows); + auto full = column.cut(begin, end - begin)->convertToFullColumnIfReplicated(); + encodeField(*full, type, 0, end - begin); return; } if (type->lowCardinality()) { /// Materialize LowCardinality to its full column (matches output_format_arrow_low_cardinality_as_dictionary=false). - auto full = column.convertToFullColumnIfLowCardinality(); - encodeField(*full, removeLowCardinality(type), num_rows); + auto full = column.cut(begin, end - begin)->convertToFullColumnIfLowCardinality(); + encodeField(*full, removeLowCardinality(type), 0, end - begin); return; } /// Variant maps to an Arrow dense union, which (unlike every other type) has no validity buffer. if (isVariant(type)) { - encodeVariant(column, type, num_rows); + encodeVariant(column, type, begin, end); return; } @@ -586,12 +636,12 @@ void RecordBatchEncoder::encodeField(const IColumn & column, const DataTypePtr & } /// `appendValidity` packs the validity buffer and returns the null count in one pass over the null map. - const Int64 null_count = appendValidity(null_map_column, num_rows); - nodes.emplace_back(static_cast(num_rows), null_count); - encodeValues(*nested, nested_type, num_rows, null_map_column); + const Int64 null_count = appendValidity(null_map_column, begin, end); + nodes.emplace_back(static_cast(end - begin), null_count); + encodeValues(*nested, nested_type, begin, end, null_map_column); } -void RecordBatchEncoder::encodeVariant(const IColumn & column, const DataTypePtr & type, size_t num_rows) +void RecordBatchEncoder::encodeVariant(const IColumn & column, const DataTypePtr & type, size_t begin, size_t end) { const auto & variant_column = assert_cast(column); const auto & variant_type = assert_cast(*type); @@ -602,56 +652,83 @@ void RecordBatchEncoder::encodeVariant(const IColumn & column, const DataTypePtr throw Exception( ErrorCodes::NOT_IMPLEMENTED, "Native Arrow IPC writer does not support a Variant with {} alternatives (maximum is 127)", num_variants); + const size_t num_rows = end - begin; const auto & local_discriminators = variant_column.getLocalDiscriminators(); const auto & ch_offsets = variant_column.getOffsets(); + /// Each alternative's rows are contiguous within a contiguous row range (see + /// `ColumnVariant::updateHashWithValueRange`), so the range maps to one contiguous child range per + /// alternative, and the union offsets are rebased against that child's start. + const size_t num_local = variant_column.getNumVariants(); + std::vector nested_begin(num_local, 0); + std::vector nested_rows(num_local, 0); + for (size_t row = begin; row != end; ++row) + { + const auto local = local_discriminators[row]; + if (local == ColumnVariant::NULL_DISCRIMINATOR) + continue; + if (nested_rows[local] == 0) + nested_begin[local] = ch_offsets[row]; + ++nested_rows[local]; + } + /// The union node itself: no nulls are reported here (NULL rows use the dedicated null child). nodes.emplace_back(static_cast(num_rows), 0); PODArray type_ids(num_rows); PODArray value_offsets(num_rows); - for (size_t row = 0; row < num_rows; ++row) + for (size_t i = 0; i < num_rows; ++i) { - const auto local = local_discriminators[row]; + const auto local = local_discriminators[begin + i]; if (local == ColumnVariant::NULL_DISCRIMINATOR) { - type_ids[row] = static_cast(num_variants); - value_offsets[row] = 0; + type_ids[i] = static_cast(num_variants); + value_offsets[i] = 0; continue; } - type_ids[row] = static_cast(variant_column.globalDiscriminatorByLocal(local)); - const UInt64 off = ch_offsets[row]; - if (off > static_cast(std::numeric_limits::max())) + type_ids[i] = static_cast(variant_column.globalDiscriminatorByLocal(local)); + const UInt64 off = ch_offsets[begin + i] - nested_begin[local]; + if (off > MAX_ARROW_BUFFER_SIZE) throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Arrow IPC dense union offset exceeds 32 bits"); - value_offsets[row] = static_cast(off); + value_offsets[i] = static_cast(off); } appendBuffer(type_ids.data(), num_rows * sizeof(Int8)); appendBuffer(value_offsets.data(), num_rows * sizeof(Int32)); /// Children in global discriminator order, then a trailing single-element null child for NULL rows. - for (size_t i = 0; i < num_variants; ++i) + for (size_t global_discr = 0; global_discr < num_variants; ++global_discr) { - const IColumn & child = variant_column.getVariantByGlobalDiscriminator(i); - encodeField(child, variant_type.getVariant(i), child.size()); + const auto local = variant_column.localDiscriminatorByGlobal(static_cast(global_discr)); + encodeField( + variant_column.getVariantByGlobalDiscriminator(global_discr), + variant_type.getVariant(global_discr), + nested_begin[local], + nested_begin[local] + nested_rows[local]); } nodes.emplace_back(1, 1); } -RecordBatchEncoder::EncodedBatch RecordBatchEncoder::encode(const Columns & columns, const DataTypes & types, size_t num_rows) +RecordBatchEncoder::EncodedBatch RecordBatchEncoder::encode( + const Columns & columns, const DataTypes & types, size_t begin, size_t end) { nodes.clear(); buffers.clear(); body.clear(); + const size_t num_rows = end - begin; + /// Reserve the body up front so `appendBuffer` does not repeatedly reallocate and copy as it grows. - /// The column byte sizes approximate the Arrow body size closely enough to serve as a capacity hint. + /// The column byte sizes approximate the Arrow body size closely enough to serve as a capacity hint, + /// scaled to the share of the chunk this batch covers. size_t body_hint = 0; for (const auto & column : columns) body_hint += column->byteSize(); + if (const size_t total_rows = columns.empty() ? 0 : columns.front()->size(); total_rows > num_rows) + body_hint = body_hint / total_rows * num_rows; body.reserve(body_hint); for (size_t i = 0; i < columns.size(); ++i) - encodeField(*columns[i], types[i], num_rows); + encodeField(*columns[i], types[i], begin, end); /// Pad the body so its total length is a multiple of 8. while (body.size() % 8 != 0) diff --git a/src/Processors/Formats/Impl/ArrowIPC/RecordBatchEncoder.h b/src/Processors/Formats/Impl/ArrowIPC/RecordBatchEncoder.h index b5f39701bc740..02173ca4a08dc 100644 --- a/src/Processors/Formats/Impl/ArrowIPC/RecordBatchEncoder.h +++ b/src/Processors/Formats/Impl/ArrowIPC/RecordBatchEncoder.h @@ -36,31 +36,35 @@ class RecordBatchEncoder std::optional codec; }; - EncodedBatch encode(const Columns & columns, const DataTypes & types, size_t num_rows); + /// Encodes rows [begin, end) of `columns`. A range rather than a whole column, so that a chunk too + /// large for one record batch is written as several without copying a slice of it per batch. + EncodedBatch encode(const Columns & columns, const DataTypes & types, size_t begin, size_t end); /// Appends a buffer to the body (8-byte aligned start) and records its {offset, length}. void appendBuffer(const void * data, size_t length); private: - void encodeField(const IColumn & column, const DataTypePtr & type, size_t num_rows); + void encodeField(const IColumn & column, const DataTypePtr & type, size_t begin, size_t end); /// `null_map_column` (the outer `ColumnNullable`'s null map, or null for a non-nullable column) lets /// leaf encoders skip work for null rows — e.g. `DateTime64` rescaling, which would otherwise overflow /// on the arbitrary value a null row may carry. - void encodeValues(const IColumn & column, const DataTypePtr & type, size_t num_rows, const IColumn * null_map_column = nullptr); + void encodeValues( + const IColumn & column, const DataTypePtr & type, size_t begin, size_t end, const IColumn * null_map_column = nullptr); /// Encodes a Variant column as an Arrow dense union (no validity buffer; a types and an offsets /// buffer, the variant children in global order, and a trailing single-element null child). - void encodeVariant(const IColumn & column, const DataTypePtr & type, size_t num_rows); + void encodeVariant(const IColumn & column, const DataTypePtr & type, size_t begin, size_t end); /// Writes a column with no first-class Arrow mapping as an Arrow `Binary` column (an int32 offsets /// buffer and the concatenated per-row `getDataAt` bytes), matching the Apache Arrow library writer's /// `output_format_arrow_unsupported_types_as_binary` fallback. Read back as `String`. `null_map_column` /// (when set) marks rows to emit as zero-length, so a NULL row's nested bytes are not written. - void encodeAsBinary(const IColumn & column, size_t num_rows, const IColumn * null_map_column = nullptr); + void encodeAsBinary(const IColumn & column, size_t begin, size_t end, const IColumn * null_map_column = nullptr); void appendEmptyBuffer(); /// Emits the validity buffer: a packed LSB-first bitmap (1 = valid) for nullable columns, or an /// empty buffer otherwise. Returns the null count. - Int64 appendValidity(const IColumn * null_map_column, size_t num_rows); - void appendOffsets(const IColumn::Offsets & ch_offsets, size_t num_rows); + Int64 appendValidity(const IColumn * null_map_column, size_t begin, size_t end); + /// Emits a `List`/`Map` offsets buffer, rebased so that the range's first row starts at 0. + void appendOffsets(const IColumn::Offsets & ch_offsets, size_t begin, size_t end); const FormatSettings & settings; VectorWithMemoryTracking nodes; diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp index f5ea7197025ce..d14d65f2231a6 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -47,6 +48,7 @@ #include #include +#include #include #define FOR_INTERNAL_NUMERIC_TYPES(M) \ @@ -89,6 +91,7 @@ namespace DB extern const int ILLEGAL_COLUMN; extern const int UNKNOWN_TYPE; extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE; + extern const int TOO_LARGE_ARRAY_SIZE; } class ArrowUUIDExtensionType : public arrow::ExtensionType @@ -182,6 +185,18 @@ namespace DB return bitmap; } + /// Arrow's `Utf8`/`Binary` builders take an int32 length and their `Append` skips the copy when it is + /// negative, so a narrowed size would store an empty value instead of failing. + static void checkStringSizeFitsArrowOffsets(size_t size, const String & column_name, const String & format_name) + { + if (size > MAX_ARROW_BUFFER_SIZE) + throw Exception( + ErrorCodes::TOO_LARGE_ARRAY_SIZE, + "Cannot write a value of {} bytes of column {} to {}: `Utf8`/`Binary` offsets are 32-bit, so a single " + "value cannot be larger than {} bytes", + size, column_name, format_name, MAX_ARROW_BUFFER_SIZE); + } + static void fillArrowArrayWithRawColumnData( ColumnPtr write_column, const PaddedPODArray * null_bytemap, @@ -198,7 +213,11 @@ namespace DB if (null_bytemap && (*null_bytemap)[value_i]) status = builder.AppendNull(); else - status = builder.Append(write_column->getDataAt(value_i)); + { + const std::string_view value = write_column->getDataAt(value_i); + checkStringSizeFitsArrowOffsets(value.size(), write_column->getName(), format_name); + status = builder.Append(value); + } checkStatus(status, write_column->getName(), format_name); } } @@ -842,7 +861,9 @@ namespace DB const auto column_offsets = assert_cast(column_array->getOffsetsColumn()).getPtr(); size_t offsets_start = start > 0 ? start - 1 : 0; size_t offsets_view_start = start > 0 ? 1 : 0; - auto offsets = extractIndexes(column_offsets, offsets_start, end, false); + /// Keep the absolute offsets 64-bit: only the value rebased against `values_start` has to fit an + /// Arrow `List` offset, and for a range that does not start at row 0 the absolute one can be larger. + auto offsets = extractIndexes(column_offsets, offsets_start, end, false); size_t values_start = start == 0 ? 0 : offsets[0]; size_t values_end = offsets.empty() ? values_start : offsets.back(); @@ -856,7 +877,14 @@ namespace DB checkStatus(status, column_name, format_name); for (size_t i = offsets_view_start; i < offsets.size(); ++i) { - status = offsets_builder.Append(static_cast(offsets[i] - values_start)); + const Int64 rebased = offsets[i] - static_cast(values_start); + if (static_cast(rebased) > MAX_ARROW_BUFFER_SIZE) + throw Exception( + ErrorCodes::TOO_LARGE_ARRAY_SIZE, + "Cannot write a row of {} elements of column '{}' to {}: `List` offsets are 32-bit, so a " + "single row cannot hold more than {} elements", + rebased, column_name, format_name, MAX_ARROW_BUFFER_SIZE); + status = offsets_builder.Append(static_cast(rebased)); checkStatus(status, column_name, format_name); } @@ -1025,6 +1053,7 @@ namespace DB else { std::string_view string_ref = internal_column.getDataAt(string_i); + checkStringSizeFitsArrowOffsets(string_ref.size(), write_column->getName(), format_name); status = builder.Append(string_ref.data(), static_cast(string_ref.size())); } checkStatus(status, write_column->getName(), format_name); @@ -1035,6 +1064,7 @@ namespace DB for (size_t string_i = start; string_i < end; ++string_i) { std::string_view string_ref = internal_column.getDataAt(string_i); + checkStringSizeFitsArrowOffsets(string_ref.size(), write_column->getName(), format_name); status = builder.Append(string_ref.data(), static_cast(string_ref.size())); checkStatus(status, write_column->getName(), format_name); } @@ -1868,47 +1898,76 @@ namespace DB for (const auto & chunk : chunks) { - /// For arrow::Table creation + /// Normalize once per chunk: `fillArrowArray` converts a constant or replicated column on every + /// call, which would otherwise repeat for each row range below. + Columns columns(columns_num); + DataTypes column_types(columns_num); for (size_t column_i = 0; column_i < columns_num; ++column_i) { - const ColumnWithTypeAndName & header_column = header_columns[column_i]; - auto column_type = header_column.type; - auto column = chunk.getColumns()[column_i]; - + auto column = chunk.getColumns()[column_i]->convertToFullColumnIfConst()->convertToFullColumnIfReplicated(); + auto column_type = header_columns[column_i].type; if (!settings.low_cardinality_as_dictionary) { column = recursiveRemoveLowCardinality(column); column_type = recursiveRemoveLowCardinality(column_type); } + columns[column_i] = std::move(column); + column_types[column_i] = std::move(column_type); + } - // Generate the unwrapped builder schema (safe for MakeBuilder) - bool is_column_nullable = false; - auto builder_type = getArrowType( - column_type, column, header_column.name, format_name, settings, &is_column_nullable, true /* for_builder */); - - std::unique_ptr array_builder; - arrow::Status status = MakeBuilder(ArrowMemoryPool::instance(), builder_type, &array_builder); - checkStatus(status, column->getName(), format_name); + /// A chunk that exceeds the 32-bit Arrow offsets becomes several arrays of the ChunkedArray. + const size_t num_rows = chunk.getNumRows(); + size_t offset = 0; + do + { + size_t rows = maxRowsFittingOneArrowBatch( + columns, + column_types, + offset, + num_rows, + settings.output_fixed_string_as_fixed_byte_array, + settings.output_max_record_batch_bytes); + /// A row that does not fit on its own cannot be represented at all; the leaf encoders reject it. + rows = std::min(std::max(rows, 1), num_rows - offset); + + /// For arrow::Table creation + for (size_t column_i = 0; column_i < columns_num; ++column_i) + { + const ColumnWithTypeAndName & header_column = header_columns[column_i]; + const ColumnPtr & column = columns[column_i]; + const DataTypePtr & column_type = column_types[column_i]; + + // Generate the unwrapped builder schema (safe for MakeBuilder) + bool is_column_nullable = false; + auto builder_type = getArrowType( + column_type, column, header_column.name, format_name, settings, &is_column_nullable, true /* for_builder */); + + std::unique_ptr array_builder; + arrow::Status status = MakeBuilder(ArrowMemoryPool::instance(), builder_type, &array_builder); + checkStatus(status, column->getName(), format_name); + + std::shared_ptr arrow_array = fillArrowArray( + header_column.name, + column, + column_type, + nullptr, + array_builder.get(), + format_name, + offset, + offset + rows, + settings, + dictionary_values); - std::shared_ptr arrow_array = fillArrowArray( - header_column.name, - column, - column_type, - nullptr, - array_builder.get(), - format_name, - 0, - column->size(), - settings, - dictionary_values); + // Zero-copy cast to the extension-rich schema (handles infinite nesting) + auto target_type = schema->field(static_cast(column_i))->type(); + if (!arrow_array->type()->Equals(*target_type)) + arrow_array = checkResult(arrow_array->View(target_type), column->getName(), format_name); - // Zero-copy cast to the extension-rich schema (handles infinite nesting) - auto target_type = schema->field(static_cast(column_i))->type(); - if (!arrow_array->type()->Equals(*target_type)) - arrow_array = checkResult(arrow_array->View(target_type), column->getName(), format_name); + table_data.at(column_i).emplace_back(std::move(arrow_array)); + } - table_data.at(column_i).emplace_back(std::move(arrow_array)); - } + offset += rows; + } while (offset < num_rows); } std::vector> columns; diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.h b/src/Processors/Formats/Impl/CHColumnToArrowColumn.h index d3a3094e021b6..17cb5d82202da 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.h +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.h @@ -4,6 +4,7 @@ #if USE_ARROW || USE_PARQUET #include +#include #include #include @@ -35,6 +36,8 @@ class CHColumnToArrowColumn bool output_date_as_uint16 = false; /// Output types having no conversion as raw binary data. If false - such types would raise UNKNOWN_TYPE exception. bool output_unsupported_types_as_binary = false; + /// Upper bound on one record batch; a chunk holding more is written as several. + UInt64 output_max_record_batch_bytes = MAX_ARROW_BUFFER_SIZE; }; static std::shared_ptr calculateArrowSchema( diff --git a/src/Processors/tests/gtest_ch_column_to_arrow_column_record_batch_split.cpp b/src/Processors/tests/gtest_ch_column_to_arrow_column_record_batch_split.cpp new file mode 100644 index 0000000000000..38367d69dc3e2 --- /dev/null +++ b/src/Processors/tests/gtest_ch_column_to_arrow_column_record_batch_split.cpp @@ -0,0 +1,127 @@ +#include "config.h" + +#if USE_ARROW || USE_PARQUET + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/// Arrow offsets its `Utf8`, `Binary` and `List` buffers with 32-bit integers, so a chunk holding more +/// than that becomes several record batches, each covering a row range. A range not starting at row 0 has +/// to rebase those offsets against the row before it. Reaching the real limit needs more than 2 GiB of +/// data, so these tests lower it. +/// +/// `CHColumnToArrowColumn` is the Apache Arrow library writer, used by `ArrowFlight`, +/// `StorageArrowFlight` and Delta Lake writes rather than by the `Arrow` output format. + +namespace DB +{ + +namespace +{ + +constexpr size_t NUM_ROWS = 8; +constexpr size_t VALUE_SIZE = 16; + +std::string valueAt(size_t row) +{ + return std::string(VALUE_SIZE, static_cast('a' + row)); +} + +CHColumnToArrowColumn::Settings settingsWithLimit(UInt64 max_record_batch_bytes) +{ + CHColumnToArrowColumn::Settings settings; + settings.output_string_as_string = true; + settings.output_max_record_batch_bytes = max_record_batch_bytes; + return settings; +} + +std::shared_ptr convertToArrow(const DataTypePtr & type, MutableColumnPtr column, UInt64 max_record_batch_bytes) +{ + ColumnsWithTypeAndName header_columns{ColumnWithTypeAndName(type->createColumn(), type, "t")}; + + Columns columns; + columns.emplace_back(std::move(column)); + + std::vector chunks; + chunks.emplace_back(std::move(columns), NUM_ROWS); + + CHColumnToArrowColumn converter(header_columns, "Arrow", settingsWithLimit(max_record_batch_bytes)); + std::shared_ptr arrow_table; + converter.chChunkToArrowTable(arrow_table, chunks, 1); + return arrow_table; +} + +} + +TEST(CHColumnToArrowColumn, StringSplitAcrossRecordBatches) +{ + auto type = std::make_shared(); + auto column = type->createColumn(); + for (size_t row = 0; row < NUM_ROWS; ++row) + column->insert(valueAt(row)); + + /// Two values per batch, so most batches start past row 0. + auto table = convertToArrow(type, std::move(column), 2 * VALUE_SIZE); + + const auto & chunked = *table->column(0); + ASSERT_GT(chunked.num_chunks(), 1); + ASSERT_EQ(chunked.length(), static_cast(NUM_ROWS)); + + size_t row = 0; + for (int chunk_i = 0; chunk_i < chunked.num_chunks(); ++chunk_i) + { + const auto & array = assert_cast(*chunked.chunk(chunk_i)); + /// Every batch addresses its own values from zero. + ASSERT_EQ(array.value_offset(0), 0); + for (int64_t i = 0; i < array.length(); ++i, ++row) + ASSERT_EQ(array.GetString(i), valueAt(row)); + } + ASSERT_EQ(row, NUM_ROWS); +} + +TEST(CHColumnToArrowColumn, ArraySplitAcrossRecordBatches) +{ + auto type = std::make_shared(std::make_shared()); + auto column = type->createColumn(); + for (size_t row = 0; row < NUM_ROWS; ++row) + column->insert(Array{Field(valueAt(row)), Field(valueAt(row))}); + + auto table = convertToArrow(type, std::move(column), 2 * VALUE_SIZE); + + const auto & chunked = *table->column(0); + ASSERT_GT(chunked.num_chunks(), 1); + ASSERT_EQ(chunked.length(), static_cast(NUM_ROWS)); + + size_t row = 0; + for (int chunk_i = 0; chunk_i < chunked.num_chunks(); ++chunk_i) + { + const auto & list = assert_cast(*chunked.chunk(chunk_i)); + ASSERT_EQ(list.value_offset(0), 0); + for (int64_t i = 0; i < list.length(); ++i, ++row) + { + /// `value_slice` returns a new array by value: keep it alive while it is inspected. + const auto values_holder = list.value_slice(i); + const auto & values = assert_cast(*values_holder); + ASSERT_EQ(values.length(), 2); + ASSERT_EQ(values.GetString(0), valueAt(row)); + ASSERT_EQ(values.GetString(1), valueAt(row)); + } + } + ASSERT_EQ(row, NUM_ROWS); +} + +} + +#endif diff --git a/src/Storages/ObjectStorage/DataLakes/DeltaLake/WriteTransaction.cpp b/src/Storages/ObjectStorage/DataLakes/DeltaLake/WriteTransaction.cpp index 8dcb11bd7e722..d51fe061cd40b 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeltaLake/WriteTransaction.cpp +++ b/src/Storages/ObjectStorage/DataLakes/DeltaLake/WriteTransaction.cpp @@ -49,17 +49,13 @@ UInt64 getCurrentTime() return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); } -void exportTable( - std::shared_ptr table, +void exportRecordBatch( + const arrow::RecordBatch & batch, ffi::FFI_ArrowArray & array, ffi::FFI_ArrowSchema & schema) { - auto batch = table->CombineChunksToBatch(); - if (!batch.ok()) - DB::throwFromArrowStatus(batch.status(), DB::ErrorCodes::UNKNOWN_EXCEPTION, "Failed to create chunks batch"); - arrow::Status status = arrow::ExportRecordBatch( - **batch, + batch, reinterpret_cast(&array), reinterpret_cast(&schema)); @@ -250,30 +246,60 @@ void WriteTransaction::commit(const std::vector & files) LOG_TEST(log, "Will commit {} files", files.size()); auto write_metadata = getWriteMetadata(files, log); - ffi::FFI_ArrowArray array{}; - ffi::FFI_ArrowSchema schema{}; - SCOPE_EXIT({ - if (schema.release) - schema.release(&schema); - }); - exportTable(write_metadata, array, schema); - - KernelEngineData engine_data; - try + /// Metadata too large for one Arrow record batch comes back as several table chunks. Recombining them + /// would hit the same 32-bit offset limit, so each record batch is added to the transaction on its own. + /// `TableBatchReader` yields nothing for an empty table, which still has to reach the kernel as before. + arrow::RecordBatchVector batches; + if (write_metadata->num_rows() == 0) + { + auto empty_batch = arrow::RecordBatch::MakeEmpty(write_metadata->schema()); + if (!empty_batch.ok()) + DB::throwFromArrowStatus(empty_batch.status(), DB::ErrorCodes::UNKNOWN_EXCEPTION, "Failed to create an empty record batch"); + batches.push_back(*std::move(empty_batch)); + } + else { - /// Takes ownership of `array` (but not`schema`) if successfully called. - engine_data = DeltaLake::KernelUtils::unwrapResult( - ffi::get_engine_data(array, &schema, &KernelUtils::allocateError), - "get_engine_data"); + arrow::TableBatchReader batch_reader(*write_metadata); + while (true) + { + std::shared_ptr batch; + arrow::Status read_status = batch_reader.ReadNext(&batch); + if (!read_status.ok()) + DB::throwFromArrowStatus(read_status, DB::ErrorCodes::UNKNOWN_EXCEPTION, "Failed to read a record batch"); + if (!batch) + break; + batches.push_back(std::move(batch)); + } } - catch (...) + + for (const auto & batch : batches) { - if (array.release) - array.release(&array); - throw; + ffi::FFI_ArrowArray array{}; + ffi::FFI_ArrowSchema schema{}; + SCOPE_EXIT({ + if (schema.release) + schema.release(&schema); + }); + exportRecordBatch(*batch, array, schema); + + KernelEngineData engine_data; + try + { + /// Takes ownership of `array` (but not`schema`) if successfully called. + engine_data = DeltaLake::KernelUtils::unwrapResult( + ffi::get_engine_data(array, &schema, &KernelUtils::allocateError), + "get_engine_data"); + } + catch (...) + { + if (array.release) + array.release(&array); + throw; + } + + ffi::add_files(transaction.get(), engine_data.release()); } - ffi::add_files(transaction.get(), engine_data.release()); using KernelCommittedTransaction = DeltaLake::KernelPointerWrapper; KernelCommittedTransaction committed(DeltaLake::KernelUtils::unwrapResult( ffi::commit(transaction.release(), engine.get()), diff --git a/tests/integration/test_arrowflight_interface/test.py b/tests/integration/test_arrowflight_interface/test.py index 0f89533db0452..03a2dd8558baa 100644 --- a/tests/integration/test_arrowflight_interface/test.py +++ b/tests/integration/test_arrowflight_interface/test.py @@ -519,6 +519,43 @@ def test_doget_multiple_chunks(): assert actual[1].column("name").equals(pa.chunked_array([pa.array(["cde"], type=pa.string())])) +# A result is split across several Arrow arrays only when it exceeds the 32-bit offsets of the +# String/Binary/List buffers, so a small one must arrive as a single chunk per column for every type. +def test_doget_single_chunk_per_column(): + node.query( + """ + CREATE TABLE mytable ( + id Int64, + s String, + ns Nullable(String), + lc LowCardinality(String), + arr Array(String), + m Map(String, String), + t Tuple(String, String), + fs FixedString(3) + ) ORDER BY id + """ + ) + node.query( + """ + INSERT INTO mytable SELECT + number, toString(number), if(number % 2, NULL, toString(number)), toString(number % 2), + [toString(number)], map('k', toString(number)), (toString(number), 'x'), toFixedString('abc', 3) + FROM numbers(4) + """ + ) + + client, options = get_client() + + descriptor = flight.FlightDescriptor.for_command("SELECT * FROM mytable ORDER BY id") + flight_info = client.get_flight_info(descriptor, options) + table = client.do_get(flight_info.endpoints[0].ticket, options).read_all() + + assert table.num_rows == 4 + for name, column in zip(table.schema.names, table.columns): + assert column.num_chunks == 1, f"column {name} was split into {column.num_chunks} chunks" + + # We don't support any formats different from "Arrow". def test_doget_cmd_select_invalid_format(): node.query("CREATE TABLE mytable (id Int64, name String) ORDER BY id") diff --git a/tests/queries/0_stateless/05055_arrow_split_large_record_batch.reference b/tests/queries/0_stateless/05055_arrow_split_large_record_batch.reference new file mode 100644 index 0000000000000..af13b7a9ddc76 --- /dev/null +++ b/tests/queries/0_stateless/05055_arrow_split_large_record_batch.reference @@ -0,0 +1,10 @@ +String: batches>1=1 64 2016 1 +Nullable(String): batches>1=1 64 2016 1 +LowCardinality: batches>1=1 64 2016 4 +Array(String): batches>1=1 64 2016 1 +Map: batches>1=1 64 2016 1 +Tuple: batches>1=1 64 2016 1 +Variant: batches>1=1 64 2016 33 +FixedString: batches>1=1 64 2016 1 +FixedString fixed-width: batches>1=0 64 2016 1 +UInt64: batches>1=0 64 2016 64 diff --git a/tests/queries/0_stateless/05055_arrow_split_large_record_batch.sh b/tests/queries/0_stateless/05055_arrow_split_large_record_batch.sh new file mode 100755 index 0000000000000..9e4dba6b14cdf --- /dev/null +++ b/tests/queries/0_stateless/05055_arrow_split_large_record_batch.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Arrow offsets its String/Binary/List buffers with 32-bit integers, so a block holding more than that has +# to be written as several record batches. `output_format_arrow_max_record_batch_size_bytes` lowers the +# bound, which exercises the same splitting with kilobytes instead of the 2 GiB a real overflow needs. +# +# Compression is off because older pyarrow cannot read the "uncompressed" sentinel. + +FILE="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_split.arrows" +trap 'rm -f "$FILE"' EXIT + +batches() { + python3 - "$FILE" <<'PY' +import sys +import pyarrow as pa +import pyarrow.ipc as ipc + +with pa.OSFile(sys.argv[1], "rb") as src: + print(sum(1 for _ in ipc.open_stream(src))) +PY +} + +run() { + local label="$1" expr="$2" limit="$3" extra="${4:-}" + $CLICKHOUSE_LOCAL --allow_suspicious_fixed_string_types 1 --query " + SELECT number AS n, ${expr} AS v + FROM numbers(64) + SETTINGS max_block_size = 64, + output_format_arrow_compression_method = 'none', + output_format_arrow_max_record_batch_size_bytes = ${limit}${extra} + FORMAT ArrowStream + " > "$FILE" + echo -n "${label}: batches>1=$([ "$(batches)" -gt 1 ] && echo 1 || echo 0) " + $CLICKHOUSE_LOCAL --query "SELECT count(), sum(n), uniqExact(v) FROM file('$FILE', ArrowStream)" +} + +# One block of 64 rows against a small bound, so each shape must split. +run "String" "repeat('x', 64)" 512 +run "Nullable(String)" "if(number % 3 = 0, NULL, repeat('y', 64))::Nullable(String)" 512 +run "LowCardinality" "toLowCardinality(repeat(toString(number % 4), 64))" 512 +run "Array(String)" "[repeat('z', 64)]" 512 +run "Map" "map('k', repeat('m', 64))" 512 +run "Tuple" "(repeat('p', 64), repeat('q', 64))" 512 +run "Variant" "if(number % 2, number, repeat('v', 64))::Variant(UInt64, String)" 512 +# Only the offsets mode gives `FixedString` a buffer the bound applies to; the default maps it to a +# fixed-width Arrow type instead. +run "FixedString" "toFixedString(repeat('f', 64), 64)" 512 ", output_format_arrow_fixed_string_as_fixed_byte_array = 0" + +# A fixed-width column has no offsets buffer to overflow, so the bound leaves it in one batch. +run "FixedString fixed-width" "toFixedString(repeat('f', 64), 64)" 512 +run "UInt64" "number" 64 diff --git a/tests/queries/0_stateless/05056_arrow_record_batch_split_boundaries.reference b/tests/queries/0_stateless/05056_arrow_record_batch_split_boundaries.reference new file mode 100644 index 0000000000000..fe1489a7d47eb --- /dev/null +++ b/tests/queries/0_stateless/05056_arrow_record_batch_split_boundaries.reference @@ -0,0 +1,4 @@ +low_cardinality_as_dictionary=0 record batches for 3 blocks: 3 +6 15 +low_cardinality_as_dictionary=1 record batches for 3 blocks: 3 +6 15 diff --git a/tests/queries/0_stateless/05056_arrow_record_batch_split_boundaries.sh b/tests/queries/0_stateless/05056_arrow_record_batch_split_boundaries.sh new file mode 100755 index 0000000000000..ed8ae006336a8 --- /dev/null +++ b/tests/queries/0_stateless/05056_arrow_record_batch_split_boundaries.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The writer splits a chunk across record batches only when it exceeds the 32-bit offsets of the Arrow +# buffers, so a small result must give exactly one batch per block for every type shape. A wrong row +# count from the split logic shows up only here: the data reads back correctly either way. +# +# `FORMAT ArrowStream` into a file rather than `INSERT INTO FUNCTION file`, whose path squashes the +# blocks into one. Compression is off because older pyarrow cannot read the "uncompressed" sentinel. + +FILE="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_batches.arrows" +trap 'rm -f "$FILE"' EXIT + +GEN="SELECT + toString(number) AS s, + if(number % 3 = 0, NULL, toString(number))::Nullable(String) AS ns, + toLowCardinality(toString(number % 2)) AS lc, + arrayMap(x -> toString(x), range(number % 3)) AS arr, + arrayMap(y -> arrayMap(x -> toString(x), range(y)), range(number % 3)) AS arr2, + map(toString(number), toString(number % 2)) AS m, + (toString(number), toString(number % 2)) AS t, + toFixedString(toString(number), 3) AS fs, + number::Variant(UInt64, String) AS v, + initializeAggregation('quantileState(0.5)', number) AS agg, + toBFloat16(number) AS bf, + number AS u +FROM numbers(6)" + +for lc_as_dict in 0 1; do + $CLICKHOUSE_LOCAL --query "${GEN} SETTINGS + max_block_size = 2, + output_format_arrow_compression_method = 'none', + output_format_arrow_fixed_string_as_fixed_byte_array = 0, + output_format_arrow_low_cardinality_as_dictionary = ${lc_as_dict} + FORMAT ArrowStream" > "$FILE" + + echo -n "low_cardinality_as_dictionary=${lc_as_dict} record batches for 3 blocks: " + python3 - "$FILE" <<'PY' +import sys +import pyarrow as pa +import pyarrow.ipc as ipc + +with pa.OSFile(sys.argv[1], "rb") as src: + print(sum(1 for _ in ipc.open_stream(src))) +PY + + $CLICKHOUSE_LOCAL --query "SELECT count(), sum(u) FROM file('$FILE', ArrowStream)" +done diff --git a/tests/queries/0_stateless/05483_arrow_record_batch_split_many_batches.reference b/tests/queries/0_stateless/05483_arrow_record_batch_split_many_batches.reference new file mode 100644 index 0000000000000..62464cbf5e5c0 --- /dev/null +++ b/tests/queries/0_stateless/05483_arrow_record_batch_split_many_batches.reference @@ -0,0 +1,2 @@ +batches: 4096 row counts: [64] +262144 34359607296 1000 16777216 diff --git a/tests/queries/0_stateless/05483_arrow_record_batch_split_many_batches.sh b/tests/queries/0_stateless/05483_arrow_record_batch_split_many_batches.sh new file mode 100755 index 0000000000000..818ccdf9b491c --- /dev/null +++ b/tests/queries/0_stateless/05483_arrow_record_batch_split_many_batches.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# ^ pyarrow is not available in the fast test + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# One large block split by a small `output_format_arrow_max_record_batch_size_bytes` into thousands of +# record batches. The `Variant` and `LowCardinality` columns never need a split themselves, but measuring +# them is linear in the rows given, so this is the shape where measuring the whole remainder of the block +# for every batch would be quadratic. Every batch is bounded by the `String` column: 64 rows of 64 bytes. + +FILE="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_many_batches.arrows" +trap 'rm -f "$FILE"' EXIT + +$CLICKHOUSE_LOCAL --query " + SELECT + number::Variant(UInt64, String) AS v, + toLowCardinality(toString(number % 1000)) AS lc, + repeat('x', 64) AS s + FROM numbers(262144) + SETTINGS + max_block_size = 262144, + output_format_arrow_max_record_batch_size_bytes = 4096, + output_format_arrow_compression_method = 'none' + FORMAT ArrowStream" > "$FILE" + +python3 - "$FILE" <<'PY' +import sys +import pyarrow as pa +import pyarrow.ipc as ipc + +with pa.OSFile(sys.argv[1], "rb") as src: + batches = list(ipc.open_stream(src)) +print("batches:", len(batches), "row counts:", sorted({batch.num_rows for batch in batches})) +PY + +$CLICKHOUSE_LOCAL --query " + SELECT count(), sum(variantElement(v, 'UInt64')), uniqExact(lc), sum(length(s)) + FROM file('$FILE', ArrowStream)" From c2c4414a2d2104a4983097309355d3965b8dbbea Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 7 Nov 2026 07:42:45 +0000 Subject: [PATCH 122/219] Backport #129193 to 26.8: Fix server abort when an Arrow Flight query throws while finishing --- src/Server/ArrowFlight/ArrowFlightServer.cpp | 136 ++++++++++--------- 1 file changed, 69 insertions(+), 67 deletions(-) diff --git a/src/Server/ArrowFlight/ArrowFlightServer.cpp b/src/Server/ArrowFlight/ArrowFlightServer.cpp index c560ba7838ed4..2548bb0884692 100644 --- a/src/Server/ArrowFlight/ArrowFlightServer.cpp +++ b/src/Server/ArrowFlight/ArrowFlightServer.cpp @@ -575,9 +575,7 @@ static arrow::Result, std::vector, std::vectorgetSettingsRef()[Setting::output_format_arrow_unsupported_types_as_binary]}); - - if (schema_modifier) { - auto status = schema_modifier(schema); - ARROW_RETURN_NOT_OK(status); - schema = status.ValueUnsafe(); - } + PullingPipelineExecutor executor{block_io.pipeline}; + schema = CHColumnToArrowColumn::calculateArrowSchema( + executor.getHeader().getColumnsWithTypeAndName(), + "Arrow", + nullptr, + {.output_string_as_string = true, .output_unsupported_types_as_binary = query_context->getSettingsRef()[Setting::output_format_arrow_unsupported_types_as_binary]}); + + if (schema_modifier) + { + auto status = schema_modifier(schema); + ARROW_RETURN_NOT_OK(status); + schema = status.ValueUnsafe(); + } - std::optional header; - std::vector chunks; - Block block; - while (executor.pull(block)) - { - if (!block.empty()) + std::optional header; + std::vector chunks; + Block block; + while (executor.pull(block)) { - if (block_modifier) - block_modifier(query_context, block); - if (!header) - header = getHeader(block.getColumnsWithTypeAndName()); - chunks.emplace_back(Chunk{block.getColumns(), block.rows()}); - if (!single_table) + if (!block.empty()) { - tables.emplace_back( - CHColumnToArrowColumn::calculateArrowTable( - *header, "Arrow", chunks, - {.output_string_as_string = true, .output_unsupported_types_as_binary = query_context->getSettingsRef()[Setting::output_format_arrow_unsupported_types_as_binary]}, - header->size(), schema)); - chunks.clear(); + if (block_modifier) + block_modifier(query_context, block); + if (!header) + header = getHeader(block.getColumnsWithTypeAndName()); + chunks.emplace_back(Chunk{block.getColumns(), block.rows()}); + if (!single_table) + { + tables.emplace_back( + CHColumnToArrowColumn::calculateArrowTable( + *header, "Arrow", chunks, + {.output_string_as_string = true, .output_unsupported_types_as_binary = query_context->getSettingsRef()[Setting::output_format_arrow_unsupported_types_as_binary]}, + header->size(), schema)); + chunks.clear(); + } } } - } - if (!header) - tables.emplace_back(getEmptyArrowTable(schema)); - else if (single_table) - tables.emplace_back( - CHColumnToArrowColumn::calculateArrowTable( - *header, "Arrow", chunks, - {.output_string_as_string = true, .output_unsupported_types_as_binary = query_context->getSettingsRef()[Setting::output_format_arrow_unsupported_types_as_binary]}, - header->size(), schema)); + if (!header) + tables.emplace_back(getEmptyArrowTable(schema)); + else if (single_table) + tables.emplace_back( + CHColumnToArrowColumn::calculateArrowTable( + *header, "Arrow", chunks, + {.output_string_as_string = true, .output_unsupported_types_as_binary = query_context->getSettingsRef()[Setting::output_format_arrow_unsupported_types_as_binary]}, + header->size(), schema)); + } + block_io.onFinish(); query_finished = true; } catch (...) @@ -790,9 +791,7 @@ arrow::Status ArrowFlightServer::GetSchema( bool query_finished = false; bool handling_exception = false; SCOPE_EXIT({ - if (query_finished) - block_io.onFinish(); - else if (!handling_exception) + if (!query_finished && !handling_exception) block_io.onCancelOrConnectionLoss(); }); @@ -801,18 +800,21 @@ arrow::Status ArrowFlightServer::GetSchema( ARROW_RETURN_NOT_OK(checkNoCustomFormat(ast)); ARROW_RETURN_NOT_OK(checkPipelineIsPulling(block_io.pipeline)); - PullingPipelineExecutor executor{block_io.pipeline}; - - schema = CHColumnToArrowColumn::calculateArrowSchema( - executor.getHeader().getColumnsWithTypeAndName(), "Arrow", nullptr, - {.output_string_as_string = true, .output_unsupported_types_as_binary = query_context->getSettingsRef()[Setting::output_format_arrow_unsupported_types_as_binary]}); - if (schema_modifier) { - auto status = schema_modifier(schema); - ARROW_RETURN_NOT_OK(status); - schema = status.ValueUnsafe(); + PullingPipelineExecutor executor{block_io.pipeline}; + + schema = CHColumnToArrowColumn::calculateArrowSchema( + executor.getHeader().getColumnsWithTypeAndName(), "Arrow", nullptr, + {.output_string_as_string = true, .output_unsupported_types_as_binary = query_context->getSettingsRef()[Setting::output_format_arrow_unsupported_types_as_binary]}); + if (schema_modifier) + { + auto status = schema_modifier(schema); + ARROW_RETURN_NOT_OK(status); + schema = status.ValueUnsafe(); + } } + block_io.onFinish(); query_finished = true; } catch (...) @@ -1087,9 +1089,7 @@ arrow::Status ArrowFlightServer::DoGet( bool query_finished = false; bool handling_exception = false; SCOPE_EXIT({ - if (query_finished) - block_io.onFinish(); - else if (!handling_exception) + if (!query_finished && !handling_exception) block_io.onCancelOrConnectionLoss(); }); @@ -1098,16 +1098,19 @@ arrow::Status ArrowFlightServer::DoGet( ARROW_RETURN_NOT_OK(checkNoCustomFormat(ast)); ARROW_RETURN_NOT_OK(checkPipelineIsPulling(block_io.pipeline)); - PullingPipelineExecutor executor{block_io.pipeline}; + { + PullingPipelineExecutor executor{block_io.pipeline}; - Block block; - while (executor.pull(block)) - chunks.emplace_back(Chunk(block.getColumns(), block.rows())); + Block block; + while (executor.pull(block)) + chunks.emplace_back(Chunk(block.getColumns(), block.rows())); - auto header = executor.getHeader(); - auto ch_to_arrow_converter = createCHToArrowConverter(header, query_context); - ch_to_arrow_converter->chChunkToArrowTable(table, chunks, header.columns()); + auto header = executor.getHeader(); + auto ch_to_arrow_converter = createCHToArrowConverter(header, query_context); + ch_to_arrow_converter->chChunkToArrowTable(table, chunks, header.columns()); + } + block_io.onFinish(); query_finished = true; } catch (...) @@ -1222,9 +1225,7 @@ arrow::Status ArrowFlightServer::DoPut( bool query_finished = false; bool handling_exception = false; SCOPE_EXIT({ - if (query_finished) - block_io.onFinish(); - else if (!handling_exception) + if (!query_finished && !handling_exception) block_io.onCancelOrConnectionLoss(); }); @@ -1252,6 +1253,7 @@ arrow::Status ArrowFlightServer::DoPut( executor.execute(); } + block_io.onFinish(); query_finished = true; } catch (...) From c608322216cd6c2fb9c0473cea5e69722459744a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 7 Nov 2026 11:02:32 +0000 Subject: [PATCH 123/219] Backport #129879 to 26.8: Fix flaky test 05457_dictionary_remote_clickhouse_invalidate_query --- .../05457_dictionary_remote_clickhouse_invalidate_query.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/05457_dictionary_remote_clickhouse_invalidate_query.sh b/tests/queries/0_stateless/05457_dictionary_remote_clickhouse_invalidate_query.sh index 59efc91c52fff..9da82e9a67558 100755 --- a/tests/queries/0_stateless/05457_dictionary_remote_clickhouse_invalidate_query.sh +++ b/tests/queries/0_stateless/05457_dictionary_remote_clickhouse_invalidate_query.sh @@ -34,8 +34,8 @@ DICT_INFO="FROM system.dictionaries WHERE database = currentDatabase() AND name CHECKS="FROM system.query_log WHERE current_database = currentDatabase() AND type != 'QueryStart' AND query LIKE 'SELECT max(upd)%'" # The first periodic check always reloads: there is no previous invalidate query response to compare with. -t0=$($CLICKHOUSE_CLIENT --query "SELECT toUnixTimestamp(last_successful_update_time) $DICT_INFO") -wait_until "SELECT toUnixTimestamp(last_successful_update_time) > $t0 $DICT_INFO" +# The reloaded dictionary is a new object, so its `query_count` is 0 again (the `dictGet` above made it 1). +wait_until "SELECT query_count = 0 $DICT_INFO" # A new row that does not change the invalidate query result. $CLICKHOUSE_CLIENT --query "INSERT INTO dict_src VALUES (2, 'b', '2019-01-01 00:00:00')" From 94aaa8aa2cb5c335ad9cb417c742d08a170c7ef9 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 7 Nov 2026 17:31:29 +0000 Subject: [PATCH 124/219] Backport #127057 to 26.8: Add AI FUNCTIONS privilege group --- .../regular-functions/ai-functions.mdx | 31 +++++- docs/reference/statements/grant.mdx | 28 ++++++ src/Access/Common/AccessType.h | 13 ++- src/Functions/FunctionBaseAI.cpp | 30 ++++++ src/Functions/FunctionBaseAI.h | 10 +- src/Functions/aiClassify.cpp | 7 +- src/Functions/aiEmbed.cpp | 12 ++- src/Functions/aiExtract.cpp | 7 +- src/Functions/aiFilter.cpp | 7 +- src/Functions/aiGenerate.cpp | 7 +- src/Functions/aiRedact.cpp | 7 +- src/Functions/aiSimilarity.cpp | 12 ++- src/Functions/aiTranslate.cpp | 7 +- src/Interpreters/Context.cpp | 1 + src/Interpreters/Context.h | 7 ++ src/Interpreters/InterpreterAlterQuery.cpp | 4 +- src/Interpreters/InterpreterDeleteQuery.cpp | 4 +- src/Storages/AlterCommands.cpp | 26 +++++ src/Storages/MergeTree/MergeTreeData.cpp | 49 +++++++++- src/Storages/MergeTree/MergeTreeData.h | 5 + .../MergeTree/registerStorageMergeTree.cpp | 8 ++ tests/integration/test_ai_functions/test.py | 14 --- .../01271_show_privileges.reference | 9 ++ ...42_ai_functions_named_collection_access.sh | 1 + .../05441_ai_functions_privileges.reference | 4 + .../05441_ai_functions_privileges.sql.j2 | 70 +++++++++++++ ...ns_privileges_stored_expressions.reference | 1 + ...unctions_privileges_stored_expressions.sql | 56 +++++++++++ .../05444_ai_functions_no_user.reference | 9 ++ .../05444_ai_functions_no_user.sql | 98 +++++++++++++++++++ ...ai_functions_background_defaults.reference | 3 + ...05459_ai_functions_background_defaults.sql | 28 ++++++ 32 files changed, 545 insertions(+), 30 deletions(-) create mode 100644 tests/queries/0_stateless/05441_ai_functions_privileges.reference create mode 100644 tests/queries/0_stateless/05441_ai_functions_privileges.sql.j2 create mode 100644 tests/queries/0_stateless/05442_ai_functions_privileges_stored_expressions.reference create mode 100644 tests/queries/0_stateless/05442_ai_functions_privileges_stored_expressions.sql create mode 100644 tests/queries/0_stateless/05444_ai_functions_no_user.reference create mode 100644 tests/queries/0_stateless/05444_ai_functions_no_user.sql create mode 100644 tests/queries/0_stateless/05459_ai_functions_background_defaults.reference create mode 100644 tests/queries/0_stateless/05459_ai_functions_background_defaults.sql diff --git a/docs/reference/functions/regular-functions/ai-functions.mdx b/docs/reference/functions/regular-functions/ai-functions.mdx index edf769b482e0f..2cb7851fbf60b 100644 --- a/docs/reference/functions/regular-functions/ai-functions.mdx +++ b/docs/reference/functions/regular-functions/ai-functions.mdx @@ -32,7 +32,7 @@ All functions are sharing a common infrastructure that provides: ## Configuration {#configuration} -AI functions reference a **named collection** that stores provider credentials and configuration. Different named collections can be created and used for different functions or functions calls. For example you may want to define a different named collection to use with the text functions (`aiGenerate`, `aiClassify`, `aiFilter`, `aiExtract`, `aiTranslate`, `aiRedact`) vs the embedding functions (`aiEmbed`, `aiSimilarity`), which require different endpoints and usually use different models. +AI functions require the `AI FUNCTIONS` grant (see [Access control](#access-control)) and reference a **named collection** that stores provider credentials and configuration. Different named collections can be created and used for different functions or functions calls. For example you may want to define a different named collection to use with the text functions (`aiGenerate`, `aiClassify`, `aiFilter`, `aiExtract`, `aiTranslate`, `aiRedact`) vs the embedding functions (`aiEmbed`, `aiSimilarity`), which require different endpoints and usually use different models. Example statement to create a named collection with provider credentials, one with a chat endpoint and another with an embedding endpoint: ```sql @@ -51,6 +51,35 @@ CREATE NAMED COLLECTION ai_embedding_credentials AS api_key = 'sk-...'; ``` +### Access control {#access-control} + +Calling an AI function requires two privileges: + +- the function's privilege: either the per-function privilege (e.g. `aiGenerate`) or the `AI FUNCTIONS` group, see [`GRANT`](/reference/statements/grant#ai-functions); +- the `NAMED COLLECTION` privilege on the selected credentials. + +```sql +GRANT AI FUNCTIONS ON *.* TO ai_user_role; +GRANT aiGenerate, aiClassify ON *.* TO ai_analyst_role; +GRANT NAMED COLLECTION ON ai_text_credentials TO ai_user_role, ai_analyst_role; +``` + +AI functions are executed only in queries run by a user. They are rejected in mutations (`ALTER TABLE ... UPDATE`, `ALTER TABLE ... DELETE`) and in the keys, TTL, skip indexes and projections of a table, which are executed in the background, and `Buffer` flushes throw an exception when they evaluate an AI function. `DEFAULT` and `MATERIALIZED` columns are evaluated by the insert; when such a column is added with `ALTER TABLE`, the existing parts compute it in the background, where AI functions are not executed. + +To fill a column of existing rows, use a lightweight [`UPDATE`](/reference/statements/update), which is executed by the query and writes only the updated column: + +```sql +UPDATE t SET y = aiEmbed(x, 'model') WHERE empty(y) SETTINGS enable_lightweight_update = 1; +``` + +For a table that does not meet the [lightweight update requirements](/reference/statements/update#lightweight-update-requirements), use `INSERT ... SELECT`, for example one partition at a time: + +```sql +CREATE TABLE t_tmp AS t; +INSERT INTO t_tmp SELECT x, aiEmbed(x, 'model') AS y FROM t WHERE _partition_id = '202610'; +ALTER TABLE t REPLACE PARTITION ID '202610' FROM t_tmp; +``` + ### Named collection parameters {#named-collection-parameters} | Parameter | Type | Default | Description | diff --git a/docs/reference/statements/grant.mdx b/docs/reference/statements/grant.mdx index de01a6ecc3a9e..645098c3c0785 100644 --- a/docs/reference/statements/grant.mdx +++ b/docs/reference/statements/grant.mdx @@ -160,6 +160,15 @@ The hierarchy of privileges in ClickHouse is shown below: - `SHOW ROW POLICIES` - `SHOW SETTINGS PROFILES` - `SHOW USERS` + - [`AI FUNCTIONS`](#ai-functions) + - `aiClassify` + - `aiEmbed` + - `aiExtract` + - `aiFilter` + - `aiGenerate` + - `aiRedact` + - `aiSimilarity` + - `aiTranslate` - [`ALTER`](#alter) - `ALTER DATABASE` - `ALTER DATABASE SETTINGS` @@ -626,6 +635,25 @@ Allows using [introspection](/concepts/features/performance/troubleshoot/samplin - `addressToSymbol`. Level: `GLOBAL` - `demangle`. Level: `GLOBAL` +### AI FUNCTIONS {#ai-functions} + +Allows using [AI functions](/reference/functions/regular-functions/ai-functions). Calling an AI function also requires the `NAMED COLLECTION` privilege on the selected credentials. + +- `AI FUNCTIONS`. Level: `GROUP` + - `aiClassify`. Level: `GLOBAL` + - `aiEmbed`. Level: `GLOBAL` + - `aiExtract`. Level: `GLOBAL` + - `aiFilter`. Level: `GLOBAL` + - `aiGenerate`. Level: `GLOBAL` + - `aiRedact`. Level: `GLOBAL` + - `aiSimilarity`. Level: `GLOBAL` + - `aiTranslate`. Level: `GLOBAL` + +Examples: + +- `GRANT AI FUNCTIONS ON *.* TO john` +- `GRANT aiGenerate, aiClassify ON *.* TO john` + ### SOURCES {#sources} Allows using external data sources. Applies to [table engines](/reference/engines/table-engines/index) and [table functions](/reference/functions/table-functions/index). diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h index c1cbb1aed671e..ba14d5e582375 100644 --- a/src/Access/Common/AccessType.h +++ b/src/Access/Common/AccessType.h @@ -150,7 +150,7 @@ ENUM_ACCESS_OBJECT(Source, APPLY_FOR_SOURCE) /// Represents an access type which can be granted on databases, tables, columns, etc. -enum class AccessType : uint8_t +enum class AccessType : uint16_t { /// Macro M should be defined as M(name, aliases, node_type, parent_group_name) /// where name is identifier with underscores (instead of spaces); @@ -446,6 +446,17 @@ enum class AccessType : uint8_t M(DISK, "", GLOBAL, ALL) \ M(SOURCES, "", GLOBAL, ALL) \ \ + /* AI functions */ \ + M(aiClassify, "", GLOBAL, AI_FUNCTIONS) /* allows to execute function aiClassify() */\ + M(aiEmbed, "", GLOBAL, AI_FUNCTIONS) /* allows to execute function aiEmbed() */\ + M(aiExtract, "", GLOBAL, AI_FUNCTIONS) /* allows to execute function aiExtract() */\ + M(aiFilter, "", GLOBAL, AI_FUNCTIONS) /* allows to execute function aiFilter() */\ + M(aiGenerate, "", GLOBAL, AI_FUNCTIONS) /* allows to execute function aiGenerate() */\ + M(aiRedact, "", GLOBAL, AI_FUNCTIONS) /* allows to execute function aiRedact() */\ + M(aiSimilarity, "", GLOBAL, AI_FUNCTIONS) /* allows to execute function aiSimilarity() */\ + M(aiTranslate, "", GLOBAL, AI_FUNCTIONS) /* allows to execute function aiTranslate() */\ + M(AI_FUNCTIONS, "", GROUP, ALL) /* allows to execute all AI functions */\ + \ /* Consts */ \ M(ALL, "ALL PRIVILEGES", GROUP, NONE) /* full access */ \ M(NONE, "USAGE, NO PRIVILEGES", GROUP, NONE) /* no access */ diff --git a/src/Functions/FunctionBaseAI.cpp b/src/Functions/FunctionBaseAI.cpp index 63a4f15ebb471..b657fffb7731a 100644 --- a/src/Functions/FunctionBaseAI.cpp +++ b/src/Functions/FunctionBaseAI.cpp @@ -58,6 +58,7 @@ namespace ErrorCodes { extern const int ACCESS_DENIED; extern const int BAD_ARGUMENTS; + extern const int NOT_IMPLEMENTED; } namespace @@ -370,6 +371,32 @@ void FunctionBaseAI::insertProcessedResult(IColumn & column, const String & proc column.insertData(processed.data(), processed.size()); } +void FunctionBaseAI::checkHasUser(const ContextPtr & context) +{ + if (context->getUserID()) + return; + + /// A background operation is retried, and its failure is visible only in the logs and system tables. + context->addOrUpdateWarningMessage( + Context::WarningType::AI_FUNCTION_IN_BACKGROUND_OPERATION, + PreformattedMessage::create( + "A background operation (merge, mutation or `Buffer` flush) failed because it evaluates an AI function, " + "which is executed only in queries run by a user. {}Remove the AI function from the table's keys, TTL, " + "skip indexes, projections, `DEFAULT` and `MATERIALIZED` columns and constraints.", + context->getCurrentQueryId().empty() ? "" : fmt::format("Last failed task: {}. ", context->getCurrentQueryId()))); + + throw Exception(ErrorCodes::ACCESS_DENIED, + "AI functions cannot be executed without a user, e.g. in background merges, mutations or `Buffer` flushes"); +} + +void FunctionBaseAI::throwIfBackgroundExpression(const ContextPtr & context) +{ + if (context->isAnalyzingBackgroundExpressions()) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "AI functions cannot be used in expressions executed in the background: mutations, keys, TTL, skip indexes " + "and projections. Use a lightweight `UPDATE` or `INSERT ... SELECT` instead"); +} + AIParamSpecs FunctionBaseAI::embeddingParams() { return { @@ -471,6 +498,9 @@ void FunctionBaseAI::embedTexts( ColumnPtr FunctionBaseAI::executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const { + /// Before resolving the credentials: their validation errors must not reveal a named collection. + checkHasUser(getContext()); + const auto & settings = getContext()->getSettingsRef(); auto params = resolveAIParams(getContext(), arguments, allParams(), settings[Setting::ai_function_text_default_credentials]); diff --git a/src/Functions/FunctionBaseAI.h b/src/Functions/FunctionBaseAI.h index 140c677ff5d70..5932413e733fc 100644 --- a/src/Functions/FunctionBaseAI.h +++ b/src/Functions/FunctionBaseAI.h @@ -52,7 +52,7 @@ using AIParamValues = std::map>; // STYLE_CHECK_ALLOW class FunctionBaseAI : public IFunction { public: - explicit FunctionBaseAI(ContextPtr context_) : context(context_) {} + explicit FunctionBaseAI(ContextPtr context_) : context(context_) { throwIfBackgroundExpression(context); } bool isStateful() const override { return true; } bool isDeterministic() const override { return false; } @@ -139,6 +139,14 @@ class FunctionBaseAI : public IFunction /// target vector size when the model supports it. static AIParamSpecs embeddingParams(); + /// Throws if the context has no user. Background work (merges, mutations, `Buffer` flushes) evaluates + /// expressions with such a context, which has full access, so the privilege checks do not apply to it. + static void checkHasUser(const ContextPtr & context); + + /// Throws if the context analyzes expressions that are executed in the background (mutations, expressions stored + /// in a table definition), where `checkHasUser` throws. + static void throwIfBackgroundExpression(const ContextPtr & context); + /// Result of `embedTexts`. `embeddings` is aligned 1:1 with the `inputs` argument of `embedTexts`. An /// entry is empty when that input was not embedded (quota exceeded, or a failed request with /// `ai_function_throw_on_error` disabled). diff --git a/src/Functions/aiClassify.cpp b/src/Functions/aiClassify.cpp index 1bc6b5f15c39b..5e2cba24bdb3c 100644 --- a/src/Functions/aiClassify.cpp +++ b/src/Functions/aiClassify.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -38,7 +39,11 @@ class FunctionAiClassify final : public FunctionBaseAI explicit FunctionAiClassify(ContextPtr context_) : FunctionBaseAI(context_) {} - static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } + static FunctionPtr create(ContextPtr context_) + { + context_->checkAccess(AccessType::aiClassify); + return std::make_shared(context_); + } String getName() const override { return name; } bool isVariadic() const override { return true; } diff --git a/src/Functions/aiEmbed.cpp b/src/Functions/aiEmbed.cpp index b7d686147381f..2bd46f24195f0 100644 --- a/src/Functions/aiEmbed.cpp +++ b/src/Functions/aiEmbed.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -53,9 +54,13 @@ class FunctionAiEmbed final : public IFunction public: static constexpr auto name = "aiEmbed"; - static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } + static FunctionPtr create(ContextPtr context) + { + context->checkAccess(AccessType::aiEmbed); + return std::make_shared(context); + } - explicit FunctionAiEmbed(ContextPtr context_) : context(context_) {} + explicit FunctionAiEmbed(ContextPtr context_) : context(context_) { FunctionBaseAI::throwIfBackgroundExpression(context); } String getName() const override { return name; } bool isVariadic() const override { return true; } @@ -93,6 +98,9 @@ class FunctionAiEmbed final : public IFunction ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { + /// Before resolving the credentials: their validation errors must not reveal a named collection. + FunctionBaseAI::checkHasUser(getContext()); + const auto & settings = getContext()->getSettingsRef(); auto params = FunctionBaseAI::resolveAIParams( getContext(), arguments, FunctionBaseAI::embeddingParams(), settings[Setting::ai_function_embedding_default_credentials]); diff --git a/src/Functions/aiExtract.cpp b/src/Functions/aiExtract.cpp index 850665d982e27..d8b456b91350f 100644 --- a/src/Functions/aiExtract.cpp +++ b/src/Functions/aiExtract.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -26,7 +27,11 @@ class FunctionAiExtract final : public FunctionBaseAI explicit FunctionAiExtract(ContextPtr context_) : FunctionBaseAI(context_) {} - static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } + static FunctionPtr create(ContextPtr context_) + { + context_->checkAccess(AccessType::aiExtract); + return std::make_shared(context_); + } String getName() const override { return name; } bool isVariadic() const override { return true; } diff --git a/src/Functions/aiFilter.cpp b/src/Functions/aiFilter.cpp index 51ac1dbffb0e9..6f593160117d3 100644 --- a/src/Functions/aiFilter.cpp +++ b/src/Functions/aiFilter.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -25,7 +26,11 @@ class FunctionAiFilter final : public FunctionBaseAI explicit FunctionAiFilter(ContextPtr context_) : FunctionBaseAI(context_) {} - static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } + static FunctionPtr create(ContextPtr context_) + { + context_->checkAccess(AccessType::aiFilter); + return std::make_shared(context_); + } String getName() const override { return name; } bool isVariadic() const override { return true; } diff --git a/src/Functions/aiGenerate.cpp b/src/Functions/aiGenerate.cpp index 9eda2c0afe5b1..102bff50c6f5c 100644 --- a/src/Functions/aiGenerate.cpp +++ b/src/Functions/aiGenerate.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -22,7 +23,11 @@ class FunctionAiGenerate final : public FunctionBaseAI explicit FunctionAiGenerate(ContextPtr context_) : FunctionBaseAI(context_) {} - static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } + static FunctionPtr create(ContextPtr context_) + { + context_->checkAccess(AccessType::aiGenerate); + return std::make_shared(context_); + } String getName() const override { return name; } bool isVariadic() const override { return true; } diff --git a/src/Functions/aiRedact.cpp b/src/Functions/aiRedact.cpp index fdadbe3507600..0276839927c3b 100644 --- a/src/Functions/aiRedact.cpp +++ b/src/Functions/aiRedact.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -36,7 +37,11 @@ class FunctionAiRedact final : public FunctionBaseAI explicit FunctionAiRedact(ContextPtr context_) : FunctionBaseAI(context_) {} - static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } + static FunctionPtr create(ContextPtr context_) + { + context_->checkAccess(AccessType::aiRedact); + return std::make_shared(context_); + } String getName() const override { return name; } bool isVariadic() const override { return true; } diff --git a/src/Functions/aiSimilarity.cpp b/src/Functions/aiSimilarity.cpp index 0536b0035d608..ace4995fd91a5 100644 --- a/src/Functions/aiSimilarity.cpp +++ b/src/Functions/aiSimilarity.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -80,9 +81,13 @@ class FunctionAiSimilarity final : public IFunction public: static constexpr auto name = "aiSimilarity"; - static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } + static FunctionPtr create(ContextPtr context) + { + context->checkAccess(AccessType::aiSimilarity); + return std::make_shared(context); + } - explicit FunctionAiSimilarity(ContextPtr context_) : context(context_) {} + explicit FunctionAiSimilarity(ContextPtr context_) : context(context_) { FunctionBaseAI::throwIfBackgroundExpression(context); } String getName() const override { return name; } bool isVariadic() const override { return true; } @@ -121,6 +126,9 @@ class FunctionAiSimilarity final : public IFunction ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { + /// Before resolving the credentials: their validation errors must not reveal a named collection. + FunctionBaseAI::checkHasUser(getContext()); + const auto & settings = getContext()->getSettingsRef(); auto params = FunctionBaseAI::resolveAIParams( getContext(), arguments, FunctionBaseAI::embeddingParams(), settings[Setting::ai_function_embedding_default_credentials]); diff --git a/src/Functions/aiTranslate.cpp b/src/Functions/aiTranslate.cpp index cbf2afcb243c2..712ce120b7b5e 100644 --- a/src/Functions/aiTranslate.cpp +++ b/src/Functions/aiTranslate.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -19,7 +20,11 @@ class FunctionAiTranslate final : public FunctionBaseAI explicit FunctionAiTranslate(ContextPtr context_) : FunctionBaseAI(context_) {} - static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } + static FunctionPtr create(ContextPtr context_) + { + context_->checkAccess(AccessType::aiTranslate); + return std::make_shared(context_); + } String getName() const override { return name; } bool isVariadic() const override { return true; } diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index ebd3e4a70ba46..e1b3e7fbceefb 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -1430,6 +1430,7 @@ ContextData::ContextData(const ContextData &o) : is_background_operation(o.is_background_operation), is_ddl_or_on_cluster_internal(o.is_ddl_or_on_cluster_internal), is_recovery_from_stored_metadata(o.is_recovery_from_stored_metadata), + is_analyzing_background_expressions(o.is_analyzing_background_expressions), is_view_inner_query(o.is_view_inner_query), positional_arguments_already_resolved(o.positional_arguments_already_resolved), join_analyze_mode(o.join_analyze_mode), diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index e74cbddab6fcc..bfbf999195581 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -610,6 +610,9 @@ class ContextData /// Set for CREATE queries a Replicated database replays from a definition it already stored. /// Such a definition describes existing state, so validation that may reject a new one must not run. bool is_recovery_from_stored_metadata = false; + /// Set while analyzing expressions that are executed later in the background, without a user: + /// mutations and the expressions stored in a table definition. + bool is_analyzing_background_expressions = false; /// True when this context belongs to the inner query of an expanded view. /// Positional arguments inside views must be resolved even on remote/secondary nodes where /// enable_positional_arguments would otherwise be skipped (views are expanded on remote nodes, @@ -814,6 +817,7 @@ class Context: public ContextData, public std::enable_shared_from_this /// present to be able to add, remove or update warnings from the table enum class WarningType { + AI_FUNCTION_IN_BACKGROUND_OPERATION, AST_FUZZER_IS_ENABLED, AVAILABLE_DISK_SPACE_TOO_LOW_FOR_DATA, AVAILABLE_DISK_SPACE_TOO_LOW_FOR_LOGS, @@ -1872,6 +1876,9 @@ class Context: public ContextData, public std::enable_shared_from_this bool isRecoveryFromStoredMetadata() const { return is_recovery_from_stored_metadata; } void setRecoveryFromStoredMetadata(bool value) { is_recovery_from_stored_metadata = value; } + bool isAnalyzingBackgroundExpressions() const { return is_analyzing_background_expressions; } + void setAnalyzingBackgroundExpressions(bool value) { is_analyzing_background_expressions = value; } + bool isViewInnerQuery() const { return is_view_inner_query; } void setIsViewInnerQuery(bool value) { is_view_inner_query = value; } diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index d49f401deadd6..e68b420f66e3a 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -382,7 +382,9 @@ BlockIO runCommandSegments(CommandSegments & segments, const StoragePtr & table, if (settings[Setting::validate_mutation_query]) { MutationsInterpreter::Settings mutation_settings(false); - MutationsInterpreter(table, metadata_snapshot, *mutation_commands, context, mutation_settings).validate(); + auto validation_context = Context::createCopy(context); + validation_context->setAnalyzingBackgroundExpressions(true); + MutationsInterpreter(table, metadata_snapshot, *mutation_commands, validation_context, mutation_settings).validate(); } table->mutate(*mutation_commands, context); } diff --git a/src/Interpreters/InterpreterDeleteQuery.cpp b/src/Interpreters/InterpreterDeleteQuery.cpp index 8d8b2b07e693c..c72f58120197e 100644 --- a/src/Interpreters/InterpreterDeleteQuery.cpp +++ b/src/Interpreters/InterpreterDeleteQuery.cpp @@ -127,7 +127,9 @@ BlockIO InterpreterDeleteQuery::execute() if (getContext()->getSettingsRef()[Setting::validate_mutation_query]) { MutationsInterpreter::Settings mutation_settings(false); - MutationsInterpreter(table, metadata_snapshot, mutation_commands, getContext(), mutation_settings).validate(); + auto validation_context = Context::createCopy(getContext()); + validation_context->setAnalyzingBackgroundExpressions(true); + MutationsInterpreter(table, metadata_snapshot, mutation_commands, validation_context, mutation_settings).validate(); } table->mutate(mutation_commands, getContext()); return {}; diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 9a8c5912c7baf..8641a82d04018 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -2039,6 +2040,8 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const /// to its target using the target metadata and never evaluates its own column defaults, so a default /// over a virtual column is inert there and is left out of this set. NameSet insert_time_default_columns; + /// New `DEFAULT` and `MATERIALIZED` expressions of a `MergeTree` table, also computed in the background for existing parts. + ASTPtr background_default_expr_list = make_intrusive(); bool defaults_evaluated_at_insert_time = true; if (const auto * mv = dynamic_cast(table.get())) defaults_evaluated_at_insert_time = mv->hasInnerTable(); @@ -2408,6 +2411,16 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const /// Collect default expressions for MODIFY and ADD commands if (command.type == AlterCommand::MODIFY_COLUMN || command.type == AlterCommand::ADD_COLUMN) { + /// A column TTL resets the column to its existing default in merges. + if (command.ttl && !command.default_expression && all_columns.has(column_name)) + { + const auto & default_desc = all_columns.get(column_name).default_desc; + if (default_desc.expression + && (default_desc.kind == ColumnDefaultKind::Default || default_desc.kind == ColumnDefaultKind::Materialized)) + background_default_expr_list->children.emplace_back( + setAlias(default_desc.expression->clone(), column_name + "_tmp_ttl_default")); + } + if (command.default_expression) { DataTypePtr data_type_ptr; @@ -2429,6 +2442,9 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const if (defaults_evaluated_at_insert_time && (command.default_kind == ColumnDefaultKind::Default || command.default_kind == ColumnDefaultKind::Materialized)) insert_time_default_columns.insert(final_column_name); + + if (command.default_kind == ColumnDefaultKind::Default || command.default_kind == ColumnDefaultKind::Materialized) + background_default_expr_list->children.emplace_back(setAlias(command.default_expression->clone(), tmp_column_name)); } /// if we change data type for column with default else if (all_columns.has(column_name) && command.data_type) { @@ -2460,6 +2476,16 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot DROP or CLEAR all columns"); validateColumnsDefaultsAndGetSampleBlock(default_expr_list, all_columns.getAll(), context, insert_time_default_columns); + + /// A replica replaying the `ALTER` of a Replicated database does not repeat the check. + auto txn = context->getZooKeeperMetadataTransaction(); + if (!background_default_expr_list->children.empty() && castStorage(table, DeferredTable::Load) + && (!txn || txn->isInitialQuery())) + { + auto background_context = Context::createCopy(context); + background_context->setAnalyzingBackgroundExpressions(true); + validateColumnsDefaultsAndGetSampleBlock(background_default_expr_list, all_columns.getAll(), background_context, {}); + } } bool AlterCommands::hasNonReplicatedAlterCommand() const diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 8c039d07acbec..e4d333c3f1faf 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -81,6 +81,7 @@ #include #include #include +#include #include #include #include @@ -5104,6 +5105,47 @@ Names expressionSourceColumns(const ASTPtr & ast, const ColumnsDescription & col } +void MergeTreeData::checkStoredExpressions(const StorageInMemoryMetadata & metadata, const ContextPtr & local_context) +{ + auto context = Context::createCopy(local_context); + context->setAnalyzingBackgroundExpressions(true); + + const auto & columns = metadata.columns; + const auto & virtuals = metadata.virtuals; + + for (const auto * key : {&metadata.partition_key, &metadata.sorting_key, &metadata.sampling_key, &metadata.unique_key}) + { + if (key->definition_ast) + KeyDescription::getKeyFromAST(key->definition_ast, columns, virtuals, context); + } + if (metadata.primary_key.definition_ast) + KeyDescription::getPrimaryKeyFromAST(metadata.primary_key.definition_ast, metadata.sorting_key, columns, virtuals, context); + + for (const auto & index : metadata.secondary_indices) + { + if (!index.isImplicitlyCreated()) + IndexDescription::getIndexFromAST(index.definition_ast, columns, false, index.escape_filenames, context); + } + + for (const auto & projection : metadata.projections) + ProjectionDescription::getProjectionFromAST(projection.definition_ast, columns, &metadata.partition_key, context); + + /// A column TTL resets the column to its `DEFAULT` or `MATERIALIZED` expression in merges. + ASTPtr ttl_default_expr_list = make_intrusive(); + for (const auto & [name, ttl_ast] : columns.getColumnTTLs()) + { + TTLDescription::getTTLFromAST(ttl_ast, columns, context, metadata.primary_key, TTLValidationMode::SkipValidation); + const auto & default_desc = columns.get(name).default_desc; + if (default_desc.expression && (default_desc.kind == ColumnDefaultKind::Default || default_desc.kind == ColumnDefaultKind::Materialized)) + ttl_default_expr_list->children.emplace_back(setAlias(default_desc.expression->clone(), name + "_tmp_ttl_default")); + } + if (!ttl_default_expr_list->children.empty()) + validateColumnsDefaultsAndGetSampleBlock(ttl_default_expr_list, columns.getAll(), context); + if (metadata.table_ttl.definition_ast) + TTLTableDescription::getTTLForTableFromAST( + metadata.table_ttl.definition_ast, columns, context, metadata.primary_key, TTLValidationMode::SkipValidation); +} + void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, ContextPtr local_context) const { /// Reject schema-changing ALTER while a streaming query holds a subscription on this storage. @@ -5383,7 +5425,12 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context } removeImplicitStatistics(new_metadata.columns); - commands.apply(new_metadata, local_context, share_nested_offsets); + /// Keys, TTL and projections, and the indices the commands add, are built with this context. They are executed in the + /// background, so AI functions reject it. A replica replaying an ALTER the initiator already committed must not fail on it. + auto apply_context = Context::createCopy(local_context); + if (auto txn = local_context->getZooKeeperMetadataTransaction(); !txn || txn->isInitialQuery()) + apply_context->setAnalyzingBackgroundExpressions(true); + commands.apply(new_metadata, apply_context, share_nested_offsets); /// The `Quantize(...)` codec is immutable via ALTER: it cannot be added, removed, or changed, and the TYPE of a /// Quantize-coded column cannot be changed either. Both reach `ColumnsDescription::modify` as metadata-only changes diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 1e48e4e85f6a7..ff655ec4c340d 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -1047,6 +1047,11 @@ class MergeTreeData : public WithMutableContext, public IStorage, public IBackgr /// If something is wrong, throws an exception. void checkAlterIsPossible(const AlterCommands & commands, ContextPtr context) const override; + /// The expressions stored in the metadata (keys, TTL, skip indices, projections) are built with the global context, + /// which has full access, and are executed in the background. Builds them again with a copy of the query context, + /// which checks the user's access to their functions and rejects functions that are not executed in the background. + static void checkStoredExpressions(const StorageInMemoryMetadata & metadata, const ContextPtr & local_context); + /// Throw exception if command is some kind of DROP command (drop column, drop index, etc) or rename command /// and we have unfinished mutation which need this column to finish. void checkDropOrRenameCommandDoesntAffectInProgressMutations( diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index 89507350ce8e1..6569cb6f28e0e 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -1130,6 +1130,14 @@ static StoragePtr create(const StorageFactory::Arguments & args) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Table TTL is not allowed for MergeTree in old syntax"); } + /// Only a fresh definition written now, not one loaded from stored metadata or replayed from another replica's DDL. + if (is_fresh_definition && !args.getLocalContext()->isRecoveryFromStoredMetadata()) + { + const auto txn = args.getLocalContext()->getZooKeeperMetadataTransaction(); + if (!txn || txn->isInitialQuery()) + MergeTreeData::checkStoredExpressions(metadata, args.getLocalContext()); + } + DataTypes data_types = metadata.partition_key.data_types; if (args.mode <= LoadingStrictnessLevel::CREATE && !(*storage_settings)[MergeTreeSetting::allow_floating_point_partition_key]) { diff --git a/tests/integration/test_ai_functions/test.py b/tests/integration/test_ai_functions/test.py index 98fd6cf9f10cb..ea4bf564c7711 100644 --- a/tests/integration/test_ai_functions/test.py +++ b/tests/integration/test_ai_functions/test.py @@ -1740,20 +1740,6 @@ def test_query_id_header(started_cluster): == qid ) - # A background mutation sends the mutation task's own `current_query_id` and has no `initial_query_id`. - instance.query("DROP TABLE IF EXISTS query_id_header SYNC") - instance.query("CREATE TABLE query_id_header (k UInt64, s String) ENGINE = MergeTree ORDER BY k") - instance.query("INSERT INTO query_id_header VALUES (1, 'a')") - instance.query( - "ALTER TABLE query_id_header UPDATE s = aiGenerate('hi', map('credentials', 'ai_mock')) WHERE 1", - settings={"mutations_sync": 2, "allow_nondeterministic_mutations": 1}, - ) - table_uuid = instance.query("SELECT uuid FROM system.tables WHERE database = 'default' AND name = 'query_id_header'").strip() - headers = last_request()["headers"] - assert headers.get("x-clickhouse-query-id") == f"{table_uuid}::all_1_1_0_2" - assert "x-clickhouse-initial-query-id" not in headers - instance.query("DROP TABLE query_id_header SYNC") - def test_query_id_header_control_characters(started_cluster): """A query id from the native protocol can contain control characters. They are replaced with spaces diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index b06d70208ba21..c5a77060b902a 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -244,5 +244,14 @@ ARROW FLIGHT [] GLOBAL ALL BIGQUERY [] GLOBAL ALL DISK [] GLOBAL ALL SOURCES [] GLOBAL ALL +aiClassify [] GLOBAL AI FUNCTIONS +aiEmbed [] GLOBAL AI FUNCTIONS +aiExtract [] GLOBAL AI FUNCTIONS +aiFilter [] GLOBAL AI FUNCTIONS +aiGenerate [] GLOBAL AI FUNCTIONS +aiRedact [] GLOBAL AI FUNCTIONS +aiSimilarity [] GLOBAL AI FUNCTIONS +aiTranslate [] GLOBAL AI FUNCTIONS +AI FUNCTIONS [] \N ALL ALL ['ALL PRIVILEGES'] \N \N NONE ['USAGE','NO PRIVILEGES'] \N \N diff --git a/tests/queries/0_stateless/04142_ai_functions_named_collection_access.sh b/tests/queries/0_stateless/04142_ai_functions_named_collection_access.sh index dcafa45c1b4de..811dd75331a9f 100755 --- a/tests/queries/0_stateless/04142_ai_functions_named_collection_access.sh +++ b/tests/queries/0_stateless/04142_ai_functions_named_collection_access.sh @@ -21,6 +21,7 @@ CREATE NAMED COLLECTION $collection_name AS api_key = 'fake-key'; CREATE USER $user_name IDENTIFIED WITH plaintext_password BY 'password'; +GRANT AI FUNCTIONS ON *.* TO $user_name; " # Runs the AI functions as the test user in one client invocation and emits one line diff --git a/tests/queries/0_stateless/05441_ai_functions_privileges.reference b/tests/queries/0_stateless/05441_ai_functions_privileges.reference new file mode 100644 index 0000000000000..5d861b92d7d0a --- /dev/null +++ b/tests/queries/0_stateless/05441_ai_functions_privileges.reference @@ -0,0 +1,4 @@ +--- no grant +--- GRANT aiGenerate +--- GRANT AI FUNCTIONS +--- REVOKE aiClassify diff --git a/tests/queries/0_stateless/05441_ai_functions_privileges.sql.j2 b/tests/queries/0_stateless/05441_ai_functions_privileges.sql.j2 new file mode 100644 index 0000000000000..133a07f3576b4 --- /dev/null +++ b/tests/queries/0_stateless/05441_ai_functions_privileges.sql.j2 @@ -0,0 +1,70 @@ +-- Tags: no-parallel +-- no-parallel: creates global users, a named collection and a function with fixed names + +-- All queries read zero rows, so no outbound request is made. + +{% set text_params = "map('credentials', 'nc_05441', 'model', 'm')" %} +{% set embed_params = "map('credentials', 'nc_05441')" %} +{% set fns = [ + ('aiClassify', "x, ['a'], " ~ text_params), + ('aiEmbed', "x, 'm', " ~ embed_params), + ('aiExtract', "x, 'i', " ~ text_params), + ('aiFilter', "x, 'c', " ~ text_params), + ('aiGenerate', "x, " ~ text_params), + ('aiRedact', "x, ['a'], " ~ text_params), + ('aiSimilarity', "x, x, 'm', " ~ embed_params), + ('aiTranslate', "x, 'fr', " ~ text_params), +] %} + +{% macro check(denied, invoker_denied) %} +{% for f, args in fns %} +EXECUTE AS u_05441 SELECT {{ f }}({{ args }}) FROM (SELECT '' AS x WHERE 0);{% if f in denied %} -- { serverError ACCESS_DENIED }{% endif %} + +{% endfor %} +EXECUTE AS u_05441 SELECT * FROM v_definer; +EXECUTE AS u_05441 SELECT * FROM v_invoker;{% if invoker_denied %} -- { serverError ACCESS_DENIED }{% endif %} + +EXECUTE AS u_05441 SELECT udf_05441(x) FROM (SELECT '' AS x WHERE 0);{% if invoker_denied %} -- { serverError ACCESS_DENIED }{% endif %} + +{% endmacro %} + +DROP USER IF EXISTS u_05441, definer_05441; +DROP NAMED COLLECTION IF EXISTS nc_05441; +DROP FUNCTION IF EXISTS udf_05441; + +CREATE NAMED COLLECTION nc_05441 AS + provider = 'openai', + endpoint = 'http://localhost:1/v1/chat/completions', + api_key = 'fake-key'; + +CREATE USER u_05441, definer_05441 IDENTIFIED WITH no_password; +GRANT NAMED COLLECTION ON nc_05441 TO u_05441, definer_05441; +GRANT SELECT ON * TO u_05441; +GRANT AI FUNCTIONS ON *.* TO definer_05441; + +CREATE VIEW v_definer DEFINER = definer_05441 SQL SECURITY DEFINER + AS SELECT aiGenerate(x, map('credentials', 'nc_05441', 'model', 'm')) FROM (SELECT '' AS x WHERE 0); +CREATE VIEW v_invoker SQL SECURITY INVOKER + AS SELECT aiGenerate(x, map('credentials', 'nc_05441', 'model', 'm')) FROM (SELECT '' AS x WHERE 0); +CREATE FUNCTION udf_05441 AS x -> aiGenerate(x, map('credentials', 'nc_05441', 'model', 'm')); + +SELECT '--- no grant'; +{{ check(fns | map(attribute=0) | list, true) }} + +SELECT '--- GRANT aiGenerate'; +GRANT aiGenerate ON *.* TO u_05441; +{{ check(fns | map(attribute=0) | reject('equalto', 'aiGenerate') | list, false) }} + +SELECT '--- GRANT AI FUNCTIONS'; +GRANT AI FUNCTIONS ON *.* TO u_05441; +{{ check([], false) }} + +SELECT '--- REVOKE aiClassify'; +REVOKE aiClassify ON *.* FROM u_05441; +{{ check(['aiClassify'], false) }} + +DROP VIEW v_definer; +DROP VIEW v_invoker; +DROP FUNCTION udf_05441; +DROP USER u_05441, definer_05441; +DROP NAMED COLLECTION nc_05441; diff --git a/tests/queries/0_stateless/05442_ai_functions_privileges_stored_expressions.reference b/tests/queries/0_stateless/05442_ai_functions_privileges_stored_expressions.reference new file mode 100644 index 0000000000000..573541ac9702d --- /dev/null +++ b/tests/queries/0_stateless/05442_ai_functions_privileges_stored_expressions.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/05442_ai_functions_privileges_stored_expressions.sql b/tests/queries/0_stateless/05442_ai_functions_privileges_stored_expressions.sql new file mode 100644 index 0000000000000..bbebd33778dc9 --- /dev/null +++ b/tests/queries/0_stateless/05442_ai_functions_privileges_stored_expressions.sql @@ -0,0 +1,56 @@ +-- Tags: no-parallel, no-replicated-database +-- no-parallel: creates a global user with a fixed name +-- no-replicated-database: DDL in a `Replicated` database is executed with a context without the user, so it is not checked + +-- Expressions stored in a table definition are checked against the user who defines the table. +-- AI functions are executed only in queries run by a user, so they are rejected there even with the privilege. + +DROP USER IF EXISTS u_05442; +CREATE USER u_05442 IDENTIFIED WITH no_password; +GRANT CREATE TABLE, INSERT, SELECT, ALTER ON * TO u_05442; +GRANT TABLE ENGINE ON MergeTree TO u_05442; + +EXECUTE AS u_05442 CREATE TABLE t1 (x String, d DateTime) ENGINE = MergeTree ORDER BY x TTL d + INTERVAL 1 DAY DELETE WHERE aiFilter(x, 'c'); -- { serverError ACCESS_DENIED } +EXECUTE AS u_05442 CREATE TABLE t2 (x String) ENGINE = MergeTree ORDER BY length(aiEmbed(x, 'm')); -- { serverError ACCESS_DENIED } +EXECUTE AS u_05442 CREATE TABLE t3 (x String) ENGINE = MergeTree PARTITION BY length(aiEmbed(x, 'm')) ORDER BY x; -- { serverError ACCESS_DENIED } +EXECUTE AS u_05442 CREATE TABLE t4 (x String, INDEX i length(aiEmbed(x, 'm')) TYPE minmax) ENGINE = MergeTree ORDER BY x; -- { serverError ACCESS_DENIED } +EXECUTE AS u_05442 CREATE TABLE t5 (x String, PROJECTION p (SELECT aiGenerate(x) ORDER BY x)) ENGINE = MergeTree ORDER BY x; -- { serverError ACCESS_DENIED } +EXECUTE AS u_05442 CREATE TABLE t8 (k String, d DateTime, x String) ENGINE = MergeTree ORDER BY (k, d) TTL d + INTERVAL 1 DAY GROUP BY k SET x = any(aiGenerate(x)); -- { serverError ACCESS_DENIED } + +EXECUTE AS u_05442 CREATE TABLE t7 (x String, d DateTime) ENGINE = MergeTree ORDER BY x; +EXECUTE AS u_05442 ALTER TABLE t7 MODIFY TTL d + INTERVAL 1 DAY DELETE WHERE aiFilter(x, 'c'); -- { serverError ACCESS_DENIED } +EXECUTE AS u_05442 ALTER TABLE t7 ADD INDEX i length(aiEmbed(x, 'm')) TYPE minmax; -- { serverError ACCESS_DENIED } +EXECUTE AS u_05442 ALTER TABLE t7 ADD PROJECTION p (SELECT aiGenerate(x) ORDER BY x); -- { serverError ACCESS_DENIED } +EXECUTE AS u_05442 ALTER TABLE t7 ADD COLUMN y String, MODIFY ORDER BY (x, length(aiEmbed(y, 'm'))); -- { serverError ACCESS_DENIED } + +-- Applies to all function privileges, e.g. `demangle`. +SET allow_introspection_functions = 1; +EXECUTE AS u_05442 CREATE TABLE t10 (x String) ENGINE = MergeTree ORDER BY demangle(x); -- { serverError ACCESS_DENIED } +GRANT demangle ON *.* TO u_05442; +EXECUTE AS u_05442 CREATE TABLE t10 (x String) ENGINE = MergeTree ORDER BY demangle(x); +DROP TABLE t10; + +GRANT AI FUNCTIONS ON *.* TO u_05442; +EXECUTE AS u_05442 CREATE TABLE t1 (x String, d DateTime) ENGINE = MergeTree ORDER BY x TTL d + INTERVAL 1 DAY DELETE WHERE aiFilter(x, 'c'); -- { serverError NOT_IMPLEMENTED } +EXECUTE AS u_05442 CREATE TABLE t2 (x String) ENGINE = MergeTree ORDER BY length(aiEmbed(x, 'm')); -- { serverError NOT_IMPLEMENTED } +EXECUTE AS u_05442 CREATE TABLE t8 (k String, d DateTime, x String) ENGINE = MergeTree ORDER BY (k, d) TTL d + INTERVAL 1 DAY GROUP BY k SET x = any(aiGenerate(x)); -- { serverError NOT_IMPLEMENTED } +EXECUTE AS u_05442 ALTER TABLE t7 MODIFY TTL d + INTERVAL 1 DAY DELETE WHERE aiFilter(x, 'c'); -- { serverError NOT_IMPLEMENTED } +EXECUTE AS u_05442 ALTER TABLE t7 ADD PROJECTION p (SELECT aiGenerate(x) ORDER BY x); -- { serverError NOT_IMPLEMENTED } +EXECUTE AS u_05442 CREATE TABLE t3 (x String) ENGINE = MergeTree PARTITION BY length(aiEmbed(x, 'm')) ORDER BY x; -- { serverError NOT_IMPLEMENTED } +EXECUTE AS u_05442 CREATE TABLE t4 (x String, INDEX i length(aiEmbed(x, 'm')) TYPE minmax) ENGINE = MergeTree ORDER BY x; -- { serverError NOT_IMPLEMENTED } +EXECUTE AS u_05442 ALTER TABLE t7 ADD INDEX i length(aiEmbed(x, 'm')) TYPE minmax; -- { serverError NOT_IMPLEMENTED } +EXECUTE AS u_05442 ALTER TABLE t7 ADD COLUMN y String, MODIFY ORDER BY (x, length(aiEmbed(y, 'm'))); -- { serverError NOT_IMPLEMENTED } + +-- Defaults computed in the background: of a column with a TTL, and added by `ALTER` for existing parts. +EXECUTE AS u_05442 CREATE TABLE t9 (x String, d DateTime, y String DEFAULT aiGenerate(x) TTL d + INTERVAL 1 DAY) ENGINE = MergeTree ORDER BY x; -- { serverError NOT_IMPLEMENTED } +EXECUTE AS u_05442 CREATE TABLE t9 (x String, d DateTime, y String MATERIALIZED aiGenerate(x) TTL d + INTERVAL 1 DAY) ENGINE = MergeTree ORDER BY x; -- { serverError NOT_IMPLEMENTED } +EXECUTE AS u_05442 ALTER TABLE t7 ADD COLUMN y String DEFAULT aiGenerate(x); -- { serverError NOT_IMPLEMENTED } +EXECUTE AS u_05442 ALTER TABLE t7 ADD COLUMN y String MATERIALIZED aiGenerate(x); -- { serverError NOT_IMPLEMENTED } +EXECUTE AS u_05442 ALTER TABLE t7 ADD COLUMN y String, MODIFY COLUMN x String DEFAULT aiGenerate(y); -- { serverError NOT_IMPLEMENTED } +EXECUTE AS u_05442 ALTER TABLE t7 ADD COLUMN y String ALIAS aiGenerate(x); +EXECUTE AS u_05442 CREATE TABLE t9 (x String, y String DEFAULT aiGenerate(x)) ENGINE = MergeTree ORDER BY x; +DROP TABLE t9; +SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = 't7'; + +DROP TABLE t7; +DROP USER u_05442; diff --git a/tests/queries/0_stateless/05444_ai_functions_no_user.reference b/tests/queries/0_stateless/05444_ai_functions_no_user.reference new file mode 100644 index 0000000000000..aaf616077c107 --- /dev/null +++ b/tests/queries/0_stateless/05444_ai_functions_no_user.reference @@ -0,0 +1,9 @@ +1 +0 +a z +0 +a +b +a +b +a diff --git a/tests/queries/0_stateless/05444_ai_functions_no_user.sql b/tests/queries/0_stateless/05444_ai_functions_no_user.sql new file mode 100644 index 0000000000000..d4d5a104c115b --- /dev/null +++ b/tests/queries/0_stateless/05444_ai_functions_no_user.sql @@ -0,0 +1,98 @@ +-- Tags: no-parallel +-- no-parallel: creates a global named collection and a global user with fixed names + +-- AI functions are not executed with a context without a user (`Buffer` flushes, mutations). +-- No request is made: the check runs before the provider is contacted. + +DROP NAMED COLLECTION IF EXISTS nc_05444; +CREATE NAMED COLLECTION nc_05444 AS + provider = 'openai', + endpoint = 'http://localhost:1/v1/chat/completions', + api_key = 'fake-key'; + +CREATE TABLE t (x String, CONSTRAINT c CHECK aiFilter(x, 'c', map('credentials', 'nc_05444', 'model', 'm'))) ENGINE = MergeTree ORDER BY x; +CREATE TABLE b (x String) ENGINE = Buffer(currentDatabase(), t, 1, 1, 1, 1, 1, 1, 1); +INSERT INTO b VALUES ('a'); -- { serverError ACCESS_DENIED } +CREATE TABLE t_embed (x String, CONSTRAINT c CHECK length(aiEmbed(x, 'm', map('credentials', 'nc_05444'))) > 0) ENGINE = MergeTree ORDER BY x; +CREATE TABLE b_embed (x String) ENGINE = Buffer(currentDatabase(), t_embed, 1, 1, 1, 1, 1, 1, 1); +INSERT INTO b_embed VALUES ('a'); -- { serverError ACCESS_DENIED } +CREATE TABLE t_similarity (x String, CONSTRAINT c CHECK aiSimilarity(x, x, 'm', map('credentials', 'nc_05444')) > 0) ENGINE = MergeTree ORDER BY x; +CREATE TABLE b_similarity (x String) ENGINE = Buffer(currentDatabase(), t_similarity, 1, 1, 1, 1, 1, 1, 1); +INSERT INTO b_similarity VALUES ('a'); -- { serverError ACCESS_DENIED } +-- The failure is reported in `system.warnings`. +SELECT count() > 0 FROM system.warnings WHERE message LIKE 'A background operation%AI function%'; + +-- The user is checked before the credentials: an inaccessible and an absent collection fail with the same error. +DROP USER IF EXISTS u_05444; +CREATE USER u_05444 IDENTIFIED WITH no_password; +GRANT CREATE TABLE, INSERT, SELECT ON * TO u_05444; +GRANT TABLE ENGINE ON MergeTree, TABLE ENGINE ON Buffer TO u_05444; +EXECUTE AS u_05444 CREATE TABLE t_hidden (x String, CONSTRAINT c CHECK aiFilter(x, 'c', map('credentials', 'nc_05444', 'model', 'm'))) ENGINE = MergeTree ORDER BY x; +EXECUTE AS u_05444 CREATE TABLE b_hidden (x String) ENGINE = Buffer(currentDatabase(), t_hidden, 1, 1, 1, 1, 1, 1, 1); +EXECUTE AS u_05444 INSERT INTO b_hidden SELECT 'a'; -- { serverError ACCESS_DENIED } +EXECUTE AS u_05444 CREATE TABLE t_absent (x String, CONSTRAINT c CHECK aiFilter(x, 'c', map('credentials', 'absent_05444', 'model', 'm'))) ENGINE = MergeTree ORDER BY x; +EXECUTE AS u_05444 CREATE TABLE b_absent (x String) ENGINE = Buffer(currentDatabase(), t_absent, 1, 1, 1, 1, 1, 1, 1); +EXECUTE AS u_05444 INSERT INTO b_absent SELECT 'a'; -- { serverError ACCESS_DENIED } + +-- Mutations are executed in the background, so AI functions in them are rejected when submitted. +CREATE TABLE t_update (x String, y String) ENGINE = MergeTree ORDER BY x; +INSERT INTO t_update VALUES ('a', ''); +ALTER TABLE t_update UPDATE y = aiGenerate(x, map('credentials', 'nc_05444', 'model', 'm')) WHERE 1 SETTINGS mutations_sync = 0, allow_nondeterministic_mutations = 1; -- { serverError NOT_IMPLEMENTED } +ALTER TABLE t_update UPDATE y = 'e' WHERE length(aiEmbed(x, 'm', map('credentials', 'nc_05444'))) > 0 SETTINGS mutations_sync = 0; -- { serverError NOT_IMPLEMENTED } +ALTER TABLE t_update DELETE WHERE aiFilter(x, 'c', map('credentials', 'nc_05444', 'model', 'm')) SETTINGS mutations_sync = 0, allow_nondeterministic_mutations = 1; -- { serverError NOT_IMPLEMENTED } +DROP FUNCTION IF EXISTS udf_05444; +CREATE FUNCTION udf_05444 AS x -> aiGenerate(x, map('credentials', 'nc_05444', 'model', 'm')); +ALTER TABLE t_update UPDATE y = udf_05444(x) WHERE 1 SETTINGS mutations_sync = 0, allow_nondeterministic_mutations = 1; -- { serverError NOT_IMPLEMENTED } +DROP FUNCTION udf_05444; +SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = 't_update'; +ALTER TABLE t_update UPDATE y = 'z' WHERE 1 SETTINGS mutations_sync = 1; +SELECT x, y FROM t_update; + +-- `DEFAULT` and `MATERIALIZED` columns are evaluated by the insert, which has a user. +-- The request fails (unreachable endpoint) and returns an empty string instead of throwing. +SET ai_function_throw_on_error = 0, ai_function_max_retries = 0; + +-- Updating a column recomputes a `MATERIALIZED` column that depends on it, which is rejected the same way. +CREATE TABLE t_materialized (k UInt8, x String, m String MATERIALIZED aiGenerate(x, map('credentials', 'nc_05444', 'model', 'm'))) ENGINE = MergeTree ORDER BY k; +INSERT INTO t_materialized (k, x) VALUES (1, 'a'); +ALTER TABLE t_materialized UPDATE x = 'b' WHERE 1 SETTINGS mutations_sync = 0, allow_nondeterministic_mutations = 1; -- { serverError NOT_IMPLEMENTED } +SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = 't_materialized'; + +CREATE TABLE t_default +( + x String, + d String DEFAULT aiGenerate(x, map('credentials', 'nc_05444', 'model', 'm')), + m String MATERIALIZED aiGenerate(x, map('credentials', 'nc_05444', 'model', 'm')) +) +ENGINE = MergeTree ORDER BY x; +INSERT INTO t_default (x) VALUES ('a'); +INSERT INTO t_default (x) VALUES ('b'); +SELECT x, d, m FROM t_default ORDER BY x; + +-- A merge does not evaluate them again. +OPTIMIZE TABLE t_default FINAL; +SELECT x, d, m FROM t_default ORDER BY x; + +-- A lightweight update is evaluated by the query, which has a user. +CREATE TABLE t_lwu (x String, y String) ENGINE = MergeTree ORDER BY x + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1; +INSERT INTO t_lwu VALUES ('a', 'old'); +UPDATE t_lwu SET y = aiGenerate(x, map('credentials', 'nc_05444', 'model', 'm')) WHERE 1 SETTINGS enable_lightweight_update = 1; +SELECT x, y FROM t_lwu; + +DROP TABLE b; +DROP TABLE t; +DROP TABLE b_embed; +DROP TABLE t_embed; +DROP TABLE b_similarity; +DROP TABLE t_similarity; +DROP TABLE t_update; +DROP TABLE t_default; +DROP TABLE t_lwu; +DROP TABLE t_materialized; +DROP TABLE b_hidden; +DROP TABLE t_hidden; +DROP TABLE b_absent; +DROP TABLE t_absent; +DROP USER u_05444; +DROP NAMED COLLECTION nc_05444; diff --git a/tests/queries/0_stateless/05459_ai_functions_background_defaults.reference b/tests/queries/0_stateless/05459_ai_functions_background_defaults.reference new file mode 100644 index 0000000000000..7b06fcf9e8f7e --- /dev/null +++ b/tests/queries/0_stateless/05459_ai_functions_background_defaults.reference @@ -0,0 +1,3 @@ +x +y DEFAULT +z ALIAS diff --git a/tests/queries/0_stateless/05459_ai_functions_background_defaults.sql b/tests/queries/0_stateless/05459_ai_functions_background_defaults.sql new file mode 100644 index 0000000000000..a625113c2e5a7 --- /dev/null +++ b/tests/queries/0_stateless/05459_ai_functions_background_defaults.sql @@ -0,0 +1,28 @@ +-- AI functions are rejected in `DEFAULT` and `MATERIALIZED` expressions computed in the background: +-- of a column with a TTL, and of a column added or modified by `ALTER`. + +CREATE TABLE t1 (x String, d DateTime, y String DEFAULT aiGenerate(x) TTL d + INTERVAL 1 DAY) ENGINE = MergeTree ORDER BY x; -- { serverError NOT_IMPLEMENTED } +CREATE TABLE t1 (x String, d DateTime, y String MATERIALIZED aiGenerate(x) TTL d + INTERVAL 1 DAY) ENGINE = MergeTree ORDER BY x; -- { serverError NOT_IMPLEMENTED } + +CREATE TABLE t2 (x String, y String DEFAULT aiGenerate(x)) ENGINE = MergeTree ORDER BY x; +ALTER TABLE t2 ADD COLUMN z String DEFAULT aiGenerate(x); -- { serverError NOT_IMPLEMENTED } +ALTER TABLE t2 ADD COLUMN z String MATERIALIZED aiGenerate(x); -- { serverError NOT_IMPLEMENTED } +ALTER TABLE t2 MODIFY COLUMN x String DEFAULT aiGenerate(y); -- { serverError NOT_IMPLEMENTED } +ALTER TABLE t2 ADD COLUMN z String ALIAS aiGenerate(x); +ALTER TABLE t2 MATERIALIZE COLUMN y; -- { serverError NOT_IMPLEMENTED } +SELECT name, default_kind FROM system.columns WHERE database = currentDatabase() AND table = 't2' ORDER BY position; + +DROP TABLE t2; + +-- Column TTL added by `ALTER`. +CREATE TABLE t3 (x String, d DateTime, y String, w String DEFAULT aiGenerate(x)) ENGINE = MergeTree ORDER BY x; +ALTER TABLE t3 MODIFY COLUMN y TTL d + toIntervalDay(length(aiGenerate(x))); -- { serverError NOT_IMPLEMENTED } +ALTER TABLE t3 MODIFY COLUMN w TTL d; -- { serverError NOT_IMPLEMENTED } +DROP TABLE t3; + +-- A lightweight `DELETE` executed as a mutation is rejected. Executed as a lightweight update, it runs in the query +-- and fails on the missing credentials. +CREATE TABLE t4 (x String) ENGINE = MergeTree ORDER BY x SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1; +DELETE FROM t4 WHERE aiFilter(x, 'c') SETTINGS lightweight_delete_mode = 'alter_update'; -- { serverError NOT_IMPLEMENTED } +DELETE FROM t4 WHERE aiFilter(x, 'c') SETTINGS lightweight_delete_mode = 'lightweight_update_force', enable_lightweight_update = 1; -- { serverError BAD_ARGUMENTS } +DROP TABLE t4; From 6e9c059fc09f8ee857328e0b82dab4aa61876fd5 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 7 Nov 2026 21:00:57 +0000 Subject: [PATCH 125/219] Backport #128824 to 26.8: Fix use-after-free of the Replicated database DDL worker --- src/Common/FailPoint.cpp | 1 + src/Databases/DatabaseReplicated.cpp | 134 ++++++++++++++---- src/Databases/DatabaseReplicated.h | 22 ++- src/Databases/DatabaseReplicatedWorker.cpp | 65 ++++++++- src/Databases/DatabaseReplicatedWorker.h | 25 ++++ .../0_stateless/03784_bad_base_backup.sh | 12 ++ ...abase_ddl_during_restore_replica.reference | 1 + ...ted_database_ddl_during_restore_replica.sh | 65 +++++++++ ..._replica_waits_for_initial_query.reference | 5 + ...restore_replica_waits_for_initial_query.sh | 58 ++++++++ ..._replica_waits_for_initial_alter.reference | 6 + ...restore_replica_waits_for_initial_alter.sh | 58 ++++++++ 12 files changed, 420 insertions(+), 32 deletions(-) create mode 100644 tests/queries/0_stateless/05483_replicated_database_ddl_during_restore_replica.reference create mode 100755 tests/queries/0_stateless/05483_replicated_database_ddl_during_restore_replica.sh create mode 100644 tests/queries/0_stateless/05484_replicated_database_restore_replica_waits_for_initial_query.reference create mode 100755 tests/queries/0_stateless/05484_replicated_database_restore_replica_waits_for_initial_query.sh create mode 100644 tests/queries/0_stateless/05546_replicated_database_restore_replica_waits_for_initial_alter.reference create mode 100755 tests/queries/0_stateless/05546_replicated_database_restore_replica_waits_for_initial_alter.sh diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index 08a3ecd04eda6..0c00fdb7849fc 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -230,6 +230,7 @@ static struct InitFiu REGULAR(database_replicated_delay_recovery) \ REGULAR(database_replicated_delay_entry_execution) \ PAUSEABLE(database_replicated_stop_entry_execution) \ + PAUSEABLE_ONCE(database_replicated_pause_before_initial_query_execution) \ PAUSEABLE_ONCE(database_replicated_pause_after_reading_log_pointer) \ PAUSEABLE_ONCE(database_replicated_pause_after_snapshot_identity_check) \ REGULAR(remove_merge_tree_part_delay) \ diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index d83f38975ccab..e432535a3e36f 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -64,6 +64,7 @@ #include #include #include +#include #include #include @@ -860,14 +861,14 @@ bool DatabaseReplicated::waitForReplicaToProcessAllEntries(UInt64 timeout_ms, Sy { auto component_guard = Coordination::setCurrentComponent("DatabaseReplicated::waitForReplicaToProcessAllEntries"); chassert(mode == SyncReplicaMode::DEFAULT || mode == SyncReplicaMode::STRICT); - { - std::lock_guard lock{ddl_worker_mutex}; - if (!ddl_worker || is_probably_dropped) - return false; - } + /// The worker can be stopped and replaced concurrently (`SYSTEM RESTORE DATABASE REPLICA`, `DROP DATABASE`): + /// the reference keeps it alive, and a stopped worker returns from the wait. + auto worker = tryGetDDLWorker(); + if (!worker) + return false; if (mode == SyncReplicaMode::DEFAULT) - return ddl_worker->waitForReplicaToProcessAllEntries(timeout_ms); + return worker->waitForReplicaToProcessAllEntries(timeout_ms); /// In STRICT mode the sync is satisfied as soon as our replication lag is within the /// max_replication_lag_to_enqueue threshold - we do not require every entry to be fully @@ -879,7 +880,7 @@ bool DatabaseReplicated::waitForReplicaToProcessAllEntries(UInt64 timeout_ms, Sy Stopwatch elapsed; while (true) { - UInt32 our_log_ptr = ddl_worker->getLogPointer(); + UInt32 our_log_ptr = worker->getLogPointer(); UInt32 max_log_ptr = parse(getZooKeeper()->get(fs::path(zookeeper_path) / "max_log_ptr")); bool became_synced = our_log_ptr + db_settings[DatabaseReplicatedSetting::max_replication_lag_to_enqueue] >= max_log_ptr; if (became_synced) @@ -893,8 +894,12 @@ bool DatabaseReplicated::waitForReplicaToProcessAllEntries(UInt64 timeout_ms, Sy /// processing (our_log_ptr == max_log_ptr) is strictly stronger than the STRICT /// threshold, so return immediately in that case; otherwise loop and re-check the /// threshold above, re-reading max_log_ptr, which might have advanced while we waited. - if (ddl_worker->waitForReplicaToProcessAllEntries(std::min(wait_step_ms, timeout_ms - elapsed_ms))) + if (worker->waitForReplicaToProcessAllEntries(std::min(wait_step_ms, timeout_ms - elapsed_ms))) return true; + + /// The worker was stopped (e.g. by `SYSTEM RESTORE DATABASE REPLICA` or `DROP DATABASE`), it will not make progress. + if (worker->isStopped()) + return false; } } @@ -1066,11 +1071,36 @@ void DatabaseReplicated::initDDLWorkerUnlocked() chassert(!is_probably_dropped.load()); chassert(!ddl_worker_initialized.load()); - ddl_worker = std::make_unique(this, getContext()); + replaceDDLWorkerUnlocked(std::make_shared(this, getContext())); ddl_worker->startup(); ddl_worker_initialized = true; } +std::shared_ptr DatabaseReplicated::tryGetDDLWorker() const +{ + std::lock_guard lock{ddl_worker_mutex}; + if (!ddl_worker || is_probably_dropped) + return nullptr; + return ddl_worker; +} + +std::shared_ptr DatabaseReplicated::replaceDDLWorkerUnlocked(std::shared_ptr new_worker) +{ + std::lock_guard lock{ddl_worker_ptr_mutex}; + std::swap(ddl_worker, new_worker); + return new_worker; +} + +bool DatabaseReplicated::isDDLWorkerCurrentlyActive() const +{ + std::shared_ptr worker; + { + std::lock_guard lock{ddl_worker_ptr_mutex}; + worker = ddl_worker; + } + return worker && worker->isCurrentlyActive(); +} + void DatabaseReplicated::restoreDatabaseNodesInKeeper(const ZooKeeperPtr & zookeeper) { auto local_context = getContext(); @@ -1129,7 +1159,7 @@ void DatabaseReplicated::reinitializeDDLWorker() ddl_worker->shutdown(); { ddl_worker_initialized = false; - ddl_worker = nullptr; + replaceDDLWorkerUnlocked(nullptr); } } @@ -1359,7 +1389,7 @@ bool DatabaseReplicated::checkDigestValid(const ContextPtr & local_context) cons LOG_TEST(log, "Current in-memory metadata digest: {}", tables_metadata_digest); /// Database is probably being dropped - if (!local_context->getZooKeeperMetadataTransaction() && (!ddl_worker || !ddl_worker->isCurrentlyActive())) + if (!local_context->getZooKeeperMetadataTransaction() && !isDDLWorkerCurrentlyActive()) return true; /// SYSTEM RESTART REPLICA temporarily removes a table from the in-memory tables map @@ -1465,13 +1495,12 @@ BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, Contex if (is_readonly) throw Exception(ErrorCodes::NO_ZOOKEEPER, "Database is in readonly mode, because it cannot connect to ZooKeeper"); - String host_fqdn_id; - { - std::lock_guard lock{ddl_worker_mutex}; - if (!ddl_worker || is_probably_dropped) - throw Exception(ErrorCodes::DATABASE_REPLICATION_FAILED, "Database is not initialized or is being dropped"); - host_fqdn_id = ddl_worker->getCommonHostID(); - } + /// Hold our own reference: `SYSTEM RESTORE DATABASE REPLICA` or `DROP DATABASE` can stop and replace the worker + /// while this query waits in `tryEnqueueAndExecuteEntry`. A stopped worker makes it throw instead of executing. + auto worker = tryGetDDLWorker(); + if (!worker) + throw Exception(ErrorCodes::DATABASE_REPLICATION_FAILED, "Database is not initialized or is being dropped"); + String host_fqdn_id = worker->getCommonHostID(); if (!flags.internal && query_context->isDDLOrOnClusterInternal()) throw Exception(ErrorCodes::INCORRECT_QUERY, "It's not initial query. ON CLUSTER is not allowed for Replicated database."); @@ -1486,7 +1515,7 @@ BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, Contex entry.tracing_context = OpenTelemetry::CurrentContext(); entry.initial_query_id = query_context->getClientInfo().initial_query_id; entry.is_backup_restore = flags.distributed_backup_restore; - String node_path = ddl_worker->tryEnqueueAndExecuteEntry(entry, query_context, flags.internal); + String node_path = worker->tryEnqueueAndExecuteEntry(entry, query_context, flags.internal); Strings hosts_to_wait; Strings unfiltered_hosts = getZooKeeper()->getChildren(zookeeper_path + "/replicas"); @@ -2375,10 +2404,63 @@ void DatabaseReplicated::restoreDatabaseInKeeper(ContextPtr) auto component_guard = Coordination::setCurrentComponent("DatabaseReplicated::restoreDatabaseMetadataInKeeper"); waitDatabaseStarted(); + /// Concurrent restores of the same database would stop and re-create the DDL worker under each other + /// and race on the replication state that `initDatabaseReplica` rewrites (e.g. `max_log_ptr_at_creation`). + std::lock_guard restore_lock{restore_in_keeper_mutex}; + auto zookeeper = getZooKeeper(); if (!zookeeper) throw Exception(ErrorCodes::NO_ZOOKEEPER, "No ZooKeeper"); + /// Stop this replica's DDL worker before the calls below re-initialize replication state: + /// its recovery reads `max_log_ptr_at_creation` and compares the Keeper metadata with the + /// local table set, which those calls rewrite, and a recovery running against a half-updated + /// view detaches local tables into `_broken_replicated_tables`. + std::shared_ptr old_worker; + { + std::lock_guard lock{ddl_worker_mutex}; + old_worker = ddl_worker; + } + + /// Initial queries admitted by the worker execute and commit on their own threads, and `shutdown` does not wait + /// for them. Wait for them before rewriting the replication state: otherwise a query could commit its metadata + /// while the recovery of the new worker compares the Keeper snapshot with the local tables, and the recovery + /// would move the just created table to `_broken_replicated_tables` (or drop it). + /// Stop admitting new initial queries, but keep the worker running while waiting: an admitted query may depend + /// on it, e.g. an `ALTER` of a `ReplicatedMergeTree` table with `alter_sync` waits for its `ALTER_METADATA` entry, + /// which the table executes only while `canExecuteReplicatedMetadataAlter` is true. + /// Not under `ddl_worker_mutex`, because these queries may need it. + if (old_worker) + { + LOG_TRACE(log, "Waiting for initial queries before restoring database metadata in Keeper."); + old_worker->stopInitialQueries(); + old_worker->waitForForegroundTasks(); + } + + { + std::lock_guard lock{ddl_worker_mutex}; + /// Only `restoreDatabaseInKeeper` (serialized by `restore_in_keeper_mutex`) creates a new worker after startup, + /// so the worker is either still the same or was removed by a concurrent `DROP DATABASE`. + chassert(!ddl_worker || ddl_worker == old_worker); + if (ddl_worker) + { + LOG_TRACE(log, "Stopping DDL worker before restoring database metadata in Keeper."); + ddl_worker->shutdown(); + ddl_worker_initialized = false; + replaceDDLWorkerUnlocked(nullptr); + } + } + old_worker.reset(); + + /// If the restore fails, reinitialize the DDL worker so the database remains functional. + /// SAFE: the body runs while an exception propagates and can itself throw (thread + /// creation), which must not replace the restore's own error. + bool need_reinitialize_ddl_worker = true; + SCOPE_EXIT_SAFE({ + if (need_reinitialize_ddl_worker) + reinitializeDDLWorker(); + }); + try { restoreDatabaseNodesInKeeper(zookeeper); @@ -2396,6 +2478,8 @@ void DatabaseReplicated::restoreDatabaseInKeeper(ContextPtr) /// Force the database to recover to update the restored metadata auto current_zookeeper = getZooKeeper(); current_zookeeper->set(replica_path + "/digest", DatabaseReplicatedDDLWorker::FORCE_AUTO_RECOVERY_DIGEST); + + need_reinitialize_ddl_worker = false; reinitializeDDLWorker(); } @@ -2490,7 +2574,7 @@ void DatabaseReplicated::shutdown() { std::lock_guard lock{ddl_worker_mutex}; ddl_worker_initialized = false; - ddl_worker = nullptr; + replaceDDLWorkerUnlocked(nullptr); } DatabaseAtomic::shutdown(); @@ -2505,7 +2589,7 @@ void DatabaseReplicated::dropTable(ContextPtr local_context, const String & tabl waitDatabaseStarted(); auto txn = local_context->getZooKeeperMetadataTransaction(); - chassert(!ddl_worker || !ddl_worker->isCurrentlyActive() || txn || startsWith(table_name, ".inner_id.") || startsWith(table_name, ".tmp.inner_id.")); + chassert(txn || !isDDLWorkerCurrentlyActive() || startsWith(table_name, ".inner_id.") || startsWith(table_name, ".tmp.inner_id.")); if (txn && txn->isInitialQuery() && !txn->isCreateOrReplaceQuery()) { String metadata_zk_path = zookeeper_path + "/metadata/" + escapeForFileName(table_name); @@ -2617,7 +2701,7 @@ void DatabaseReplicated::commitCreateTable(const ASTCreateQuery & query, const S ContextPtr query_context) { auto txn = query_context->getZooKeeperMetadataTransaction(); - chassert(!ddl_worker->isCurrentlyActive() || txn); + chassert(txn || !isDDLWorkerCurrentlyActive()); String statement = getObjectDefinitionFromCreateQuery(query.clone()); @@ -2654,7 +2738,7 @@ void DatabaseReplicated::commitAlterTable(const StorageID & table_id, const String & statement, ContextPtr query_context) { auto txn = query_context->getZooKeeperMetadataTransaction(); - chassert(!ddl_worker || !ddl_worker->isCurrentlyActive() || txn); + chassert(txn || !isDDLWorkerCurrentlyActive()); if (txn && txn->isInitialQuery()) { String metadata_zk_path = zookeeper_path + "/metadata/" + escapeForFileName(table_id.table_name); @@ -2694,7 +2778,7 @@ void DatabaseReplicated::detachTablePermanently(ContextPtr local_context, const throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Support for DETACH TABLE PERMANENTLY is disabled"); auto txn = local_context->getZooKeeperMetadataTransaction(); - chassert(!ddl_worker->isCurrentlyActive() || txn); + chassert(txn || !isDDLWorkerCurrentlyActive()); if (txn && txn->isInitialQuery()) { /// We have to remove metadata from zookeeper, because we do not distinguish permanently detached tables @@ -2719,7 +2803,7 @@ void DatabaseReplicated::removeDetachedPermanentlyFlag(ContextPtr local_context, waitDatabaseStarted(); auto txn = local_context->getZooKeeperMetadataTransaction(); - chassert(!ddl_worker->isCurrentlyActive() || txn); + chassert(txn || !isDDLWorkerCurrentlyActive()); if (txn && txn->isInitialQuery() && attach) { String metadata_zk_path = zookeeper_path + "/metadata/" + escapeForFileName(table_name); diff --git a/src/Databases/DatabaseReplicated.h b/src/Databases/DatabaseReplicated.h index 68fd64c8e93b2..adb9063fe751c 100644 --- a/src/Databases/DatabaseReplicated.h +++ b/src/Databases/DatabaseReplicated.h @@ -249,6 +249,15 @@ class DatabaseReplicated : public DatabaseAtomic void stopLoading() override; void initDDLWorkerUnlocked() TSA_REQUIRES(ddl_worker_mutex); + /// Returns a reference to the DDL worker taken under `ddl_worker_mutex`, or nullptr if the database + /// is not initialized yet or is being dropped. + std::shared_ptr tryGetDDLWorker() const; + /// Assigns `ddl_worker` and returns the previous worker. Requires both mutexes, see `ddl_worker_ptr_mutex`. + std::shared_ptr replaceDDLWorkerUnlocked(std::shared_ptr new_worker) + TSA_REQUIRES(ddl_worker_mutex); + /// Whether the current DDL worker exists and is active. Does not take `ddl_worker_mutex`, + /// so it is safe to call from the callbacks of DDL queries executed by the worker. + bool isDDLWorkerCurrentlyActive() const; void restoreDatabaseNodesInKeeper(const ZooKeeperPtr & zookeeper); void reinitializeDDLWorker(); @@ -276,9 +285,20 @@ class DatabaseReplicated : public DatabaseAtomic std::atomic_bool is_probably_dropped = false; std::atomic_bool is_recovering = false; std::atomic_bool ddl_worker_initialized = false; - std::unique_ptr ddl_worker; + /// Shared, so that a query thread that uses the worker outside of `ddl_worker_mutex` (e.g. waiting in + /// `tryEnqueueAndExecuteEntry`) keeps it alive while `shutdown`, `restoreDatabaseInKeeper` or + /// `reinitializeDDLWorker` concurrently stop and replace it. Such a thread must take its own reference + /// under the mutex with `tryGetDDLWorker` and never dereference this member without the lock. + std::shared_ptr ddl_worker; mutable std::mutex ddl_worker_mutex; + /// Protects only the `ddl_worker` pointer itself: it is assigned under both mutexes and can be read under either. + /// Unlike `ddl_worker_mutex`, it is never held while the worker is shut down (and its threads are joined), + /// so the callbacks of DDL queries (`commitCreateTable`, `dropTable`, ...) can take it with `isDDLWorkerCurrentlyActive` + /// even when they are executed by the worker's threads or concurrently with `SYSTEM RESTORE DATABASE REPLICA`. + mutable std::mutex ddl_worker_ptr_mutex; UInt32 max_log_ptr_at_creation = 0; + /// Serializes `restoreDatabaseInKeeper`. Taken before `ddl_worker_mutex`. + std::mutex restore_in_keeper_mutex; /// Usually operation with metadata are single-threaded because of the way replication works, /// but StorageReplicatedMergeTree may call alterTable outside from DatabaseReplicatedDDLWorker causing race conditions. diff --git a/src/Databases/DatabaseReplicatedWorker.cpp b/src/Databases/DatabaseReplicatedWorker.cpp index 363a2452e8204..bcc650f292e92 100644 --- a/src/Databases/DatabaseReplicatedWorker.cpp +++ b/src/Databases/DatabaseReplicatedWorker.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -52,6 +53,7 @@ namespace FailPoints extern const char database_replicated_delay_recovery[]; extern const char database_replicated_delay_entry_execution[]; extern const char database_replicated_stop_entry_execution[]; + extern const char database_replicated_pause_before_initial_query_execution[]; } @@ -163,9 +165,35 @@ void DatabaseReplicatedDDLWorker::shutdown() active_node_holder.reset(); active_node_holder_zookeeper.reset(); + /// Waiters in `tryEnqueueAndExecuteEntry` and `waitForReplicaToProcessAllEntries` (which may outlive the + /// shutdown, holding a reference to this worker) check `stop_flag` under `mutex`. Taking it before notifying + /// guarantees that each of them either sees the flag or receives the notification. + { + std::lock_guard lock{mutex}; + for (const auto & event : log_ptr_waiters) + event->set(); + } + wait_current_task_change.notify_all(); +} + +void DatabaseReplicatedDDLWorker::stopInitialQueries() +{ + { + std::lock_guard lock{mutex}; + initial_queries_stopped = true; + } wait_current_task_change.notify_all(); } +void DatabaseReplicatedDDLWorker::waitForForegroundTasks() +{ + std::unique_lock lock{mutex}; + chassert(stop_flag || initial_queries_stopped); + if (foreground_tasks) + LOG_TRACE(log, "Waiting for {} admitted initial queries", foreground_tasks); + foreground_tasks_finished.wait(lock, [&] { return foreground_tasks == 0; }); +} + void DatabaseReplicatedDDLWorker::initializeReplication() { /// Check if we need to recover replica. @@ -367,19 +395,31 @@ bool DatabaseReplicatedDDLWorker::waitForReplicaToProcessAllEntries(UInt64 timeo return zookeeper->expired() || current_task >= max_log || stop_flag; }); - if (!processed) + if (!processed || stop_flag) return false; } - /// Lets now wait for max_log_ptr to be processed - Coordination::Stat stat; + /// Lets now wait for max_log_ptr to be processed. + /// The event is set either by the watch or by `shutdown`, so a stopped worker does not wait for the whole timeout. auto event_ptr = std::make_shared(); + { + std::lock_guard lock{mutex}; + if (stop_flag) + return false; + log_ptr_waiters.insert(event_ptr); + } + SCOPE_EXIT({ + std::lock_guard lock{mutex}; + log_ptr_waiters.erase(event_ptr); + }); + + Coordination::Stat stat; auto new_log = zookeeper->get(our_log_ptr_path, &stat, event_ptr); if (new_log == toString(max_log_ptr)) return true; - return event_ptr->tryWait(timeout_ms); + return event_ptr->tryWait(timeout_ms) && !stop_flag; } @@ -483,9 +523,9 @@ String DatabaseReplicatedDDLWorker::tryEnqueueAndExecuteEntry(DDLLogEntry & entr { chassert(zookeeper->expired() || current_task <= entry_name); - if (zookeeper->expired() || stop_flag) + if (zookeeper->expired() || stop_flag || initial_queries_stopped) { - LOG_TRACE(log, "Not enqueueing query: {}", stop_flag ? "replication stopped" : "ZooKeeper session expired"); + LOG_TRACE(log, "Not enqueueing query: {}", zookeeper->expired() ? "ZooKeeper session expired" : "replication stopped"); throw Exception(ErrorCodes::DATABASE_REPLICATION_FAILED, "ZooKeeper session expired or replication stopped, try again"); } @@ -501,8 +541,21 @@ String DatabaseReplicatedDDLWorker::tryEnqueueAndExecuteEntry(DDLLogEntry & entr if (!processed) throw Exception(ErrorCodes::UNFINISHED, "Timeout: Cannot enqueue query on this replica, " "most likely because replica is busy with previous queue entries"); + + /// The predicate above has checked `stop_flag` and `initial_queries_stopped` under `mutex`, + /// so `waitForForegroundTasks` will wait for this query. + ++foreground_tasks; } + SCOPE_EXIT({ + std::lock_guard lock{mutex}; + --foreground_tasks; + if (foreground_tasks == 0) + foreground_tasks_finished.notify_all(); + }); + + FailPointInjection::pauseFailPoint(FailPoints::database_replicated_pause_before_initial_query_execution); + if (entry.parent_table_uuid.has_value() && !checkParentTableExists(entry.parent_table_uuid.value())) throw Exception(ErrorCodes::TABLE_IS_DROPPED, "Parent table doesn't exist"); diff --git a/src/Databases/DatabaseReplicatedWorker.h b/src/Databases/DatabaseReplicatedWorker.h index 5583f112662ff..1f03d07229552 100644 --- a/src/Databases/DatabaseReplicatedWorker.h +++ b/src/Databases/DatabaseReplicatedWorker.h @@ -2,6 +2,9 @@ #include #include #include +#include + +#include namespace DB { @@ -31,8 +34,21 @@ class DatabaseReplicatedDDLWorker : public DDLWorker void shutdown() override; + /// Returns false on timeout or if the worker is stopped before or during the wait. bool waitForReplicaToProcessAllEntries(UInt64 timeout_ms); + /// The worker was shut down. A stopped worker is never restarted, the database replaces it with a new one. + bool isStopped() const { return stop_flag; } + + /// Makes `tryEnqueueAndExecuteEntry` reject new initial queries (as if the worker was stopped), + /// while the worker itself keeps running. Irreversible: the database replaces such a worker afterwards. + void stopInitialQueries(); + + /// Waits until the initial queries that `tryEnqueueAndExecuteEntry` admitted for execution finish. + /// They execute and commit on the submitting threads, so `shutdown` (which joins only the worker's + /// own threads) does not wait for them. Requires `stopInitialQueries` or `shutdown` to be called before. + void waitForForegroundTasks(); + static String enqueueQueryImpl(const ZooKeeperPtr & zookeeper, DDLLogEntry & entry, DatabaseReplicated * const database, bool committed = false, Coordination::Requests additional_checks = {}); /// NOLINT @@ -68,6 +84,15 @@ class DatabaseReplicatedDDLWorker : public DDLWorker std::condition_variable wait_current_task_change; String current_task; + /// The number of initial queries being executed by `tryEnqueueAndExecuteEntry` on the submitting threads. + /// Protected by `mutex`, see `waitForForegroundTasks`. + size_t foreground_tasks = 0; + std::condition_variable foreground_tasks_finished; + /// Set by `stopInitialQueries`. Protected by `mutex`. + bool initial_queries_stopped = false; + /// Events of `waitForReplicaToProcessAllEntries` waiting for our `log_ptr` in Keeper to change. + /// `shutdown` sets them, so that the waiters return without waiting for the timeout. + std::unordered_set> log_ptr_waiters; std::atomic logs_to_keep = std::numeric_limits::max(); std::atomic_bool unsynced_after_recovery = false; diff --git a/tests/queries/0_stateless/03784_bad_base_backup.sh b/tests/queries/0_stateless/03784_bad_base_backup.sh index ef0cdd18a1102..6ac1401f2feae 100755 --- a/tests/queries/0_stateless/03784_bad_base_backup.sh +++ b/tests/queries/0_stateless/03784_bad_base_backup.sh @@ -48,3 +48,15 @@ thread & thread & wait + +# The databases are created outside of the test database, so they must be dropped here: +# leftover `Replicated` databases break tests that enumerate all `Replicated` databases of the server, +# such as `05061_system_drop_database_replica_access`. Wait for the `ASYNC` backups to finish first. +for _ in $(seq 1 600) +do + [[ $($CLICKHOUSE_CLIENT --query "SELECT count() FROM system.backups WHERE status = 'CREATING_BACKUP' AND name LIKE '%${CLICKHOUSE_DATABASE}%'") == 0 ]] && break + sleep 0.1 +done + +$CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS d1_$CLICKHOUSE_DATABASE SYNC" +$CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS d2_$CLICKHOUSE_DATABASE SYNC" diff --git a/tests/queries/0_stateless/05483_replicated_database_ddl_during_restore_replica.reference b/tests/queries/0_stateless/05483_replicated_database_ddl_during_restore_replica.reference new file mode 100644 index 0000000000000..d00491fd7e5bb --- /dev/null +++ b/tests/queries/0_stateless/05483_replicated_database_ddl_during_restore_replica.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/05483_replicated_database_ddl_during_restore_replica.sh b/tests/queries/0_stateless/05483_replicated_database_ddl_during_restore_replica.sh new file mode 100755 index 0000000000000..eec443dcca491 --- /dev/null +++ b/tests/queries/0_stateless/05483_replicated_database_ddl_during_restore_replica.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Tags: zookeeper + +# `SYSTEM RESTORE DATABASE REPLICA` stops and replaces the DDL worker of a `Replicated` database. +# A DDL query that was concurrently waiting in the worker used to keep a raw pointer to it +# and dereference the destroyed worker (segmentation fault). Now such a query either succeeds +# or fails with an exception, and the server stays alive. + +# The racing queries fail with various errors by design; they are not interesting. +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=none + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +DB="rdb_${CLICKHOUSE_TEST_UNIQUE_NAME}" +ZK_PATH="/test/${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}/${CLICKHOUSE_TEST_UNIQUE_NAME}" + +$CLICKHOUSE_CLIENT --query "CREATE DATABASE ${DB} ENGINE = Replicated('${ZK_PATH}', 'shard1', 'replica1')" + +TIMEOUT=10 + +function ddl_thread() +{ + local i=0 + local end=$((SECONDS + TIMEOUT)) + while [ $SECONDS -lt $end ]; do + i=$((i + 1)) + $CLICKHOUSE_CLIENT --query "CREATE TABLE ${DB}.t_${1}_${i} (x UInt64) ENGINE = MergeTree ORDER BY x" >/dev/null 2>&1 + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS ${DB}.t_${1}_${i} SYNC" >/dev/null 2>&1 + done +} + +function sync_thread() +{ + local end=$((SECONDS + TIMEOUT)) + while [ $SECONDS -lt $end ]; do + $CLICKHOUSE_CLIENT --query "SYSTEM SYNC DATABASE REPLICA ${DB}" >/dev/null 2>&1 + done +} + +function restore_thread() +{ + local end=$((SECONDS + TIMEOUT)) + while [ $SECONDS -lt $end ]; do + $CLICKHOUSE_CLIENT --query "SYSTEM RESTORE DATABASE REPLICA ${DB}" >/dev/null 2>&1 + done +} + +ddl_thread 1 & +ddl_thread 2 & +ddl_thread 3 & +sync_thread & +restore_thread & +restore_thread & +wait + +# The server is alive and the database still accepts DDL. +for _ in $(seq 1 60); do + $CLICKHOUSE_CLIENT --query "CREATE TABLE ${DB}.t_final (x UInt64) ENGINE = MergeTree ORDER BY x" >/dev/null 2>&1 && break + sleep 1 +done +$CLICKHOUSE_CLIENT --query "EXISTS TABLE ${DB}.t_final" + +$CLICKHOUSE_CLIENT --query "DROP DATABASE ${DB} SYNC" diff --git a/tests/queries/0_stateless/05484_replicated_database_restore_replica_waits_for_initial_query.reference b/tests/queries/0_stateless/05484_replicated_database_restore_replica_waits_for_initial_query.reference new file mode 100644 index 0000000000000..587aee15552be --- /dev/null +++ b/tests/queries/0_stateless/05484_replicated_database_restore_replica_waits_for_initial_query.reference @@ -0,0 +1,5 @@ +restore is waiting 1 +create succeeded +restore finished +1 +0 diff --git a/tests/queries/0_stateless/05484_replicated_database_restore_replica_waits_for_initial_query.sh b/tests/queries/0_stateless/05484_replicated_database_restore_replica_waits_for_initial_query.sh new file mode 100755 index 0000000000000..6ef139d50d2d2 --- /dev/null +++ b/tests/queries/0_stateless/05484_replicated_database_restore_replica_waits_for_initial_query.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Tags: zookeeper, no-fasttest, no-parallel +# no-parallel: the `database_replicated_pause_before_initial_query_execution` failpoint fires globally, +# it would pause (or be consumed by) a DDL query of another `Replicated` database. + +# An initial DDL query of a `Replicated` database executes and commits on the submitting thread. +# `SYSTEM RESTORE DATABASE REPLICA` must wait for such a query that was already admitted by the old +# DDL worker before the new worker recovers the replica. Otherwise the query could commit its metadata +# while the recovery compares the Keeper snapshot with the local tables, and the recovery would move +# the just created table to the `_broken_replicated_tables` database. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +DB="rdb_${CLICKHOUSE_TEST_UNIQUE_NAME}" +ZK_PATH="/test/${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}/${CLICKHOUSE_TEST_UNIQUE_NAME}" +FAILPOINT="database_replicated_pause_before_initial_query_execution" +RESTORE_QUERY_ID="restore_${CLICKHOUSE_TEST_UNIQUE_NAME}" + +function cleanup() +{ + $CLICKHOUSE_CLIENT --query "SYSTEM DISABLE FAILPOINT $FAILPOINT" >/dev/null 2>&1 ||: + $CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS ${DB} SYNC" >/dev/null 2>&1 ||: + $CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS ${DB}_broken_replicated_tables SYNC" >/dev/null 2>&1 ||: +} +trap cleanup EXIT + +$CLICKHOUSE_CLIENT --query "CREATE DATABASE ${DB} ENGINE = Replicated('${ZK_PATH}', 'shard1', 'replica1')" + +# Hold the initial `CREATE TABLE` after the DDL worker has admitted it, before it executes and commits. +$CLICKHOUSE_CLIENT --query "SYSTEM ENABLE FAILPOINT $FAILPOINT" +$CLICKHOUSE_CLIENT --distributed_ddl_output_mode=none --query "CREATE TABLE ${DB}.t (x UInt64) ENGINE = MergeTree ORDER BY x" & +create_pid=$! +$CLICKHOUSE_CLIENT --query "SYSTEM WAIT FAILPOINT $FAILPOINT PAUSE" + +$CLICKHOUSE_CLIENT --query_id "$RESTORE_QUERY_ID" --query "SYSTEM RESTORE DATABASE REPLICA ${DB}" >/dev/null 2>&1 & +restore_pid=$! + +# The restore stops the old DDL worker and then has to wait for the paused `CREATE TABLE`. +for _ in $(seq 1 600); do + [ "$($CLICKHOUSE_CLIENT --query "SELECT count() FROM system.processes WHERE query_id = '$RESTORE_QUERY_ID'")" = 1 ] && break + sleep 0.1 +done +sleep 2 +$CLICKHOUSE_CLIENT --query "SELECT 'restore is waiting', count() FROM system.processes WHERE query_id = '$RESTORE_QUERY_ID'" + +$CLICKHOUSE_CLIENT --query "SYSTEM NOTIFY FAILPOINT $FAILPOINT" +wait $create_pid && echo "create succeeded" +# The restore proceeds only after the query has committed. The replica is not lost then (its digest is not empty), +# so the restore may refuse with `REPLICA_ALREADY_EXISTS`; either way the DDL worker is re-created. +wait $restore_pid ||: +echo "restore finished" + +# The table created by the successful query is still there after the recovery of the replica. +$CLICKHOUSE_CLIENT --query "SYSTEM SYNC DATABASE REPLICA ${DB}" +$CLICKHOUSE_CLIENT --query "EXISTS TABLE ${DB}.t" +$CLICKHOUSE_CLIENT --query "SELECT count() FROM system.tables WHERE database = '${DB}_broken_replicated_tables'" diff --git a/tests/queries/0_stateless/05546_replicated_database_restore_replica_waits_for_initial_alter.reference b/tests/queries/0_stateless/05546_replicated_database_restore_replica_waits_for_initial_alter.reference new file mode 100644 index 0000000000000..36b34a909261b --- /dev/null +++ b/tests/queries/0_stateless/05546_replicated_database_restore_replica_waits_for_initial_alter.reference @@ -0,0 +1,6 @@ +restore is waiting 1 +alter succeeded +restore finished +x +y +0 diff --git a/tests/queries/0_stateless/05546_replicated_database_restore_replica_waits_for_initial_alter.sh b/tests/queries/0_stateless/05546_replicated_database_restore_replica_waits_for_initial_alter.sh new file mode 100755 index 0000000000000..4083ae7715ff8 --- /dev/null +++ b/tests/queries/0_stateless/05546_replicated_database_restore_replica_waits_for_initial_alter.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Tags: zookeeper, no-fasttest, no-parallel +# no-parallel: the `database_replicated_pause_before_initial_query_execution` failpoint fires globally, +# it would pause (or be consumed by) a DDL query of another `Replicated` database. + +# `SYSTEM RESTORE DATABASE REPLICA` waits for the initial DDL queries that the DDL worker has already admitted. +# An `ALTER` of a `ReplicatedMergeTree` table with `alter_sync = 1` waits until the table executes its +# `ALTER_METADATA` entry, which requires an active DDL worker of the database (`canExecuteReplicatedMetadataAlter`). +# The restore must not stop the worker before such a query finishes, otherwise both queries wait for each other. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +DB="rdb_${CLICKHOUSE_TEST_UNIQUE_NAME}" +ZK_PATH="/test/${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}/${CLICKHOUSE_TEST_UNIQUE_NAME}" +FAILPOINT="database_replicated_pause_before_initial_query_execution" +RESTORE_QUERY_ID="restore_${CLICKHOUSE_TEST_UNIQUE_NAME}" + +function cleanup() +{ + $CLICKHOUSE_CLIENT --query "SYSTEM DISABLE FAILPOINT $FAILPOINT" >/dev/null 2>&1 ||: + $CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS ${DB} SYNC" >/dev/null 2>&1 ||: + $CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS ${DB}_broken_replicated_tables SYNC" >/dev/null 2>&1 ||: +} +trap cleanup EXIT + +$CLICKHOUSE_CLIENT --query "CREATE DATABASE ${DB} ENGINE = Replicated('${ZK_PATH}', 'shard1', 'replica1')" +$CLICKHOUSE_CLIENT --distributed_ddl_output_mode=none --query "CREATE TABLE ${DB}.t (x UInt64) ENGINE = ReplicatedMergeTree ORDER BY x" + +# Hold the initial `ALTER` after the DDL worker has admitted it, before it executes. +$CLICKHOUSE_CLIENT --query "SYSTEM ENABLE FAILPOINT $FAILPOINT" +$CLICKHOUSE_CLIENT --alter_sync 1 --distributed_ddl_output_mode=none --query "ALTER TABLE ${DB}.t ADD COLUMN y UInt64" & +alter_pid=$! +$CLICKHOUSE_CLIENT --query "SYSTEM WAIT FAILPOINT $FAILPOINT PAUSE" + +$CLICKHOUSE_CLIENT --query_id "$RESTORE_QUERY_ID" --query "SYSTEM RESTORE DATABASE REPLICA ${DB}" >/dev/null 2>&1 & +restore_pid=$! + +# The restore has to wait for the paused `ALTER`. +for _ in $(seq 1 600); do + [ "$($CLICKHOUSE_CLIENT --query "SELECT count() FROM system.processes WHERE query_id = '$RESTORE_QUERY_ID'")" = 1 ] && break + sleep 0.1 +done +sleep 2 +$CLICKHOUSE_CLIENT --query "SELECT 'restore is waiting', count() FROM system.processes WHERE query_id = '$RESTORE_QUERY_ID'" + +# The `ALTER` finishes, including the wait for its `ALTER_METADATA` entry, and then the restore proceeds. +$CLICKHOUSE_CLIENT --query "SYSTEM NOTIFY FAILPOINT $FAILPOINT" +wait $alter_pid && echo "alter succeeded" +# The replica is not lost (its digest is not empty), so the restore may refuse with `REPLICA_ALREADY_EXISTS`; +# either way the DDL worker is re-created. +wait $restore_pid ||: +echo "restore finished" + +$CLICKHOUSE_CLIENT --query "SYSTEM SYNC DATABASE REPLICA ${DB}" +$CLICKHOUSE_CLIENT --query "SELECT name FROM system.columns WHERE database = '${DB}' AND table = 't' ORDER BY name" +$CLICKHOUSE_CLIENT --query "SELECT count() FROM system.tables WHERE database = '${DB}_broken_replicated_tables'" From e249ecd7e06c3e1d228149470d982b7ec5e1b267 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 7 Nov 2026 22:08:07 +0000 Subject: [PATCH 126/219] Backport #129205 to 26.8: Check access rights of table functions in DESCRIBE as in SELECT --- src/TableFunctions/ITableFunction.cpp | 16 ++- src/TableFunctions/ITableFunction.h | 9 +- .../TableFunctionViewIfPermitted.cpp | 3 + tests/integration/test_storage_url/test.py | 3 +- .../03822_file_function_read_on_file_grant.sh | 2 +- ...on_create_temporary_table_access.reference | 31 +++++ ..._function_create_temporary_table_access.sh | 110 ++++++++++++++++++ 7 files changed, 167 insertions(+), 7 deletions(-) create mode 100644 tests/queries/0_stateless/05493_describe_table_function_create_temporary_table_access.reference create mode 100755 tests/queries/0_stateless/05493_describe_table_function_create_temporary_table_access.sh diff --git a/src/TableFunctions/ITableFunction.cpp b/src/TableFunctions/ITableFunction.cpp index f55ab64d7ce64..a1009901d26a0 100644 --- a/src/TableFunctions/ITableFunction.cpp +++ b/src/TableFunctions/ITableFunction.cpp @@ -45,6 +45,13 @@ void ITableFunction::checkSourceAccess(ContextPtr context, bool is_insert_query) } } +void ITableFunction::checkCreateTemporaryTableAccess(ContextPtr context, bool is_insert_query) const +{ + auto table_function_properties = TableFunctionFactory::instance().tryGetProperties(getName()); + if (is_insert_query || !(table_function_properties && table_function_properties->allow_readonly)) + context->checkAccess(AccessType::CREATE_TEMPORARY_TABLE); +} + void ITableFunction::checkEngineAccess(ContextPtr context) const { if (!requiresTableEngineGrant()) @@ -60,6 +67,10 @@ ColumnsDescription ITableFunction::getActualTableStructureWithAccess(ContextPtr /// Resolving table structure is always a read operation. checkSourceAccess(context, /*is_insert_query*/ false); checkEngineAccess(context); + /// Resolving the structure may access the source (e.g. schema inference fetches a URL), so it must be + /// gated in the same way as `execute`: otherwise `DESCRIBE TABLE url(...)` would let a user who cannot + /// run the table function (e.g. under `readonly = 1`) make the server request an endpoint. + checkCreateTemporaryTableAccess(context, /*is_insert_query*/ false); return getActualTableStructure(context, is_insert_query); } @@ -74,9 +85,8 @@ StoragePtr ITableFunction::execute(const ASTPtr & ast_function, ContextPtr conte /// Deliberately not under `check_source_access`: no caller takes over the engine check. checkEngineAccess(context); - auto table_function_properties = TableFunctionFactory::instance().tryGetProperties(getName()); - if (check_create_temporary_table && (is_insert_query || !(table_function_properties && table_function_properties->allow_readonly))) - context->checkAccess(AccessType::CREATE_TEMPORARY_TABLE); + if (check_create_temporary_table) + checkCreateTemporaryTableAccess(context, is_insert_query); auto context_to_use = use_global_context ? context->getGlobalContext() : context; diff --git a/src/TableFunctions/ITableFunction.h b/src/TableFunctions/ITableFunction.h index f34337773e668..5cbfb803c94f9 100644 --- a/src/TableFunctions/ITableFunction.h +++ b/src/TableFunctions/ITableFunction.h @@ -107,13 +107,18 @@ class ITableFunction : public std::enable_shared_from_this StoragePtr execute(const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, ColumnsDescription cached_columns_ = {}, bool use_global_context = false, bool is_insert_query = false, bool check_create_temporary_table = true, bool check_source_access = true) const; - /// Returns actual table structure after enforcing source access checks. - /// Use this instead of getActualTableStructure() from outside execute(). + /// Returns actual table structure after enforcing the same access checks as `execute`: source, engine, + /// and `CREATE TEMPORARY TABLE` (for table functions not allowed in readonly mode). + /// Use this instead of `getActualTableStructure` from outside `execute`. ColumnsDescription getActualTableStructureWithAccess(ContextPtr context, bool is_insert_query) const; /// Check that the user has the required source access (e.g. READ ON MYSQL, WRITE ON S3). void checkSourceAccess(ContextPtr context, bool is_insert_query) const; + /// Check the `CREATE TEMPORARY TABLE` grant, required to use a table function unless it is allowed + /// in readonly mode and the query is not an insert. + void checkCreateTemporaryTableAccess(ContextPtr context, bool is_insert_query) const; + /// The URI of the function for permission checking. Can be an empty string if not applicable. /// For example, for url('https://foo.bar') the URI would be 'https://foo.bar'. virtual const String & getFunctionURI() const diff --git a/src/TableFunctions/TableFunctionViewIfPermitted.cpp b/src/TableFunctions/TableFunctionViewIfPermitted.cpp index a30e3ac004695..610bb6f63f094 100644 --- a/src/TableFunctions/TableFunctionViewIfPermitted.cpp +++ b/src/TableFunctions/TableFunctionViewIfPermitted.cpp @@ -91,6 +91,9 @@ void TableFunctionViewIfPermitted::parseArguments(const ASTPtr & ast_function, C ColumnsDescription TableFunctionViewIfPermitted::getActualTableStructure(ContextPtr context, bool is_insert_query) const { + /// Resolving the structure of the `ELSE` table function may access its source (e.g. schema inference + /// fetches a URL), so it requires the same `CREATE TEMPORARY TABLE` grant as executing it. + else_table_function->checkCreateTemporaryTableAccess(context, /*is_insert_query=*/ false); return else_table_function->getActualTableStructure(context, is_insert_query); } diff --git a/tests/integration/test_storage_url/test.py b/tests/integration/test_storage_url/test.py index cd72d82b2fc10..f0bbc2824b716 100644 --- a/tests/integration/test_storage_url/test.py +++ b/tests/integration/test_storage_url/test.py @@ -807,7 +807,8 @@ def test_table_function_url_access_rights(): "DESCRIBE TABLE url('http://nginx:80/test_1', 'TSV')", user="u1" ) - node1.query("GRANT READ ON URL TO u1") + # Like `SELECT`, `DESCRIBE` of a table function also needs `CREATE TEMPORARY TABLE`. + node1.query("GRANT READ ON URL, CREATE TEMPORARY TABLE ON *.* TO u1") assert node1.query( "DESCRIBE TABLE url('http://nginx:80/test_1', 'TSV')", user="u1", diff --git a/tests/queries/0_stateless/03822_file_function_read_on_file_grant.sh b/tests/queries/0_stateless/03822_file_function_read_on_file_grant.sh index 44023bb901007..90b5207bf2d70 100755 --- a/tests/queries/0_stateless/03822_file_function_read_on_file_grant.sh +++ b/tests/queries/0_stateless/03822_file_function_read_on_file_grant.sh @@ -19,7 +19,7 @@ EOF (( $(${CLICKHOUSE_CLIENT} --user $user --query "SELECT file('$missing_txt')" 2>&1 | grep -c "ACCESS_DENIED") >= 1 )) && echo "ACCESS_DENIED" || echo "UNEXPECTED"; (( $(${CLICKHOUSE_CLIENT} --user $user --query "DESCRIBE TABLE file('$missing_csv', 'CSV')" 2>&1 | grep -c "ACCESS_DENIED") >= 1 )) && echo "ACCESS_DENIED" || echo "UNEXPECTED"; -${CLICKHOUSE_CLIENT} --query "GRANT READ ON FILE TO $user"; +${CLICKHOUSE_CLIENT} --query "GRANT READ ON FILE, CREATE TEMPORARY TABLE ON *.* TO $user"; (( $(${CLICKHOUSE_CLIENT} --user $user --query "SELECT file('$missing_txt')" 2>&1 | grep -c "FILE_DOESNT_EXIST") >= 1 )) && echo "FILE_DOESNT_EXIST" || echo "UNEXPECTED"; (( $(${CLICKHOUSE_CLIENT} --user $user --query "DESCRIBE TABLE file('$missing_csv', 'CSV')" 2>&1 | grep -c "CANNOT_STAT") >= 1 )) && echo "CANNOT_STAT" || echo "UNEXPECTED"; diff --git a/tests/queries/0_stateless/05493_describe_table_function_create_temporary_table_access.reference b/tests/queries/0_stateless/05493_describe_table_function_create_temporary_table_access.reference new file mode 100644 index 0000000000000..8a264f0dc70c0 --- /dev/null +++ b/tests/queries/0_stateless/05493_describe_table_function_create_temporary_table_access.reference @@ -0,0 +1,31 @@ +--- readonly_1 +READONLY +READONLY +READONLY +READONLY +READONLY +READONLY +READONLY +OK +--- readonly_2 +OK +OK +OK +OK +OK +OK +--- readonly_0 +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +OK +--- requests to the endpoint +readonly_2_cluster 1 +readonly_2_describe 1 +readonly_2_remote 1 +readonly_2_select 1 +readonly_2_view_if_permitted 1 diff --git a/tests/queries/0_stateless/05493_describe_table_function_create_temporary_table_access.sh b/tests/queries/0_stateless/05493_describe_table_function_create_temporary_table_access.sh new file mode 100755 index 0000000000000..c8437fd1f3aea --- /dev/null +++ b/tests/queries/0_stateless/05493_describe_table_function_create_temporary_table_access.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Resolving the structure of a table function (`DESCRIBE TABLE url(...)`, or a table function nested in +# `cluster` or `viewIfPermitted`) must require the same `CREATE TEMPORARY TABLE` grant as reading from it, and must be refused +# before the source is contacted. Otherwise a user who cannot run `url` (e.g. under `readonly = 1`) can make +# the server request an arbitrary endpoint and read the response from the schema inference error. + +USER_READONLY_1="readonly_1_${CLICKHOUSE_TEST_UNIQUE_NAME}" +USER_READONLY_2="readonly_2_${CLICKHOUSE_TEST_UNIQUE_NAME}" +USER_READONLY_0="readonly_0_${CLICKHOUSE_TEST_UNIQUE_NAME}" +TEMP_CREDENTIALS="unknown" + +# Create users with different readonly settings. +function create_users() { + ${CLICKHOUSE_CLIENT} -m -q " + DROP USER IF EXISTS ${USER_READONLY_1}, ${USER_READONLY_2}, ${USER_READONLY_0}; + CREATE USER ${USER_READONLY_1} IDENTIFIED WITH PLAINTEXT_PASSWORD BY '${TEMP_CREDENTIALS}' SETTINGS readonly = 1; + CREATE USER ${USER_READONLY_2} IDENTIFIED WITH PLAINTEXT_PASSWORD BY '${TEMP_CREDENTIALS}' SETTINGS readonly = 2; + -- \`readonly = 0\`, but without the \`CREATE TEMPORARY TABLE\` grant. + CREATE USER ${USER_READONLY_0} IDENTIFIED WITH PLAINTEXT_PASSWORD BY '${TEMP_CREDENTIALS}'; + GRANT SELECT ON *.*, READ ON URL, READ ON REMOTE TO ${USER_READONLY_1}, ${USER_READONLY_2}, ${USER_READONLY_0}; + GRANT CREATE TEMPORARY TABLE ON *.* TO ${USER_READONLY_1}, ${USER_READONLY_2}; + " +} + +function drop_users() { + ${CLICKHOUSE_CLIENT} -q "DROP USER ${USER_READONLY_1}, ${USER_READONLY_2}, ${USER_READONLY_0}" +} + +# Instead of a mock server, use this server's HTTP interface: every request to it is logged in `system.query_log` +# with the marker given as the argument, so `check_endpoint_requests` can tell whether the source was contacted. +# It always responds in `JSONEachRow`, like the JSON metadata service in the original report: the format passed +# to `url` is how the response is parsed, so `Values` fails to parse it and would echo the body in the error. +function endpoint() { + local marker="$1" + echo "${CLICKHOUSE_PORT_HTTP_PROTO}://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/?database=${CLICKHOUSE_DATABASE}&query=SELECT+1+AS+x,+%27marker_${marker}%27+AS+m+FORMAT+JSONEachRow" +} + +# Run a query as the given user and print `READONLY`, `ACCESS_DENIED`, `OK`, or the full unexpected error. +# `CLICKHOUSE_CLIENT` passes randomized settings, which a `readonly = 1` user is not allowed to change. +function run() { + local user="$1" + local query="$2" + local output + output=$(${CLICKHOUSE_CLIENT_BINARY} --host "${CLICKHOUSE_HOST}" --port "${CLICKHOUSE_PORT_TCP}" \ + --user "${user}" --password "${TEMP_CREDENTIALS}" -q "${query}" 2>&1) + if [[ "${output}" =~ \((READONLY|ACCESS_DENIED)\) ]]; then + echo "${BASH_REMATCH[1]}" + elif [[ "${output}" == *Exception* ]]; then + echo "${output}" + else + echo "OK" + fi +} + +# Run all the queries as one user. The user name without the unique suffix (e.g. `readonly_1`) +# labels the output and the endpoint markers. +function check_user() { + local user="$1" + local label="${user%_"${CLICKHOUSE_TEST_UNIQUE_NAME}"}" + echo "--- ${label}" + run "${user}" "SELECT * FROM url('$(endpoint "${label}_select")', 'JSONEachRow')" + run "${user}" "DESCRIBE TABLE url('$(endpoint "${label}_describe")', 'JSONEachRow')" + # With a format that fails to parse the response, the schema inference error would contain the response body. + # It is skipped for `readonly_2`, which is allowed to read the URL: its error text depends on the server version. + if [[ "${label}" != "readonly_2" ]]; then + run "${user}" "DESCRIBE TABLE url('$(endpoint "${label}_values")', 'Values')" + fi + + # `remote` is not allowed in readonly mode, so it is refused before the nested `url` is resolved. + run "${user}" "DESCRIBE TABLE remote('127.0.0.1', url('$(endpoint "${label}_remote")', 'JSONEachRow'))" + # `cluster` is allowed in readonly mode, but it resolves the structure of the nested `url` on the local shard. + run "${user}" "DESCRIBE TABLE cluster('test_shard_localhost', url('$(endpoint "${label}_cluster")', 'JSONEachRow'))" + + # `viewIfPermitted` is allowed in readonly mode, but it resolves the structure of its `ELSE` table function. + run "${user}" "DESCRIBE TABLE viewIfPermitted(SELECT 1 ELSE url('$(endpoint "${label}_view_if_permitted")', 'JSONEachRow'))" + # `SELECT` from `viewIfPermitted` resolves the structure of its `ELSE` table function too, before choosing a branch. + # It is skipped for `readonly_2`: `SELECT 1` is permitted for it and does not match the structure of the `ELSE` branch. + if [[ "${label}" != "readonly_2" ]]; then + run "${user}" "SELECT * FROM viewIfPermitted(SELECT 1 ELSE url('$(endpoint "${label}_select_view_if_permitted")', 'JSONEachRow'))" + fi + + # Table functions allowed in readonly mode stay available without the grant. + run "${user}" "DESCRIBE TABLE numbers(1)" +} + +# Print the markers of the requests that reached the endpoint, i.e. which queries contacted the source. +# Only the queries of the `readonly = 2` user may appear: a marker without requests has no row. +function check_endpoint_requests() { + echo "--- requests to the endpoint" + ${CLICKHOUSE_CLIENT} -m -q " + SYSTEM FLUSH LOGS query_log; + SELECT extract(query, 'marker_([a-z0-9_]+)') AS marker, count() > 0 + FROM system.query_log + WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND query LIKE 'SELECT 1 AS x, %marker_%' + GROUP BY marker + ORDER BY marker; + " +} + +create_users +for user in "${USER_READONLY_1}" "${USER_READONLY_2}" "${USER_READONLY_0}"; do + check_user "${user}" +done +check_endpoint_requests +drop_users From 942307d0746541998aba20b478f552e297a21a90 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 7 Nov 2026 22:12:19 +0000 Subject: [PATCH 127/219] Backport #117174 to 26.8: Fix lightweight update pruning of empty partitions --- src/Storages/MergeTree/MergeTreeData.cpp | 10 ++ src/Storages/MergeTree/MergeTreeData.h | 5 + src/Storages/StorageMergeTree.cpp | 5 + src/Storages/StorageReplicatedMergeTree.cpp | 8 + ...runing_empty_partition_zookeeper.reference | 6 + ...wu_58_pruning_empty_partition_zookeeper.sh | 140 ++++++++++++++++++ 6 files changed, 174 insertions(+) create mode 100644 tests/queries/0_stateless/03100_lwu_58_pruning_empty_partition_zookeeper.reference create mode 100755 tests/queries/0_stateless/03100_lwu_58_pruning_empty_partition_zookeeper.sh diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index e4d333c3f1faf..86b56fc5fc33d 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -12915,6 +12915,16 @@ StorageSnapshotPtr MergeTreeData::getStorageSnapshot(const StorageMetadataPtr & return cache->emplace(this, createStorageSnapshot(metadata_snapshot, query_context, false)).first->second; } +void MergeTreeData::discardSharedStorageSnapshot(const ContextPtr & query_context) const +{ + auto query_metadata_cache = query_context->getQueryMetadataCache(); + if (!query_metadata_cache) + return; + + auto [cache, lock] = query_metadata_cache->getStorageSnapshotCache(); + cache->erase(this); +} + StorageSnapshotPtr MergeTreeData::getStorageSnapshotWithoutData(const StorageMetadataPtr & metadata_snapshot, ContextPtr query_context) const { diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index ff655ec4c340d..147e3a115495a 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -702,6 +702,11 @@ class MergeTreeData : public WithMutableContext, public IStorage, public IBackgr /// The same as above but does not hold vector of data parts. StorageSnapshotPtr getStorageSnapshotWithoutData(const StorageMetadataPtr & metadata_snapshot, ContextPtr query_context) const override; + /// Removes the snapshot of this table from the query-scoped snapshot cache of + /// `enable_shared_storage_snapshot_in_query`, so that the next `getStorageSnapshot` in the query + /// takes a fresh one. Holders of the removed snapshot keep it. + void discardSharedStorageSnapshot(const ContextPtr & query_context) const; + /// Load the set of data parts from disk. Call once - immediately after the object is created. void loadDataParts(bool skip_sanity_checks, std::optional> expected_parts); diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 4b05914c42094..39ed4ac29ddb1 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1249,6 +1249,11 @@ QueryPipeline StorageMergeTree::updateLightweight(const MutationCommands & comma /// Updates currently don't work with parallel replicas. context_copy->setSetting("max_parallel_replicas", Field(1)); + /// A snapshot cached by `enable_shared_storage_snapshot_in_query` before the wait for committing + /// inserts above (e.g. by the validation of `IN` operands in `InterpreterUpdateQuery`) would hide + /// the parts committed during the wait from the update. See `StorageReplicatedMergeTree::updateLightweight`. + discardSharedStorageSnapshot(context_copy); + auto [pipeline, patch_metadata] = updateLightweightImpl(commands, context_copy); auto sink = std::make_shared( diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index bd1d2f6321ea9..fcdb29f957217 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -8786,6 +8786,14 @@ QueryPipeline StorageReplicatedMergeTree::updateLightweight(const MutationComman waitForCommittingOpsToFinish(zookeeper, context_copy->getPartitionIdToMaxBlock(getStorageID().uuid), ops, backoff_ms, sync_timeout); } + /// With `enable_shared_storage_snapshot_in_query` the update reads with the snapshot of this table + /// cached for the whole query. Anything analyzed before this point may have cached one: the pruning + /// analysis in `allocateBlockNumbersInAffectedPartitions`, or the validation of `IN` operands in + /// `InterpreterUpdateQuery`. Such a snapshot predates the wait for committing inserts above, so the + /// parts that arrived during the wait would be invisible to the update and their rows would stay + /// unchanged. Discard it, so that the update and its subqueries share a snapshot taken now. + discardSharedStorageSnapshot(context_copy); + auto [pipeline, patch_metadata] = updateLightweightImpl(commands, context_copy); auto sink = std::make_shared( diff --git a/tests/queries/0_stateless/03100_lwu_58_pruning_empty_partition_zookeeper.reference b/tests/queries/0_stateless/03100_lwu_58_pruning_empty_partition_zookeeper.reference new file mode 100644 index 0000000000000..8c21354a38e1b --- /dev/null +++ b/tests/queries/0_stateless/03100_lwu_58_pruning_empty_partition_zookeeper.reference @@ -0,0 +1,6 @@ +1 +0 +0 +1 +0 +0 diff --git a/tests/queries/0_stateless/03100_lwu_58_pruning_empty_partition_zookeeper.sh b/tests/queries/0_stateless/03100_lwu_58_pruning_empty_partition_zookeeper.sh new file mode 100755 index 0000000000000..bc088b046271c --- /dev/null +++ b/tests/queries/0_stateless/03100_lwu_58_pruning_empty_partition_zookeeper.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# Tags: zookeeper, no-fasttest, no-parallel, no-replicated-database, no-shared-merge-tree +# no-parallel: uses a global failpoint and two replicas with stopped replication queues. +# no-fasttest: a test that arms a global failpoint must run alone, and such tests are kept out of the fast test. +# no-replicated-database: the test controls replication queues of individual tables. +# no-shared-merge-tree: the failpoint is in the `ReplicatedMergeTree` block allocation path. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +set -e + +function cleanup_tables() +{ + $CLICKHOUSE_CLIENT --query "SYSTEM DISABLE FAILPOINT rmt_lightweight_update_sleep_after_block_allocation" 2>/dev/null || true + $CLICKHOUSE_CLIENT --query "SYSTEM START REPLICATION QUEUES $R1" 2>/dev/null || true + $CLICKHOUSE_CLIENT --query "SYSTEM START REPLICATION QUEUES $R2" 2>/dev/null || true + wait || true + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS $R2 SYNC" 2>/dev/null || true + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS $R1 SYNC" 2>/dev/null || true +} + +function cleanup() +{ + cleanup_tables +} +trap cleanup EXIT + +# Runs a lightweight update of partition 2 whose rows arrive from the other replica while the update +# waits for committing inserts, and checks that the update has patched all of them. +function run_case() +{ + local name=$1 + local assignment=$2 + R1=t_lwu_prune_empty_03100_58_${name}_r1 + R2=t_lwu_prune_empty_03100_58_${name}_r2 + + $CLICKHOUSE_CLIENT --query " + SET insert_keeper_fault_injection_probability = 0.0; + + DROP TABLE IF EXISTS $R2 SYNC; + DROP TABLE IF EXISTS $R1 SYNC; + + CREATE TABLE $R1 (p UInt8, x UInt64, v UInt64) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/t_lwu_prune_empty_03100_58_$name', 'r1') + PARTITION BY p ORDER BY x + SETTINGS + enable_block_number_column = 1, + enable_block_offset_column = 1, + remove_empty_parts = 0; + + CREATE TABLE $R2 (p UInt8, x UInt64, v UInt64) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/t_lwu_prune_empty_03100_58_$name', 'r2') + PARTITION BY p ORDER BY x + SETTINGS + enable_block_number_column = 1, + enable_block_offset_column = 1, + remove_empty_parts = 0; + + INSERT INTO $R1 SELECT 1, number, 1 FROM numbers(50); + INSERT INTO $R1 SELECT 2, number, 1 FROM numbers(50); + SYSTEM SYNC REPLICA $R2; + + ALTER TABLE $R1 DELETE WHERE p = 2 SETTINGS mutations_sync = 2; + + SYSTEM STOP REPLICATION QUEUES $R1; + + SELECT count() FROM system.parts + WHERE database = currentDatabase() AND table = '$R1' AND partition_id = '2' AND active AND rows = 0; + " + + $CLICKHOUSE_CLIENT --query "INSERT INTO $R2 SELECT 2, 1000000 + number, 1 FROM numbers(30)" + + block_numbers_path="/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/t_lwu_prune_empty_03100_58_$name/block_numbers/2" + block_count_before=$($CLICKHOUSE_CLIENT --query " + SYSTEM FLUSH LOGS zookeeper_log; + SELECT count() FROM system.zookeeper_log + WHERE type = 'Response' AND op_num = 'Create' AND path = '$block_numbers_path/block-' + AND startsWith(path_created, '$block_numbers_path/block-')") + + $CLICKHOUSE_CLIENT --query "SYSTEM ENABLE FAILPOINT rmt_lightweight_update_sleep_after_block_allocation" + + $CLICKHOUSE_CLIENT --query "UPDATE $R1 SET v = $assignment WHERE p = 2 SETTINGS enable_lightweight_update = 1, enable_shared_storage_snapshot_in_query = 1" & + + for _ in {0..300}; do + sleep 0.3 + block_count_after=$($CLICKHOUSE_CLIENT --query " + SYSTEM FLUSH LOGS zookeeper_log; + SELECT count() FROM system.zookeeper_log + WHERE type = 'Response' AND op_num = 'Create' AND path = '$block_numbers_path/block-' + AND startsWith(path_created, '$block_numbers_path/block-')") + if [[ "$block_count_after" -gt "$block_count_before" ]]; then + break + fi + done + + if [[ "$block_count_after" -le "$block_count_before" ]]; then + echo "Failed to observe a new block allocation for lightweight update at $block_numbers_path" >&2 + exit 2 + fi + + $CLICKHOUSE_CLIENT --query "SYSTEM START REPLICATION QUEUES $R1" + + wait + + + result=$($CLICKHOUSE_CLIENT --query " + SYSTEM SYNC REPLICA $R1; + SYSTEM SYNC REPLICA $R2; + + SELECT count() FROM $R1 WHERE p = 2 AND v = 1 SETTINGS apply_patch_parts = 1; + SELECT count() FROM $R2 WHERE p = 2 AND v = 1 SETTINGS apply_patch_parts = 1") + + if [[ "$result" != $'0\n0' ]]; then + echo "Unexpected unpatched-row counts in case $name:" >&2 + echo "$result" >&2 + $CLICKHOUSE_CLIENT --query " + SELECT 'patch parts', name, rows FROM system.parts + WHERE database = currentDatabase() AND table IN ('$R1', '$R2') AND startsWith(name, 'patch') + ORDER BY table, name; + SELECT 'partition 2 parts', table, name, rows, min_block_number, max_block_number FROM system.parts + WHERE database = currentDatabase() AND table IN ('$R1', '$R2') AND partition_id = '2' + ORDER BY table, name; + SELECT 'frozen mutation version', table, mutation_id, block_numbers.partition_id, block_numbers.number + FROM system.mutations + WHERE database = currentDatabase() AND table = '$R1' + ORDER BY create_time DESC;" >&2 + exit 1 + fi + + echo "$result" + + cleanup_tables +} + +run_case plain 999 +# The `IN` operand names the updated table, so validating it in `InterpreterUpdateQuery` takes a storage +# snapshot of the table before the update waits for committing inserts. +run_case in_assignment "if((p, x, v) IN t_lwu_prune_empty_03100_58_in_assignment_r1, 999, 998)" From ad2800e0f5db4eb9c1efedda1a437af83e418365 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 6 Nov 2026 17:29:15 +0000 Subject: [PATCH 128/219] Backport #127690 to 26.8: Replace the apache/orc library with a native ORC reader and writer --- .gitmodules | 3 - cmake/target.cmake | 2 +- contrib/CMakeLists.txt | 2 +- contrib/arrow-cmake/CMakeLists.txt | 185 +- contrib/arrow-cmake/orc_check.cmake | 14 - contrib/hive-metastore-cmake/CMakeLists.txt | 8 +- contrib/orc | 1 - src/CMakeLists.txt | 9 +- .../Formats/Impl/ArrowBufferedStreams.cpp | 2 +- .../Formats/Impl/ArrowBufferedStreams.h | 2 +- .../Formats/Impl/ArrowColumnToCHColumn.cpp | 2 +- .../Formats/Impl/ArrowColumnToCHColumn.h | 2 +- .../Impl/NativeORCBlockInputFormat.cpp | 820 ++++----- .../Formats/Impl/NativeORCBlockInputFormat.h | 50 +- .../Formats/Impl/ORC/ColumnReader.cpp | 1213 ++++++++++++++ .../Formats/Impl/ORC/ColumnReader.h | 63 + .../Formats/Impl/ORC/Compression.cpp | 437 +++++ src/Processors/Formats/Impl/ORC/Compression.h | 42 + src/Processors/Formats/Impl/ORC/Proto.cpp | 944 +++++++++++ src/Processors/Formats/Impl/ORC/Proto.h | 343 ++++ src/Processors/Formats/Impl/ORC/RLE.cpp | 1129 +++++++++++++ src/Processors/Formats/Impl/ORC/RLE.h | 151 ++ src/Processors/Formats/Impl/ORC/Reader.cpp | 842 ++++++++++ src/Processors/Formats/Impl/ORC/Reader.h | 188 +++ .../Formats/Impl/ORC/SearchArgument.cpp | 1075 ++++++++++++ .../Formats/Impl/ORC/SearchArgument.h | 198 +++ .../Formats/Impl/ORC/Statistics.cpp | 462 +++++ src/Processors/Formats/Impl/ORC/Statistics.h | 71 + src/Processors/Formats/Impl/ORC/Stream.cpp | 248 +++ src/Processors/Formats/Impl/ORC/Stream.h | 140 ++ src/Processors/Formats/Impl/ORC/Type.cpp | 380 +++++ src/Processors/Formats/Impl/ORC/Type.h | 114 ++ src/Processors/Formats/Impl/ORC/Vector.cpp | 143 ++ src/Processors/Formats/Impl/ORC/Vector.h | 174 ++ src/Processors/Formats/Impl/ORC/Writer.cpp | 1492 +++++++++++++++++ src/Processors/Formats/Impl/ORC/Writer.h | 78 + .../Formats/Impl/ORCBlockOutputFormat.cpp | 490 +----- .../Formats/Impl/ORCBlockOutputFormat.h | 40 +- .../gtest_write_orc_iceberg_required.cpp | 16 +- src/Storages/Hive/HiveFile.cpp | 93 +- src/Storages/Hive/HiveFile.h | 8 +- src/configure_config.cmake | 3 +- .../01308_orc_output_format_arrays.reference | 2 +- .../0_stateless/02426_orc_bug.reference | 2 +- ...337_orc_schema_inference_deep_recursion.sh | 23 +- .../04707_tuple_subcolumn_pushdown_orc.sh | 2 +- .../04856_orc_corrupt_stripe_footer.reference | 1 - .../04856_orc_corrupt_stripe_footer.sh | 10 +- ...orc_unimplemented_codec_no_abort.reference | 14 + .../05036_orc_unimplemented_codec_no_abort.sh | 59 + ...rupt_stripe_footer_stream_column.reference | 3 + ...orc_corrupt_stripe_footer_stream_column.sh | 49 + ...407_orc_timezone_traversal_abort.reference | 5 + .../05407_orc_timezone_traversal_abort.sh | 134 ++ .../05447_orc_native_reader_writer.reference | 14 + .../05447_orc_native_reader_writer.sh | 46 + .../05448_orc_date_after_2059.reference | 12 + .../0_stateless/05448_orc_date_after_2059.sh | 20 + .../05453_orc_bad_bloom_filter.reference | 8 + .../0_stateless/05453_orc_bad_bloom_filter.sh | 27 + .../05473_orc_footer_content_length.reference | 2 + .../05473_orc_footer_content_length.sh | 53 + .../05474_orc_decimal64_v2.reference | 20 + .../0_stateless/05474_orc_decimal64_v2.sh | 13 + .../05507_orc_malformed_timestamps.reference | 16 + .../05507_orc_malformed_timestamps.sh | 19 + ...8_orc_writer_out_of_range_values.reference | 7 + .../05508_orc_writer_out_of_range_values.sh | 26 + ...05509_orc_malformed_dictionaries.reference | 16 + .../05509_orc_malformed_dictionaries.sh | 21 + ...510_orc_allocation_amplification.reference | 32 + .../05510_orc_allocation_amplification.sh | 27 + ...rc_malformed_stripes_and_streams.reference | 16 + ...05511_orc_malformed_stripes_and_streams.sh | 19 + .../05512_orc_malformed_row_groups.reference | 32 + .../05512_orc_malformed_row_groups.sh | 26 + ...c_metadata_repeated_field_limits.reference | 16 + ...5513_orc_metadata_repeated_field_limits.sh | 17 + ...imestamp_statistics_before_epoch.reference | 21 + ...4_orc_timestamp_statistics_before_epoch.sh | 45 + .../05515_orc_nullable_floats.reference | 2 + .../0_stateless/05515_orc_nullable_floats.sh | 30 + .../data_orc/bad_bloom_filter_1.6.0.orc | Bin 0 -> 709 bytes .../data_orc/bad_bloom_filter_1.6.11.orc | Bin 0 -> 717 bytes .../0_stateless/data_orc/decimal64_v2.orc | Bin 0 -> 738 bytes .../malformed_compression_block_1g.orc | Bin 0 -> 414 bytes .../data_orc/malformed_dictionary_empty.orc | Bin 0 -> 170 bytes .../malformed_dictionary_size_max.orc | Bin 0 -> 174 bytes .../malformed_footer_too_many_statistics.orc | Bin 0 -> 353 bytes .../malformed_metadata_too_many_stripes.orc | Bin 0 -> 673 bytes ...med_patched_base_terminal_continuation.orc | Bin 0 -> 83 bytes .../malformed_row_group_count_overflow.orc | Bin 0 -> 237 bytes .../malformed_row_index_seek_past_stream.orc | Bin 0 -> 213 bytes .../data_orc/malformed_snappy_chunk_size.orc | Bin 0 -> 31 bytes .../malformed_stream_offset_overflow.orc | Bin 0 -> 301 bytes .../malformed_stripe_offset_overflow.orc | Bin 0 -> 301 bytes .../malformed_timestamp_nanoseconds.orc | Bin 0 -> 238 bytes ...ormed_timestamp_statistics_nanoseconds.orc | Bin 0 -> 264 bytes .../data_orc/malformed_zstd_chunk_size.orc | Bin 0 -> 38 bytes .../data_orc/row_index_seek_control.orc | Bin 0 -> 213 bytes .../timestamps_before_epoch_arrow.orc | Bin 0 -> 294 bytes ...amps_before_epoch_pyarrow_bloom_filter.orc | Bin 0 -> 517 bytes tests/queries/0_stateless/orc_malformed.lib | 55 + 103 files changed, 11357 insertions(+), 1284 deletions(-) delete mode 100644 contrib/arrow-cmake/orc_check.cmake delete mode 160000 contrib/orc create mode 100644 src/Processors/Formats/Impl/ORC/ColumnReader.cpp create mode 100644 src/Processors/Formats/Impl/ORC/ColumnReader.h create mode 100644 src/Processors/Formats/Impl/ORC/Compression.cpp create mode 100644 src/Processors/Formats/Impl/ORC/Compression.h create mode 100644 src/Processors/Formats/Impl/ORC/Proto.cpp create mode 100644 src/Processors/Formats/Impl/ORC/Proto.h create mode 100644 src/Processors/Formats/Impl/ORC/RLE.cpp create mode 100644 src/Processors/Formats/Impl/ORC/RLE.h create mode 100644 src/Processors/Formats/Impl/ORC/Reader.cpp create mode 100644 src/Processors/Formats/Impl/ORC/Reader.h create mode 100644 src/Processors/Formats/Impl/ORC/SearchArgument.cpp create mode 100644 src/Processors/Formats/Impl/ORC/SearchArgument.h create mode 100644 src/Processors/Formats/Impl/ORC/Statistics.cpp create mode 100644 src/Processors/Formats/Impl/ORC/Statistics.h create mode 100644 src/Processors/Formats/Impl/ORC/Stream.cpp create mode 100644 src/Processors/Formats/Impl/ORC/Stream.h create mode 100644 src/Processors/Formats/Impl/ORC/Type.cpp create mode 100644 src/Processors/Formats/Impl/ORC/Type.h create mode 100644 src/Processors/Formats/Impl/ORC/Vector.cpp create mode 100644 src/Processors/Formats/Impl/ORC/Vector.h create mode 100644 src/Processors/Formats/Impl/ORC/Writer.cpp create mode 100644 src/Processors/Formats/Impl/ORC/Writer.h create mode 100644 tests/queries/0_stateless/05036_orc_unimplemented_codec_no_abort.reference create mode 100755 tests/queries/0_stateless/05036_orc_unimplemented_codec_no_abort.sh create mode 100644 tests/queries/0_stateless/05317_orc_corrupt_stripe_footer_stream_column.reference create mode 100755 tests/queries/0_stateless/05317_orc_corrupt_stripe_footer_stream_column.sh create mode 100644 tests/queries/0_stateless/05407_orc_timezone_traversal_abort.reference create mode 100755 tests/queries/0_stateless/05407_orc_timezone_traversal_abort.sh create mode 100644 tests/queries/0_stateless/05447_orc_native_reader_writer.reference create mode 100755 tests/queries/0_stateless/05447_orc_native_reader_writer.sh create mode 100644 tests/queries/0_stateless/05448_orc_date_after_2059.reference create mode 100755 tests/queries/0_stateless/05448_orc_date_after_2059.sh create mode 100644 tests/queries/0_stateless/05453_orc_bad_bloom_filter.reference create mode 100755 tests/queries/0_stateless/05453_orc_bad_bloom_filter.sh create mode 100644 tests/queries/0_stateless/05473_orc_footer_content_length.reference create mode 100755 tests/queries/0_stateless/05473_orc_footer_content_length.sh create mode 100644 tests/queries/0_stateless/05474_orc_decimal64_v2.reference create mode 100755 tests/queries/0_stateless/05474_orc_decimal64_v2.sh create mode 100644 tests/queries/0_stateless/05507_orc_malformed_timestamps.reference create mode 100755 tests/queries/0_stateless/05507_orc_malformed_timestamps.sh create mode 100644 tests/queries/0_stateless/05508_orc_writer_out_of_range_values.reference create mode 100755 tests/queries/0_stateless/05508_orc_writer_out_of_range_values.sh create mode 100644 tests/queries/0_stateless/05509_orc_malformed_dictionaries.reference create mode 100755 tests/queries/0_stateless/05509_orc_malformed_dictionaries.sh create mode 100644 tests/queries/0_stateless/05510_orc_allocation_amplification.reference create mode 100755 tests/queries/0_stateless/05510_orc_allocation_amplification.sh create mode 100644 tests/queries/0_stateless/05511_orc_malformed_stripes_and_streams.reference create mode 100755 tests/queries/0_stateless/05511_orc_malformed_stripes_and_streams.sh create mode 100644 tests/queries/0_stateless/05512_orc_malformed_row_groups.reference create mode 100755 tests/queries/0_stateless/05512_orc_malformed_row_groups.sh create mode 100644 tests/queries/0_stateless/05513_orc_metadata_repeated_field_limits.reference create mode 100755 tests/queries/0_stateless/05513_orc_metadata_repeated_field_limits.sh create mode 100644 tests/queries/0_stateless/05514_orc_timestamp_statistics_before_epoch.reference create mode 100755 tests/queries/0_stateless/05514_orc_timestamp_statistics_before_epoch.sh create mode 100644 tests/queries/0_stateless/05515_orc_nullable_floats.reference create mode 100755 tests/queries/0_stateless/05515_orc_nullable_floats.sh create mode 100644 tests/queries/0_stateless/data_orc/bad_bloom_filter_1.6.0.orc create mode 100644 tests/queries/0_stateless/data_orc/bad_bloom_filter_1.6.11.orc create mode 100644 tests/queries/0_stateless/data_orc/decimal64_v2.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_compression_block_1g.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_dictionary_empty.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_dictionary_size_max.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_footer_too_many_statistics.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_metadata_too_many_stripes.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_patched_base_terminal_continuation.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_row_group_count_overflow.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_row_index_seek_past_stream.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_snappy_chunk_size.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_stream_offset_overflow.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_stripe_offset_overflow.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_timestamp_nanoseconds.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_timestamp_statistics_nanoseconds.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_zstd_chunk_size.orc create mode 100644 tests/queries/0_stateless/data_orc/row_index_seek_control.orc create mode 100644 tests/queries/0_stateless/data_orc/timestamps_before_epoch_arrow.orc create mode 100644 tests/queries/0_stateless/data_orc/timestamps_before_epoch_pyarrow_bloom_filter.orc create mode 100644 tests/queries/0_stateless/orc_malformed.lib diff --git a/.gitmodules b/.gitmodules index 26fec9523a636..8644bac77ee9d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -76,9 +76,6 @@ [submodule "contrib/fastops"] path = contrib/fastops url = https://github.com/ClickHouse/fastops -[submodule "contrib/orc"] - path = contrib/orc - url = https://github.com/ClickHouse/orc [submodule "contrib/sparsehash-c11"] path = contrib/sparsehash-c11 url = https://github.com/sparsehash/sparsehash-c11 diff --git a/cmake/target.cmake b/cmake/target.cmake index 84f1aaaf4a447..53a8ce14b4a44 100644 --- a/cmake/target.cmake +++ b/cmake/target.cmake @@ -56,7 +56,7 @@ if (OS_WASM) set (ENABLE_GRPC OFF CACHE INTERNAL "") # Protobuf needs a `protoc` that runs on the host, and the nested native configure at the # bottom of the top-level `CMakeLists.txt` would be handed `emcc` as its host compiler. - # Arrow, Parquet and ORC hard-depend on it, so they go with it. + # Arrow and Parquet hard-depend on it, so they go with it. set (ENABLE_PROTOBUF OFF CACHE INTERNAL "") # Its `kj` library uses `fallocate` and friends unconditionally in its POSIX branch, # which the Emscripten libc does not provide. diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 537a9f057984d..0a2ee36dfce6b 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -103,7 +103,7 @@ add_contrib (minizip-ng-cmake minizip-ng) add_contrib (snappy-cmake snappy) add_contrib (thrift-cmake thrift) add_contrib (StringZilla-cmake StringZilla) -# parquet/arrow/orc +# parquet/arrow add_contrib (xsimd-cmake xsimd) # required by arrow's SIMD codepaths add_contrib (arrow-cmake arrow) # requires: snappy, thrift, double-conversion, xsimd add_contrib (avro-cmake avro) # requires: snappy diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 52dd396142ac0..37c97f40e8090 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -24,14 +24,6 @@ if (NOT ENABLE_PARQUET) return() endif() -# ORC lives here because Arrow has an adapter for it, but nothing about reading Parquet needs it. -# It is also the only thing in the whole tree that runs `protoc`, and on a cross-compile that means -# bootstrapping a native protobuf first, so turning it off is what makes a Parquet-only build cheap. -option (ENABLE_ORC "Enable ORC (pulls in protobuf and, when cross-compiling, a native protoc)" ON) -if (NOT ENABLE_ORC) - message(STATUS "Not using ORC") -endif() - # Support C11 if(NOT DEFINED CMAKE_C_STANDARD) set(CMAKE_C_STANDARD 11) @@ -65,38 +57,8 @@ else() endif() -# === orc -set(ORC_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/orc/c++") -set(ORC_INCLUDE_DIR "${ORC_SOURCE_DIR}/include") -set(ORC_SOURCE_SRC_DIR "${ORC_SOURCE_DIR}/src") -# set(ORC_SOURCE_WRAP_DIR "${ORC_SOURCE_DIR}/wrap") - -set(ORC_BUILD_SRC_DIR "${CMAKE_CURRENT_BINARY_DIR}/../orc/c++/src") -set(ORC_BUILD_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/../orc/c++/include") - -set(ORC_ADDITION_SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}) set(ARROW_SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src") -if (NOT CMAKE_HOST_SYSTEM_NAME STREQUAL CMAKE_SYSTEM_NAME OR NOT CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL CMAKE_SYSTEM_PROCESSOR OR USE_MUSL) - set(PROTOBUF_EXECUTABLE "${PROJECT_BINARY_DIR}/native/contrib/google-protobuf-cmake/protoc") - set(GRPC_EXECUTABLE "${PROJECT_BINARY_DIR}/native/contrib/grpc-cmake/grpc_cpp_plugin") -else () - set(PROTOBUF_EXECUTABLE $) - set(GRPC_EXECUTABLE $) -endif() -set(PROTO_DIR "${ORC_SOURCE_DIR}/../proto") - -if (ENABLE_ORC) - add_custom_command(OUTPUT orc_proto.pb.h orc_proto.pb.cc - COMMAND ${PROTOBUF_EXECUTABLE} - -I ${PROTO_DIR} - --cpp_out="${CMAKE_CURRENT_BINARY_DIR}" - "${PROTO_DIR}/orc_proto.proto" - # Without these, the generated sources are not regenerated when the schema or - # `protoc` itself changes, and a stale `orc_proto.pb.h` is compiled instead. - DEPENDS "${PROTO_DIR}/orc_proto.proto" ${PROTOBUF_EXECUTABLE}) -endif() - # === flatbuffers set(FLATBUFFERS_SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/flatbuffers") set(FLATBUFFERS_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/flatbuffers") @@ -130,134 +92,6 @@ add_library(ch_contrib::arrow_ipc_metadata ALIAS _arrow_ipc_metadata) # NOTE: cannot use ch_contrib::hdfs since it's INCLUDE_DIRECTORIES does not includes trailing "hdfs/" set(HDFS_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include/hdfs/") -# Everything down to the end of the `_orc` target below is ORC. Left un-indented inside the -# condition to keep it comparable with the Arrow build description it was transcribed from. -if (ENABLE_ORC) - -# arrow-cmake cmake file calling orc cmake subroutine which detects certain compiler features. -# Apple Clang compiler failed to compile this code without specifying c++11 standard. -# As result these compiler features detected as absent. In result it failed to compile orc itself. -# In orc makefile there is code that sets flags, but arrow-cmake ignores these flags. -if (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") - set(CXX11_FLAGS "-std=c++0x") -endif () - -set (ORC_CXX_HAS_INITIALIZER_LIST 1) -set (ORC_CXX_HAS_NOEXCEPT 1) -set (ORC_CXX_HAS_NULLPTR 1) -set (ORC_CXX_HAS_OVERRIDE 1) -set (ORC_CXX_HAS_UNIQUE_PTR 1) -set (ORC_CXX_HAS_CSTDINT 1) -set (ORC_CXX_HAS_THREAD_LOCAL 1) - -include(orc_check.cmake) -configure_file("${ORC_SOURCE_SRC_DIR}/Adaptor.hh.in" "${ORC_BUILD_INCLUDE_DIR}/Adaptor.hh") - - -# ARROW_ORC + adapters/orc/CMakefiles -set(ORC_SRCS - "${CMAKE_CURRENT_BINARY_DIR}/orc_proto.pb.h" - "${ORC_ADDITION_SOURCE_DIR}/orc_proto.pb.cc" - "${ORC_SOURCE_SRC_DIR}/Adaptor.cc" - "${ORC_SOURCE_SRC_DIR}/Adaptor.hh.in" - "${ORC_SOURCE_SRC_DIR}/BlockBuffer.cc" - "${ORC_SOURCE_SRC_DIR}/BlockBuffer.hh" - "${ORC_SOURCE_SRC_DIR}/BloomFilter.cc" - "${ORC_SOURCE_SRC_DIR}/BloomFilter.hh" - "${ORC_SOURCE_SRC_DIR}/Bpacking.hh" - "${ORC_SOURCE_SRC_DIR}/BpackingDefault.cc" - "${ORC_SOURCE_SRC_DIR}/BpackingDefault.hh" - "${ORC_SOURCE_SRC_DIR}/ByteRLE.cc" - "${ORC_SOURCE_SRC_DIR}/ByteRLE.hh" - "${ORC_SOURCE_SRC_DIR}/ColumnPrinter.cc" - "${ORC_SOURCE_SRC_DIR}/ColumnReader.cc" - "${ORC_SOURCE_SRC_DIR}/ColumnReader.hh" - "${ORC_SOURCE_SRC_DIR}/ColumnWriter.cc" - "${ORC_SOURCE_SRC_DIR}/ColumnWriter.hh" - "${ORC_SOURCE_SRC_DIR}/Common.cc" - "${ORC_SOURCE_SRC_DIR}/Dictionary.cc" - "${ORC_SOURCE_SRC_DIR}/DictionaryLoader.cc" - "${ORC_SOURCE_SRC_DIR}/Compression.cc" - "${ORC_SOURCE_SRC_DIR}/Compression.hh" - "${ORC_SOURCE_SRC_DIR}/ConvertColumnReader.cc" - "${ORC_SOURCE_SRC_DIR}/ConvertColumnReader.hh" - "${ORC_SOURCE_SRC_DIR}/CpuInfoUtil.cc" - "${ORC_SOURCE_SRC_DIR}/CpuInfoUtil.hh" - "${ORC_SOURCE_SRC_DIR}/Dispatch.hh" - "${ORC_SOURCE_SRC_DIR}/Exceptions.cc" - "${ORC_SOURCE_SRC_DIR}/Geospatial.cc" - "${ORC_SOURCE_SRC_DIR}/Int128.cc" - "${ORC_SOURCE_SRC_DIR}/LzoDecompressor.cc" - "${ORC_SOURCE_SRC_DIR}/LzoDecompressor.hh" - "${ORC_SOURCE_SRC_DIR}/MemoryPool.cc" - "${ORC_SOURCE_SRC_DIR}/Murmur3.cc" - "${ORC_SOURCE_SRC_DIR}/Murmur3.hh" - "${ORC_SOURCE_SRC_DIR}/Options.hh" - "${ORC_SOURCE_SRC_DIR}/OrcFile.cc" - "${ORC_SOURCE_SRC_DIR}/RLE.cc" - "${ORC_SOURCE_SRC_DIR}/RLE.hh" - "${ORC_SOURCE_SRC_DIR}/RLEV2Util.cc" - "${ORC_SOURCE_SRC_DIR}/RLEV2Util.hh" - "${ORC_SOURCE_SRC_DIR}/RLEv1.cc" - "${ORC_SOURCE_SRC_DIR}/RLEv1.hh" - "${ORC_SOURCE_SRC_DIR}/RLEv2.hh" - "${ORC_SOURCE_SRC_DIR}/Reader.cc" - "${ORC_SOURCE_SRC_DIR}/Reader.hh" - "${ORC_SOURCE_SRC_DIR}/RleDecoderV2.cc" - "${ORC_SOURCE_SRC_DIR}/RleEncoderV2.cc" - "${ORC_SOURCE_SRC_DIR}/SchemaEvolution.cc" - "${ORC_SOURCE_SRC_DIR}/SchemaEvolution.hh" - "${ORC_SOURCE_SRC_DIR}/Statistics.cc" - "${ORC_SOURCE_SRC_DIR}/Statistics.hh" - "${ORC_SOURCE_SRC_DIR}/StripeStream.cc" - "${ORC_SOURCE_SRC_DIR}/StripeStream.hh" - "${ORC_SOURCE_SRC_DIR}/Timezone.cc" - "${ORC_SOURCE_SRC_DIR}/Timezone.hh" - "${ORC_SOURCE_SRC_DIR}/TypeImpl.cc" - "${ORC_SOURCE_SRC_DIR}/TypeImpl.hh" - "${ORC_SOURCE_SRC_DIR}/Utils.hh" - "${ORC_SOURCE_SRC_DIR}/Vector.cc" - "${ORC_SOURCE_SRC_DIR}/Writer.cc" - "${ORC_SOURCE_SRC_DIR}/io/InputStream.cc" - "${ORC_SOURCE_SRC_DIR}/io/InputStream.hh" - "${ORC_SOURCE_SRC_DIR}/io/OutputStream.cc" - "${ORC_SOURCE_SRC_DIR}/io/OutputStream.hh" - "${ORC_SOURCE_SRC_DIR}/io/Cache.cc" - "${ORC_SOURCE_SRC_DIR}/io/Cache.hh" - "${ORC_SOURCE_SRC_DIR}/sargs/ExpressionTree.cc" - "${ORC_SOURCE_SRC_DIR}/sargs/ExpressionTree.hh" - "${ORC_SOURCE_SRC_DIR}/sargs/Literal.cc" - "${ORC_SOURCE_SRC_DIR}/sargs/PredicateLeaf.cc" - "${ORC_SOURCE_SRC_DIR}/sargs/PredicateLeaf.hh" - "${ORC_SOURCE_SRC_DIR}/sargs/SargsApplier.cc" - "${ORC_SOURCE_SRC_DIR}/sargs/SargsApplier.hh" - "${ORC_SOURCE_SRC_DIR}/sargs/SearchArgument.cc" - "${ORC_SOURCE_SRC_DIR}/sargs/SearchArgument.hh" - "${ORC_SOURCE_SRC_DIR}/sargs/TruthValue.cc" - ) - -add_library(_orc ${ORC_SRCS}) -target_link_libraries(_orc PRIVATE - ch_contrib::protobuf - ch_contrib::lz4 - ch_contrib::snappy - ch_contrib::zlib - ch_contrib::zstd) -target_include_directories(_orc SYSTEM BEFORE PUBLIC - ${ORC_INCLUDE_DIR} - "${ClickHouse_SOURCE_DIR}/contrib/arrow-cmake/cpp/src/orc/c++/include") -target_compile_options(_orc PRIVATE - -include "${ClickHouse_SOURCE_DIR}/contrib/google-protobuf/src/google/protobuf/stubs/port.h" -) -target_include_directories(_orc SYSTEM BEFORE PUBLIC ${ORC_BUILD_INCLUDE_DIR}) -target_include_directories(_orc SYSTEM PRIVATE - ${ORC_SOURCE_SRC_DIR} - ${ORC_SOURCE_WRAP_DIR} - ${ORC_BUILD_SRC_DIR} - ${ORC_ADDITION_SOURCE_DIR} - ${ARROW_SRC_DIR}) - -endif() # ENABLE_ORC # === arrow @@ -269,11 +103,8 @@ configure_file("${LIBRARY_DIR}/util/config_internal.h.cmake" "${ARROW_GENERATED_ # arrow/cpp/src/arrow/CMakeLists.txt (ARROW_SRCS + ARROW_COMPUTE + ARROW_IPC) # find . \( -iname \*.cc -o -iname \*.cpp -o -iname \*.c \) | sort | awk '{print "\"${LIBRARY_DIR}" substr($1,2) "\"" }' | grep -v 'test.cc' | grep -v 'json' | grep -v 'flight' \| -# grep -v 'csv' | grep -v 'acero' | grep -v 'dataset' | grep -v 'testing' | grep -v 'gpu' | grep -v 'engine' | grep -v 'filesystem' | grep -v 'benchmark.cc' +# grep -v 'csv' | grep -v 'acero' | grep -v 'dataset' | grep -v 'testing' | grep -v 'gpu' | grep -v 'engine' | grep -v 'filesystem' | grep -v 'benchmark.cc' | grep -v 'adapters/orc' set(ARROW_SRCS - "${LIBRARY_DIR}/adapters/orc/adapter.cc" - "${LIBRARY_DIR}/adapters/orc/options.cc" - "${LIBRARY_DIR}/adapters/orc/util.cc" "${LIBRARY_DIR}/array/array_base.cc" "${LIBRARY_DIR}/array/array_binary.cc" "${LIBRARY_DIR}/array/array_decimal.cc" @@ -495,16 +326,8 @@ set(ARROW_SRCS "${LIBRARY_DIR}/vendored/uriparser/UriResolve.c" "${LIBRARY_DIR}/vendored/uriparser/UriShorten.c" "${LIBRARY_DIR}/visitor.cc" - - "${ARROW_SRC_DIR}/arrow/adapters/orc/adapter.cc" - "${ARROW_SRC_DIR}/arrow/adapters/orc/util.cc" - "${ARROW_SRC_DIR}/arrow/adapters/orc/options.cc" ) -if (NOT ENABLE_ORC) - # The Arrow-to-ORC adapter is the one part of Arrow that includes `orc/OrcFile.hh`. - list (FILTER ARROW_SRCS EXCLUDE REGEX "/adapters/orc/") -endif() add_definitions(-DARROW_WITH_BROTLI) SET(ARROW_SRCS "${LIBRARY_DIR}/util/compression_brotli.cc" ${ARROW_SRCS}) @@ -575,12 +398,6 @@ target_link_libraries(_arrow PRIVATE # PUBLIC so the xsimd headers are visible to _parquet too (it links _arrow PUBLIC). target_link_libraries(_arrow PUBLIC ch_contrib::xsimd) -if (ENABLE_ORC) - target_link_libraries(_arrow PUBLIC _orc) - # ORC is the only thing here with generated protobuf sources. - add_dependencies(_arrow protoc) -endif() - target_include_directories(_arrow SYSTEM BEFORE PUBLIC ${ARROW_SRC_DIR}) target_include_directories(_arrow SYSTEM BEFORE PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/arrow-cmake/cpp/src") diff --git a/contrib/arrow-cmake/orc_check.cmake b/contrib/arrow-cmake/orc_check.cmake deleted file mode 100644 index 523e1cf1d86bc..0000000000000 --- a/contrib/arrow-cmake/orc_check.cmake +++ /dev/null @@ -1,14 +0,0 @@ -set (HAS_PREAD 1) -set (HAS_STRPTIME 1) -set (HAS_STOLL 1) -set (INT64_IS_LL 1) -set (HAS_DIAGNOSTIC_PUSH 1) -set (HAS_STD_ISNAN 1) -set (HAS_STD_MUTEX 1) -set (NEEDS_REDUNDANT_MOVE 1) -set (HAS_PRE_1970 1) -set (HAS_POST_2038 1) -set (NEEDS_Z_PREFIX 0) - -set(CMAKE_REQUIRED_INCLUDES) -set(CMAKE_REQUIRED_LIBRARIES) diff --git a/contrib/hive-metastore-cmake/CMakeLists.txt b/contrib/hive-metastore-cmake/CMakeLists.txt index c9d698264c4a6..bf92db62d1b92 100644 --- a/contrib/hive-metastore-cmake/CMakeLists.txt +++ b/contrib/hive-metastore-cmake/CMakeLists.txt @@ -2,11 +2,11 @@ if (NOT TARGET ch_contrib::hdfs) if (ENABLE_HIVE) message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use Hive without HDFS") endif() -# `Storages/Hive/HiveFile.h` includes `orc/OrcFile.hh` and `parquet/arrow/reader.h` outright, -# so Hive needs both readers, not just the metastore client. -elseif (NOT TARGET _orc) +# `Storages/Hive/HiveFile.h` uses the ORC reader and `parquet/arrow/reader.h` outright, +# so Hive needs both readers, not just the metastore client. The ORC reader needs only snappy. +elseif (NOT TARGET ch_contrib::snappy OR NOT TARGET _parquet) if (ENABLE_HIVE) - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use Hive without ORC") + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use Hive without ORC and Parquet") endif() else() option(ENABLE_HIVE "Enable Hive" ${ENABLE_LIBRARIES}) diff --git a/contrib/orc b/contrib/orc deleted file mode 160000 index 6a2fe65eb16ce..0000000000000 --- a/contrib/orc +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6a2fe65eb16ce4c760964dca619260226c1b418e diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 784fbf9f13cb9..e0e76478534e0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -511,6 +511,10 @@ if (USE_ARROW) add_object_library(clickhouse_processors_formats_impl_arrow_ipc Processors/Formats/Impl/ArrowIPC) endif() +if (USE_ORC) + add_object_library(clickhouse_processors_formats_impl_orc Processors/Formats/Impl/ORC) +endif() + if (TARGET ch_contrib::nuraft) add_object_library(clickhouse_coordination Coordination) add_object_library(clickhouse_coordination_storage Coordination/Storage) @@ -824,11 +828,6 @@ endif() target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::fast_float) -if (USE_ORC) - target_link_libraries(dbms PUBLIC ${ORC_LIBRARIES}) - target_include_directories(dbms SYSTEM BEFORE PUBLIC ${ORC_INCLUDE_DIR} "${PROJECT_BINARY_DIR}/contrib/orc/c++/include") -endif () - if (TARGET ch_contrib::rocksdb) target_link_libraries(dbms PUBLIC ch_contrib::rocksdb) endif() diff --git a/src/Processors/Formats/Impl/ArrowBufferedStreams.cpp b/src/Processors/Formats/Impl/ArrowBufferedStreams.cpp index ac12c3baa4ffd..08b8015edb8fd 100644 --- a/src/Processors/Formats/Impl/ArrowBufferedStreams.cpp +++ b/src/Processors/Formats/Impl/ArrowBufferedStreams.cpp @@ -1,7 +1,7 @@ #pragma clang diagnostic ignored "-Wreserved-identifier" #include -#if USE_ARROW || USE_ORC || USE_PARQUET +#if USE_ARROW || USE_PARQUET #include #include #include diff --git a/src/Processors/Formats/Impl/ArrowBufferedStreams.h b/src/Processors/Formats/Impl/ArrowBufferedStreams.h index f2c29a832841d..de54491968800 100644 --- a/src/Processors/Formats/Impl/ArrowBufferedStreams.h +++ b/src/Processors/Formats/Impl/ArrowBufferedStreams.h @@ -3,7 +3,7 @@ #include -#if USE_ARROW || USE_ORC || USE_PARQUET +#if USE_ARROW || USE_PARQUET #include diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index a5acaf6aaa625..c8e52e542a2ae 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -1,7 +1,7 @@ #include #include -#if USE_ARROW || USE_ORC || USE_PARQUET +#if USE_ARROW || USE_PARQUET #include #include diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h index a11e7297ad7bb..f1817118c8299 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h @@ -5,7 +5,7 @@ #include "config.h" -#if USE_ARROW || USE_ORC || USE_PARQUET +#if USE_ARROW || USE_PARQUET #include #include diff --git a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp index aa7115021e73a..08779ffb6304e 100644 --- a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp @@ -3,6 +3,7 @@ #if USE_ORC #include +#include #include #include #include @@ -47,8 +48,6 @@ #include #include #include -#include -#include #include #include #include @@ -57,7 +56,6 @@ #include #include -#include #include @@ -92,11 +90,6 @@ UInt64 ORCInputStream::getLength() const return file_size; } -UInt64 ORCInputStream::getNaturalReadSize() const -{ - return 128 * 1024; -} - void ORCInputStream::read(void * buf, UInt64 length, UInt64 offset) { if (use_offset_based_read) @@ -152,7 +145,7 @@ std::future ORCInputStream::readAsync(void * buf, uint64_t length, uint64_ } } -std::unique_ptr +std::unique_ptr asORCInputStream(ReadBuffer & in, const FormatSettings & settings, bool use_prefetch, std::atomic & is_cancelled) { bool has_file_size = isBufferWithFileSize(in); @@ -165,14 +158,15 @@ asORCInputStream(ReadBuffer & in, const FormatSettings & settings, bool use_pref return asORCInputStreamLoadIntoMemory(in, is_cancelled); } -std::unique_ptr asORCInputStreamLoadIntoMemory(ReadBuffer & in, std::atomic & is_cancelled) +std::unique_ptr asORCInputStreamLoadIntoMemory(ReadBuffer & in, std::atomic & is_cancelled) { - size_t magic_size = strlen(ORC_MAGIC_BYTES); + static constexpr std::string_view orc_magic_bytes = "ORC"; + size_t magic_size = orc_magic_bytes.size(); std::string file_data(magic_size, '\0'); /// Avoid loading the whole file if it doesn't seem to even be in the correct format. size_t bytes_read = in.read(file_data.data(), magic_size); - if (bytes_read < magic_size || file_data != ORC_MAGIC_BYTES) + if (bytes_read < magic_size || file_data != orc_magic_bytes) throw Exception(ErrorCodes::INCORRECT_DATA, "Not an ORC file"); { @@ -186,25 +180,8 @@ std::unique_ptr asORCInputStreamLoadIntoMemory(ReadBuffer & in /// Defined below, next to the read path that also uses it. The sargs path resolves predicate column /// names through the same recursive walk, so pruning and reading always agree on the column. -static const orc::Type * -traverseDownORCTypeByName(const std::string & target, const orc::Type * orc_type, DataTypePtr & type, bool ignore_case); - -/// The default ORC memory pool allocates with `std::malloc`, which returns a null pointer when it -/// fails - and the library dereferences it - and which is invisible to the memory tracker. A -/// malformed file can ask for an arbitrarily large buffer, so allocate through `operator new` -/// instead: it is accounted for and it throws instead of returning null. -class ORCMemoryPool : public orc::MemoryPool -{ -public: - char * malloc(uint64_t size) override { return new char[size]; } - void free(char * p) override { delete[] p; } -}; - -orc::MemoryPool & getORCMemoryPool() -{ - static ORCMemoryPool pool; - return pool; -} +static const ORC::Type * +traverseDownORCTypeByName(const std::string & target, const ORC::Type * orc_type, DataTypePtr & type, bool ignore_case); /// Resolves the CH type of `name` against `header`, following dots into tuple elements when the /// header carries only the parent column (which is the case for ORC, whose reader is not asked for @@ -298,28 +275,36 @@ static Block buildORCKeyConditionBlock(const Block & header, const ActionsDAG * return keys; } -static bool isDictionaryEncoded(const orc::StripeInformation * stripe_info, const orc::Type * orc_type) +static bool isDictionaryEncoded(const ORC::Proto::StripeFooter * stripe_footer, const ORC::Type * orc_type) { - if (!stripe_info) + if (!stripe_footer) return false; - auto encoding = stripe_info->getColumnEncoding(orc_type->getColumnId()); - return encoding == orc::ColumnEncodingKind_DICTIONARY || encoding == orc::ColumnEncodingKind_DICTIONARY_V2; + UInt64 column_id = orc_type->getColumnId(); + if (column_id >= stripe_footer->columns.size()) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Failed to read ORC schema: the first stripe has no encoding for column {}, it has {} column encodings", + column_id, + stripe_footer->columns.size()); + + auto encoding = stripe_footer->columns[column_id].kind; + return encoding == ORC::Proto::ColumnEncodingKind::DICTIONARY || encoding == ORC::Proto::ColumnEncodingKind::DICTIONARY_V2; } static DataTypePtr parseORCType( - const orc::Type * orc_type, + const ORC::Type * orc_type, bool skip_columns_with_unsupported_types, bool dictionary_as_low_cardinality, - const orc::StripeInformation * stripe_info, + const ORC::Proto::StripeFooter * stripe_info, bool & skipped, size_t max_depth = DBMS_DEFAULT_MAX_PARSER_DEPTH, size_t depth = 0) { chassert(orc_type != nullptr); - /// ORC LIST/MAP/STRUCT types can be nested arbitrarily deep and the ORC library does not bound - /// the nesting, so reject deep nesting early (before building the type) with an explicit limit. + /// ORC LIST/MAP/STRUCT types can be nested deep (the ORC reader allows up to 2000 levels) + /// so reject deep nesting early (before building the type) with an explicit limit. /// This keeps schema inference cheap and interruptible instead of recursing over (and later /// walking) a pathologically deep type. checkStackSize is a last-resort backstop. /// max_depth == 0 means unlimited (matching the SQL parser), leaving only checkStackSize. @@ -339,7 +324,7 @@ static DataTypePtr parseORCType( /// branches with identical types cannot be represented as a Variant; reject them explicitly /// instead of silently squashing them. A branch that is itself a union is rejected too: it /// would map to a Variant, which Variant does not allow to nest. - if (orc_type->getKind() == orc::TypeKind::UNION) + if (orc_type->getKind() == ORC::TypeKind::UNION) { /// A union with more branches than Variant can hold (ColumnVariant::MAX_NESTED_COLUMNS) /// is valid ORC but not representable; reject it through the normal unsupported-type / @@ -370,7 +355,7 @@ static DataTypePtr parseORCType( /// here, before the outer DataTypeVariant is constructed - otherwise its constructor /// throws a confusing BAD_ARGUMENTS ("Nested Variant types are not allowed") and the /// skip setting is never consulted because the inner union has already been parsed. - if (subtype->getKind() == orc::TypeKind::UNION) + if (subtype->getKind() == ORC::TypeKind::UNION) { if (skip_columns_with_unsupported_types) { @@ -411,32 +396,32 @@ static DataTypePtr parseORCType( switch (orc_type->getKind()) { - case orc::TypeKind::BOOLEAN: + case ORC::TypeKind::BOOLEAN: return DataTypeFactory::instance().get("Bool"); - case orc::TypeKind::BYTE: + case ORC::TypeKind::BYTE: return std::make_shared(); - case orc::TypeKind::SHORT: + case ORC::TypeKind::SHORT: return std::make_shared(); - case orc::TypeKind::INT: + case ORC::TypeKind::INT: return std::make_shared(); - case orc::TypeKind::LONG: + case ORC::TypeKind::LONG: return std::make_shared(); - case orc::TypeKind::FLOAT: + case ORC::TypeKind::FLOAT: return std::make_shared(); - case orc::TypeKind::DOUBLE: + case ORC::TypeKind::DOUBLE: return std::make_shared(); - case orc::TypeKind::DATE: + case ORC::TypeKind::DATE: return std::make_shared(); - case orc::TypeKind::TIMESTAMP: + case ORC::TypeKind::TIMESTAMP: return std::make_shared(9); - case orc::TypeKind::TIMESTAMP_INSTANT: + case ORC::TypeKind::TIMESTAMP_INSTANT: return std::make_shared(9, "UTC"); - case orc::TypeKind::CHAR: - case orc::TypeKind::VARCHAR: - case orc::TypeKind::BINARY: - case orc::TypeKind::STRING: { + case ORC::TypeKind::CHAR: + case ORC::TypeKind::VARCHAR: + case ORC::TypeKind::BINARY: + case ORC::TypeKind::STRING: { DataTypePtr type; - if (orc_type->getKind() == orc::TypeKind::CHAR) + if (orc_type->getKind() == ORC::TypeKind::CHAR) type = std::make_shared(orc_type->getMaximumLength()); else type = std::make_shared(); @@ -447,7 +432,7 @@ static DataTypePtr parseORCType( return type; } - case orc::TypeKind::DECIMAL: { + case ORC::TypeKind::DECIMAL: { UInt64 precision = orc_type->getPrecision(); UInt64 scale = orc_type->getScale(); if (precision == 0) @@ -457,7 +442,7 @@ static DataTypePtr parseORCType( } return createDecimal(precision, scale); } - case orc::TypeKind::LIST: { + case ORC::TypeKind::LIST: { if (subtype_count != 1) throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid Orc List type {}", orc_type->toString()); @@ -468,7 +453,7 @@ static DataTypePtr parseORCType( return std::make_shared(nested_type); } - case orc::TypeKind::MAP: { + case ORC::TypeKind::MAP: { if (subtype_count != 2) throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid Orc Map type {}", orc_type->toString()); @@ -484,7 +469,7 @@ static DataTypePtr parseORCType( return std::make_shared(key_type, value_type); } - case orc::TypeKind::STRUCT: { + case ORC::TypeKind::STRUCT: { DataTypes nested_types; Strings nested_names; nested_types.reserve(subtype_count); @@ -519,30 +504,30 @@ static DataTypePtr parseORCType( } } -static std::optional convertORCTypeToPredicateType(const orc::Type & orc_type) +static std::optional convertORCTypeToPredicateType(const ORC::Type & orc_type) { switch (orc_type.getKind()) { - case orc::BOOLEAN: - return orc::PredicateDataType::BOOLEAN; - case orc::BYTE: - case orc::SHORT: - case orc::INT: - case orc::LONG: - return orc::PredicateDataType::LONG; - case orc::FLOAT: - case orc::DOUBLE: - return orc::PredicateDataType::FLOAT; - case orc::VARCHAR: - case orc::CHAR: - case orc::STRING: - return orc::PredicateDataType::STRING; - case orc::DATE: - return orc::PredicateDataType::DATE; - case orc::TIMESTAMP: - return orc::PredicateDataType::TIMESTAMP; - case orc::DECIMAL: - return orc::PredicateDataType::DECIMAL; + case ORC::BOOLEAN: + return ORC::PredicateDataType::BOOLEAN; + case ORC::BYTE: + case ORC::SHORT: + case ORC::INT: + case ORC::LONG: + return ORC::PredicateDataType::LONG; + case ORC::FLOAT: + case ORC::DOUBLE: + return ORC::PredicateDataType::FLOAT; + case ORC::VARCHAR: + case ORC::CHAR: + case ORC::STRING: + return ORC::PredicateDataType::STRING; + case ORC::DATE: + return ORC::PredicateDataType::DATE; + case ORC::TIMESTAMP: + return ORC::PredicateDataType::TIMESTAMP; + case ORC::DECIMAL: + return ORC::PredicateDataType::DECIMAL; default: return {}; } @@ -559,23 +544,23 @@ static String getColumnNameFromKeyCondition(const KeyCondition & key_condition, throw Exception(ErrorCodes::LOGICAL_ERROR, "Can't get column from KeyCondition with indice {}", indice); } -static std::optional -convertFieldToORCLiteral(const orc::Type & orc_type, const Field & field, DataTypePtr type_hint = nullptr) +static std::optional +convertFieldToORCLiteral(const ORC::Type & orc_type, const Field & field, DataTypePtr type_hint = nullptr) { try { /// We always fallback to return null if possible CH type hint not consistent with ORC type switch (orc_type.getKind()) { - case orc::BOOLEAN: { + case ORC::BOOLEAN: { /// May throw exception auto val = field.safeGet(); - return orc::Literal(val != 0); + return ORC::Literal(val != 0); } - case orc::BYTE: - case orc::SHORT: - case orc::INT: - case orc::LONG: { + case ORC::BYTE: + case ORC::SHORT: + case ORC::INT: + case ORC::LONG: { /// May throw exception. /// /// In particular, it'll throw if we request the column as unsigned, like this: @@ -583,30 +568,30 @@ convertFieldToORCLiteral(const orc::Type & orc_type, const Field & field, DataTy /// We have to reject this, otherwise it would miss values > 127 (because /// they're treated as negative by ORC). auto val = field.safeGet(); - return orc::Literal(val); + return ORC::Literal(val); } - case orc::FLOAT: - case orc::DOUBLE: { + case ORC::FLOAT: + case ORC::DOUBLE: { Float64 val = 0; if (field.tryGet(val)) - return orc::Literal(val); + return ORC::Literal(val); break; } - case orc::VARCHAR: - case orc::CHAR: - case orc::STRING: { + case ORC::VARCHAR: + case ORC::CHAR: + case ORC::STRING: { String str; if (field.tryGet(str)) - return orc::Literal(str.data(), str.size()); + return ORC::Literal(str.data(), str.size()); break; } - case orc::DATE: { + case ORC::DATE: { Int64 val = 0; if (field.tryGet(val)) - return orc::Literal(orc::PredicateDataType::DATE, val); + return ORC::Literal(ORC::PredicateDataType::DATE, val); break; } - case orc::TIMESTAMP: { + case ORC::TIMESTAMP: { if (type_hint && isDateTime64(type_hint)) { const auto * datetime64_type = typeid_cast(type_hint.get()); @@ -615,15 +600,23 @@ convertFieldToORCLiteral(const orc::Type & orc_type, const Field & field, DataTy } DecimalField ts; - if (field.tryGet(ts)) + if (field.tryGet(ts) && ts.getScale() <= 9) { - Int64 secs = (ts.getValue() / ts.getScaleMultiplier()).convertTo(); - Int32 nanos = (ts.getValue() - (ts.getValue() / ts.getScaleMultiplier()) * ts.getScaleMultiplier()).convertTo(); - return orc::Literal(secs, nanos); + /// The nanoseconds are not negative, as in the reference implementation and in the statistics. + const Int64 value = ts.getValue().value; + const Int64 multiplier = ts.getScaleMultiplier().value; + Int64 secs = value / multiplier; + Int64 rest = value % multiplier; + if (rest < 0) + { + rest += multiplier; + --secs; + } + return ORC::Literal(secs, static_cast(rest * (1'000'000'000 / multiplier))); } break; } - case orc::DECIMAL: { + case ORC::DECIMAL: { auto precision = orc_type.getPrecision(); if (precision == 0) precision = 38; @@ -634,8 +627,8 @@ convertFieldToORCLiteral(const orc::Type & orc_type, const Field & field, DataTy if (field.tryGet(val)) { Int64 right = val.getValue().convertTo(); - return orc::Literal( - orc::Int128(right), static_cast(orc_type.getPrecision()), static_cast(orc_type.getScale())); + return ORC::Literal( + Int128(right), static_cast(orc_type.getPrecision()), static_cast(orc_type.getScale())); } } else if (precision <= DecimalUtils::max_precision) @@ -644,8 +637,8 @@ convertFieldToORCLiteral(const orc::Type & orc_type, const Field & field, DataTy if (field.tryGet(val)) { Int64 right = val.getValue().convertTo(); - return orc::Literal( - orc::Int128(right), static_cast(orc_type.getPrecision()), static_cast(orc_type.getScale())); + return ORC::Literal( + Int128(right), static_cast(orc_type.getPrecision()), static_cast(orc_type.getScale())); } } else if (precision <= DecimalUtils::max_precision) @@ -653,10 +646,8 @@ convertFieldToORCLiteral(const orc::Type & orc_type, const Field & field, DataTy DecimalField val; if (field.tryGet(val)) { - Int64 high = val.getValue().value.items[1]; - UInt64 low = static_cast(val.getValue().value.items[0]); - return orc::Literal( - orc::Int128(high, low), static_cast(orc_type.getPrecision()), static_cast(orc_type.getScale())); + return ORC::Literal( + val.getValue().value, static_cast(orc_type.getPrecision()), static_cast(orc_type.getScale())); } } break; @@ -727,9 +718,9 @@ static bool evaluateRPNElement(const Field & field, const KeyCondition::RPNEleme static void buildORCSearchArgumentImpl( const KeyCondition & key_condition, const Block & header, - const orc::Type & schema, + const ORC::Type & schema, KeyCondition::RPN & rpn_stack, - orc::SearchArgumentBuilder & builder, + ORC::SearchArgumentBuilder & builder, const FormatSettings & format_settings) { if (rpn_stack.empty()) @@ -761,7 +752,7 @@ static void buildORCSearchArgumentImpl( /// Key filter expressions like "func(col) > 100" are not supported for ORC filter push down if (!curr.monotonic_functions_chain.empty()) { - builder.literal(orc::TruthValue::YES_NO_NULL); + builder.literal(ORC::TruthValue::YES_NO_NULL); break; } @@ -772,7 +763,7 @@ static void buildORCSearchArgumentImpl( { if (!set_index || set_index->getOrderedSet().size() != 1 || set_index->hasMonotonicFunctionsChain()) { - builder.literal(orc::TruthValue::YES_NO_NULL); + builder.literal(ORC::TruthValue::YES_NO_NULL); break; } } @@ -785,7 +776,7 @@ static void buildORCSearchArgumentImpl( auto column_type = findHeaderTypeByPath(header, column_name, ignore_case); if (!column_type) { - builder.literal(orc::TruthValue::YES_NO_NULL); + builder.literal(ORC::TruthValue::YES_NO_NULL); break; } @@ -796,7 +787,7 @@ static void buildORCSearchArgumentImpl( const auto * orc_type = traverseDownORCTypeByName(column_name, &schema, resolved_type, ignore_case); if (!orc_type) { - builder.literal(orc::TruthValue::YES_NO_NULL); + builder.literal(ORC::TruthValue::YES_NO_NULL); break; } @@ -813,7 +804,7 @@ static void buildORCSearchArgumentImpl( auto expect_type = makeNullableRecursively(parseORCType(orc_type, true, false, nullptr, skipped, format_settings.max_parser_depth), format_settings); if (!expect_type) { - builder.literal(orc::TruthValue::YES_NO_NULL); + builder.literal(ORC::TruthValue::YES_NO_NULL); break; } @@ -821,7 +812,7 @@ static void buildORCSearchArgumentImpl( auto expect_nested_type = removeNullable(expect_type); if (!nested_type->equals(*expect_nested_type)) { - builder.literal(orc::TruthValue::YES_NO_NULL); + builder.literal(ORC::TruthValue::YES_NO_NULL); break; } @@ -833,7 +824,7 @@ static void buildORCSearchArgumentImpl( bool match_if_default = evaluateRPNElement(column_type->getDefault(), curr); if (match_if_default != match_if_null) { - builder.literal(orc::TruthValue::YES_NO_NULL); + builder.literal(ORC::TruthValue::YES_NO_NULL); break; } } @@ -841,7 +832,7 @@ static void buildORCSearchArgumentImpl( auto predicate_type = convertORCTypeToPredicateType(*orc_type); if (!predicate_type.has_value()) { - builder.literal(orc::TruthValue::YES_NO_NULL); + builder.literal(ORC::TruthValue::YES_NO_NULL); break; } @@ -859,8 +850,8 @@ static void buildORCSearchArgumentImpl( bool has_right_bound = !range.right.isPositiveInfinity(); if (!has_left_bound && !has_right_bound) { - /// Transform whole range orc::TruthValue::YES_NULL - builder.literal(orc::TruthValue::YES_NULL); + /// Transform whole range ORC::TruthValue::YES_NULL + builder.literal(ORC::TruthValue::YES_NULL); } else if (has_left_bound && has_right_bound && range.left_included && range.right_included && range.left == range.right) { @@ -869,15 +860,15 @@ static void buildORCSearchArgumentImpl( if (literal.has_value()) builder.equals(orc_type->getColumnId(), *predicate_type, *literal); else - builder.literal(orc::TruthValue::YES_NO_NULL); + builder.literal(ORC::TruthValue::YES_NO_NULL); } else { - std::optional left_literal; + std::optional left_literal; if (has_left_bound) left_literal = convertFieldToORCLiteral(*orc_type, range.left); - std::optional right_literal; + std::optional right_literal; if (has_right_bound) right_literal = convertFieldToORCLiteral(*orc_type, range.right); @@ -897,7 +888,7 @@ static void buildORCSearchArgumentImpl( builder.end(); } else - builder.literal(orc::TruthValue::YES_NO_NULL); + builder.literal(ORC::TruthValue::YES_NO_NULL); } if (has_right_bound) @@ -910,7 +901,7 @@ static void buildORCSearchArgumentImpl( builder.lessThan(orc_type->getColumnId(), *predicate_type, *right_literal); } else - builder.literal(orc::TruthValue::YES_NO_NULL); + builder.literal(ORC::TruthValue::YES_NO_NULL); } if (has_left_bound && has_right_bound) @@ -924,7 +915,7 @@ static void buildORCSearchArgumentImpl( const auto & set_column = ordered_set[0]; bool fail = false; - std::vector literals; + std::vector literals; literals.reserve(set_column->size()); for (size_t i = 0; i < set_column->size(); ++i) { @@ -940,9 +931,9 @@ static void buildORCSearchArgumentImpl( /// set has zero element if (literals.empty()) - builder.literal(orc::TruthValue::YES); + builder.literal(ORC::TruthValue::YES); else if (fail) - builder.literal(orc::TruthValue::YES_NO_NULL); + builder.literal(ORC::TruthValue::YES_NO_NULL); else builder.in(orc_type->getColumnId(), *predicate_type, literals); } @@ -958,7 +949,7 @@ static void buildORCSearchArgumentImpl( case KeyCondition::RPNElement::FUNCTION_POINT_IN_POLYGON: case KeyCondition::RPNElement::FUNCTION_UNKNOWN: { - builder.literal(orc::TruthValue::YES_NO_NULL); + builder.literal(ORC::TruthValue::YES_NO_NULL); rpn_stack.pop_back(); break; } @@ -990,35 +981,35 @@ static void buildORCSearchArgumentImpl( } case KeyCondition::RPNElement::ALWAYS_FALSE: { - builder.literal(orc::TruthValue::NO); + builder.literal(ORC::TruthValue::NO); rpn_stack.pop_back(); break; } case KeyCondition::RPNElement::ALWAYS_TRUE: { - builder.literal(orc::TruthValue::YES); + builder.literal(ORC::TruthValue::YES); rpn_stack.pop_back(); break; } } } -std::unique_ptr buildORCSearchArgument( - const KeyCondition & key_condition, const Block & header, const orc::Type & schema, const FormatSettings & format_settings) +std::unique_ptr buildORCSearchArgument( + const KeyCondition & key_condition, const Block & header, const ORC::Type & schema, const FormatSettings & format_settings) { auto rpn_stack = key_condition.getRPN(); if (rpn_stack.empty()) return nullptr; - auto builder = orc::SearchArgumentFactory::newBuilder(); + auto builder = std::make_unique(); buildORCSearchArgumentImpl(key_condition, header, schema, rpn_stack, *builder, format_settings); return builder->build(); } -std::unique_ptr createORCReader(std::unique_ptr stream, const orc::ReaderOptions & options) +std::unique_ptr createORCReader(std::unique_ptr stream, const ORC::ReaderOptions & options) { - auto reader = orc::createReader(std::move(stream), options); - if (reader->getType().getKind() != orc::STRUCT) + auto reader = std::make_unique(std::move(stream), options); + if (reader->getType().getKind() != ORC::STRUCT) throw Exception( ErrorCodes::INCORRECT_DATA, "ORC files whose root type is not a struct are not supported, the file has root type {}", @@ -1028,7 +1019,7 @@ std::unique_ptr createORCReader(std::unique_ptr s static void getFileReader( ReadBuffer & in, - std::unique_ptr & file_reader, + std::unique_ptr & file_reader, const FormatSettings & format_settings, bool use_prefetch, size_t min_bytes_for_seek, @@ -1037,21 +1028,18 @@ static void getFileReader( if (is_stopped) return; - orc::ReaderOptions options; - options.setMemoryPool(getORCMemoryPool()); - /// ORC library requires rangeSizeLimit > holeSizeLimit. - static constexpr uint64_t default_range_size_limit = 10 * 1024 * 1024UL; - /// Clamp to avoid overflow when computing holeSizeLimit + 1. - uint64_t hole_size_limit = std::min(min_bytes_for_seek, std::numeric_limits::max() - 1); - uint64_t range_size_limit = std::max(default_range_size_limit, hole_size_limit + 1); - options.setCacheOptions(orc::CacheOptions{.holeSizeLimit = hole_size_limit, .rangeSizeLimit = range_size_limit}); + /// The ranges with the gaps up to `min_bytes_for_seek` are read with one request. + static constexpr UInt64 default_range_size_limit = 10 * 1024 * 1024UL; + ORC::ReaderOptions options; + options.hole_size_limit = min_bytes_for_seek; + options.range_size_limit = std::max(default_range_size_limit, min_bytes_for_seek); auto input_stream = asORCInputStream(in, format_settings, use_prefetch, is_stopped); file_reader = createORCReader(std::move(input_stream), options); } -static const orc::Type * -traverseDownORCTypeByName(const std::string & target, const orc::Type * orc_type, DataTypePtr & type, bool ignore_case) +static const ORC::Type * +traverseDownORCTypeByName(const std::string & target, const ORC::Type * orc_type, DataTypePtr & type, bool ignore_case) { /// Recurses the file-controlled ORC type tree. The matching CH type (and the requested column /// name) bound the depth in practice, but keep a stack backstop here too, since this runs in @@ -1061,7 +1049,7 @@ traverseDownORCTypeByName(const std::string & target, const orc::Type * orc_type if (target.empty()) return orc_type; - auto search_struct_field = [&](const std::string & target_, const orc::Type * type_) -> std::pair + auto search_struct_field = [&](const std::string & target_, const ORC::Type * type_) -> std::pair { auto target_copy = target_; if (ignore_case) @@ -1081,19 +1069,19 @@ traverseDownORCTypeByName(const std::string & target, const orc::Type * orc_type return {"", nullptr}; }; - if (orc::STRUCT == orc_type->getKind()) + if (ORC::STRUCT == orc_type->getKind()) { const auto [next_target, next_orc_type] = search_struct_field(target, orc_type); return next_orc_type ? traverseDownORCTypeByName(next_target, next_orc_type, type, ignore_case) : nullptr; } - if (orc::LIST == orc_type->getKind()) + if (ORC::LIST == orc_type->getKind()) { /// For cases in which header contains subcolumns flattened from nested columns. /// For example, "a Nested(x String, y Int64)" is flattened to "a.x Array(String), a.y Array(Int64)", and ORC file schema is still "a array>". /// In this case, we should skip possible array type and traverse down to its nested struct type. const auto * array_type = typeid_cast(removeNullable(type).get()); const auto * orc_nested_type = orc_type->getSubtype(0); - if (array_type && orc::STRUCT == orc_nested_type->getKind()) + if (array_type && ORC::STRUCT == orc_nested_type->getKind()) { auto next_type_and_target = search_struct_field(target, orc_nested_type); const auto & next_target = next_type_and_target.first; @@ -1112,12 +1100,12 @@ traverseDownORCTypeByName(const std::string & target, const orc::Type * orc_type /// Forward declarations: updateIncludeTypeIds needs the union branch-hint machinery (defined /// below, near readColumnFromORCColumn, which is where it is also used) to prune struct-branch /// fields the same way the read path does. See computeOrcUnionBranchHints. -static bool orcUnionBranchMatchesType(const orc::Type * orc_branch_type, const DataTypePtr & target_type, bool case_insensitive); -static bool orcUnionBranchPrefersType(const orc::Type * orc_branch_type, const DataTypePtr & target_type); -static DataTypes computeOrcUnionBranchHints(const orc::Type * orc_type, const DataTypePtr & type_hint, bool case_insensitive_matching); +static bool orcUnionBranchMatchesType(const ORC::Type * orc_branch_type, const DataTypePtr & target_type, bool case_insensitive); +static bool orcUnionBranchPrefersType(const ORC::Type * orc_branch_type, const DataTypePtr & target_type); +static DataTypes computeOrcUnionBranchHints(const ORC::Type * orc_type, const DataTypePtr & type_hint, bool case_insensitive_matching); static void -updateIncludeTypeIds(DataTypePtr type, const orc::Type * orc_type, bool ignore_case, std::unordered_set & include_typeids) +updateIncludeTypeIds(DataTypePtr type, const ORC::Type * orc_type, bool ignore_case, std::unordered_set & include_typeids) { /// Recurses the file-controlled ORC type tree in lockstep with the (parser-bounded) CH type. /// Keep a stack backstop here too: this runs in prepareFileReader, before the @@ -1134,7 +1122,7 @@ updateIncludeTypeIds(DataTypePtr type, const orc::Type * orc_type, bool ignore_c auto non_nullable_type = removeNullable(type); switch (orc_type->getKind()) { - case orc::LIST: { + case ORC::LIST: { const auto * array_type = typeid_cast(non_nullable_type.get()); if (array_type) { @@ -1142,7 +1130,7 @@ updateIncludeTypeIds(DataTypePtr type, const orc::Type * orc_type, bool ignore_c } return; } - case orc::MAP: { + case ORC::MAP: { const auto * map_type = typeid_cast(non_nullable_type.get()); if (map_type) { @@ -1151,7 +1139,7 @@ updateIncludeTypeIds(DataTypePtr type, const orc::Type * orc_type, bool ignore_c } return; } - case orc::STRUCT: { + case ORC::STRUCT: { /// To make sure tuple field pruning work fine, we should include only the fields of ORC struct type which are also contained in CH tuple types, instead of all fields of ORC struct type. /// For example, CH tupe type in header is "x Tuple(a String)", ORC struct type is "x struct", then only type id of field "x.a" should be included. /// For tuple field pruning purpose, we should never include "x.b" for it is not required in format header. @@ -1196,7 +1184,7 @@ updateIncludeTypeIds(DataTypePtr type, const orc::Type * orc_type, bool ignore_c } return; } - case orc::UNION: { + case ORC::UNION: { /// ORC union maps to the ClickHouse Variant type. A branch that gets a forced type hint /// (the same way readColumnFromORCColumn computes it - see computeOrcUnionBranchHints) /// is selected by recursing into the hint, so a STRUCT branch keeps the named-tuple @@ -1252,10 +1240,6 @@ void NativeORCBlockInputFormat::prepareFileReader() format_filter_info->filter_actions_dag.get(), format_settings.orc.case_insensitive_column_matching)); - std::unique_ptr stripe_info; - if (file_reader->getNumberOfStripes()) - stripe_info = file_reader->getStripe(0); - orc_column_to_ch_column = std::make_unique( getPort().getHeader(), format_settings.orc.allow_missing_columns, @@ -1300,7 +1284,7 @@ void NativeORCBlockInputFormat::prefetchStripes() int stripe = selected_stripes[prefetch_iterator]; stripes.push_back(stripe); - total_stripe_size += file_reader->getStripe(stripe)->getLength(); + total_stripe_size += ORC::Reader::getStripeLength(file_reader->getStripe(stripe)); ++prefetch_iterator; } @@ -1337,26 +1321,25 @@ bool NativeORCBlockInputFormat::prepareStripeReader() return false; int current_stripe = selected_stripes[read_iterator]; - current_stripe_info = file_reader->getStripe(current_stripe); - if (!current_stripe_info->getNumberOfRows()) + current_stripe_info = &file_reader->getStripe(current_stripe); + if (!current_stripe_info->number_of_rows) throw Exception(ErrorCodes::INCORRECT_DATA, "ORC stripe {} has no rows", current_stripe); - orc::RowReaderOptions row_reader_options; - row_reader_options.setEnableLazyDecoding(format_settings.orc.dictionary_as_low_cardinality); - row_reader_options.includeTypes(include_indices); - row_reader_options.setTimezoneName(format_settings.orc.reader_time_zone_name); - row_reader_options.range(current_stripe_info->getOffset(), current_stripe_info->getLength()); + ORC::RowReaderOptions row_reader_options; + row_reader_options.enable_lazy_decoding = format_settings.orc.dictionary_as_low_cardinality; + row_reader_options.include_types = include_indices; + row_reader_options.timezone_name = format_settings.orc.reader_time_zone_name; + row_reader_options.range_offset = current_stripe_info->offset; + row_reader_options.range_length = ORC::Reader::getStripeLength(*current_stripe_info); if (format_settings.orc.filter_push_down && sargs) - { - row_reader_options.searchArgument(sargs); - } + row_reader_options.search_argument = sargs; stripe_reader = file_reader->createRowReader(row_reader_options); ++read_iterator; if (use_prefetch) { /// Release outdated buffer before boundary to avoid OOM - file_reader->releaseBuffer(current_stripe_info->getOffset()); + file_reader->releaseBuffer(current_stripe_info->offset); /// Prefetch next selected stripe prefetchStripes(); @@ -1379,7 +1362,7 @@ Chunk NativeORCBlockInputFormat::read() int current_stripe = selected_stripes[read_iterator]; ++read_iterator; - return getChunkForCount(file_reader->getStripe(current_stripe)->getNumberOfRows()); + return getChunkForCount(file_reader->getStripe(current_stripe).number_of_rows); } if (!stripe_reader) @@ -1391,25 +1374,12 @@ Chunk NativeORCBlockInputFormat::read() if (is_stopped) return {}; - /// TODO: figure out why reuse batch would cause asan fatals in https://s3.amazonaws.com/clickhouse-test-reports/55330/be39d23af2d7e27f5ec7f168947cf75aeaabf674/stateless_tests__asan__[4_4].html - /// Not sure if it is a false positive case. Notice that reusing batch will speed up reading ORC by 1.15x. + /// The values are decoded directly into the columns of the batch, which are then handed over to the chunk. Stopwatch time; - auto batch = stripe_reader->createRowBatch(format_settings.orc.row_batch_size); + auto batch = stripe_reader->createRowBatch(); while (true) { - bool ok = false; - try - { - ok = stripe_reader->next(*batch); - } - catch (const orc::ParseError & e) - { - /// The ORC library throws ParseError when the encoded data of a stripe is corrupt (for - /// example, a union tag that is out of range for the union's branches). Surface it as - /// INCORRECT_DATA instead of letting it propagate as a generic std::exception. - throw Exception(ErrorCodes::INCORRECT_DATA, "Failed to read ORC data: {}", e.what()); - } - if (ok) + if (stripe_reader->next(*batch, static_cast(format_settings.orc.row_batch_size))) break; /// No more rows to read in current stripe, continue to prepare reading next stripe @@ -1426,7 +1396,7 @@ Chunk NativeORCBlockInputFormat::read() orc_column_to_ch_column->orcTableToCHChunk(res, &schema, batch.get(), num_rows, &block_missing_values); LOG_TEST(getLogger("NativeORCBlockInputFormat"), "Convert {} rows take {} ms", num_rows, time.elapsedMilliseconds()); - approx_bytes_read_for_chunk = num_rows * current_stripe_info->getLength() / current_stripe_info->getNumberOfRows(); + approx_bytes_read_for_chunk = num_rows * ORC::Reader::getStripeLength(*current_stripe_info) / current_stripe_info->number_of_rows; return res; } @@ -1465,37 +1435,28 @@ NamesAndTypesList NativeORCSchemaReader::readSchema() const auto & schema = file_reader->getType(); Block header; - std::unique_ptr stripe_info; + std::optional first_stripe_footer; if (file_reader->getNumberOfStripes()) - stripe_info = file_reader->getStripe(0); + first_stripe_footer = file_reader->readStripeFooter(0); - try { for (size_t i = 0; i < schema.getSubtypeCount(); ++i) { const std::string & name = schema.getFieldName(i); - const orc::Type * orc_type = schema.getSubtype(i); + const ORC::Type * orc_type = schema.getSubtype(i); bool skipped = false; DataTypePtr type = parseORCType( orc_type, format_settings.orc.skip_columns_with_unsupported_types_in_schema_inference, format_settings.orc.dictionary_as_low_cardinality, - stripe_info.get(), + first_stripe_footer ? &*first_stripe_footer : nullptr, skipped, format_settings.max_parser_depth); if (!skipped) header.insert(ColumnWithTypeAndName{type, name}); } } - catch (const orc::ParseError & e) - { - /// The ORC library throws ParseError when the stripe footer is corrupt (for example, a - /// column id that has no matching column encoding, which is consulted here to detect - /// dictionary encoding). Surface it as INCORRECT_DATA instead of letting it propagate as a - /// generic std::exception. - throw Exception(ErrorCodes::INCORRECT_DATA, "Failed to read ORC schema: {}", e.what()); - } /// ORC doesn't have non-nullable data types. if (format_settings.schema_inference_make_columns_nullable != 0) @@ -1526,11 +1487,11 @@ ORCColumnToCHColumn::ORCColumnToCHColumn( } void ORCColumnToCHColumn::orcTableToCHChunk( - Chunk & res, const orc::Type * schema, const orc::ColumnVectorBatch * table, size_t num_rows, BlockMissingValues * block_missing_values) + Chunk & res, const ORC::Type * schema, const ORC::ColumnVectorBatch * table, size_t num_rows, BlockMissingValues * block_missing_values) { - const auto * struct_batch = dynamic_cast(table); + const auto * struct_batch = dynamic_cast(table); if (!struct_batch) - throw Exception(ErrorCodes::LOGICAL_ERROR, "ORC table must be StructVectorBatch but is {}", table->toString()); + throw Exception(ErrorCodes::LOGICAL_ERROR, "ORC table must be StructVectorBatch but is {}", demangle(typeid(*table).name())); if (schema->getSubtypeCount() != struct_batch->fields.size()) throw Exception( @@ -1555,7 +1516,7 @@ void ORCColumnToCHColumn::orcTableToCHChunk( } /// Creates a null bytemap from ORC's not-null bytemap -static ColumnPtr readByteMapFromORCColumn(const orc::ColumnVectorBatch * orc_column) +static ColumnPtr readByteMapFromORCColumn(const ORC::ColumnVectorBatch * orc_column) { if (!orc_column->hasNulls) return ColumnUInt8::create(orc_column->numElements, static_cast(0)); @@ -1598,70 +1559,39 @@ static ColumnPtr replaceNullPayloadRowsWithValidRow(const ColumnPtr & column, co } -static const orc::ColumnVectorBatch * getNestedORCColumn(const orc::ListVectorBatch * orc_column) +static const ORC::ColumnVectorBatch * getNestedORCColumn(const ORC::ListVectorBatch * orc_column) { return orc_column->elements.get(); } +/// The reader decodes the lengths directly into the end offsets, as in `ColumnArray`. template static ColumnPtr readOffsetsFromORCListColumn(const BatchType * orc_column) { - auto offsets_column = ColumnUInt64::create(); - ColumnArray::Offsets & offsets_data = assert_cast &>(*offsets_column).getData(); - offsets_data.reserve(orc_column->numElements); - - for (size_t i = 0; i < orc_column->numElements; ++i) - offsets_data.push_back(orc_column->offsets[i + 1]); - - return offsets_column; + return orc_column->offsets; } +/// The values of the primitive types are decoded directly into the columns (with zeros for the nulls). static ColumnWithTypeAndName -readColumnWithBooleanData(const orc::ColumnVectorBatch * orc_column, const String & column_name) +readColumnWithBooleanData(const ORC::ColumnVectorBatch * orc_column, const String & column_name) { - const auto * orc_bool_column = dynamic_cast(orc_column); - auto internal_type = DataTypeFactory::instance().get("Bool"); - auto internal_column = internal_type->createColumn(); - auto & column_data = assert_cast &>(*internal_column).getData(); - - column_data.reserve(orc_bool_column->numElements); - for (size_t i = 0; i < orc_bool_column->numElements; ++i) - { - if (!orc_bool_column->hasNulls || orc_bool_column->notNull[i]) - column_data.push_back(static_cast(orc_bool_column->data[i])); - else - column_data.push_back(static_cast(0)); - } - - return {std::move(internal_column), internal_type, column_name}; + const auto & orc_bool_column = typeid_cast(*orc_column); + return {orc_bool_column.column, DataTypeFactory::instance().get("Bool"), column_name}; } -template +template static ColumnWithTypeAndName -readColumnWithNumericData(const orc::ColumnVectorBatch * orc_column, const String & column_name) +readColumnWithNumericData(const ORC::ColumnVectorBatch * orc_column, const String & column_name) { - auto internal_type = std::make_shared>(); - auto internal_column = internal_type->createColumn(); - auto & column_data = static_cast &>(*internal_column).getData(); - const auto * orc_int_column = dynamic_cast(orc_column); - - column_data.reserve(orc_int_column->numElements); - for (size_t i = 0; i < orc_int_column->numElements; ++i) - { - if (!orc_int_column->hasNulls || orc_int_column->notNull[i]) - column_data.push_back(NumericType(orc_int_column->data[i])); - else - column_data.push_back(NumericType{}); - } - - return {std::move(internal_column), internal_type, column_name}; + const auto & orc_numeric_column = typeid_cast &>(*orc_column); + return {orc_numeric_column.column, std::make_shared>(), column_name}; } template static ColumnWithTypeAndName readColumnWithEncodedStringOrFixedStringData( - const orc::ColumnVectorBatch * orc_column, const orc::Type * orc_type, const String & column_name, bool nullable) + const ORC::ColumnVectorBatch * orc_column, const ORC::Type * orc_type, const String & column_name, bool nullable) { /// Fill CH holder_column with ORC dictionary /// Note that holder_column is always a ColumnString or ColumnFixedstring whether nullable is true or false, because ORC dictionary doesn't contain null values. @@ -1674,60 +1604,45 @@ static ColumnWithTypeAndName readColumnWithEncodedStringOrFixedStringData( DataTypePtr nested_type = nullable ? std::make_shared(holder_type) : holder_type; auto internal_type = std::make_shared(std::move(nested_type)); - const auto & orc_str_column = dynamic_cast(*orc_column); + const auto & orc_str_column = dynamic_cast(*orc_column); size_t rows = orc_str_column.numElements; - const auto & orc_dict = *orc_str_column.dictionary; - if (orc_dict.dictionaryOffset.size() <= 1) + const auto & orc_dict = assert_cast(*orc_str_column.dictionary->column); + size_t dict_size = orc_dict.size(); + if (dict_size == 0) { + /// Only nulls can be encoded with an empty dictionary. + for (size_t i = 0; i < rows; ++i) + { + if (!orc_str_column.hasNulls || orc_str_column.notNull[i]) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "ORC dictionary index {} at row {} is out of range [0, 0)", + orc_str_column.index[i], i); + } auto result_column = internal_type->createColumn(); result_column->insertManyDefaults(rows); return {std::move(result_column), internal_type, column_name}; } - size_t dict_size = orc_dict.dictionaryOffset.size() - 1; - auto holder_column = holder_type->createColumn(); + /// The dictionary of ORC is already a `ColumnString`, which is the holder for `String`. + ColumnPtr holder_column = orc_str_column.dictionary->column; if constexpr (fixed_string) { const size_t n = orc_type->getMaximumLength(); - auto & concrete_holder_column = assert_cast(*holder_column); - PaddedPODArray & column_chars = concrete_holder_column.getChars(); - size_t reserve_size = dict_size * n; - column_chars.resize_exact(reserve_size); - size_t curr_offset = 0; + auto fixed_holder_column = ColumnFixedString::create(n); + PaddedPODArray & column_chars = fixed_holder_column->getChars(); + column_chars.resize_fill(dict_size * n); for (size_t i = 0; i < dict_size; ++i) { - const auto * buf = orc_dict.dictionaryBlob.data() + orc_dict.dictionaryOffset[i]; - size_t buf_size = orc_dict.dictionaryOffset[i + 1] - orc_dict.dictionaryOffset[i]; - if (buf_size > n) + auto value = orc_dict.getDataAt(i); + if (value.size() > n) throw Exception( ErrorCodes::INCORRECT_DATA, "ORC dictionary entry {} has size {} that exceeds the declared FixedString length {}", - i, buf_size, n); - memcpy(&column_chars[curr_offset], buf, buf_size); - /// resize_exact does not zero-initialize, so pad shorter entries to keep FixedString values - /// deterministic and to avoid leaking uninitialized heap memory. - if (buf_size < n) - memset(&column_chars[curr_offset + buf_size], 0, n - buf_size); - curr_offset += n; - } - } - else - { - auto & concrete_holder_column = assert_cast(*holder_column); - PaddedPODArray & column_chars = concrete_holder_column.getChars(); - PaddedPODArray & column_offsets = concrete_holder_column.getOffsets(); - - column_chars.resize_exact(orc_dict.dictionaryBlob.size()); - column_offsets.resize_exact(dict_size); - size_t curr_offset = 0; - for (size_t i = 0; i < dict_size; ++i) - { - const auto * buf = orc_dict.dictionaryBlob.data() + orc_dict.dictionaryOffset[i]; - size_t buf_size = orc_dict.dictionaryOffset[i + 1] - orc_dict.dictionaryOffset[i]; - memcpy(&column_chars[curr_offset], buf, buf_size); - curr_offset += buf_size; - column_offsets[i] = curr_offset; + i, value.size(), n); + memcpy(&column_chars[i * n], value.data(), value.size()); } + holder_column = std::move(fixed_holder_column); } /// Insert CH dictionary_column from holder_column @@ -1791,57 +1706,22 @@ static ColumnWithTypeAndName readColumnWithEncodedStringOrFixedStringData( return {std::move(internal_column), internal_type, column_name}; } -static ColumnWithTypeAndName -readColumnWithStringData(const orc::ColumnVectorBatch * orc_column, const String & column_name) +static const ColumnString & getStringColumnFromBatch(const ORC::ColumnVectorBatch * orc_column, ColumnPtr & holder) { - auto internal_type = std::make_shared(); - auto internal_column = internal_type->createColumn(); - PaddedPODArray & column_chars = assert_cast(*internal_column).getChars(); - PaddedPODArray & column_offsets = assert_cast(*internal_column).getOffsets(); - - const auto * orc_str_column = dynamic_cast(orc_column); - size_t reserve_size = 0; - for (size_t i = 0; i < orc_str_column->numElements; ++i) - { - if (!orc_str_column->hasNulls || orc_str_column->notNull[i]) - reserve_size += orc_str_column->length[i]; - } - - column_chars.resize_exact(reserve_size); - column_offsets.resize_exact(orc_str_column->numElements); - - size_t curr_offset = 0; - if (!orc_str_column->hasNulls) - { - for (size_t i = 0; i < orc_str_column->numElements; ++i) - { - const auto * buf = orc_str_column->data[i]; - size_t buf_size = orc_str_column->length[i]; - memcpy(&column_chars[curr_offset], buf, buf_size); - curr_offset += buf_size; - column_offsets[i] = curr_offset; - } - } - else - { - for (size_t i = 0; i < orc_str_column->numElements; ++i) - { - if (orc_str_column->notNull[i]) - { - const auto * buf = orc_str_column->data[i]; - size_t buf_size = orc_str_column->length[i]; - memcpy(&column_chars[curr_offset], buf, buf_size); - curr_offset += buf_size; - } + holder = ORC::getStringColumn(dynamic_cast(*orc_column)); + return assert_cast(*holder); +} - column_offsets[i] = curr_offset; - } - } - return {std::move(internal_column), internal_type, column_name}; +static ColumnWithTypeAndName +readColumnWithStringData(const ORC::ColumnVectorBatch * orc_column, const String & column_name) +{ + ColumnPtr column; + getStringColumnFromBatch(orc_column, column); + return {column, std::make_shared(), column_name}; } static ColumnWithTypeAndName -readColumnWithFixedStringData(const orc::ColumnVectorBatch * orc_column, const orc::Type * orc_type, const String & column_name) +readColumnWithFixedStringData(const ORC::ColumnVectorBatch * orc_column, const ORC::Type * orc_type, const String & column_name) { size_t fixed_len = orc_type->getMaximumLength(); auto internal_type = std::make_shared(fixed_len); @@ -1849,25 +1729,22 @@ readColumnWithFixedStringData(const orc::ColumnVectorBatch * orc_column, const o PaddedPODArray & column_chars = assert_cast(*internal_column).getChars(); column_chars.reserve(orc_column->numElements * fixed_len); - const auto * orc_str_column = dynamic_cast(orc_column); - for (size_t i = 0; i < orc_str_column->numElements; ++i) + ColumnPtr holder; + const auto & strings = getStringColumnFromBatch(orc_column, holder); + for (size_t i = 0; i < orc_column->numElements; ++i) { - if (!orc_str_column->hasNulls || orc_str_column->notNull[i]) - { - const Int64 length = orc_str_column->length[i]; - if (length < 0 || static_cast(length) > fixed_len) - throw Exception( - ErrorCodes::INCORRECT_DATA, - "ORC string value at row {} has size {} that doesn't fit into FixedString({})", - i, length, fixed_len); + /// The null values are empty strings. + auto value = strings.getDataAt(i); + if (value.size() > fixed_len) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "ORC string value at row {} has size {} that doesn't fit into FixedString({})", + i, value.size(), fixed_len); - column_chars.insert_assume_reserved(orc_str_column->data[i], orc_str_column->data[i] + length); - /// Zero-pad shorter values to the fixed width to keep the FixedString layout consistent. - if (static_cast(length) < fixed_len) - column_chars.resize_fill(column_chars.size() + (fixed_len - static_cast(length))); - } - else - column_chars.resize_fill(column_chars.size() + fixed_len); + column_chars.insert_assume_reserved(value.data(), value.data() + value.size()); + /// Zero-pad shorter values to the fixed width to keep the FixedString layout consistent. + if (value.size() < fixed_len) + column_chars.resize_fill(column_chars.size() + (fixed_len - value.size())); } return {std::move(internal_column), internal_type, column_name}; @@ -1876,10 +1753,10 @@ readColumnWithFixedStringData(const orc::ColumnVectorBatch * orc_column, const o template static ColumnWithTypeAndName readColumnWithDecimalDataCast( - const orc::ColumnVectorBatch * orc_column, const String & column_name, DataTypePtr internal_type) + const ORC::ColumnVectorBatch * orc_column, const String & column_name, DataTypePtr internal_type) { using NativeType = typename DecimalType::NativeType; - static_assert(std::is_same_v || std::is_same_v); + static_assert(std::is_same_v || std::is_same_v); auto internal_column = internal_type->createColumn(); auto & column_data = static_cast &>(*internal_column).getData(); @@ -1891,12 +1768,9 @@ static ColumnWithTypeAndName readColumnWithDecimalDataCast( if (!orc_decimal_column->hasNulls || orc_decimal_column->notNull[i]) { DecimalType decimal_value{}; - if constexpr (std::is_same_v) + if constexpr (std::is_same_v) { - Int128 int128_value; - int128_value.items[0] = orc_decimal_column->values[i].getLowBits(); - int128_value.items[1] = orc_decimal_column->values[i].getHighBits(); - decimal_value.value = static_cast(int128_value); + decimal_value.value = static_cast(orc_decimal_column->values[i]); } else decimal_value.value = static_cast(orc_decimal_column->values[i]); @@ -1911,26 +1785,30 @@ static ColumnWithTypeAndName readColumnWithDecimalDataCast( } static ColumnWithTypeAndName -readIPv6ColumnFromBinaryData(const orc::ColumnVectorBatch * orc_column, const String & column_name) +readIPv6ColumnFromBinaryData(const ORC::ColumnVectorBatch * orc_column, const String & column_name) { - const auto * orc_str_column = dynamic_cast(orc_column); + ColumnPtr holder; + const auto & strings = getStringColumnFromBatch(orc_column, holder); - for (size_t i = 0; i < orc_str_column->numElements; ++i) + for (size_t i = 0; i < orc_column->numElements; ++i) { /// If at least one value size is not 16 bytes, fallback to reading String column and further cast to IPv6. - if ((!orc_str_column->hasNulls || orc_str_column->notNull[i]) && orc_str_column->length[i] != sizeof(IPv6)) - return readColumnWithStringData(orc_column, column_name); + if ((!orc_column->hasNulls || orc_column->notNull[i]) && strings.getDataAt(i).size() != sizeof(IPv6)) + return {holder, std::make_shared(), column_name}; } auto internal_type = std::make_shared(); auto internal_column = internal_type->createColumn(); auto & ipv6_column = assert_cast(*internal_column); - ipv6_column.reserve(orc_str_column->numElements); + ipv6_column.reserve(orc_column->numElements); - for (size_t i = 0; i < orc_str_column->numElements; ++i) + for (size_t i = 0; i < orc_column->numElements; ++i) { - if (!orc_str_column->hasNulls || orc_str_column->notNull[i]) - ipv6_column.insertData(orc_str_column->data[i], orc_str_column->length[i]); + if (!orc_column->hasNulls || orc_column->notNull[i]) + { + auto value = strings.getDataAt(i); + ipv6_column.insertData(value.data(), value.size()); + } else ipv6_column.insertDefault(); } @@ -1939,49 +1817,46 @@ readIPv6ColumnFromBinaryData(const orc::ColumnVectorBatch * orc_column, const St } static ColumnWithTypeAndName -readIPv4ColumnWithInt32Data(const orc::ColumnVectorBatch * orc_column, const String & column_name) +readIPv4ColumnWithInt32Data(const ORC::ColumnVectorBatch * orc_column, const String & column_name) { - const auto * orc_int_column = dynamic_cast(orc_column); + const auto & data = typeid_cast(*orc_column).getData(); auto internal_type = std::make_shared(); auto internal_column = internal_type->createColumn(); auto & column_data = assert_cast(*internal_column).getData(); - column_data.reserve(orc_int_column->numElements); - - for (size_t i = 0; i < orc_int_column->numElements; ++i) - { - if (!orc_int_column->hasNulls || orc_int_column->notNull[i]) - column_data.push_back(static_cast(orc_int_column->data[i])); - else - column_data.push_back(0); - } + column_data.resize(data.size()); + /// The null values are zeros. + for (size_t i = 0; i < data.size(); ++i) + column_data[i] = static_cast(data[i]); return {std::move(internal_column), internal_type, column_name}; } template static ColumnWithTypeAndName readColumnWithBigNumberFromBinaryData( - const orc::ColumnVectorBatch * orc_column, const String & column_name, const DataTypePtr & column_type) + const ORC::ColumnVectorBatch * orc_column, const String & column_name, const DataTypePtr & column_type) { - const auto * orc_str_column = dynamic_cast(orc_column); + ColumnPtr holder; + const auto & strings = getStringColumnFromBatch(orc_column, holder); auto internal_column = column_type->createColumn(); auto & integer_column = assert_cast(*internal_column); - integer_column.reserve(orc_str_column->numElements); + integer_column.reserve(orc_column->numElements); - for (size_t i = 0; i < orc_str_column->numElements; ++i) + for (size_t i = 0; i < orc_column->numElements; ++i) { - if (!orc_str_column->hasNulls || orc_str_column->notNull[i]) + if (!orc_column->hasNulls || orc_column->notNull[i]) { - if (sizeof(typename ColumnType::ValueType) != orc_str_column->length[i]) + auto value = strings.getDataAt(i); + if (sizeof(typename ColumnType::ValueType) != value.size()) throw Exception( ErrorCodes::INCORRECT_DATA, "ValueType size {} of column {} is not equal to size of binary data {}", sizeof(typename ColumnType::ValueType), integer_column.getName(), - orc_str_column->length[i]); + value.size()); - integer_column.insertData(orc_str_column->data[i], orc_str_column->length[i]); + integer_column.insertData(value.data(), value.size()); } else { @@ -1992,7 +1867,7 @@ static ColumnWithTypeAndName readColumnWithBigNumberFromBinaryData( } static ColumnWithTypeAndName readColumnWithDateData( - const orc::ColumnVectorBatch * orc_column, + const ORC::ColumnVectorBatch * orc_column, const String & column_name, const DataTypePtr & raw_type_hint, FormatSettings::DateTimeOverflowBehavior date_time_overflow_behavior) @@ -2053,7 +1928,8 @@ static ColumnWithTypeAndName readColumnWithDateData( internal_type = std::make_shared(); } - const auto * orc_int_column = dynamic_cast(orc_column); + const auto * orc_int_column = dynamic_cast(orc_column); + const auto & orc_int_data = orc_int_column->getData(); auto internal_column = internal_type->createColumn(); PaddedPODArray & column_data = assert_cast &>(*internal_column).getData(); column_data.reserve(orc_int_column->numElements); @@ -2064,7 +1940,7 @@ static ColumnWithTypeAndName readColumnWithDateData( { /// Range-check the original Int64 day count before narrowing to Int32, /// otherwise a malformed value could wrap into an apparently valid date. - Int64 days_num = orc_int_column->data[i]; + Int64 days_num = orc_int_data[i]; if (check_date32_range && (days_num > DATE_LUT_MAX_EXTEND_DAY_NUM || days_num < DATE_LUT_MIN_EXTEND_DAY_NUM)) { if (date_time_overflow_behavior == FormatSettings::DateTimeOverflowBehavior::Saturate) @@ -2132,12 +2008,12 @@ static ColumnWithTypeAndName readColumnWithDateData( } static ColumnWithTypeAndName readColumnWithTimestampData( - const orc::ColumnVectorBatch * orc_column, + const ORC::ColumnVectorBatch * orc_column, const String & column_name, const DataTypePtr & type_hint, FormatSettings::DateTimeOverflowBehavior date_time_overflow_behavior) { - const auto * orc_ts_column = dynamic_cast(orc_column); + const auto * orc_ts_column = dynamic_cast(orc_column); /// ORC stores timestamps as (seconds, nanoseconds). ClickHouse used to convert them through a fixed /// DateTime64(9) intermediate (seconds * 1e9 + nanoseconds), which overflows Int64 for timestamps beyond @@ -2254,7 +2130,7 @@ static bool orcIntegerBranchWidensTo(const DataTypePtr & type, size_t orc_size) /// fields (the per-branch repair cast then projects/reorders the read tuple to exactly the target), /// while an unnamed target tuple is matched positionally with the same arity. `case_insensitive` /// mirrors `input_format_orc_case_insensitive_column_matching` for that name-based matching. -static bool orcUnionBranchMatchesType(const orc::Type * orc_branch_type, const DataTypePtr & target_type, bool case_insensitive) +static bool orcUnionBranchMatchesType(const ORC::Type * orc_branch_type, const DataTypePtr & target_type, bool case_insensitive) { checkStackSize(); @@ -2263,48 +2139,48 @@ static bool orcUnionBranchMatchesType(const orc::Type * orc_branch_type, const D switch (orc_branch_type->getKind()) { - case orc::TypeKind::BOOLEAN: + case ORC::TypeKind::BOOLEAN: /// A boolean holds only 0 and 1, so every integer type is a widening of it. return which.isUInt8() || which.isEnum8() || orcIntegerBranchWidensTo(type, 0); - case orc::TypeKind::BYTE: + case ORC::TypeKind::BYTE: return which.isInt8() || (which.isUInt8() && !isBool(type)) || which.isEnum8() || orcIntegerBranchWidensTo(type, 1); - case orc::TypeKind::SHORT: + case ORC::TypeKind::SHORT: return which.isInt16() || which.isUInt16() || which.isEnum16() || orcIntegerBranchWidensTo(type, 2); - case orc::TypeKind::INT: + case ORC::TypeKind::INT: return which.isInt32() || which.isUInt32() || which.isIPv4() || orcIntegerBranchWidensTo(type, 4); - case orc::TypeKind::LONG: + case ORC::TypeKind::LONG: return which.isInt64() || which.isUInt64() || orcIntegerBranchWidensTo(type, 8); - case orc::TypeKind::FLOAT: + case ORC::TypeKind::FLOAT: return which.isFloat32() || which.isFloat64(); - case orc::TypeKind::DOUBLE: + case ORC::TypeKind::DOUBLE: return which.isFloat64(); - case orc::TypeKind::DATE: + case ORC::TypeKind::DATE: return which.isDate32() || which.isDate(); - case orc::TypeKind::TIMESTAMP: + case ORC::TypeKind::TIMESTAMP: /// Permissive: matches any DateTime/DateTime64 alternative. The repair cast relabels the /// branch to the alternative's timezone, which is value-preserving for DateTime64. return which.isDateTime64() || which.isDateTime(); - case orc::TypeKind::TIMESTAMP_INSTANT: + case ORC::TypeKind::TIMESTAMP_INSTANT: /// Matches only an explicitly time-zoned target so it stays distinct from a TIMESTAMP /// branch in the same union (both read back as DateTime64(9)); see the comment above the /// function. return (which.isDateTime64() || which.isDateTime()) && orcTimestampTargetHasExplicitTimeZone(type); - case orc::TypeKind::DECIMAL: + case ORC::TypeKind::DECIMAL: return which.isDecimal(); - case orc::TypeKind::STRING: - case orc::TypeKind::VARCHAR: + case ORC::TypeKind::STRING: + case ORC::TypeKind::VARCHAR: return which.isStringOrFixedString(); - case orc::TypeKind::BINARY: + case ORC::TypeKind::BINARY: return which.isStringOrFixedString() || which.isIPv6() || which.isInt128() || which.isUInt128() || which.isInt256() || which.isUInt256() || which.isDecimal256(); - case orc::TypeKind::CHAR: + case ORC::TypeKind::CHAR: return which.isStringOrFixedString() || which.isInt128() || which.isUInt128() || which.isInt256() || which.isUInt256() || which.isDecimal256(); - case orc::TypeKind::LIST: + case ORC::TypeKind::LIST: return which.isArray() && orc_branch_type->getSubtypeCount() == 1 && orcUnionBranchMatchesType( orc_branch_type->getSubtype(0), assert_cast(*type).getNestedType(), case_insensitive); - case orc::TypeKind::MAP: + case ORC::TypeKind::MAP: { if (!which.isMap() || orc_branch_type->getSubtypeCount() != 2) return false; @@ -2312,7 +2188,7 @@ static bool orcUnionBranchMatchesType(const orc::Type * orc_branch_type, const D return orcUnionBranchMatchesType(orc_branch_type->getSubtype(0), map_type.getKeyType(), case_insensitive) && orcUnionBranchMatchesType(orc_branch_type->getSubtype(1), map_type.getValueType(), case_insensitive); } - case orc::TypeKind::STRUCT: + case ORC::TypeKind::STRUCT: { if (!which.isTuple()) return false; @@ -2324,7 +2200,7 @@ static bool orcUnionBranchMatchesType(const orc::Type * orc_branch_type, const D /// a subset of the ORC struct's fields, in any order. Each target field must map to /// an ORC field with a recursively matching type; extra ORC fields are projected out /// by the repair cast. Build a name -> ORC subtype map for the lookup. - std::unordered_map orc_field_by_name; + std::unordered_map orc_field_by_name; orc_field_by_name.reserve(orc_branch_type->getSubtypeCount()); for (size_t i = 0; i < orc_branch_type->getSubtypeCount(); ++i) { @@ -2355,10 +2231,10 @@ static bool orcUnionBranchMatchesType(const orc::Type * orc_branch_type, const D return false; return true; } - case orc::TypeKind::UNION: + case ORC::TypeKind::UNION: return which.isVariant(); - case orc::TypeKind::GEOMETRY: - case orc::TypeKind::GEOGRAPHY: + case ORC::TypeKind::GEOMETRY: + case ORC::TypeKind::GEOGRAPHY: /// Added in ORC 2.3. We do not read these, so no target type can match such a branch. return false; } @@ -2383,32 +2259,32 @@ static bool orcUnionBranchMatchesType(const orc::Type * orc_branch_type, const D /// schema `Variant(Int8, Int64)` leaves the `BYTE` branch with two candidates. Preferring the /// natural pairing keeps the exact-width alternative with its own branch and lets the wider /// alternative go to the branch that needs it. -static bool orcUnionBranchPrefersType(const orc::Type * orc_branch_type, const DataTypePtr & target_type) +static bool orcUnionBranchPrefersType(const ORC::Type * orc_branch_type, const DataTypePtr & target_type) { const DataTypePtr type = removeLowCardinality(removeNullableOrLowCardinalityNullable(target_type)); const WhichDataType which(type); switch (orc_branch_type->getKind()) { - case orc::TypeKind::BOOLEAN: + case ORC::TypeKind::BOOLEAN: return isBool(type); - case orc::TypeKind::BYTE: + case ORC::TypeKind::BYTE: return which.isInt8(); - case orc::TypeKind::SHORT: + case ORC::TypeKind::SHORT: return which.isInt16(); - case orc::TypeKind::INT: + case ORC::TypeKind::INT: return which.isInt32(); - case orc::TypeKind::LONG: + case ORC::TypeKind::LONG: return which.isInt64(); - case orc::TypeKind::FLOAT: + case ORC::TypeKind::FLOAT: return which.isFloat32(); - case orc::TypeKind::DOUBLE: + case ORC::TypeKind::DOUBLE: return which.isFloat64(); - case orc::TypeKind::CHAR: + case ORC::TypeKind::CHAR: return which.isFixedString() && assert_cast(*type).getN() == orc_branch_type->getMaximumLength(); - case orc::TypeKind::STRING: - case orc::TypeKind::VARCHAR: + case ORC::TypeKind::STRING: + case ORC::TypeKind::VARCHAR: return which.isString(); default: return false; @@ -2424,7 +2300,7 @@ static bool orcUnionBranchPrefersType(const orc::Type * orc_branch_type, const D /// hint (nullptr in the result), so no guess is ever made. Shared between /// readColumnFromORCColumn (drives the branch conversions) and updateIncludeTypeIds (prunes a /// hinted STRUCT branch's fields the same way the read path does), so the two stay in sync. -static DataTypes computeOrcUnionBranchHints(const orc::Type * orc_type, const DataTypePtr & type_hint, bool case_insensitive_matching) +static DataTypes computeOrcUnionBranchHints(const ORC::Type * orc_type, const DataTypePtr & type_hint, bool case_insensitive_matching) { const size_t num_children = orc_type->getSubtypeCount(); DataTypes branch_hints(num_children); @@ -2511,8 +2387,8 @@ static DataTypes computeOrcUnionBranchHints(const orc::Type * orc_type, const Da } ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( - const orc::ColumnVectorBatch * orc_column, - const orc::Type * orc_type, + const ORC::ColumnVectorBatch * orc_column, + const ORC::Type * orc_type, const std::string & column_name, bool inside_nullable, DataTypePtr type_hint) const @@ -2524,7 +2400,7 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( bool skipped = false; if (!inside_nullable && (orc_column->hasNulls || (type_hint && isNullableOrLowCardinalityNullable(type_hint))) && !orc_column->isEncoded - && (orc_type->getKind() != orc::LIST && orc_type->getKind() != orc::MAP && orc_type->getKind() != orc::UNION)) + && (orc_type->getKind() != ORC::LIST && orc_type->getKind() != ORC::MAP && orc_type->getKind() != ORC::UNION)) { DataTypePtr nested_type_hint; if (type_hint) @@ -2540,9 +2416,9 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( /// ORC union maps to the ClickHouse Variant type. Handled before the switch below so the switch /// stays non-exhaustive (its default handles unsupported types). - if (orc_type->getKind() == orc::UNION) + if (orc_type->getKind() == ORC::UNION) { - const auto * orc_union_column = dynamic_cast(orc_column); + const auto * orc_union_column = dynamic_cast(orc_column); if (!orc_union_column) throw Exception(ErrorCodes::LOGICAL_ERROR, "ORC column for union type must be a UnionVectorBatch"); @@ -2586,7 +2462,7 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( /// forbids; reject it explicitly (matching the schema-inference path) instead of letting /// the DataTypeVariant below throw a confusing BAD_ARGUMENTS. Only reachable with an /// explicit structure, since schema inference already rejects nested unions. - if (orc_type->getSubtype(i)->getKind() == orc::UNION) + if (orc_type->getSubtype(i)->getKind() == ORC::UNION) throw Exception( ErrorCodes::UNKNOWN_TYPE, "ORC union type '{}' has a nested union branch, which is not supported, while reading column {}", @@ -2686,7 +2562,7 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( auto & offsets_data = offsets->getData(); offsets_data.resize_exact(num_rows); - const unsigned char * tags = orc_union_column->tags.data(); + const UInt8 * tags = orc_union_column->tags.data(); const uint64_t * orc_offsets = orc_union_column->offsets.data(); const char * not_null = orc_union_column->hasNulls ? orc_union_column->notNull.data() : nullptr; @@ -2738,9 +2614,9 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( switch (orc_type->getKind()) { - case orc::STRING: - case orc::BINARY: - case orc::VARCHAR: + case ORC::STRING: + case ORC::BINARY: + case ORC::VARCHAR: { if (type_hint) { @@ -2772,7 +2648,7 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( else return readColumnWithStringData(orc_column, column_name); } - case orc::CHAR: + case ORC::CHAR: { if (type_hint) { @@ -2798,32 +2674,32 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( else return readColumnWithFixedStringData(orc_column, orc_type, column_name); } - case orc::BOOLEAN: + case ORC::BOOLEAN: return readColumnWithBooleanData(orc_column, column_name); - case orc::BYTE: - return readColumnWithNumericData(orc_column, column_name); - case orc::SHORT: - return readColumnWithNumericData(orc_column, column_name); - case orc::INT: + case ORC::BYTE: + return readColumnWithNumericData(orc_column, column_name); + case ORC::SHORT: + return readColumnWithNumericData(orc_column, column_name); + case ORC::INT: { /// ORC format doesn't have unsigned integers and we output IPv4 as Int32. /// We should allow to read it back from Int32. if (type_hint && isIPv4(type_hint)) return readIPv4ColumnWithInt32Data(orc_column, column_name); - return readColumnWithNumericData(orc_column, column_name); + return readColumnWithNumericData(orc_column, column_name); } - case orc::LONG: - return readColumnWithNumericData(orc_column, column_name); - case orc::FLOAT: - return readColumnWithNumericData(orc_column, column_name); - case orc::DOUBLE: - return readColumnWithNumericData(orc_column, column_name); - case orc::DATE: + case ORC::LONG: + return readColumnWithNumericData(orc_column, column_name); + case ORC::FLOAT: + return readColumnWithNumericData(orc_column, column_name); + case ORC::DOUBLE: + return readColumnWithNumericData(orc_column, column_name); + case ORC::DATE: return readColumnWithDateData(orc_column, column_name, type_hint, date_time_overflow_behavior); - case orc::TIMESTAMP: [[fallthrough]]; - case orc::TIMESTAMP_INSTANT: + case ORC::TIMESTAMP: [[fallthrough]]; + case ORC::TIMESTAMP_INSTANT: return readColumnWithTimestampData(orc_column, column_name, type_hint, date_time_overflow_behavior); - case orc::DECIMAL: + case ORC::DECIMAL: { auto interal_type = parseORCType(orc_type, false, false, nullptr, skipped); @@ -2832,15 +2708,15 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( precision = 38; if (precision <= DecimalUtils::max_precision) - return readColumnWithDecimalDataCast(orc_column, column_name, interal_type); + return readColumnWithDecimalDataCast(orc_column, column_name, interal_type); if (precision <= DecimalUtils::max_precision) - return readColumnWithDecimalDataCast(orc_column, column_name, interal_type); + return readColumnWithDecimalDataCast(orc_column, column_name, interal_type); if (precision <= DecimalUtils::max_precision) - return readColumnWithDecimalDataCast(orc_column, column_name, interal_type); + return readColumnWithDecimalDataCast(orc_column, column_name, interal_type); throw Exception( ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Decimal precision {} in ORC type {} is out of bound", precision, orc_type->toString()); } - case orc::MAP: + case ORC::MAP: { DataTypePtr key_type_hint; DataTypePtr value_type_hint; @@ -2854,7 +2730,7 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( } } - const auto * orc_map_column = dynamic_cast(orc_column); + const auto * orc_map_column = dynamic_cast(orc_column); const auto * orc_key_column = orc_map_column->keys.get(); const auto * orc_value_column = orc_map_column->elements.get(); const auto * orc_key_type = orc_type->getSubtype(0); @@ -2878,7 +2754,7 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( auto map_type = std::make_shared(key_column.type, value_column.type); return {map_column, map_type, column_name}; } - case orc::LIST: + case ORC::LIST: { DataTypePtr nested_type_hint; if (type_hint) @@ -2888,7 +2764,7 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( nested_type_hint = array_type_hint->getNestedType(); } - const auto * orc_list_column = dynamic_cast(orc_column); + const auto * orc_list_column = dynamic_cast(orc_column); const auto * orc_nested_column = getNestedORCColumn(orc_list_column); const auto * orc_nested_type = orc_type->getSubtype(0); auto nested_column = readColumnFromORCColumn(orc_nested_column, orc_nested_type, column_name, false, nested_type_hint); @@ -2917,14 +2793,14 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( auto array_column = ColumnArray::create(array_data_column, offsets_column); return {array_column, array_type, column_name}; } - case orc::STRUCT: + case ORC::STRUCT: { Columns tuple_elements; DataTypes tuple_types; std::vector tuple_names; const auto * tuple_type_hint = type_hint ? typeid_cast(type_hint.get()) : nullptr; - const auto * orc_struct_column = dynamic_cast(orc_column); + const auto * orc_struct_column = dynamic_cast(orc_column); for (size_t i = 0; i < orc_type->getSubtypeCount(); ++i) { diff --git a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h index 49a29a7d85179..c50e2d388176d 100644 --- a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h +++ b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include namespace DB @@ -19,15 +19,14 @@ namespace DB class KeyCondition; -class ORCInputStream : public orc::InputStream +class ORCInputStream : public ORC::InputStream { public: ORCInputStream(SeekableReadBuffer & in_, size_t file_size_, bool use_prefetch); - uint64_t getLength() const override; - uint64_t getNaturalReadSize() const override; - void read(void * buf, uint64_t length, uint64_t offset) override; - std::future readAsync(void * buf, uint64_t length, uint64_t offset) override; + UInt64 getLength() const override; + void read(void * buf, UInt64 length, UInt64 offset) override; + std::future readAsync(void * buf, UInt64 length, UInt64 offset) override; const std::string & getName() const override { return name; } protected: @@ -52,22 +51,17 @@ class ORCInputStreamFromString : public ReadBufferFromOwnString, public ORCInput } }; -std::unique_ptr +std::unique_ptr asORCInputStream(ReadBuffer & in, const FormatSettings & settings, bool use_prefetch, std::atomic & is_cancelled); // Reads the whole file into a memory buffer, owned by the returned RandomAccessFile. -std::unique_ptr asORCInputStreamLoadIntoMemory(ReadBuffer & in, std::atomic & is_cancelled); - -/// The memory pool to set on the options of every ORC reader and writer we create: unlike the -/// default one of the library, it is accounted for by the memory tracker and it throws on failure -/// instead of returning a null pointer that the library dereferences. -orc::MemoryPool & getORCMemoryPool(); +std::unique_ptr asORCInputStreamLoadIntoMemory(ReadBuffer & in, std::atomic & is_cancelled); /// Creates an ORC file reader; throws INCORRECT_DATA if the root type of the file is not a struct. -std::unique_ptr createORCReader(std::unique_ptr stream, const orc::ReaderOptions & options); +std::unique_ptr createORCReader(std::unique_ptr stream, const ORC::ReaderOptions & options); -std::unique_ptr buildORCSearchArgument( - const KeyCondition & key_condition, const Block & header, const orc::Type & schema, const FormatSettings & format_settings); +std::unique_ptr buildORCSearchArgument( + const KeyCondition & key_condition, const Block & header, const ORC::Type & schema, const FormatSettings & format_settings); class ORCColumnToCHColumn; class NativeORCBlockInputFormat final : public IInputFormat @@ -101,14 +95,14 @@ class NativeORCBlockInputFormat final : public IInputFormat bool prepareStripeReader(); void prefetchStripes(); - std::unique_ptr file_reader; - std::unique_ptr stripe_reader; + std::unique_ptr file_reader; + std::unique_ptr stripe_reader; std::unique_ptr orc_column_to_ch_column; - std::shared_ptr sargs; + std::shared_ptr sargs; // indices of columns to read from ORC file - std::list include_indices; + std::vector include_indices; BlockMissingValues block_missing_values; size_t approx_bytes_read_for_chunk = 0; @@ -123,7 +117,7 @@ class NativeORCBlockInputFormat final : public IInputFormat size_t read_iterator{}; size_t prefetch_iterator{}; - std::unique_ptr current_stripe_info; + const ORC::Proto::StripeInformation * current_stripe_info = nullptr; std::atomic is_stopped{0}; }; @@ -140,7 +134,7 @@ class NativeORCSchemaReader final : public ISchemaReader void initializeIfNeeded(); const FormatSettings format_settings; - std::unique_ptr file_reader; + std::unique_ptr file_reader; std::atomic is_stopped{0}; bool initialized = false; }; @@ -148,8 +142,8 @@ class NativeORCSchemaReader final : public ISchemaReader class ORCColumnToCHColumn { public: - using ORCColumnPtr = const orc::ColumnVectorBatch *; - using ORCTypePtr = const orc::Type *; + using ORCColumnPtr = const ORC::ColumnVectorBatch *; + using ORCTypePtr = const ORC::Type *; using ORCColumnWithType = std::pair; using NameToColumnPtr = std::unordered_map; @@ -163,8 +157,8 @@ class ORCColumnToCHColumn void orcTableToCHChunk( Chunk & res, - const orc::Type * schema, - const orc::ColumnVectorBatch * table, + const ORC::Type * schema, + const ORC::ColumnVectorBatch * table, size_t num_rows, BlockMissingValues * block_missing_values = nullptr); @@ -173,8 +167,8 @@ class ORCColumnToCHColumn private: ColumnWithTypeAndName readColumnFromORCColumn( - const orc::ColumnVectorBatch * orc_column, - const orc::Type * orc_type, + const ORC::ColumnVectorBatch * orc_column, + const ORC::Type * orc_type, const std::string & column_name, bool inside_nullable, DataTypePtr type_hint = nullptr) const; diff --git a/src/Processors/Formats/Impl/ORC/ColumnReader.cpp b/src/Processors/Formats/Impl/ORC/ColumnReader.cpp new file mode 100644 index 0000000000000..b04bb53500921 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/ColumnReader.cpp @@ -0,0 +1,1213 @@ +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int INCORRECT_DATA; + extern const int NOT_IMPLEMENTED; +} +} + +namespace DB::ORC +{ + +namespace +{ + +RleVersion getRleVersion(const Proto::ColumnEncoding & encoding) +{ + switch (encoding.kind) + { + case Proto::ColumnEncodingKind::DIRECT: + case Proto::ColumnEncodingKind::DICTIONARY: + return RleVersion::V1; + case Proto::ColumnEncodingKind::DIRECT_V2: + case Proto::ColumnEncodingKind::DICTIONARY_V2: + return RleVersion::V2; + } + throw Exception(ErrorCodes::INCORRECT_DATA, "Unknown ORC column encoding {}", static_cast(encoding.kind)); +} + +std::unique_ptr getRequiredStream(StripeStreams & stripe, UInt64 column_id, Proto::StreamKind kind, const char * stream_name, const char * column_kind) +{ + auto stream = stripe.getStream(column_id, kind); + if (!stream) + throw Exception(ErrorCodes::INCORRECT_DATA, "{} stream not found in ORC {} column {}", stream_name, column_kind, column_id); + return stream; +} + +/// Converts the bytes read into the beginning of a buffer of integers into the integers, in place. +/// BOOLEAN and BYTE: the values are decoded into the bytes of the column. +template +class ByteColumnReaderImpl final : public ColumnReader +{ +public: + ByteColumnReaderImpl(const Type & type, StripeStreams & stripe) + : ColumnReader(type, stripe) + , rle(getRequiredStream(stripe, column_id, Proto::StreamKind::DATA, "DATA", type.toString().c_str())) + { + } + + size_t skip(size_t num_values) override + { + num_values = ColumnReader::skip(num_values); + rle.skip(num_values); + return num_values; + } + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + ColumnReader::next(batch, num_values, incoming_mask); + auto & data = typeid_cast(batch).prepareColumn(num_values); + rle.next(reinterpret_cast(data.data()), num_values, batch.hasNulls ? batch.notNull.data() : nullptr); + } + + void seekToRowGroup(std::unordered_map & positions) override + { + ColumnReader::seekToRowGroup(positions); + rle.seek(positions.at(column_id)); + } + +private: + Decoder rle; +}; + +using BooleanColumnReader = ByteColumnReaderImpl; +using ByteColumnReader = ByteColumnReaderImpl; + +/// SHORT, INT, LONG and DATE: the values are decoded into a column of the matching width. +template +class IntegerColumnReader final : public ColumnReader +{ +public: + IntegerColumnReader(const Type & type, StripeStreams & stripe) + : ColumnReader(type, stripe) + { + rle = createIntRleDecoder( + getRequiredStream(stripe, column_id, Proto::StreamKind::DATA, "DATA", "integer"), + true, getRleVersion(stripe.getEncoding(column_id))); + } + + size_t skip(size_t num_values) override + { + num_values = ColumnReader::skip(num_values); + rle->skip(num_values); + return num_values; + } + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + ColumnReader::next(batch, num_values, incoming_mask); + auto & data = typeid_cast(batch).prepareColumn(num_values); + rle->next(data.data(), num_values, batch.hasNulls ? batch.notNull.data() : nullptr); + } + + void seekToRowGroup(std::unordered_map & positions) override + { + ColumnReader::seekToRowGroup(positions); + rle->seek(positions.at(column_id)); + } + +private: + std::unique_ptr rle; +}; + +/// FLOAT and DOUBLE: the values are stored as is (little endian), so they are copied into the column. +template +class FloatingColumnReader final : public ColumnReader +{ +public: + FloatingColumnReader(const Type & type, StripeStreams & stripe) + : ColumnReader(type, stripe) + , stream(getRequiredStream(stripe, column_id, Proto::StreamKind::DATA, "DATA", "floating point")) + { + } + + size_t skip(size_t num_values) override + { + num_values = ColumnReader::skip(num_values); + stream->skipBytes(num_values * sizeof(ItemType)); + return num_values; + } + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + ColumnReader::next(batch, num_values, incoming_mask); + const char * not_null = batch.hasNulls ? batch.notNull.data() : nullptr; + auto & data = typeid_cast &>(batch).prepareColumn(num_values); + ItemType * out = data.data(); + + /// The stream has only the non-null values: read them at once to the beginning of the column. + size_t num_present = num_values; + if (not_null) + num_present = std::count_if(not_null, not_null + num_values, [](char c) { return c != 0; }); + + stream->readBytes(reinterpret_cast(out), num_present * sizeof(ItemType)); + if constexpr (std::endian::native == std::endian::big) + for (size_t i = 0; i < num_present; ++i) + out[i] = std::bit_cast(std::byteswap(std::bit_cast(out[i]))); + + if (!not_null) + return; + + /// Move the values to their rows from the back: the k-th value goes to a row i >= k, so it is not overwritten before it is moved. + size_t next_value = num_present; + for (size_t i = num_values; i-- > 0;) + out[i] = not_null[i] ? out[--next_value] : 0; + } + + void seekToRowGroup(std::unordered_map & positions) override + { + ColumnReader::seekToRowGroup(positions); + stream->seek(positions.at(column_id)); + } + +private: + using Bits = std::conditional_t; + + std::unique_ptr stream; +}; + +/// Converts the lengths in `offsets` into the end offsets (as in `ColumnString` and `ColumnArray`), starting +/// from `total`, and returns the total length. The lengths of the null values are zeros. +UInt64 lengthsToEndOffsets(UInt64 * offsets, size_t num_values, UInt64 column_id, UInt64 total = 0) +{ + for (size_t i = 0; i < num_values; ++i) + { + /// The lengths are non-negative, a negative one is huge as unsigned. + if (offsets[i] > static_cast(std::numeric_limits::max()) || common::addOverflow(total, offsets[i], total) + || total > static_cast(std::numeric_limits::max())) + throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid length {} in ORC column {}", static_cast(offsets[i]), column_id); + offsets[i] = total; + } + return total; +} + +/// Reads the strings with their lengths in a separate stream into a `ColumnString`. +ColumnString::MutablePtr readStrings( + IntRleDecoder & length_rle, InputByteStream & blob_stream, size_t num_values, const char * not_null, UInt64 column_id) +{ + /// The buffers grow as the data is decoded: a malformed count or length must not allocate memory + /// before the streams prove to contain the data. + static constexpr size_t MIN_STEP = 64 * 1024; + auto column = ColumnString::create(); + auto & offsets = column->getOffsets(); + UInt64 total = 0; + while (offsets.size() < num_values) + { + size_t done = offsets.size(); + size_t step = std::min(num_values - done, std::max(MIN_STEP, done)); + offsets.resize(done + step); + length_rle.next(offsets.data() + done, step, not_null ? not_null + done : nullptr); + total = lengthsToEndOffsets(offsets.data() + done, step, column_id, total); + } + auto & chars = column->getChars(); + while (chars.size() < total) + { + size_t done = chars.size(); + size_t step = std::min(total - done, std::max(MIN_STEP, done)); + chars.resize(done + step); + blob_stream.readBytes(reinterpret_cast(chars.data() + done), step); + } + return column; +} + +std::shared_ptr loadStringDictionary(UInt64 column_id, StripeStreams & stripe) +{ + const auto & encoding = stripe.getEncoding(column_id); + /// The dictionary is not bounded by the number of rows of the stripe: a column nested in a list or a map + /// can have more values than the stripe has rows. + UInt32 dictionary_size = encoding.dictionary_size; + + auto dictionary = std::make_shared(); + if (dictionary_size == 0) + { + dictionary->column = ColumnString::create(); + return dictionary; + } + + auto length_rle = createIntRleDecoder( + getRequiredStream(stripe, column_id, Proto::StreamKind::LENGTH, "LENGTH", "string dictionary"), false, getRleVersion(encoding)); + auto blob_stream = stripe.getStream(column_id, Proto::StreamKind::DICTIONARY_DATA); + if (!blob_stream) + { + /// All entries can be empty strings, then the stream is not needed. + blob_stream = std::make_unique(std::string_view{}, CompressionKind::NONE, 0, "empty dictionary"); + } + dictionary->column = readStrings(*length_rle, *blob_stream, dictionary_size, nullptr, column_id); + return dictionary; +} + +class StringDictionaryColumnReader final : public ColumnReader +{ +public: + StringDictionaryColumnReader(const Type & type, StripeStreams & stripe) + : ColumnReader(type, stripe) + { + rle = createIntRleDecoder( + getRequiredStream(stripe, column_id, Proto::StreamKind::DATA, "DATA", "string dictionary"), + false, getRleVersion(stripe.getEncoding(column_id))); + dictionary = loadStringDictionary(column_id, stripe); + } + + size_t skip(size_t num_values) override + { + num_values = ColumnReader::skip(num_values); + rle->skip(num_values); + return num_values; + } + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + ColumnReader::next(batch, num_values, incoming_mask); + const char * not_null = batch.hasNulls ? batch.notNull.data() : nullptr; + auto & string_batch = dynamic_cast(batch); + index.resize(num_values); + rle->next(index.data(), num_values, not_null); + string_batch.isEncoded = false; + string_batch.column = decodeDictionary(assert_cast(*dictionary->column), index.data(), not_null, num_values); + } + + void nextEncoded(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + readIndex(dynamic_cast(batch), num_values, incoming_mask); + } + + void seekToRowGroup(std::unordered_map & positions) override + { + ColumnReader::seekToRowGroup(positions); + rle->seek(positions.at(column_id)); + } + +private: + void readIndex(EncodedStringVectorBatch & batch, size_t num_values, const char * incoming_mask) + { + ColumnReader::next(batch, num_values, incoming_mask); + batch.isEncoded = true; + batch.dictionary = dictionary; + batch.column = nullptr; + batch.index.resize(num_values); + rle->next(batch.index.data(), num_values, batch.hasNulls ? batch.notNull.data() : nullptr); + } + + std::shared_ptr dictionary; + std::unique_ptr rle; + PaddedPODArray index; +}; + +class StringDirectColumnReader final : public ColumnReader +{ +public: + StringDirectColumnReader(const Type & type, StripeStreams & stripe) + : ColumnReader(type, stripe) + { + length_rle = createIntRleDecoder( + getRequiredStream(stripe, column_id, Proto::StreamKind::LENGTH, "LENGTH", "string"), + false, getRleVersion(stripe.getEncoding(column_id))); + blob_stream = getRequiredStream(stripe, column_id, Proto::StreamKind::DATA, "DATA", "string"); + } + + size_t skip(size_t num_values) override + { + num_values = ColumnReader::skip(num_values); + static constexpr size_t BUFFER_SIZE = 1024; + UInt64 buffer[BUFFER_SIZE]; + size_t done = 0; + size_t total_bytes = 0; + while (done < num_values) + { + size_t step = std::min(BUFFER_SIZE, num_values - done); + length_rle->next(buffer, step, nullptr); + total_bytes += lengthsToEndOffsets(buffer, step, column_id); + done += step; + } + blob_stream->skipBytes(total_bytes); + return num_values; + } + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + ColumnReader::next(batch, num_values, incoming_mask); + auto & string_batch = dynamic_cast(batch); + string_batch.isEncoded = false; + string_batch.column = readStrings(*length_rle, *blob_stream, num_values, batch.hasNulls ? batch.notNull.data() : nullptr, column_id); + } + + void seekToRowGroup(std::unordered_map & positions) override + { + ColumnReader::seekToRowGroup(positions); + auto & provider = positions.at(column_id); + blob_stream->seek(provider); + length_rle->seek(provider); + } + +private: + std::unique_ptr length_rle; + std::unique_ptr blob_stream; +}; + +class TimestampColumnReader final : public ColumnReader +{ +public: + TimestampColumnReader(const Type & type, StripeStreams & stripe, bool is_instant) + : ColumnReader(type, stripe) + , writer_timezone(is_instant ? DateLUT::instance("UTC") : stripe.getWriterTimezone()) + , reader_timezone(is_instant ? DateLUT::instance("UTC") : stripe.getReaderTimezone()) + , same_timezone(&writer_timezone == &reader_timezone || writer_timezone.getTimeZone() == reader_timezone.getTimeZone()) + { + /// The base of the seconds is 2015-01-01 00:00:00 in the time zone of the writer. + static constexpr Int64 UTC_2015_01_01 = 1420070400; + epoch_offset = UTC_2015_01_01 - writer_timezone.timezoneOffset(UTC_2015_01_01); + + RleVersion version = getRleVersion(stripe.getEncoding(column_id)); + seconds_rle = createIntRleDecoder(getRequiredStream(stripe, column_id, Proto::StreamKind::DATA, "DATA", "timestamp"), true, version); + nanos_rle = createIntRleDecoder(getRequiredStream(stripe, column_id, Proto::StreamKind::SECONDARY, "SECONDARY", "timestamp"), false, version); + } + + size_t skip(size_t num_values) override + { + num_values = ColumnReader::skip(num_values); + seconds_rle->skip(num_values); + nanos_rle->skip(num_values); + return num_values; + } + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + ColumnReader::next(batch, num_values, incoming_mask); + const char * not_null = batch.hasNulls ? batch.notNull.data() : nullptr; + auto & timestamp_batch = typeid_cast(batch); + timestamp_batch.data.resize(num_values); + timestamp_batch.nanoseconds.resize(num_values); + Int64 * seconds = timestamp_batch.data.data(); + Int64 * nanos = timestamp_batch.nanoseconds.data(); + seconds_rle->next(seconds, num_values, not_null); + nanos_rle->next(nanos, num_values, not_null); + + for (size_t i = 0; i < num_values; ++i) + { + if (not_null && !not_null[i]) + continue; + + const Int64 encoded_nanos = nanos[i]; + /// The lowest 3 bits are the number of the trailing zeros (minus one) that are removed. + UInt64 zeros = nanos[i] & 0x7; + /// An arithmetic shift, as in the reference implementation: some writers (the Arrow + /// adapter) store negative nanoseconds for the timestamps before the epoch. + nanos[i] >>= 3; + if (zeros != 0 && common::mulOverflow(nanos[i], POWERS_OF_TEN_FOR_NANOS[zeros + 1], nanos[i])) + throwInvalidTimestamp(seconds[i], encoded_nanos); + if (nanos[i] < -999'999'999 || nanos[i] > 999'999'999) + throwInvalidTimestamp(seconds[i], encoded_nanos); + + Int64 writer_time = 0; + if (common::addOverflow(seconds[i], epoch_offset, writer_time)) + throwInvalidTimestamp(seconds[i], encoded_nanos); + if (writer_time < 0 && nanos[i] > 999999 && common::subOverflow(writer_time, Int64{1}, writer_time)) + throwInvalidTimestamp(seconds[i], encoded_nanos); + + if (!same_timezone) + { + /// Keep the same wall clock time when the writer and the reader time zones differ. + Int64 writer_offset = writer_timezone.timezoneOffset(writer_time); + Int64 reader_offset = reader_timezone.timezoneOffset(writer_time); + if (writer_offset != reader_offset) + { + /// If the adjustment crosses a DST boundary, the offset has to be reevaluated. + Int64 adjusted_time = 0; + if (common::addOverflow(writer_time, writer_offset - reader_offset, adjusted_time)) + throwInvalidTimestamp(seconds[i], encoded_nanos); + Int64 adjusted_reader_offset = reader_timezone.timezoneOffset(adjusted_time); + if (common::addOverflow(writer_time, writer_offset - adjusted_reader_offset, writer_time)) + throwInvalidTimestamp(seconds[i], encoded_nanos); + } + } + seconds[i] = writer_time; + } + } + + void seekToRowGroup(std::unordered_map & positions) override + { + ColumnReader::seekToRowGroup(positions); + auto & provider = positions.at(column_id); + seconds_rle->seek(provider); + nanos_rle->seek(provider); + } + +private: + static constexpr Int64 POWERS_OF_TEN_FOR_NANOS[9] = {1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000}; + + [[noreturn]] void throwInvalidTimestamp(Int64 seconds, Int64 encoded_nanos) const + { + throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid timestamp in ORC column {}: seconds={}, encoded nanoseconds={}", column_id, seconds, encoded_nanos); + } + + const DateLUTImpl & writer_timezone; + const DateLUTImpl & reader_timezone; + const bool same_timezone; + Int64 epoch_offset = 0; + std::unique_ptr seconds_rle; + std::unique_ptr nanos_rle; +}; + +constexpr UInt32 MAX_PRECISION_64 = 18; +constexpr UInt32 MAX_PRECISION_128 = 38; + +constexpr Int64 POWERS_OF_TEN[MAX_PRECISION_64 + 1] = { + 1LL, + 10LL, + 100LL, + 1000LL, + 10000LL, + 100000LL, + 1000000LL, + 10000000LL, + 100000000LL, + 1000000000LL, + 10000000000LL, + 100000000000LL, + 1000000000000LL, + 10000000000000LL, + 100000000000000LL, + 1000000000000000LL, + 10000000000000000LL, + 100000000000000000LL, + 1000000000000000000LL}; + +/// The DECIMAL columns: an unbounded signed varint per value in the DATA stream and the scale of +/// each value in the SECONDARY stream. The values are rescaled to the scale of the type. +class DecimalColumnReader : public ColumnReader +{ +public: + DecimalColumnReader(const Type & type, StripeStreams & stripe) + : ColumnReader(type, stripe) + , precision(static_cast(type.getPrecision())) + , scale(static_cast(type.getScale())) + { + if (type.getPrecision() > MAX_PRECISION_128 || type.getScale() > MAX_PRECISION_128) + throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid precision {} or scale {} of decimal ORC column {}", + type.getPrecision(), type.getScale(), column_id); + value_stream = getRequiredStream(stripe, column_id, Proto::StreamKind::DATA, "DATA", "decimal"); + scale_decoder = createIntRleDecoder( + getRequiredStream(stripe, column_id, Proto::StreamKind::SECONDARY, "SECONDARY", "decimal"), + true, getRleVersion(stripe.getEncoding(column_id))); + } + + size_t skip(size_t num_values) override + { + num_values = ColumnReader::skip(num_values); + size_t skipped = 0; + while (skipped < num_values) + if (!(value_stream->readByte() & 0x80)) + ++skipped; + scale_decoder->skip(num_values); + return num_values; + } + + void seekToRowGroup(std::unordered_map & positions) override + { + ColumnReader::seekToRowGroup(positions); + auto & provider = positions.at(column_id); + value_stream->seek(provider); + scale_decoder->seek(provider); + } + +protected: + /// Reads the zigzag-encoded varint of up to 128 bits. + Int128 readInt128() + { + UInt128 value = 0; + UInt32 shift = 0; + while (true) + { + UInt8 byte = value_stream->readByte(); + if (shift < 128) + value |= static_cast(byte & 0x7F) << shift; + shift += 7; + if (!(byte & 0x80)) + break; + } + return static_cast((value >> 1) ^ (UInt128(0) - (value & 1))); + } + + /// The scale of each value is read from the file. The valid scales are from 0 to 38, the bound + /// also keeps the differences of the scales and the rescaling work small. + Int32 getValueScale(Int64 value_scale) const + { + if (value_scale < -Int64(MAX_PRECISION_128) || value_scale > Int64(MAX_PRECISION_128)) + throw Exception(ErrorCodes::INCORRECT_DATA, "Decimal scale {} is out of range in ORC column {}", value_scale, column_id); + return static_cast(value_scale); + } + + static void scaleInt128(Int128 & value, Int32 target_scale, Int32 current_scale) + { + while (target_scale > current_scale) + { + Int32 adjust = std::min(MAX_PRECISION_64, target_scale - current_scale); + value = static_cast(static_cast(value) * static_cast(POWERS_OF_TEN[adjust])); + current_scale += adjust; + } + while (current_scale > target_scale) + { + Int32 adjust = std::min(MAX_PRECISION_64, current_scale - target_scale); + value /= POWERS_OF_TEN[adjust]; + current_scale -= adjust; + } + } + + const Int32 precision; + Int32 scale; + std::unique_ptr value_stream; + std::unique_ptr scale_decoder; + PaddedPODArray scales; +}; + +class Decimal64ColumnReader final : public DecimalColumnReader +{ +public: + using DecimalColumnReader::DecimalColumnReader; + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + ColumnReader::next(batch, num_values, incoming_mask); + const char * not_null = batch.hasNulls ? batch.notNull.data() : nullptr; + auto & decimal_batch = typeid_cast(batch); + scales.resize(num_values); + scale_decoder->next(scales.data(), num_values, not_null); + decimal_batch.precision = precision; + decimal_batch.scale = scale; + decimal_batch.values.resize(num_values); + Int64 * values = decimal_batch.values.data(); + for (size_t i = 0; i < num_values; ++i) + { + if (not_null && !not_null[i]) + continue; + values[i] = readInt64(getValueScale(scales[i])); + } + } + +private: + Int64 readInt64(Int32 current_scale) + { + UInt64 encoded = readVarint(*value_stream); + Int64 value = static_cast((encoded >> 1) ^ (~(encoded & 1) + 1)); + if (scale > current_scale && static_cast(scale - current_scale) <= MAX_PRECISION_64) + value = static_cast(static_cast(value) * static_cast(POWERS_OF_TEN[scale - current_scale])); + else if (scale < current_scale && static_cast(current_scale - scale) <= MAX_PRECISION_64) + value /= POWERS_OF_TEN[current_scale - scale]; + else if (scale != current_scale) + throw Exception(ErrorCodes::INCORRECT_DATA, "Decimal scale {} is out of range in ORC column {}", current_scale, column_id); + return value; + } +}; + +class Decimal128ColumnReader final : public DecimalColumnReader +{ +public: + using DecimalColumnReader::DecimalColumnReader; + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + ColumnReader::next(batch, num_values, incoming_mask); + const char * not_null = batch.hasNulls ? batch.notNull.data() : nullptr; + auto & decimal_batch = typeid_cast(batch); + scales.resize(num_values); + scale_decoder->next(scales.data(), num_values, not_null); + decimal_batch.precision = precision; + decimal_batch.scale = scale; + decimal_batch.values.resize(num_values); + Int128 * values = decimal_batch.values.data(); + for (size_t i = 0; i < num_values; ++i) + { + if (not_null && !not_null[i]) + continue; + values[i] = readInt128(); + scaleInt128(values[i], scale, getValueScale(scales[i])); + } + } +}; + +/// The DECIMAL columns with the precision up to 18 in the files of the format version 1.9999 +/// (`UNSTABLE_PRE_2_0`): the unscaled values are signed RLEv2 integers in the DATA stream, +/// all with the scale of the type, and there is no SECONDARY stream. +class Decimal64V2ColumnReader final : public ColumnReader +{ +public: + Decimal64V2ColumnReader(const Type & type, StripeStreams & stripe) + : ColumnReader(type, stripe) + , precision(static_cast(type.getPrecision())) + , scale(static_cast(type.getScale())) + { + if (type.getScale() > type.getPrecision()) + throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid precision {} or scale {} of decimal ORC column {}", + type.getPrecision(), type.getScale(), column_id); + /// As in the reference implementation, RLEv2 is used regardless of the column encoding. + value_decoder = createIntRleDecoder( + getRequiredStream(stripe, column_id, Proto::StreamKind::DATA, "DATA", "decimal"), true, RleVersion::V2); + } + + size_t skip(size_t num_values) override + { + num_values = ColumnReader::skip(num_values); + value_decoder->skip(num_values); + return num_values; + } + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + ColumnReader::next(batch, num_values, incoming_mask); + auto & decimal_batch = typeid_cast(batch); + decimal_batch.precision = precision; + decimal_batch.scale = scale; + decimal_batch.values.resize(num_values); + value_decoder->next(decimal_batch.values.data(), num_values, batch.hasNulls ? batch.notNull.data() : nullptr); + } + + void seekToRowGroup(std::unordered_map & positions) override + { + ColumnReader::seekToRowGroup(positions); + value_decoder->seek(positions.at(column_id)); + } + +private: + const Int32 precision; + const Int32 scale; + std::unique_ptr value_decoder; +}; + +/// The DECIMAL columns written by Hive 0.11, without the precision and the scale in the type. +/// As in the reference implementation, the values are read with the scale 6 and must have at most 38 digits. +class DecimalHive11ColumnReader final : public DecimalColumnReader +{ +public: + DecimalHive11ColumnReader(const Type & type, StripeStreams & stripe) + : DecimalColumnReader(type, stripe) + { + scale = 6; + } + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + ColumnReader::next(batch, num_values, incoming_mask); + const char * not_null = batch.hasNulls ? batch.notNull.data() : nullptr; + auto & decimal_batch = typeid_cast(batch); + scales.resize(num_values); + scale_decoder->next(scales.data(), num_values, not_null); + decimal_batch.precision = precision; + decimal_batch.scale = scale; + decimal_batch.values.resize(num_values); + Int128 * values = decimal_batch.values.data(); + + static const Int128 max_value = Int128(10000000000000000000ULL) * Int128(10000000000000000000ULL) - 1; + for (size_t i = 0; i < num_values; ++i) + { + if (not_null && !not_null[i]) + continue; + bool overflow = false; + UInt128 value = 0; + UInt32 shift = 0; + while (true) + { + UInt8 byte = value_stream->readByte(); + UInt32 bits = byte & 0x7F; + if (shift > 128 || (shift == 126 && bits > 3)) + overflow = true; + if (shift < 128) + value |= static_cast(bits) << shift; + shift += 7; + if (!(byte & 0x80)) + break; + } + Int128 decoded = static_cast((value >> 1) ^ (UInt128(0) - (value & 1))); + if (!overflow) + { + scaleInt128(decoded, scale, getValueScale(scales[i])); + overflow = decoded > max_value || decoded < -max_value; + } + if (overflow) + throw Exception(ErrorCodes::INCORRECT_DATA, "Hive 0.11 decimal was more than 38 digits in ORC column {}", column_id); + values[i] = decoded; + } + } +}; + +class StructColumnReader final : public ColumnReader +{ +public: + StructColumnReader(const Type & type, StripeStreams & stripe) + : ColumnReader(type, stripe) + { + const auto & encoding = stripe.getEncoding(column_id); + if (encoding.kind != Proto::ColumnEncodingKind::DIRECT) + throw Exception(ErrorCodes::INCORRECT_DATA, "Unknown encoding {} of ORC struct column {}", static_cast(encoding.kind), column_id); + const auto & selected = stripe.getSelectedColumns(); + for (UInt64 i = 0; i < type.getSubtypeCount(); ++i) + { + const Type & child = *type.getSubtype(i); + if (selected[child.getColumnId()]) + children.push_back(buildColumnReader(child, stripe)); + } + } + + size_t skip(size_t num_values) override + { + num_values = ColumnReader::skip(num_values); + for (auto & child : children) + child->skip(num_values); + return num_values; + } + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + nextImpl(batch, num_values, incoming_mask); + } + + void nextEncoded(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + nextImpl(batch, num_values, incoming_mask); + } + + void seekToRowGroup(std::unordered_map & positions) override + { + ColumnReader::seekToRowGroup(positions); + for (auto & child : children) + child->seekToRowGroup(positions); + } + +private: + template + void nextImpl(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) + { + ColumnReader::next(batch, num_values, incoming_mask); + const char * not_null = batch.hasNulls ? batch.notNull.data() : nullptr; + auto & struct_batch = typeid_cast(batch); + for (size_t i = 0; i < children.size(); ++i) + { + if constexpr (encoded) + children[i]->nextEncoded(*struct_batch.fields[i], num_values, not_null); + else + children[i]->next(*struct_batch.fields[i], num_values, not_null); + } + } + + std::vector> children; +}; + +/// Reads the lengths of the lists or maps as the end offsets, returns the total number of the children. +UInt64 readListOffsets(IntRleDecoder & rle, ColumnPtr & result, size_t num_values, const char * not_null, UInt64 column_id) +{ + auto column = ColumnUInt64::create(num_values); + auto & offsets = column->getData(); + rle.next(offsets.data(), num_values, not_null); + UInt64 total = lengthsToEndOffsets(offsets.data(), num_values, column_id); + result = std::move(column); + return total; +} + +UInt64 sumLengths(IntRleDecoder & rle, size_t num_values, UInt64 column_id) +{ + static constexpr size_t BUFFER_SIZE = 1024; + Int64 buffer[BUFFER_SIZE]; + UInt64 total = 0; + size_t read = 0; + while (read < num_values) + { + size_t chunk = std::min(num_values - read, BUFFER_SIZE); + rle.next(buffer, chunk, nullptr); + for (size_t i = 0; i < chunk; ++i) + { + if (buffer[i] < 0) + throw Exception(ErrorCodes::INCORRECT_DATA, "Negative length {} in ORC column {}", buffer[i], column_id); + total += static_cast(buffer[i]); + } + read += chunk; + } + return total; +} + +class ListColumnReader final : public ColumnReader +{ +public: + ListColumnReader(const Type & type, StripeStreams & stripe) + : ColumnReader(type, stripe) + { + rle = createIntRleDecoder( + getRequiredStream(stripe, column_id, Proto::StreamKind::LENGTH, "LENGTH", "list"), + false, getRleVersion(stripe.getEncoding(column_id))); + const Type & child_type = *type.getSubtype(0); + if (stripe.getSelectedColumns()[child_type.getColumnId()]) + child = buildColumnReader(child_type, stripe); + } + + size_t skip(size_t num_values) override + { + num_values = ColumnReader::skip(num_values); + if (child) + child->skip(sumLengths(*rle, num_values, column_id)); + else + rle->skip(num_values); + return num_values; + } + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + nextImpl(batch, num_values, incoming_mask); + } + + void nextEncoded(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + nextImpl(batch, num_values, incoming_mask); + } + + void seekToRowGroup(std::unordered_map & positions) override + { + ColumnReader::seekToRowGroup(positions); + rle->seek(positions.at(column_id)); + if (child) + child->seekToRowGroup(positions); + } + +private: + template + void nextImpl(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) + { + ColumnReader::next(batch, num_values, incoming_mask); + auto & list_batch = typeid_cast(batch); + const char * not_null = list_batch.hasNulls ? list_batch.notNull.data() : nullptr; + UInt64 total = readListOffsets(*rle, list_batch.offsets, num_values, not_null, column_id); + if (child) + { + if constexpr (encoded) + child->nextEncoded(*list_batch.elements, total, nullptr); + else + child->next(*list_batch.elements, total, nullptr); + } + } + + std::unique_ptr rle; + std::unique_ptr child; +}; + +class MapColumnReader final : public ColumnReader +{ +public: + MapColumnReader(const Type & type, StripeStreams & stripe) + : ColumnReader(type, stripe) + { + rle = createIntRleDecoder( + getRequiredStream(stripe, column_id, Proto::StreamKind::LENGTH, "LENGTH", "map"), + false, getRleVersion(stripe.getEncoding(column_id))); + const auto & selected = stripe.getSelectedColumns(); + const Type & key_type = *type.getSubtype(0); + if (selected[key_type.getColumnId()]) + key_reader = buildColumnReader(key_type, stripe); + const Type & value_type = *type.getSubtype(1); + if (selected[value_type.getColumnId()]) + value_reader = buildColumnReader(value_type, stripe); + } + + size_t skip(size_t num_values) override + { + num_values = ColumnReader::skip(num_values); + if (key_reader || value_reader) + { + UInt64 total = sumLengths(*rle, num_values, column_id); + if (key_reader) + key_reader->skip(total); + if (value_reader) + value_reader->skip(total); + } + else + rle->skip(num_values); + return num_values; + } + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + nextImpl(batch, num_values, incoming_mask); + } + + void nextEncoded(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + nextImpl(batch, num_values, incoming_mask); + } + + void seekToRowGroup(std::unordered_map & positions) override + { + ColumnReader::seekToRowGroup(positions); + rle->seek(positions.at(column_id)); + if (key_reader) + key_reader->seekToRowGroup(positions); + if (value_reader) + value_reader->seekToRowGroup(positions); + } + +private: + template + void nextImpl(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) + { + ColumnReader::next(batch, num_values, incoming_mask); + auto & map_batch = typeid_cast(batch); + const char * not_null = map_batch.hasNulls ? map_batch.notNull.data() : nullptr; + UInt64 total = readListOffsets(*rle, map_batch.offsets, num_values, not_null, column_id); + if (key_reader) + { + if constexpr (encoded) + key_reader->nextEncoded(*map_batch.keys, total, nullptr); + else + key_reader->next(*map_batch.keys, total, nullptr); + } + if (value_reader) + { + if constexpr (encoded) + value_reader->nextEncoded(*map_batch.elements, total, nullptr); + else + value_reader->next(*map_batch.elements, total, nullptr); + } + } + + std::unique_ptr rle; + std::unique_ptr key_reader; + std::unique_ptr value_reader; +}; + +class UnionColumnReader final : public ColumnReader +{ +public: + UnionColumnReader(const Type & type, StripeStreams & stripe) + : ColumnReader(type, stripe) + , rle(getRequiredStream(stripe, column_id, Proto::StreamKind::DATA, "DATA", "union")) + , num_children(type.getSubtypeCount()) + { + children.resize(num_children); + counts.resize(num_children); + const auto & selected = stripe.getSelectedColumns(); + for (size_t i = 0; i < num_children; ++i) + { + const Type & child = *type.getSubtype(i); + if (selected[child.getColumnId()]) + children[i] = buildColumnReader(child, stripe); + } + } + + size_t skip(size_t num_values) override + { + num_values = ColumnReader::skip(num_values); + std::fill(counts.begin(), counts.end(), 0); + static constexpr size_t BUFFER_SIZE = 1024; + char buffer[BUFFER_SIZE]; + size_t read = 0; + while (read < num_values) + { + size_t chunk = std::min(num_values - read, BUFFER_SIZE); + rle.next(buffer, chunk, nullptr); + for (size_t i = 0; i < chunk; ++i) + ++counts[checkedTag(static_cast(buffer[i]))]; + read += chunk; + } + for (size_t i = 0; i < num_children; ++i) + if (counts[i] && children[i]) + children[i]->skip(counts[i]); + return num_values; + } + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + nextImpl(batch, num_values, incoming_mask); + } + + void nextEncoded(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + nextImpl(batch, num_values, incoming_mask); + } + + void seekToRowGroup(std::unordered_map & positions) override + { + ColumnReader::seekToRowGroup(positions); + rle.seek(positions.at(column_id)); + for (auto & child : children) + if (child) + child->seekToRowGroup(positions); + } + +private: + size_t checkedTag(UInt8 tag) const + { + if (tag >= num_children) + throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid union tag {} for union with {} children in ORC column {}", static_cast(tag), num_children, column_id); + return tag; + } + + template + void nextImpl(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) + { + ColumnReader::next(batch, num_values, incoming_mask); + auto & union_batch = typeid_cast(batch); + const char * not_null = union_batch.hasNulls ? union_batch.notNull.data() : nullptr; + union_batch.tags.resize(num_values); + union_batch.offsets.resize(num_values); + UInt8 * tags = union_batch.tags.data(); + UInt64 * offsets = union_batch.offsets.data(); + memset(tags, 0, num_values); + rle.next(reinterpret_cast(tags), num_values, not_null); + std::fill(counts.begin(), counts.end(), 0); + for (size_t i = 0; i < num_values; ++i) + { + if (not_null && !not_null[i]) + continue; + offsets[i] = counts[checkedTag(tags[i])]++; + } + for (size_t i = 0; i < num_children; ++i) + { + if (!children[i]) + continue; + if constexpr (encoded) + children[i]->nextEncoded(*union_batch.children[i], counts[i], nullptr); + else + children[i]->next(*union_batch.children[i], counts[i], nullptr); + } + } + + ByteRleDecoder rle; + const size_t num_children; + std::vector> children; + std::vector counts; +}; + +} + + +ColumnReader::ColumnReader(const Type & type, StripeStreams & stripe) + : column_id(type.getColumnId()) +{ + if (auto stream = stripe.getStream(column_id, Proto::StreamKind::PRESENT)) + not_null_decoder = std::make_unique(std::move(stream)); +} + +void ColumnReader::next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) +{ + batch.reset(num_values); + + if (not_null_decoder) + { + char * not_null = batch.notNull.data(); + not_null_decoder->next(not_null, num_values, incoming_mask); + batch.hasNulls = memchr(not_null, 0, num_values) != nullptr; + return; + } + if (incoming_mask) + { + batch.hasNulls = true; + memcpy(batch.notNull.data(), incoming_mask, num_values); + return; + } + batch.hasNulls = false; +} + +size_t ColumnReader::skip(size_t num_values) +{ + if (!not_null_decoder) + return num_values; + + static constexpr size_t BUFFER_SIZE = 32768; + char buffer[BUFFER_SIZE]; + size_t remaining = num_values; + while (remaining) + { + size_t chunk = std::min(remaining, BUFFER_SIZE); + not_null_decoder->next(buffer, chunk, nullptr); + for (size_t i = 0; i < chunk; ++i) + if (!buffer[i]) + --num_values; + remaining -= chunk; + } + return num_values; +} + +void ColumnReader::seekToRowGroup(std::unordered_map & positions) +{ + if (not_null_decoder) + { + auto it = positions.find(column_id); + if (it == positions.end()) + throw Exception(ErrorCodes::INCORRECT_DATA, "No row index for ORC column {}", column_id); + not_null_decoder->seek(it->second); + } +} + +std::unique_ptr buildColumnReader(const Type & type, StripeStreams & stripe) +{ + checkStackSize(); + + switch (type.getKind()) + { + case BOOLEAN: + return std::make_unique(type, stripe); + case BYTE: + return std::make_unique(type, stripe); + case SHORT: + return std::make_unique>(type, stripe); + case INT: + return std::make_unique>(type, stripe); + case LONG: + case DATE: + return std::make_unique>(type, stripe); + case FLOAT: + return std::make_unique>(type, stripe); + case DOUBLE: + return std::make_unique>(type, stripe); + case STRING: + case BINARY: + case CHAR: + case VARCHAR: + case GEOMETRY: + case GEOGRAPHY: + switch (stripe.getEncoding(type.getColumnId()).kind) + { + case Proto::ColumnEncodingKind::DICTIONARY: + case Proto::ColumnEncodingKind::DICTIONARY_V2: + return std::make_unique(type, stripe); + case Proto::ColumnEncodingKind::DIRECT: + case Proto::ColumnEncodingKind::DIRECT_V2: + return std::make_unique(type, stripe); + } + break; + case TIMESTAMP: + return std::make_unique(type, stripe, false); + case TIMESTAMP_INSTANT: + return std::make_unique(type, stripe, true); + case DECIMAL: + if (type.getPrecision() == 0) + return std::make_unique(type, stripe); + if (type.getPrecision() <= MAX_PRECISION_64) + { + if (stripe.isDecimalAsLong()) + return std::make_unique(type, stripe); + return std::make_unique(type, stripe); + } + return std::make_unique(type, stripe); + case STRUCT: + return std::make_unique(type, stripe); + case LIST: + return std::make_unique(type, stripe); + case MAP: + return std::make_unique(type, stripe); + case UNION: + return std::make_unique(type, stripe); + } + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Reading ORC type {} is not supported", type.toString()); +} + +} diff --git a/src/Processors/Formats/Impl/ORC/ColumnReader.h b/src/Processors/Formats/Impl/ORC/ColumnReader.h new file mode 100644 index 0000000000000..1abaedb6fff58 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/ColumnReader.h @@ -0,0 +1,63 @@ +#pragma once + +#include +#include + +#include + +class DateLUTImpl; + +namespace DB::ORC +{ + +/// Access to the streams of one stripe for the column readers. +class StripeStreams +{ +public: + virtual ~StripeStreams() = default; + + virtual const Proto::ColumnEncoding & getEncoding(UInt64 column_id) const = 0; + /// Returns nullptr if the stripe does not have this stream. + virtual std::unique_ptr getStream(UInt64 column_id, Proto::StreamKind kind) const = 0; + virtual const std::vector & getSelectedColumns() const = 0; + /// The time zones for TIMESTAMP (not TIMESTAMP_INSTANT) columns. + virtual const DateLUTImpl & getWriterTimezone() const = 0; + virtual const DateLUTImpl & getReaderTimezone() const = 0; + virtual UInt64 getNumberOfRows() const = 0; + /// Whether the file has the format version 1.9999 (`UNSTABLE_PRE_2_0` in the reference implementation), + /// where DECIMAL columns with the precision up to 18 are written as signed RLEv2 integers in the DATA stream + /// without the SECONDARY stream of scales. + virtual bool isDecimalAsLong() const = 0; +}; + +class ColumnReader +{ +public: + ColumnReader(const Type & type, StripeStreams & stripe); + virtual ~ColumnReader() = default; + + /// Reads the next `num_values` values into `batch`. `incoming_mask` is the mask of the + /// non-null values of the parent (or nullptr). + virtual void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask); + + /// Same, but the string columns with dictionary encoding fill `EncodedStringVectorBatch` + /// without decoding the dictionary. + virtual void nextEncoded(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) + { + next(batch, num_values, incoming_mask); + } + + /// Skips `num_values` values, returns the number of the skipped non-null values. + virtual size_t skip(size_t num_values); + + /// Moves to the beginning of a row group, by the positions from the row indexes of the columns. + virtual void seekToRowGroup(std::unordered_map & positions); + +protected: + const UInt64 column_id; + std::unique_ptr not_null_decoder; +}; + +std::unique_ptr buildColumnReader(const Type & type, StripeStreams & stripe); + +} diff --git a/src/Processors/Formats/Impl/ORC/Compression.cpp b/src/Processors/Formats/Impl/ORC/Compression.cpp new file mode 100644 index 0000000000000..cfacdc431ca8d --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Compression.cpp @@ -0,0 +1,437 @@ +#include + +#include + +#include +#include +#include +/// For `ZSTD_decompressBound`, which is not in the stable ABI of zstd. It is safe, as zstd is linked statically from contrib. +#define ZSTD_STATIC_LINKING_ONLY +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int INCORRECT_DATA; + extern const int CANNOT_COMPRESS; + extern const int NOT_IMPLEMENTED; +} +} + +namespace DB::ORC +{ + +namespace +{ + +[[noreturn]] void throwMalformedLzo(size_t offset) +{ + throw Exception(ErrorCodes::INCORRECT_DATA, "Malformed LZO-compressed data in ORC file at offset {}", offset); +} + +/// LZO1X decompressor, as written by the Hadoop codecs (and the ORC Java writer). A chunk can be a +/// concatenation of several LZO blocks, each terminated with the end-of-stream marker. +size_t decompressLzo(const char * src, size_t src_size, char * dst, size_t dst_capacity) +{ + const UInt8 * input = reinterpret_cast(src); + const UInt8 * const input_begin = input; + const UInt8 * const input_end = input + src_size; + char * output = dst; + char * const output_end = dst + dst_capacity; + + auto offset = [&] { return static_cast(input - input_begin); }; + + auto read_byte = [&]() -> UInt32 + { + if (input >= input_end) + throwMalformedLzo(offset()); + return *input++; + }; + + /// A run of zero bytes extends a length by 255 each, then the next byte is added. + auto read_variable_length = [&](UInt32 base) -> size_t + { + size_t length = base; + UInt32 next = 0; + while ((next = read_byte()) == 0) + { + length += 0xFF; + if (length > dst_capacity) + throwMalformedLzo(offset()); + } + return length + next; + }; + + while (input < input_end) + { + bool first_command = true; + size_t last_literal_length = 0; + + while (true) + { + UInt32 command = read_byte(); + if (command == 0x11) + break; + + size_t match_length = 0; + size_t match_offset = 0; + size_t literal_length = 0; + + if ((command & 0xF0) == 0) + { + if (last_literal_length == 0) + { + /// A long run of literals. + literal_length = command & 0xF; + if (literal_length == 0) + literal_length = read_variable_length(0xF); + literal_length += 3; + } + else if (last_literal_length <= 3) + { + match_length = 3; + match_offset = (command & 0xC) >> 2; + match_offset |= read_byte() << 2; + match_offset |= 0x800; + literal_length = command & 0x3; + } + else + { + match_length = 2; + match_offset = (command & 0xC) >> 2; + match_offset |= read_byte() << 2; + literal_length = command & 0x3; + } + } + else if (first_command) + { + literal_length = command - 17; + } + else if ((command & 0xF0) == 0x10) + { + match_length = command & 0x7; + if (match_length == 0) + match_length = read_variable_length(0x7); + match_length += 2; + UInt32 trailer = read_byte(); + trailer |= read_byte() << 8; + match_offset = trailer >> 2; + match_offset |= (command & 0x8) ? 0x8000 : 0x4000; + --match_offset; + literal_length = trailer & 0x3; + } + else if ((command & 0xE0) == 0x20) + { + match_length = command & 0x1F; + if (match_length == 0) + match_length = read_variable_length(0x1F); + match_length += 2; + UInt32 trailer = read_byte(); + trailer |= read_byte() << 8; + match_offset = trailer >> 2; + literal_length = trailer & 0x3; + } + else if ((command & 0xC0) != 0) + { + match_length = ((command & 0xE0) >> 5) + 1; + match_offset = (command & 0x1C) >> 2; + match_offset |= read_byte() << 3; + literal_length = command & 0x3; + } + else + throwMalformedLzo(offset()); + + first_command = false; + + if (match_length != 0) + { + /// LZO encodes the match offset minus one. + ++match_offset; + if (match_offset > static_cast(output - dst) || match_length > static_cast(output_end - output)) + throwMalformedLzo(offset()); + const char * match = output - match_offset; + /// The source can overlap the destination, so copy byte by byte. + for (size_t i = 0; i < match_length; ++i) + output[i] = match[i]; + output += match_length; + } + + if (literal_length > static_cast(output_end - output) || literal_length > static_cast(input_end - input)) + throwMalformedLzo(offset()); + memcpy(output, input, literal_length); + output += literal_length; + input += literal_length; + last_literal_length = literal_length; + } + + /// The end-of-stream marker is 0x11 0x00 0x00. + if (input_end - input < 2 || input[0] != 0 || input[1] != 0) + throwMalformedLzo(offset()); + input += 2; + } + + return output - dst; +} + +size_t decompressZlib(const char * src, size_t src_size, char * dst, size_t dst_capacity) +{ + z_stream stream{}; + if (inflateInit2(&stream, -15) != Z_OK) + throw Exception(ErrorCodes::INCORRECT_DATA, "Cannot initialize zlib decompression for ORC file"); + + stream.next_in = reinterpret_cast(const_cast(src)); + stream.avail_in = static_cast(src_size); + stream.next_out = reinterpret_cast(dst); + stream.avail_out = static_cast(dst_capacity); + + int ret = inflate(&stream, Z_FINISH); + size_t result = dst_capacity - stream.avail_out; + inflateEnd(&stream); + + if (ret != Z_STREAM_END) + { + if (ret == Z_BUF_ERROR && stream.avail_out == 0) + throw Exception(ErrorCodes::INCORRECT_DATA, + "A zlib-compressed chunk of ORC file decompresses to more than the {} bytes allowed for the chunk", dst_capacity); + throw Exception(ErrorCodes::INCORRECT_DATA, "Malformed zlib-compressed data in ORC file (zlib error {})", ret); + } + return result; +} + +size_t decompressSnappy(const char * src, size_t src_size, char * dst, size_t dst_capacity) +{ + size_t uncompressed_size = 0; + if (!snappy::GetUncompressedLength(src, src_size, &uncompressed_size)) + throw Exception(ErrorCodes::INCORRECT_DATA, "Malformed snappy-compressed data in ORC file"); + if (uncompressed_size > dst_capacity) + throw Exception(ErrorCodes::INCORRECT_DATA, + "A snappy-compressed chunk of ORC file decompresses to {} bytes, more than the {} bytes allowed for the chunk", + uncompressed_size, dst_capacity); + if (!snappy::RawUncompress(src, src_size, dst)) + throw Exception(ErrorCodes::INCORRECT_DATA, "Malformed snappy-compressed data in ORC file"); + return uncompressed_size; +} + +size_t decompressLz4(const char * src, size_t src_size, char * dst, size_t dst_capacity) +{ + if (src_size > static_cast(std::numeric_limits::max()) || dst_capacity > static_cast(std::numeric_limits::max())) + throw Exception(ErrorCodes::INCORRECT_DATA, "LZ4-compressed chunk of ORC file is too large"); + int res = LZ4_decompress_safe(src, dst, static_cast(src_size), static_cast(dst_capacity)); + if (res < 0) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Malformed LZ4-compressed data in ORC file, or it decompresses to more than the {} bytes allowed for the chunk", dst_capacity); + return static_cast(res); +} + +size_t decompressZstd(const char * src, size_t src_size, char * dst, size_t dst_capacity) +{ + size_t res = ZSTD_decompress(dst, dst_capacity, src, src_size); + if (ZSTD_isError(res)) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Malformed zstd-compressed data in ORC file, or it decompresses to more than the {} bytes allowed for the chunk: {}", + dst_capacity, ZSTD_getErrorName(res)); + return res; +} + + +class ZlibCompressor final : public Compressor +{ +public: + ZlibCompressor() + { + /// The level and the parameters are the same as in the C++ ORC library with its default + /// compression strategy, "speed". + if (deflateInit2(&stream, Z_BEST_SPEED + 1, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY) != Z_OK) + throw Exception(ErrorCodes::CANNOT_COMPRESS, "Cannot initialize zlib compression for ORC"); + } + + ~ZlibCompressor() override { deflateEnd(&stream); } + + size_t compress(const char * src, size_t src_size, std::string & out) override + { + if (deflateReset(&stream) != Z_OK) + throw Exception(ErrorCodes::CANNOT_COMPRESS, "Cannot reset zlib compression for ORC"); + + size_t old_size = out.size(); + /// Allow only the output smaller than the input. + out.resize(old_size + src_size); + stream.next_in = reinterpret_cast(const_cast(src)); + stream.avail_in = static_cast(src_size); + stream.next_out = reinterpret_cast(out.data() + old_size); + stream.avail_out = static_cast(src_size); + + int ret = deflate(&stream, Z_FINISH); + if (ret == Z_STREAM_END && stream.avail_out > 0) + { + size_t compressed = src_size - stream.avail_out; + out.resize(old_size + compressed); + return compressed; + } + out.resize(old_size); + if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) + throw Exception(ErrorCodes::CANNOT_COMPRESS, "zlib compression failed with error {}", ret); + return 0; + } + +private: + z_stream stream{}; +}; + +class SnappyCompressor final : public Compressor +{ +public: + size_t compress(const char * src, size_t src_size, std::string & out) override + { + size_t old_size = out.size(); + out.resize(old_size + snappy::MaxCompressedLength(src_size)); + size_t compressed = 0; + snappy::RawCompress(src, src_size, out.data() + old_size, &compressed); + return finish(out, old_size, compressed, src_size); + } + + static size_t finish(std::string & out, size_t old_size, size_t compressed, size_t src_size) + { + if (compressed >= src_size) + { + out.resize(old_size); + return 0; + } + out.resize(old_size + compressed); + return compressed; + } +}; + +class Lz4Compressor final : public Compressor +{ +public: + size_t compress(const char * src, size_t src_size, std::string & out) override + { + size_t old_size = out.size(); + int bound = LZ4_compressBound(static_cast(src_size)); + out.resize(old_size + bound); + int compressed = LZ4_compress_default(src, out.data() + old_size, static_cast(src_size), bound); + if (compressed <= 0) + throw Exception(ErrorCodes::CANNOT_COMPRESS, "LZ4 compression failed"); + return SnappyCompressor::finish(out, old_size, compressed, src_size); + } +}; + +class ZstdCompressor final : public Compressor +{ +public: + ZstdCompressor() : context(ZSTD_createCCtx()) + { + if (!context) + throw Exception(ErrorCodes::CANNOT_COMPRESS, "Cannot create zstd compression context"); + } + + ~ZstdCompressor() override { ZSTD_freeCCtx(context); } + + size_t compress(const char * src, size_t src_size, std::string & out) override + { + size_t old_size = out.size(); + size_t bound = ZSTD_compressBound(src_size); + out.resize(old_size + bound); + /// Level 1, as in the C++ ORC library with its default compression strategy, "speed". + size_t compressed = ZSTD_compressCCtx(context, out.data() + old_size, bound, src, src_size, 1); + if (ZSTD_isError(compressed)) + throw Exception(ErrorCodes::CANNOT_COMPRESS, "zstd compression failed: {}", ZSTD_getErrorName(compressed)); + return SnappyCompressor::finish(out, old_size, compressed, src_size); + } + +private: + ZSTD_CCtx * context; +}; + +} + + +size_t maxDecompressedSize(CompressionKind kind, const char * src, size_t src_size) +{ + /// The ratios are the theoretical maximums of the formats, rounded up, so that valid chunks are never rejected. + /// The sizes stored in the chunks are only trusted below these bounds. + static constexpr size_t SLACK = 1024; + switch (kind) + { + case CompressionKind::NONE: + return src_size; + case CompressionKind::SNAPPY: + { + size_t uncompressed_size = 0; + if (!snappy::GetUncompressedLength(src, src_size, &uncompressed_size)) + throw Exception(ErrorCodes::INCORRECT_DATA, "Malformed snappy-compressed data in ORC file"); + /// A copy of 64 bytes takes at least 3 bytes. + return std::min(uncompressed_size, src_size * 22 + SLACK); + } + case CompressionKind::ZSTD: + { + /// Sums the content sizes of all frames, or 128 KiB per block when a frame omits its size, as in `ArrowIPC/BufferCompression.cpp`. + UInt64 bound = ZSTD_decompressBound(src, src_size); + if (bound == ZSTD_CONTENTSIZE_ERROR) + throw Exception(ErrorCodes::INCORRECT_DATA, "Malformed zstd-compressed data in ORC file"); + /// A block of at most 128 KiB takes at least 4 bytes (an RLE block). + return std::min(bound, src_size * 32768 + SLACK); + } + /// A deflate match of 258 bytes takes at least 2 bits. + case CompressionKind::ZLIB: + return src_size * 1032 + SLACK; + /// Every byte of a match length extension adds at most 255 bytes. + case CompressionKind::LZ4: + case CompressionKind::LZO: + return src_size * 256 + SLACK; + default: + break; + } + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "ORC compression {} is not supported", compressionKindToString(kind)); +} + +size_t decompressChunk(CompressionKind kind, const char * src, size_t src_size, char * dst, size_t dst_capacity) +{ + switch (kind) + { + case CompressionKind::ZLIB: return decompressZlib(src, src_size, dst, dst_capacity); + case CompressionKind::SNAPPY: return decompressSnappy(src, src_size, dst, dst_capacity); + case CompressionKind::LZO: return decompressLzo(src, src_size, dst, dst_capacity); + case CompressionKind::LZ4: return decompressLz4(src, src_size, dst, dst_capacity); + case CompressionKind::ZSTD: return decompressZstd(src, src_size, dst, dst_capacity); + case CompressionKind::NONE: + case CompressionKind::BROTLI: + break; + } + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "ORC compression {} is not supported", compressionKindToString(kind)); +} + +std::unique_ptr createCompressor(CompressionKind kind) +{ + switch (kind) + { + case CompressionKind::NONE: return nullptr; + case CompressionKind::ZLIB: return std::make_unique(); + case CompressionKind::SNAPPY: return std::make_unique(); + case CompressionKind::LZ4: return std::make_unique(); + case CompressionKind::ZSTD: return std::make_unique(); + case CompressionKind::LZO: + case CompressionKind::BROTLI: + break; + } + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Writing ORC with compression {} is not supported", compressionKindToString(kind)); +} + +String compressionKindToString(CompressionKind kind) +{ + switch (kind) + { + case CompressionKind::NONE: return "NONE"; + case CompressionKind::ZLIB: return "ZLIB"; + case CompressionKind::SNAPPY: return "SNAPPY"; + case CompressionKind::LZO: return "LZO"; + case CompressionKind::LZ4: return "LZ4"; + case CompressionKind::ZSTD: return "ZSTD"; + case CompressionKind::BROTLI: return "BROTLI"; + } + return "UNKNOWN(" + std::to_string(static_cast(kind)) + ")"; +} + +} diff --git a/src/Processors/Formats/Impl/ORC/Compression.h b/src/Processors/Formats/Impl/ORC/Compression.h new file mode 100644 index 0000000000000..7e0ceddb72a79 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Compression.h @@ -0,0 +1,42 @@ +#pragma once + +#include + +#include +#include + +namespace DB::ORC +{ + +using Proto::CompressionKind; + +/// Every compressed chunk of an ORC stream is prefixed with a 3-byte little-endian header: +/// `(chunk_length << 1) | is_original`, where `is_original` means the chunk is stored as is. +static constexpr size_t COMPRESSION_HEADER_SIZE = 3; + +/// Decompresses one chunk into `dst`, which has room for `dst_capacity` bytes (the compression block +/// size of the file). Returns the decompressed size. Throws `INCORRECT_DATA` on malformed input, +/// including a chunk that decompresses to more than `dst_capacity` bytes. +size_t decompressChunk(CompressionKind kind, const char * src, size_t src_size, char * dst, size_t dst_capacity); + +/// An upper bound of the decompressed size of a chunk, from the maximum compression ratio of the codec +/// (and the size stored in the chunk), to avoid allocating the whole compression block for a small chunk. +size_t maxDecompressedSize(CompressionKind kind, const char * src, size_t src_size); + +/// Compresses chunks of an ORC stream. +class Compressor +{ +public: + virtual ~Compressor() = default; + + /// Appends the compressed representation of `src` to `out` and returns its size, + /// or appends nothing and returns 0 if it would not be smaller than the input. + virtual size_t compress(const char * src, size_t src_size, std::string & out) = 0; +}; + +/// Returns nullptr for `NONE`. +std::unique_ptr createCompressor(CompressionKind kind); + +String compressionKindToString(CompressionKind kind); + +} diff --git a/src/Processors/Formats/Impl/ORC/Proto.cpp b/src/Processors/Formats/Impl/ORC/Proto.cpp new file mode 100644 index 0000000000000..19e504dfd365d --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Proto.cpp @@ -0,0 +1,944 @@ +#include + +#include + +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int INCORRECT_DATA; +} +} + +namespace DB::ORC::Proto +{ + +void WireReader::check(size_t n) const +{ + if (static_cast(end - pos) < n) + throw Exception(ErrorCodes::INCORRECT_DATA, "Truncated protobuf message in ORC metadata"); +} + +UInt64 WireReader::readVarint() +{ + UInt64 result = 0; + for (size_t shift = 0; shift < 64; shift += 7) + { + check(1); + UInt8 byte = static_cast(*pos++); + result |= static_cast(byte & 0x7F) << shift; + if (!(byte & 0x80)) + return result; + } + throw Exception(ErrorCodes::INCORRECT_DATA, "Malformed varint in ORC metadata"); +} + +UInt64 WireReader::readFixed64() +{ + check(8); + UInt64 v = 0; + memcpy(&v, pos, 8); + pos += 8; + if constexpr (std::endian::native == std::endian::big) + v = std::byteswap(v); + return v; +} + +UInt32 WireReader::readFixed32() +{ + check(4); + UInt32 v = 0; + memcpy(&v, pos, 4); + pos += 4; + if constexpr (std::endian::native == std::endian::big) + v = std::byteswap(v); + return v; +} + +double WireReader::readDouble() +{ + return std::bit_cast(readFixed64()); +} + +std::string_view WireReader::readBytes() +{ + UInt64 size = readVarint(); + check(size); + std::string_view res(pos, size); + pos += size; + return res; +} + +bool WireReader::nextField(UInt32 & field_number, WireType & wire_type) +{ + if (pos == end) + return false; + UInt64 key = readVarint(); + field_number = static_cast(key >> 3); + wire_type = static_cast(key & 7); + if (field_number == 0 || (key >> 3) > std::numeric_limits::max()) + throw Exception(ErrorCodes::INCORRECT_DATA, "Malformed field key in ORC metadata"); + return true; +} + +void WireReader::skip(WireType wire_type) +{ + switch (wire_type) + { + case VARINT: readVarint(); return; + case FIXED64: check(8); pos += 8; return; + case LENGTH_DELIMITED: readBytes(); return; + case FIXED32: check(4); pos += 4; return; + case START_GROUP: + case END_GROUP: + break; + } + throw Exception(ErrorCodes::INCORRECT_DATA, "Unsupported protobuf wire type {} in ORC metadata", static_cast(wire_type)); +} + +bool WireReader::readRepeatedVarint(WireType wire_type, std::vector & out) +{ + if (wire_type == LENGTH_DELIMITED) + { + WireReader packed(readBytes()); + while (packed.pos != packed.end) + out.push_back(packed.readVarint()); + return true; + } + if (wire_type == VARINT) + { + out.push_back(readVarint()); + return true; + } + return false; +} + + +void WireWriter::appendRawVarint(std::string & s, UInt64 value) +{ + while (value >= 0x80) + { + s.push_back(static_cast((value & 0x7F) | 0x80)); + value >>= 7; + } + s.push_back(static_cast(value)); +} + +void WireWriter::writeVarint(UInt32 field, UInt64 value) +{ + writeKey(field, WireReader::VARINT); + appendRawVarint(out, value); +} + +void WireWriter::writeDouble(UInt32 field, double value) +{ + writeKey(field, WireReader::FIXED64); + UInt64 v = std::bit_cast(value); + if constexpr (std::endian::native == std::endian::big) + v = std::byteswap(v); + out.append(reinterpret_cast(&v), 8); +} + +void WireWriter::writeBytes(UInt32 field, std::string_view value) +{ + writeKey(field, WireReader::LENGTH_DELIMITED); + appendRawVarint(out, value.size()); + out.append(value); +} + +void WireWriter::writePackedVarint(UInt32 field, const std::vector & values) +{ + if (values.empty()) + return; + std::string packed; + for (UInt64 v : values) + appendRawVarint(packed, v); + writeBytes(field, packed); +} + +void WireWriter::writePackedFixed64(UInt32 field, const std::vector & values) +{ + if (values.empty()) + return; + std::string packed; + for (UInt64 v : values) + { + if constexpr (std::endian::native == std::endian::big) + v = std::byteswap(v); + packed.append(reinterpret_cast(&v), 8); + } + writeBytes(field, packed); +} + + +namespace +{ + +using WireType = WireReader::WireType; + +/// A field with an unexpected wire type is an unknown field, which the Protocol Buffers rules +/// require to skip. It is detected before anything is read, and handled in `parseMessage`. +struct WireTypeMismatch : std::exception +{ +}; + +void expectWireType(WireType actual, WireType expected) +{ + if (actual != expected) + throw WireTypeMismatch{}; +} + +UInt64 readVarintField(WireReader & r, WireType t) +{ + expectWireType(t, WireReader::VARINT); + return r.readVarint(); +} + +Int64 readSignedVarintField(WireReader & r, WireType t) +{ + expectWireType(t, WireReader::VARINT); + return r.readSignedVarint(); +} + +UInt32 readUInt32Field(WireReader & r, WireType t) +{ + /// Protobuf truncates the 64-bit varint for 32-bit fields. + return static_cast(readVarintField(r, t)); +} + +/// For `sint32` fields. +Int32 readSignedVarint32Field(WireReader & r, WireType t) +{ + expectWireType(t, WireReader::VARINT); + UInt32 v = static_cast(r.readVarint()); + return static_cast((v >> 1) ^ (~(v & 1) + 1)); +} + +double readDoubleField(WireReader & r, WireType t) +{ + expectWireType(t, WireReader::FIXED64); + return r.readDouble(); +} + +std::string_view readBytesField(WireReader & r, WireType t) +{ + expectWireType(t, WireReader::LENGTH_DELIMITED); + return r.readBytes(); +} + +void checkCount(size_t count, size_t max_count, std::string_view what) +{ + if (count >= max_count) + throw Exception(ErrorCodes::INCORRECT_DATA, "Too many {} in ORC metadata, at most {} expected", what, max_count); +} + +template +void parseMessage(std::string_view data, F && handle_field) +{ + WireReader r(data); + UInt32 field = 0; + WireType type = WireReader::VARINT; + while (r.nextField(field, type)) + { + bool handled = false; + try + { + handled = handle_field(r, field, type); + } + catch (const WireTypeMismatch &) + { + handled = false; + } + if (!handled) + r.skip(type); + } +} + +IntegerStatistics parseIntegerStatistics(std::string_view data) +{ + IntegerStatistics res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: res.minimum = readSignedVarintField(r, type); return true; + case 2: res.maximum = readSignedVarintField(r, type); return true; + case 3: res.sum = readSignedVarintField(r, type); return true; + default: return false; + } + }); + return res; +} + +DoubleStatistics parseDoubleStatistics(std::string_view data) +{ + DoubleStatistics res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: res.minimum = readDoubleField(r, type); return true; + case 2: res.maximum = readDoubleField(r, type); return true; + case 3: res.sum = readDoubleField(r, type); return true; + default: return false; + } + }); + return res; +} + +StringStatistics parseStringStatistics(std::string_view data) +{ + StringStatistics res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: res.minimum = std::string(readBytesField(r, type)); return true; + case 2: res.maximum = std::string(readBytesField(r, type)); return true; + case 3: res.sum = readSignedVarintField(r, type); return true; + case 4: res.lower_bound = std::string(readBytesField(r, type)); return true; + case 5: res.upper_bound = std::string(readBytesField(r, type)); return true; + default: return false; + } + }); + return res; +} + +BucketStatistics parseBucketStatistics(std::string_view data) +{ + BucketStatistics res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + if (field != 1) + return false; + return r.readRepeatedVarint(type, res.count); + }); + return res; +} + +DecimalStatistics parseDecimalStatistics(std::string_view data) +{ + DecimalStatistics res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: res.minimum = std::string(readBytesField(r, type)); return true; + case 2: res.maximum = std::string(readBytesField(r, type)); return true; + case 3: res.sum = std::string(readBytesField(r, type)); return true; + default: return false; + } + }); + return res; +} + +DateStatistics parseDateStatistics(std::string_view data) +{ + DateStatistics res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: res.minimum = readSignedVarint32Field(r, type); return true; + case 2: res.maximum = readSignedVarint32Field(r, type); return true; + default: return false; + } + }); + return res; +} + +TimestampStatistics parseTimestampStatistics(std::string_view data) +{ + TimestampStatistics res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: res.minimum = readSignedVarintField(r, type); return true; + case 2: res.maximum = readSignedVarintField(r, type); return true; + case 3: res.minimum_utc = readSignedVarintField(r, type); return true; + case 4: res.maximum_utc = readSignedVarintField(r, type); return true; + case 5: res.minimum_nanos = readVarintField(r, type); return true; + case 6: res.maximum_nanos = readVarintField(r, type); return true; + default: return false; + } + }); + return res; +} + +BinaryStatistics parseBinaryStatistics(std::string_view data) +{ + BinaryStatistics res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + if (field != 1) + return false; + res.sum = readSignedVarintField(r, type); + return true; + }); + return res; +} + +CollectionStatistics parseCollectionStatistics(std::string_view data) +{ + CollectionStatistics res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: res.min_children = readVarintField(r, type); return true; + case 2: res.max_children = readVarintField(r, type); return true; + case 3: res.total_children = readVarintField(r, type); return true; + default: return false; + } + }); + return res; +} + +Stream parseStream(std::string_view data) +{ + Stream res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: res.kind = readUInt32Field(r, type); return true; + case 2: res.column = readUInt32Field(r, type); return true; + case 3: res.length = readVarintField(r, type); return true; + default: return false; + } + }); + return res; +} + +ColumnEncoding parseColumnEncoding(std::string_view data) +{ + ColumnEncoding res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: + { + UInt32 kind = readUInt32Field(r, type); + if (kind > static_cast(ColumnEncodingKind::DICTIONARY_V2)) + throw Exception(ErrorCodes::INCORRECT_DATA, "Unknown ORC column encoding kind {}", kind); + res.kind = static_cast(kind); + return true; + } + case 2: res.dictionary_size = readUInt32Field(r, type); return true; + case 3: res.bloom_encoding = readUInt32Field(r, type); return true; + default: return false; + } + }); + return res; +} + +Type parseType(std::string_view data) +{ + Type res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: res.kind = readUInt32Field(r, type); return true; + case 2: return r.readRepeatedVarint(type, res.subtypes); + case 3: res.field_names.emplace_back(readBytesField(r, type)); return true; + case 4: res.maximum_length = readUInt32Field(r, type); return true; + case 5: res.precision = readUInt32Field(r, type); return true; + case 6: res.scale = readUInt32Field(r, type); return true; + case 7: + { + StringPair pair; + parseMessage(readBytesField(r, type), [&](WireReader & rr, UInt32 f, WireType t) + { + switch (f) + { + case 1: pair.key = std::string(readBytesField(rr, t)); return true; + case 2: pair.value = std::string(readBytesField(rr, t)); return true; + default: return false; + } + }); + res.attributes.push_back(std::move(pair)); + return true; + } + case 8: res.crs = std::string(readBytesField(r, type)); return true; + case 9: res.algorithm = readUInt32Field(r, type); return true; + default: return false; + } + }); + return res; +} + +StripeInformation parseStripeInformation(std::string_view data) +{ + StripeInformation res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: res.offset = readVarintField(r, type); return true; + case 2: res.index_length = readVarintField(r, type); return true; + case 3: res.data_length = readVarintField(r, type); return true; + case 4: res.footer_length = readVarintField(r, type); return true; + case 5: res.number_of_rows = readVarintField(r, type); return true; + default: return false; + } + }); + return res; +} + +UserMetadataItem parseUserMetadataItem(std::string_view data) +{ + UserMetadataItem res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: res.name = std::string(readBytesField(r, type)); return true; + case 2: res.value = std::string(readBytesField(r, type)); return true; + default: return false; + } + }); + return res; +} + +std::string serializeIntegerStatistics(const IntegerStatistics & s) +{ + std::string out; + WireWriter w(out); + if (s.minimum) w.writeSignedVarint(1, *s.minimum); + if (s.maximum) w.writeSignedVarint(2, *s.maximum); + if (s.sum) w.writeSignedVarint(3, *s.sum); + return out; +} + +std::string serializeDoubleStatistics(const DoubleStatistics & s) +{ + std::string out; + WireWriter w(out); + if (s.minimum) w.writeDouble(1, *s.minimum); + if (s.maximum) w.writeDouble(2, *s.maximum); + if (s.sum) w.writeDouble(3, *s.sum); + return out; +} + +std::string serializeStringStatistics(const StringStatistics & s) +{ + std::string out; + WireWriter w(out); + if (s.minimum) w.writeBytes(1, *s.minimum); + if (s.maximum) w.writeBytes(2, *s.maximum); + if (s.sum) w.writeSignedVarint(3, *s.sum); + if (s.lower_bound) w.writeBytes(4, *s.lower_bound); + if (s.upper_bound) w.writeBytes(5, *s.upper_bound); + return out; +} + +std::string serializeDecimalStatistics(const DecimalStatistics & s) +{ + std::string out; + WireWriter w(out); + if (s.minimum) w.writeBytes(1, *s.minimum); + if (s.maximum) w.writeBytes(2, *s.maximum); + if (s.sum) w.writeBytes(3, *s.sum); + return out; +} + +std::string serializeDateStatistics(const DateStatistics & s) +{ + std::string out; + WireWriter w(out); + auto zigzag32 = [](Int32 v) { return static_cast((static_cast(v) << 1) ^ static_cast(v >> 31)); }; + if (s.minimum) w.writeVarint(1, zigzag32(*s.minimum)); + if (s.maximum) w.writeVarint(2, zigzag32(*s.maximum)); + return out; +} + +std::string serializeTimestampStatistics(const TimestampStatistics & s) +{ + std::string out; + WireWriter w(out); + if (s.minimum) w.writeSignedVarint(1, *s.minimum); + if (s.maximum) w.writeSignedVarint(2, *s.maximum); + if (s.minimum_utc) w.writeSignedVarint(3, *s.minimum_utc); + if (s.maximum_utc) w.writeSignedVarint(4, *s.maximum_utc); + /// `int32` fields: negative values are sign-extended to 64 bits. + if (s.minimum_nanos) w.writeVarint(5, *s.minimum_nanos); + if (s.maximum_nanos) w.writeVarint(6, *s.maximum_nanos); + return out; +} + +std::string serializeCollectionStatistics(const CollectionStatistics & s) +{ + std::string out; + WireWriter w(out); + if (s.min_children) w.writeVarint(1, *s.min_children); + if (s.max_children) w.writeVarint(2, *s.max_children); + if (s.total_children) w.writeVarint(3, *s.total_children); + return out; +} + +std::string serializeType(const Type & t) +{ + std::string out; + WireWriter w(out); + w.writeVarint(1, t.kind); + w.writePackedVarint(2, t.subtypes); + for (const auto & name : t.field_names) + w.writeBytes(3, name); + if (t.maximum_length) w.writeVarint(4, *t.maximum_length); + if (t.precision) w.writeVarint(5, *t.precision); + if (t.scale) w.writeVarint(6, *t.scale); + for (const auto & attr : t.attributes) + { + std::string pair; + WireWriter pw(pair); + pw.writeBytes(1, attr.key); + pw.writeBytes(2, attr.value); + w.writeBytes(7, pair); + } + if (t.crs) w.writeBytes(8, *t.crs); + if (t.algorithm) w.writeVarint(9, *t.algorithm); + return out; +} + +std::string serializeStripeInformation(const StripeInformation & s) +{ + std::string out; + WireWriter w(out); + w.writeVarint(1, s.offset); + w.writeVarint(2, s.index_length); + w.writeVarint(3, s.data_length); + w.writeVarint(4, s.footer_length); + w.writeVarint(5, s.number_of_rows); + return out; +} + +} + + +void PostScript::parse(std::string_view data) +{ + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: footer_length = readVarintField(r, type); return true; + case 2: + { + UInt64 kind = readVarintField(r, type); + if (kind > static_cast(CompressionKind::BROTLI)) + throw Exception(ErrorCodes::INCORRECT_DATA, "Unknown ORC compression kind {}", kind); + compression = static_cast(kind); + return true; + } + case 3: compression_block_size = readVarintField(r, type); return true; + case 4: return r.readRepeatedVarint(type, version); + case 5: metadata_length = readVarintField(r, type); return true; + case 6: writer_version = readUInt32Field(r, type); return true; + case 8000: magic = std::string(readBytesField(r, type)); return true; + default: return false; + } + }); +} + +std::string PostScript::serialize() const +{ + std::string out; + WireWriter w(out); + w.writeVarint(1, footer_length); + w.writeVarint(2, static_cast(compression)); + w.writeVarint(3, compression_block_size); + w.writePackedVarint(4, version); + w.writeVarint(5, metadata_length); + w.writeVarint(6, writer_version); + w.writeBytes(8000, magic); + return out; +} + +void ColumnStatistics::parse(std::string_view data) +{ + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: number_of_values = readVarintField(r, type); return true; + case 2: int_statistics = parseIntegerStatistics(readBytesField(r, type)); return true; + case 3: double_statistics = parseDoubleStatistics(readBytesField(r, type)); return true; + case 4: string_statistics = parseStringStatistics(readBytesField(r, type)); return true; + case 5: bucket_statistics = parseBucketStatistics(readBytesField(r, type)); return true; + case 6: decimal_statistics = parseDecimalStatistics(readBytesField(r, type)); return true; + case 7: date_statistics = parseDateStatistics(readBytesField(r, type)); return true; + case 8: binary_statistics = parseBinaryStatistics(readBytesField(r, type)); return true; + case 9: timestamp_statistics = parseTimestampStatistics(readBytesField(r, type)); return true; + case 10: has_null = readVarintField(r, type) != 0; return true; + case 11: bytes_on_disk = readVarintField(r, type); return true; + case 12: collection_statistics = parseCollectionStatistics(readBytesField(r, type)); return true; + default: return false; + } + }); +} + +std::string ColumnStatistics::serialize() const +{ + std::string out; + WireWriter w(out); + if (number_of_values) w.writeVarint(1, *number_of_values); + if (int_statistics) w.writeBytes(2, serializeIntegerStatistics(*int_statistics)); + if (double_statistics) w.writeBytes(3, serializeDoubleStatistics(*double_statistics)); + if (string_statistics) w.writeBytes(4, serializeStringStatistics(*string_statistics)); + if (bucket_statistics) + { + std::string bucket; + WireWriter(bucket).writePackedVarint(1, bucket_statistics->count); + w.writeBytes(5, bucket); + } + if (decimal_statistics) w.writeBytes(6, serializeDecimalStatistics(*decimal_statistics)); + if (date_statistics) w.writeBytes(7, serializeDateStatistics(*date_statistics)); + if (binary_statistics) + { + std::string binary; + if (binary_statistics->sum) + WireWriter(binary).writeSignedVarint(1, *binary_statistics->sum); + w.writeBytes(8, binary); + } + if (timestamp_statistics) w.writeBytes(9, serializeTimestampStatistics(*timestamp_statistics)); + if (has_null) w.writeVarint(10, *has_null); + if (bytes_on_disk) w.writeVarint(11, *bytes_on_disk); + if (collection_statistics) w.writeBytes(12, serializeCollectionStatistics(*collection_statistics)); + return out; +} + +void RowIndex::parse(std::string_view data, size_t max_entries) +{ + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + if (field != 1) + return false; + std::string_view entry_data = readBytesField(r, type); + checkCount(entry.size(), max_entries, "row index entries"); + auto & e = entry.emplace_back(); + parseMessage(entry_data, [&](WireReader & rr, UInt32 f, WireType t) + { + switch (f) + { + case 1: return rr.readRepeatedVarint(t, e.positions); + case 2: e.statistics.emplace().parse(readBytesField(rr, t)); return true; + default: return false; + } + }); + return true; + }); +} + +std::string RowIndex::serialize() const +{ + std::string out; + WireWriter w(out); + for (const auto & e : entry) + { + std::string entry_data; + WireWriter ew(entry_data); + ew.writePackedVarint(1, e.positions); + if (e.statistics) + ew.writeBytes(2, e.statistics->serialize()); + w.writeBytes(1, entry_data); + } + return out; +} + +void BloomFilterIndex::parse(std::string_view data, size_t max_entries) +{ + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + if (field != 1) + return false; + checkCount(bloom_filter.size(), max_entries, "bloom filters"); + BloomFilter bf; + parseMessage(readBytesField(r, type), [&](WireReader & rr, UInt32 f, WireType t) + { + switch (f) + { + case 1: bf.num_hash_functions = readUInt32Field(rr, t); return true; + case 2: + { + if (t == WireReader::LENGTH_DELIMITED) + { + /// Packed fixed64: a sequence of 8-byte little-endian values. + std::string_view bytes = rr.readBytes(); + if (bytes.size() % 8 != 0) + throw Exception(ErrorCodes::INCORRECT_DATA, "Malformed bloom filter bitset in ORC file"); + WireReader packed(bytes); + for (size_t i = 0; i < bytes.size() / 8; ++i) + bf.bitset.push_back(packed.readFixed64()); + } + else + { + expectWireType(t, WireReader::FIXED64); + bf.bitset.push_back(rr.readFixed64()); + } + return true; + } + case 3: bf.utf8bitset = std::string(readBytesField(rr, t)); return true; + default: return false; + } + }); + bloom_filter.push_back(std::move(bf)); + return true; + }); +} + +void StripeFooter::parse(std::string_view data, size_t max_columns) +{ + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: streams.push_back(parseStream(readBytesField(r, type))); return true; + case 2: + { + std::string_view encoding_data = readBytesField(r, type); + checkCount(columns.size(), max_columns, "column encodings"); + columns.push_back(parseColumnEncoding(encoding_data)); + return true; + } + case 3: writer_timezone = std::string(readBytesField(r, type)); return true; + case 4: readBytesField(r, type); ++encryption_variants; return true; + default: return false; + } + }); +} + +std::string StripeFooter::serialize() const +{ + std::string out; + WireWriter w(out); + for (const auto & s : streams) + { + std::string stream; + WireWriter sw(stream); + if (s.kind) + sw.writeVarint(1, *s.kind); + sw.writeVarint(2, s.column); + sw.writeVarint(3, s.length); + w.writeBytes(1, stream); + } + for (const auto & c : columns) + { + std::string encoding; + WireWriter ew(encoding); + ew.writeVarint(1, static_cast(c.kind)); + if (c.kind == ColumnEncodingKind::DICTIONARY || c.kind == ColumnEncodingKind::DICTIONARY_V2) + ew.writeVarint(2, c.dictionary_size); + w.writeBytes(2, encoding); + } + if (writer_timezone) + w.writeBytes(3, *writer_timezone); + return out; +} + +void Metadata::parse(std::string_view data, size_t max_stripes, size_t max_columns) +{ + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + if (field != 1) + return false; + std::string_view stripe_data = readBytesField(r, type); + checkCount(stripe_stats.size(), max_stripes, "stripe statistics"); + StripeStatistics stats; + parseMessage(stripe_data, [&](WireReader & rr, UInt32 f, WireType t) + { + if (f != 1) + return false; + std::string_view column_data = readBytesField(rr, t); + checkCount(stats.col_stats.size(), max_columns, "column statistics"); + stats.col_stats.emplace_back().parse(column_data); + return true; + }); + stripe_stats.push_back(std::move(stats)); + return true; + }); +} + +std::string Metadata::serialize() const +{ + std::string out; + WireWriter w(out); + for (const auto & stats : stripe_stats) + { + std::string stripe; + WireWriter sw(stripe); + for (const auto & col : stats.col_stats) + sw.writeBytes(1, col.serialize()); + w.writeBytes(1, stripe); + } + return out; +} + +void Footer::parse(std::string_view data) +{ + /// The statistics are parsed after the types, to limit their number by the number of columns. + std::vector statistics_data; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: header_length = readVarintField(r, type); return true; + case 2: content_length = readVarintField(r, type); return true; + case 3: stripes.push_back(parseStripeInformation(readBytesField(r, type))); return true; + case 4: types.push_back(parseType(readBytesField(r, type))); return true; + case 5: metadata.push_back(parseUserMetadataItem(readBytesField(r, type))); return true; + case 6: number_of_rows = readVarintField(r, type); return true; + case 7: statistics_data.push_back(readBytesField(r, type)); return true; + case 8: row_index_stride = readUInt32Field(r, type); return true; + case 9: writer = readUInt32Field(r, type); return true; + case 10: readBytesField(r, type); has_encryption = true; return true; + case 12: software_version = std::string(readBytesField(r, type)); return true; + default: return false; + } + }); + if (statistics_data.size() > types.size()) + throw Exception(ErrorCodes::INCORRECT_DATA, "Too many column statistics in ORC footer: {}, the file has {} columns", + statistics_data.size(), types.size()); + statistics.resize(statistics_data.size()); + for (size_t i = 0; i < statistics_data.size(); ++i) + statistics[i].parse(statistics_data[i]); +} + +std::string Footer::serialize() const +{ + std::string out; + WireWriter w(out); + w.writeVarint(1, header_length); + w.writeVarint(2, content_length); + for (const auto & s : stripes) + w.writeBytes(3, serializeStripeInformation(s)); + for (const auto & t : types) + w.writeBytes(4, serializeType(t)); + for (const auto & item : metadata) + { + std::string m; + WireWriter mw(m); + mw.writeBytes(1, item.name); + mw.writeBytes(2, item.value); + w.writeBytes(5, m); + } + w.writeVarint(6, number_of_rows); + for (const auto & s : statistics) + w.writeBytes(7, s.serialize()); + w.writeVarint(8, row_index_stride); + if (writer) + w.writeVarint(9, *writer); + if (software_version) + w.writeBytes(12, *software_version); + return out; +} + +} diff --git a/src/Processors/Formats/Impl/ORC/Proto.h b/src/Processors/Formats/Impl/ORC/Proto.h new file mode 100644 index 0000000000000..999839f6ab5df --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Proto.h @@ -0,0 +1,343 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include + +/// Structures of the ORC file metadata, which is serialized with Protocol Buffers. +/// See `orc_proto.proto` in the ORC specification: https://orc.apache.org/specification/ORCv1/ +/// +/// Only the messages and the fields that are used by the reader and the writer are represented. +/// The parser skips the unknown fields, as required by the Protocol Buffers rules, so the files +/// written by newer writers can be read. + +namespace DB::ORC::Proto +{ + +/// Low-level reader of the Protocol Buffers wire format over a contiguous buffer. +/// Throws `INCORRECT_DATA` on a malformed or truncated message. +class WireReader +{ +public: + enum WireType : UInt8 + { + VARINT = 0, + FIXED64 = 1, + LENGTH_DELIMITED = 2, + START_GROUP = 3, + END_GROUP = 4, + FIXED32 = 5, + }; + + WireReader(const char * begin_, const char * end_) : pos(begin_), end(end_) {} + explicit WireReader(std::string_view data) : pos(data.data()), end(data.data() + data.size()) {} + + /// Reads the next field key. Returns false at the end of the message. + bool nextField(UInt32 & field_number, WireType & wire_type); + + UInt64 readVarint(); + Int64 readSignedVarint() { UInt64 v = readVarint(); return static_cast((v >> 1) ^ (~(v & 1) + 1)); } + UInt64 readFixed64(); + UInt32 readFixed32(); + double readDouble(); + std::string_view readBytes(); + + /// Skips a field of the given wire type. + void skip(WireType wire_type); + + /// Reads a scalar field that can be either packed or not packed. + /// Returns false (and reads nothing) if the wire type is not suitable. + [[nodiscard]] bool readRepeatedVarint(WireType wire_type, std::vector & out); + +private: + void check(size_t n) const; + + const char * pos; + const char * end; +}; + +/// Low-level writer of the Protocol Buffers wire format. +class WireWriter +{ +public: + explicit WireWriter(std::string & out_) : out(out_) {} + + void writeVarint(UInt32 field, UInt64 value); + void writeSignedVarint(UInt32 field, Int64 value) { writeVarint(field, (static_cast(value) << 1) ^ static_cast(value >> 63)); } + void writeDouble(UInt32 field, double value); + void writeBytes(UInt32 field, std::string_view value); + void writePackedVarint(UInt32 field, const std::vector & values); + void writePackedFixed64(UInt32 field, const std::vector & values); + + static void appendRawVarint(std::string & s, UInt64 value); + +private: + void writeKey(UInt32 field, UInt8 wire_type) { appendRawVarint(out, (static_cast(field) << 3) | wire_type); } + + std::string & out; +}; + + +enum class CompressionKind : UInt8 +{ + NONE = 0, + ZLIB = 1, + SNAPPY = 2, + LZO = 3, + LZ4 = 4, + ZSTD = 5, + BROTLI = 6, +}; + +struct PostScript +{ + UInt64 footer_length = 0; + CompressionKind compression = CompressionKind::NONE; + /// The default value in the specification. + UInt64 compression_block_size = 256 * 1024; + std::vector version; + UInt64 metadata_length = 0; + UInt32 writer_version = 0; + std::string magic; + + void parse(std::string_view data); + std::string serialize() const; +}; + +struct IntegerStatistics +{ + std::optional minimum; + std::optional maximum; + std::optional sum; +}; + +struct DoubleStatistics +{ + std::optional minimum; + std::optional maximum; + std::optional sum; +}; + +struct StringStatistics +{ + std::optional minimum; + std::optional maximum; + std::optional sum; + std::optional lower_bound; + std::optional upper_bound; +}; + +struct BucketStatistics +{ + std::vector count; +}; + +struct DecimalStatistics +{ + std::optional minimum; + std::optional maximum; + std::optional sum; +}; + +struct DateStatistics +{ + std::optional minimum; + std::optional maximum; +}; + +struct TimestampStatistics +{ + /// In milliseconds, in the local time zone of the writer (deprecated). + std::optional minimum; + std::optional maximum; + /// In milliseconds since the epoch in UTC. + std::optional minimum_utc; + std::optional maximum_utc; + /// The sub-millisecond part of the nanoseconds plus one, if not zero. Unvalidated when read from a file. + std::optional minimum_nanos; + std::optional maximum_nanos; +}; + +struct BinaryStatistics +{ + std::optional sum; +}; + +struct CollectionStatistics +{ + std::optional min_children; + std::optional max_children; + std::optional total_children; +}; + +struct ColumnStatistics +{ + std::optional number_of_values; + std::optional int_statistics; + std::optional double_statistics; + std::optional string_statistics; + std::optional bucket_statistics; + std::optional decimal_statistics; + std::optional date_statistics; + std::optional binary_statistics; + std::optional timestamp_statistics; + std::optional has_null; + std::optional bytes_on_disk; + std::optional collection_statistics; + + void parse(std::string_view data); + std::string serialize() const; +}; + +struct RowIndexEntry +{ + std::vector positions; + std::optional statistics; +}; + +struct RowIndex +{ + std::vector entry; + + /// The limits of the number of the repeated objects keep a small malformed message from allocating much memory. + void parse(std::string_view data, size_t max_entries); + std::string serialize() const; +}; + +struct BloomFilter +{ + UInt32 num_hash_functions = 0; + std::vector bitset; + std::optional utf8bitset; +}; + +struct BloomFilterIndex +{ + std::vector bloom_filter; + + void parse(std::string_view data, size_t max_entries); +}; + +enum class StreamKind : UInt32 +{ + PRESENT = 0, + DATA = 1, + LENGTH = 2, + DICTIONARY_DATA = 3, + DICTIONARY_COUNT = 4, + SECONDARY = 5, + ROW_INDEX = 6, + BLOOM_FILTER = 7, + BLOOM_FILTER_UTF8 = 8, + ENCRYPTED_INDEX = 9, + ENCRYPTED_DATA = 10, + STRIPE_STATISTICS = 100, + FILE_STATISTICS = 101, +}; + +struct Stream +{ + std::optional kind; + UInt32 column = 0; + UInt64 length = 0; +}; + +enum class ColumnEncodingKind : UInt32 +{ + DIRECT = 0, + DICTIONARY = 1, + DIRECT_V2 = 2, + DICTIONARY_V2 = 3, +}; + +struct ColumnEncoding +{ + ColumnEncodingKind kind = ColumnEncodingKind::DIRECT; + UInt32 dictionary_size = 0; + /// 1 means the bloom filters use UTF-8 for the strings and UTC for the timestamps. + std::optional bloom_encoding; +}; + +struct StripeFooter +{ + std::vector streams; + std::vector columns; + std::optional writer_timezone; + /// Non-zero if the stripe has encrypted columns, which are not supported. + size_t encryption_variants = 0; + + void parse(std::string_view data, size_t max_columns); + std::string serialize() const; +}; + +struct StringPair +{ + std::string key; + std::string value; +}; + +struct Type +{ + UInt32 kind = 0; + std::vector subtypes; + std::vector field_names; + std::optional maximum_length; + std::optional precision; + std::optional scale; + std::vector attributes; + /// Only for GEOMETRY and GEOGRAPHY. + std::optional crs; + std::optional algorithm; +}; + +struct StripeInformation +{ + UInt64 offset = 0; + UInt64 index_length = 0; + UInt64 data_length = 0; + UInt64 footer_length = 0; + UInt64 number_of_rows = 0; +}; + +struct UserMetadataItem +{ + std::string name; + std::string value; +}; + +struct StripeStatistics +{ + std::vector col_stats; +}; + +struct Metadata +{ + std::vector stripe_stats; + + void parse(std::string_view data, size_t max_stripes, size_t max_columns); + std::string serialize() const; +}; + +struct Footer +{ + UInt64 header_length = 0; + UInt64 content_length = 0; + std::vector stripes; + std::vector types; + std::vector metadata; + UInt64 number_of_rows = 0; + std::vector statistics; + UInt32 row_index_stride = 0; + std::optional writer; + bool has_encryption = false; + std::optional software_version; + + void parse(std::string_view data); + std::string serialize() const; +}; + +} diff --git a/src/Processors/Formats/Impl/ORC/RLE.cpp b/src/Processors/Formats/Impl/ORC/RLE.cpp new file mode 100644 index 0000000000000..7b9762e32de33 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/RLE.cpp @@ -0,0 +1,1129 @@ +#include + +#include + +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int INCORRECT_DATA; +} +} + +namespace DB::ORC +{ + +namespace +{ + +inline Int64 unZigZag(UInt64 value) +{ + return static_cast((value >> 1) ^ (~(value & 1) + 1)); +} + +inline UInt64 zigZag(Int64 value) +{ + return (static_cast(value) << 1) ^ static_cast(value >> 63); +} + +/// The 5-bit codes of the bit widths in RLE v2. +constexpr UInt8 code_to_bit_width[32] = { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 26, 28, 30, 32, 40, 48, 56, 64}; + +/// The smallest bit width representable in RLE v2 that fits the given number of bits. +constexpr UInt8 closest_fixed_bits[65] = { + 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 26, 26, 28, 28, 30, 30, 32, 32, 40, 40, 40, 40, 40, 40, 40, 40, 48, 48, 48, + 48, 48, 48, 48, 48, 56, 56, 56, 56, 56, 56, 56, 56, 64, 64, 64, 64, 64, 64, 64, 64}; + +inline UInt32 decodeBitWidth(UInt32 code) +{ + return code_to_bit_width[code & 0x1F]; +} + +inline UInt32 getClosestFixedBits(UInt32 n) +{ + return n <= 64 ? closest_fixed_bits[n] : 64; +} + +inline UInt32 encodeBitWidth(UInt32 width) +{ + width = getClosestFixedBits(width); + if (width <= 24) + return width - 1; + switch (width) + { + case 26: return 24; + case 28: return 25; + case 30: return 26; + case 32: return 27; + case 40: return 28; + case 48: return 29; + case 56: return 30; + default: return 31; + } +} + +/// The number of bits required to represent the value, rounded to a representable bit width. +/// Negative values require 64 bits. +inline UInt32 findClosestNumBits(Int64 value) +{ + if (value < 0) + return 64; + UInt32 count = 64 - static_cast(std::countl_zero(static_cast(value))); + return getClosestFixedBits(count); +} + +} + + +UInt64 readVarint(InputByteStream & stream) +{ + UInt64 result = 0; + for (size_t shift = 0; shift < 64; shift += 7) + { + UInt8 byte = stream.readByte(); + result |= static_cast(byte & 0x7F) << shift; + if (!(byte & 0x80)) + return result; + } + throw Exception(ErrorCodes::INCORRECT_DATA, "Malformed varint in ORC stream {}", stream.getName()); +} + +Int64 readSignedVarint(InputByteStream & stream) +{ + return unZigZag(readVarint(stream)); +} + + +void ByteRleDecoder::readHeader() +{ + Int8 header = static_cast(stream->readByte()); + if (header >= 0) + { + repeating = true; + remaining_values = static_cast(header) + 3; + value = static_cast(stream->readByte()); + } + else + { + repeating = false; + remaining_values = static_cast(-static_cast(header)); + } +} + +void ByteRleDecoder::next(char * data, size_t num_values, const char * not_null) +{ + if (not_null) + memset(data, 0, num_values); + size_t position = 0; + while (position < num_values) + { + if (not_null) + { + while (position < num_values && !not_null[position]) + ++position; + if (position == num_values) + return; + } + + if (remaining_values == 0) + readHeader(); + + size_t count = std::min(num_values - position, remaining_values); + size_t consumed = 0; + if (repeating) + { + if (not_null) + { + for (size_t i = 0; i < count; ++i) + { + if (not_null[position + i]) + { + data[position + i] = value; + ++consumed; + } + } + } + else + { + memset(data + position, value, count); + consumed = count; + } + } + else + { + if (not_null) + { + for (size_t i = 0; i < count; ++i) + { + if (not_null[position + i]) + { + data[position + i] = static_cast(stream->readByte()); + ++consumed; + } + } + } + else + { + stream->readBytes(data + position, count); + consumed = count; + } + } + remaining_values -= consumed; + position += count; + } +} + +void ByteRleDecoder::skip(size_t num_values) +{ + while (num_values) + { + if (remaining_values == 0) + readHeader(); + size_t count = std::min(num_values, remaining_values); + remaining_values -= count; + num_values -= count; + if (!repeating) + stream->skipBytes(count); + } +} + +void ByteRleDecoder::seek(PositionProvider & positions) +{ + stream->seek(positions); + remaining_values = 0; + skip(positions.next()); +} + + +void BooleanRleDecoder::next(char * data, size_t num_values, const char * not_null) +{ + for (size_t i = 0; i < num_values; ++i) + { + if (not_null && !not_null[i]) + { + /// The PRESENT stream of a child column is read with the mask of its parent, + /// and the masked positions must be null. + data[i] = 0; + continue; + } + if (remaining_bits == 0) + { + char byte = 0; + byte_decoder.next(&byte, 1, nullptr); + last_byte = static_cast(byte); + remaining_bits = 8; + } + --remaining_bits; + data[i] = (last_byte >> remaining_bits) & 1; + } +} + +void BooleanRleDecoder::skip(size_t num_values) +{ + if (num_values <= remaining_bits) + { + remaining_bits -= num_values; + return; + } + num_values -= remaining_bits; + byte_decoder.skip(num_values / 8); + if (num_values % 8) + { + char byte = 0; + byte_decoder.next(&byte, 1, nullptr); + last_byte = static_cast(byte); + remaining_bits = 8 - num_values % 8; + } + else + remaining_bits = 0; +} + +void BooleanRleDecoder::seek(PositionProvider & positions) +{ + byte_decoder.seek(positions); + UInt64 consumed = positions.next(); + remaining_bits = 0; + if (consumed > 8) + throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid seek position {} of a boolean stream in ORC file", consumed); + if (consumed) + { + char byte = 0; + byte_decoder.next(&byte, 1, nullptr); + last_byte = static_cast(byte); + remaining_bits = 8 - consumed; + } +} + + +namespace +{ + +class IntRleDecoderV1 final : public IntRleDecoder +{ +public: + IntRleDecoderV1(std::unique_ptr stream_, bool is_signed_) : stream(std::move(stream_)), is_signed(is_signed_) {} + + void next(Int64 * data, size_t num_values, const char * not_null) override { nextImpl(data, num_values, not_null); } + void next(UInt64 * data, size_t num_values, const char * not_null) override { nextImpl(data, num_values, not_null); } + void next(Int32 * data, size_t num_values, const char * not_null) override { nextImpl(data, num_values, not_null); } + void next(Int16 * data, size_t num_values, const char * not_null) override { nextImpl(data, num_values, not_null); } + + template + void nextImpl(T * data, size_t num_values, const char * not_null) + { + if (not_null) + memset(static_cast(data), 0, num_values * sizeof(T)); + size_t position = 0; + while (position < num_values) + { + if (not_null) + { + while (position < num_values && !not_null[position]) + ++position; + if (position == num_values) + return; + } + + if (remaining_values == 0) + readHeader(); + + size_t count = std::min(num_values - position, remaining_values); + size_t consumed = 0; + for (size_t i = 0; i < count; ++i) + { + if (not_null && !not_null[position + i]) + continue; + if (repeating) + { + data[position + i] = static_cast(value); + value = static_cast(static_cast(value) + static_cast(delta)); + } + else + data[position + i] = static_cast(readValue()); + ++consumed; + } + remaining_values -= consumed; + position += count; + } + } + + void skip(size_t num_values) override + { + while (num_values) + { + if (remaining_values == 0) + readHeader(); + size_t count = std::min(num_values, remaining_values); + remaining_values -= count; + num_values -= count; + if (repeating) + value = static_cast(static_cast(value) + static_cast(delta) * count); + else + for (size_t i = 0; i < count; ++i) + readVarint(*stream); + } + } + + void seek(PositionProvider & positions) override + { + stream->seek(positions); + remaining_values = 0; + skip(positions.next()); + } + +private: + Int64 readValue() + { + return is_signed ? readSignedVarint(*stream) : static_cast(readVarint(*stream)); + } + + void readHeader() + { + Int8 header = static_cast(stream->readByte()); + if (header >= 0) + { + repeating = true; + remaining_values = static_cast(header) + 3; + delta = static_cast(stream->readByte()); + value = readValue(); + } + else + { + repeating = false; + remaining_values = static_cast(-static_cast(header)); + } + } + + std::unique_ptr stream; + const bool is_signed; + size_t remaining_values = 0; + bool repeating = false; + Int64 value = 0; + Int64 delta = 0; +}; + + +class IntRleDecoderV2 final : public IntRleDecoder +{ +public: + IntRleDecoderV2(std::unique_ptr stream_, bool is_signed_) : stream(std::move(stream_)), is_signed(is_signed_) {} + + void next(Int64 * data, size_t num_values, const char * not_null) override { nextImpl(data, num_values, not_null); } + void next(UInt64 * data, size_t num_values, const char * not_null) override { nextImpl(data, num_values, not_null); } + void next(Int32 * data, size_t num_values, const char * not_null) override { nextImpl(data, num_values, not_null); } + void next(Int16 * data, size_t num_values, const char * not_null) override { nextImpl(data, num_values, not_null); } + + template + void nextImpl(T * data, size_t num_values, const char * not_null) + { + if (not_null) + memset(static_cast(data), 0, num_values * sizeof(T)); + size_t position = 0; + while (position < num_values) + { + if (not_null) + { + while (position < num_values && !not_null[position]) + ++position; + if (position == num_values) + return; + } + + if (run_read == run_length) + readRun(); + + size_t count = std::min(num_values - position, run_length - run_read); + if (not_null) + { + for (size_t i = position; i < position + count; ++i) + if (not_null[i]) + data[i] = static_cast(literals[run_read++]); + } + else if constexpr (sizeof(T) == sizeof(Int64)) + { + memcpy(static_cast(data + position), literals.data() + run_read, count * sizeof(Int64)); + run_read += count; + } + else + { + for (size_t i = 0; i < count; ++i) + data[position + i] = static_cast(literals[run_read + i]); + run_read += count; + } + position += count; + } + } + + void skip(size_t num_values) override + { + while (num_values) + { + if (run_read == run_length) + readRun(); + size_t count = std::min(num_values, run_length - run_read); + run_read += count; + num_values -= count; + } + } + + void seek(PositionProvider & positions) override + { + stream->seek(positions); + run_read = run_length = 0; + skip(positions.next()); + } + +private: + enum EncodingType : UInt8 + { + SHORT_REPEAT = 0, + DIRECT = 1, + PATCHED_BASE = 2, + DELTA = 3, + }; + + Int64 readLongBE(size_t num_bytes) + { + UInt64 result = 0; + for (size_t i = 0; i < num_bytes; ++i) + result = (result << 8) | stream->readByte(); + return static_cast(result); + } + + template + static void decodeBigEndian(const UInt8 * in, Int64 * out, size_t count) + { + for (size_t i = 0; i < count; ++i, in += num_bytes) + { + UInt64 value = 0; + for (size_t j = 0; j < num_bytes; ++j) + value = (value << 8) | in[j]; + out[i] = static_cast(value); + } + } + + static void decodeBigEndian(const UInt8 * in, Int64 * out, size_t count, size_t num_bytes) + { + switch (num_bytes) + { + case 1: decodeBigEndian<1>(in, out, count); return; + case 2: decodeBigEndian<2>(in, out, count); return; + case 3: decodeBigEndian<3>(in, out, count); return; + case 4: decodeBigEndian<4>(in, out, count); return; + case 5: decodeBigEndian<5>(in, out, count); return; + case 6: decodeBigEndian<6>(in, out, count); return; + case 7: decodeBigEndian<7>(in, out, count); return; + default: decodeBigEndian<8>(in, out, count); return; + } + } + + /// The only bit widths that are not a multiple of 8 are at most 30. + template + static void decodeBitPacked(Int64 * out, size_t count, UInt32 bit_width, ReadByte && read_byte) + { + UInt64 buffer = 0; + UInt32 bits_in_buffer = 0; + const UInt64 mask = (1ULL << bit_width) - 1; + for (size_t i = 0; i < count; ++i) + { + while (bits_in_buffer < bit_width) + { + buffer = (buffer << 8) | read_byte(); + bits_in_buffer += 8; + } + bits_in_buffer -= bit_width; + out[i] = static_cast((buffer >> bits_in_buffer) & mask); + } + } + + /// Reads `count` bit-packed values of `bit_width` bits, starting and ending at a byte boundary. + void readInts(Int64 * out, size_t count, UInt32 bit_width) + { + if (bit_width % 8 == 0) + { + const size_t num_bytes = bit_width / 8; + while (count) + { + /// Decode the values that are entirely in the current chunk directly from it. + size_t in_chunk = std::min(count, static_cast(stream->end - stream->pos) / num_bytes); + if (in_chunk == 0) + { + *out = readLongBE(num_bytes); + ++out; + --count; + continue; + } + decodeBigEndian(reinterpret_cast(stream->pos), out, in_chunk, num_bytes); + stream->pos += in_chunk * num_bytes; + out += in_chunk; + count -= in_chunk; + } + return; + } + + const size_t total_bytes = (count * bit_width + 7) / 8; + if (static_cast(stream->end - stream->pos) >= total_bytes) + { + const UInt8 * in = reinterpret_cast(stream->pos); + decodeBitPacked(out, count, bit_width, [&] { return *in++; }); + stream->pos += total_bytes; + } + else + { + decodeBitPacked(out, count, bit_width, [&] { return stream->readByte(); }); + } + } + + void readRun() + { + UInt8 first_byte = stream->readByte(); + auto encoding = static_cast(first_byte >> 6); + run_read = 0; + switch (encoding) + { + case SHORT_REPEAT: + { + size_t num_bytes = ((first_byte >> 3) & 0x07) + 1; + run_length = (first_byte & 0x07) + 3; + Int64 value = readLongBE(num_bytes); + if (is_signed) + value = unZigZag(static_cast(value)); + std::fill(literals.begin(), literals.begin() + run_length, value); + break; + } + case DIRECT: + { + UInt32 bit_width = decodeBitWidth((first_byte >> 1) & 0x1F); + run_length = ((static_cast(first_byte & 0x01) << 8) | stream->readByte()) + 1; + readInts(literals.data(), run_length, bit_width); + if (is_signed) + for (size_t i = 0; i < run_length; ++i) + literals[i] = unZigZag(static_cast(literals[i])); + break; + } + case PATCHED_BASE: + readPatchedBase(first_byte); + break; + case DELTA: + readDelta(first_byte); + break; + } + } + + void readPatchedBase(UInt8 first_byte) + { + UInt32 bit_width = decodeBitWidth((first_byte >> 1) & 0x1F); + run_length = ((static_cast(first_byte & 0x01) << 8) | stream->readByte()) + 1; + + UInt8 third_byte = stream->readByte(); + size_t base_bytes = ((third_byte >> 5) & 0x07) + 1; + UInt32 patch_bit_width = decodeBitWidth(third_byte & 0x1F); + + UInt8 fourth_byte = stream->readByte(); + UInt32 patch_gap_width = ((fourth_byte >> 5) & 0x07) + 1; + size_t patch_list_length = fourth_byte & 0x1F; + if (patch_list_length == 0) + throw Exception(ErrorCodes::INCORRECT_DATA, "Corrupt PATCHED_BASE encoded data in ORC stream {} (empty patch list)", stream->getName()); + + Int64 base = readLongBE(base_bytes); + Int64 sign_mask = static_cast(1ULL << (base_bytes * 8 - 1)); + if (base & sign_mask) + { + base &= ~sign_mask; + base = -base; + } + + readInts(literals.data(), run_length, bit_width); + + if (patch_bit_width + patch_gap_width > 64) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Corrupt PATCHED_BASE encoded data in ORC stream {} (patch width + patch gap width > 64)", stream->getName()); + + Int64 patches[32]; + readInts(patches, patch_list_length, getClosestFixedBits(patch_bit_width + patch_gap_width)); + + const Int64 patch_mask = patch_bit_width >= 64 ? -1 : static_cast((1ULL << patch_bit_width) - 1); + size_t patch_index = 0; + Int64 gap = 0; + Int64 patch = 0; + + auto adjust_gap_and_patch = [&] + { + UInt64 current_gap = static_cast(patches[patch_index]) >> patch_bit_width; + patch = patches[patch_index] & patch_mask; + Int64 actual_gap = 0; + /// A gap of 255 with a zero patch means that the gap continues in the next entry. + while (current_gap == 255 && patch == 0) + { + actual_gap += 255; + ++patch_index; + if (patch_index >= patch_list_length) + throw Exception(ErrorCodes::INCORRECT_DATA, "Corrupt PATCHED_BASE encoded data in ORC stream {} (unterminated gap)", stream->getName()); + current_gap = static_cast(patches[patch_index]) >> patch_bit_width; + patch = patches[patch_index] & patch_mask; + } + gap = actual_gap + static_cast(current_gap); + }; + + adjust_gap_and_patch(); + for (size_t i = 0; i < run_length; ++i) + { + if (static_cast(i) != gap) + { + literals[i] = static_cast(static_cast(literals[i]) + static_cast(base)); + } + else + { + UInt64 patched = static_cast(literals[i]) | (bit_width < 64 ? static_cast(patch) << bit_width : 0); + literals[i] = static_cast(static_cast(base) + patched); + ++patch_index; + if (patch_index < patch_list_length) + { + adjust_gap_and_patch(); + /// The next gap is relative to the current position. + gap += static_cast(i); + } + } + } + } + + void readDelta(UInt8 first_byte) + { + UInt32 code = (first_byte >> 1) & 0x1F; + UInt32 bit_width = code ? decodeBitWidth(code) : 0; + run_length = ((static_cast(first_byte & 0x01) << 8) | stream->readByte()) + 1; + + Int64 first_value = is_signed ? readSignedVarint(*stream) : static_cast(readVarint(*stream)); + literals[0] = first_value; + Int64 delta_base = readSignedVarint(*stream); + + if (bit_width == 0) + { + /// Fixed delta. + for (size_t i = 1; i < run_length; ++i) + literals[i] = static_cast(static_cast(literals[i - 1]) + static_cast(delta_base)); + return; + } + + if (run_length < 2) + throw Exception(ErrorCodes::INCORRECT_DATA, "Illegal run length {} for delta encoding in ORC stream {}", run_length, stream->getName()); + + Int64 previous = static_cast(static_cast(first_value) + static_cast(delta_base)); + literals[1] = previous; + readInts(literals.data() + 2, run_length - 2, bit_width); + for (size_t i = 2; i < run_length; ++i) + { + if (delta_base < 0) + previous = static_cast(static_cast(previous) - static_cast(literals[i])); + else + previous = static_cast(static_cast(previous) + static_cast(literals[i])); + literals[i] = previous; + } + } + + std::unique_ptr stream; + const bool is_signed; + std::array literals{}; + size_t run_length = 0; + size_t run_read = 0; +}; + +} + +std::unique_ptr createIntRleDecoder(std::unique_ptr stream, bool is_signed, RleVersion version) +{ + if (version == RleVersion::V1) + return std::make_unique(std::move(stream), is_signed); + return std::make_unique(std::move(stream), is_signed); +} + + +void ByteRleEncoder::writeValues() +{ + if (num_literals == 0) + return; + if (repeat) + { + stream.writeByte(static_cast(num_literals - MIN_REPEAT)); + stream.writeByte(static_cast(literals[0])); + } + else + { + stream.writeByte(static_cast(-static_cast(num_literals))); + stream.write(literals, num_literals); + } + repeat = false; + tail_run_length = 0; + num_literals = 0; +} + +void ByteRleEncoder::write(char value) +{ + if (num_literals == 0) + { + literals[num_literals++] = value; + tail_run_length = 1; + } + else if (repeat) + { + if (value == literals[0]) + { + ++num_literals; + if (num_literals == MAX_REPEAT_SIZE) + writeValues(); + } + else + { + writeValues(); + literals[num_literals++] = value; + tail_run_length = 1; + } + } + else + { + if (value == literals[num_literals - 1]) + ++tail_run_length; + else + tail_run_length = 1; + + if (tail_run_length == MIN_REPEAT) + { + if (num_literals + 1 == MIN_REPEAT) + { + repeat = true; + ++num_literals; + } + else + { + num_literals -= MIN_REPEAT - 1; + writeValues(); + literals[0] = value; + repeat = true; + num_literals = MIN_REPEAT; + } + } + else + { + literals[num_literals++] = value; + if (num_literals == MAX_LITERAL_SIZE) + writeValues(); + } + } +} + +void ByteRleEncoder::recordPosition(std::vector & positions) const +{ + stream.recordPosition(positions); + positions.push_back(num_literals); +} + +void ByteRleEncoder::flush() +{ + writeValues(); +} + + +void BooleanRleEncoder::write(bool value) +{ + current = static_cast((current << 1) | (value ? 1 : 0)); + ++bits_in_current; + if (bits_in_current == 8) + { + byte_encoder.write(static_cast(current)); + current = 0; + bits_in_current = 0; + } +} + +void BooleanRleEncoder::recordPosition(std::vector & positions) const +{ + byte_encoder.recordPosition(positions); + positions.push_back(bits_in_current); +} + +void BooleanRleEncoder::flush() +{ + if (bits_in_current) + { + byte_encoder.write(static_cast(current << (8 - bits_in_current))); + current = 0; + bits_in_current = 0; + } + byte_encoder.flush(); +} + + +void IntRleEncoderV2::recordPosition(std::vector & positions) const +{ + stream.recordPosition(positions); + positions.push_back(num_literals); +} + +void IntRleEncoderV2::flush() +{ + size_t literal_start = 0; + size_t i = 0; + while (i < num_literals) + { + size_t j = i + 1; + while (j < num_literals && literals[j] == literals[i]) + ++j; + if (j - i >= 3) + { + if (literal_start < i) + encodeLiterals(literals + literal_start, i - literal_start); + encodeRepeat(literals[i], j - i); + literal_start = j; + } + i = j; + } + if (literal_start < num_literals) + encodeLiterals(literals + literal_start, num_literals - literal_start); + num_literals = 0; +} + +void IntRleEncoderV2::writeVarint(UInt64 value) +{ + while (value >= 0x80) + { + stream.writeByte(static_cast((value & 0x7F) | 0x80)); + value >>= 7; + } + stream.writeByte(static_cast(value)); +} + +void IntRleEncoderV2::writeInts(const Int64 * values, size_t count, UInt32 bit_width) +{ + if (bit_width % 8 == 0) + { + size_t num_bytes = bit_width / 8; + for (size_t i = 0; i < count; ++i) + { + UInt64 v = static_cast(values[i]); + for (size_t b = num_bytes; b > 0; --b) + stream.writeByte(static_cast(v >> ((b - 1) * 8))); + } + return; + } + + UInt64 buffer = 0; + UInt32 bits_in_buffer = 0; + const UInt64 mask = (1ULL << bit_width) - 1; + for (size_t i = 0; i < count; ++i) + { + buffer = (buffer << bit_width) | (static_cast(values[i]) & mask); + bits_in_buffer += bit_width; + while (bits_in_buffer >= 8) + { + bits_in_buffer -= 8; + stream.writeByte(static_cast(buffer >> bits_in_buffer)); + } + } + if (bits_in_buffer) + stream.writeByte(static_cast(buffer << (8 - bits_in_buffer))); +} + +void IntRleEncoderV2::encodeRepeat(Int64 value, size_t count) +{ + UInt64 encoded = is_signed ? zigZag(value) : static_cast(value); + if (count <= 10) + { + /// SHORT_REPEAT + UInt32 bits = findClosestNumBits(static_cast(encoded)); + UInt32 num_bytes = (bits + 7) / 8; + stream.writeByte(static_cast(((num_bytes - 1) << 3) | (count - 3))); + for (UInt32 b = num_bytes; b > 0; --b) + stream.writeByte(static_cast(encoded >> ((b - 1) * 8))); + return; + } + + /// DELTA with zero fixed delta. + size_t length = count - 1; + stream.writeByte(static_cast((3 << 6) | ((length >> 8) & 1))); + stream.writeByte(static_cast(length & 0xFF)); + writeVarint(encoded); + writeSignedVarint(0); +} + +void IntRleEncoderV2::writeDirect(const Int64 * values, size_t count) +{ + Int64 encoded[MAX_LITERAL_SIZE]; + UInt32 bit_width = 1; + for (size_t i = 0; i < count; ++i) + { + encoded[i] = is_signed ? static_cast(zigZag(values[i])) : values[i]; + bit_width = std::max(bit_width, findClosestNumBits(encoded[i])); + } + size_t length = count - 1; + stream.writeByte(static_cast((1 << 6) | (encodeBitWidth(bit_width) << 1) | ((length >> 8) & 1))); + stream.writeByte(static_cast(length & 0xFF)); + writeInts(encoded, count, bit_width); +} + +namespace +{ + +/// The bit width that is enough for the given fraction `p` of the values (as in the reference implementation). +UInt32 percentileBits(const Int64 * values, size_t count, double p) +{ + UInt32 histogram[32] = {}; + for (size_t i = 0; i < count; ++i) + ++histogram[encodeBitWidth(findClosestNumBits(values[i]))]; + + Int64 per_len = static_cast(static_cast(count) * (1.0 - p)); + for (Int32 i = 31; i >= 0; --i) + { + per_len -= histogram[i]; + if (per_len < 0) + return decodeBitWidth(i); + } + return 0; +} + +} + +void IntRleEncoderV2::encodeLiterals(const Int64 * values, size_t count) +{ + if (count <= 3) + { + writeDirect(values, count); + return; + } + + Int64 min = values[0]; + Int64 max = values[0]; + bool is_increasing = true; + bool is_decreasing = true; + bool is_fixed_delta = true; + bool deltas_overflow = false; + Int64 initial_delta = 0; + if (__builtin_sub_overflow(values[1], values[0], &initial_delta)) + deltas_overflow = true; + + /// Absolute values of the deltas, starting from the second one. + Int64 abs_deltas[MAX_LITERAL_SIZE]; + UInt64 max_abs_delta = 0; + + for (size_t i = 1; i < count; ++i) + { + min = std::min(min, values[i]); + max = std::max(max, values[i]); + is_increasing &= values[i - 1] <= values[i]; + is_decreasing &= values[i - 1] >= values[i]; + + Int64 delta = 0; + if (__builtin_sub_overflow(values[i], values[i - 1], &delta)) + { + deltas_overflow = true; + is_fixed_delta = false; + continue; + } + is_fixed_delta &= (delta == initial_delta); + if (i > 1) + { + UInt64 abs_delta = delta < 0 ? (0 - static_cast(delta)) : static_cast(delta); + abs_deltas[i - 2] = static_cast(abs_delta); + max_abs_delta = std::max(max_abs_delta, abs_delta); + } + } + + Int64 range = 0; + if (__builtin_sub_overflow(max, min, &range)) + { + writeDirect(values, count); + return; + } + + if (!deltas_overflow && is_fixed_delta) + { + size_t length = count - 1; + stream.writeByte(static_cast((3 << 6) | ((length >> 8) & 1))); + stream.writeByte(static_cast(length & 0xFF)); + if (is_signed) + writeSignedVarint(values[0]); + else + writeVarint(static_cast(values[0])); + writeSignedVarint(initial_delta); + return; + } + + if (!deltas_overflow && initial_delta != 0 && (is_increasing || is_decreasing) + && max_abs_delta <= static_cast(std::numeric_limits::max())) + { + UInt32 bit_width = findClosestNumBits(static_cast(max_abs_delta)); + /// The bit width code 0 means a fixed delta, so the width 1 is not representable. + if (bit_width == 1) + bit_width = 2; + size_t length = count - 1; + stream.writeByte(static_cast((3 << 6) | (encodeBitWidth(bit_width) << 1) | ((length >> 8) & 1))); + stream.writeByte(static_cast(length & 0xFF)); + if (is_signed) + writeSignedVarint(values[0]); + else + writeVarint(static_cast(values[0])); + writeSignedVarint(initial_delta); + writeInts(abs_deltas, count - 2, bit_width); + return; + } + + /// DIRECT or PATCHED_BASE. + Int64 encoded[MAX_LITERAL_SIZE]; + for (size_t i = 0; i < count; ++i) + encoded[i] = is_signed ? static_cast(zigZag(values[i])) : values[i]; + + UInt32 bits_100p = percentileBits(encoded, count, 1.0); + UInt32 bits_90p = percentileBits(encoded, count, 0.9); + + static constexpr Int64 BASE_VALUE_LIMIT = 1LL << 56; + if (bits_100p - bits_90p <= 1 || min <= -BASE_VALUE_LIMIT || min >= BASE_VALUE_LIMIT) + { + writeDirect(values, count); + return; + } + + Int64 base_reduced[MAX_LITERAL_SIZE]; + for (size_t i = 0; i < count; ++i) + base_reduced[i] = values[i] - min; + + UInt32 br_bits_95p = percentileBits(base_reduced, count, 0.95); + UInt32 br_bits_100p = percentileBits(base_reduced, count, 1.0); + if (br_bits_100p == br_bits_95p) + { + writeDirect(values, count); + return; + } + + UInt32 patch_width = getClosestFixedBits(br_bits_100p - br_bits_95p); + if (patch_width == 64) + { + patch_width = 56; + br_bits_95p = 8; + } + const Int64 mask = static_cast((1ULL << br_bits_95p) - 1); + + size_t gaps[MAX_LITERAL_SIZE]; + Int64 patches[MAX_LITERAL_SIZE]; + size_t num_patches = 0; + size_t previous = 0; + size_t max_gap = 0; + for (size_t i = 0; i < count; ++i) + { + if (base_reduced[i] > mask) + { + size_t gap = i - previous; + max_gap = std::max(max_gap, gap); + previous = i; + gaps[num_patches] = gap; + patches[num_patches] = base_reduced[i] >> br_bits_95p; + ++num_patches; + base_reduced[i] &= mask; + } + } + + UInt32 patch_gap_width = (max_gap == 0) ? 1 : findClosestNumBits(static_cast(max_gap)); + patch_gap_width = std::min(patch_gap_width, 8u); + + Int64 patch_list[MAX_LITERAL_SIZE]; + size_t patch_list_length = 0; + for (size_t p = 0; p < num_patches; ++p) + { + size_t gap = gaps[p]; + while (gap > 255) + { + patch_list[patch_list_length++] = static_cast(255ULL << patch_width); + gap -= 255; + } + patch_list[patch_list_length++] = static_cast((static_cast(gap) << patch_width) | static_cast(patches[p])); + } + + /// The length of the patch list is stored in 5 bits. + if (patch_list_length > 31) + { + writeDirect(values, count); + return; + } + + bool is_negative = min < 0; + UInt64 base = is_negative ? static_cast(-min) : static_cast(min); + UInt32 base_width = findClosestNumBits(static_cast(base)) + 1; + UInt32 base_bytes = (base_width + 7) / 8; + if (is_negative) + base |= 1ULL << (base_bytes * 8 - 1); + + size_t length = count - 1; + stream.writeByte(static_cast((2 << 6) | (encodeBitWidth(br_bits_95p) << 1) | ((length >> 8) & 1))); + stream.writeByte(static_cast(length & 0xFF)); + stream.writeByte(static_cast(((base_bytes - 1) << 5) | encodeBitWidth(patch_width))); + stream.writeByte(static_cast(((patch_gap_width - 1) << 5) | patch_list_length)); + for (UInt32 b = base_bytes; b > 0; --b) + stream.writeByte(static_cast(base >> ((b - 1) * 8))); + writeInts(base_reduced, count, getClosestFixedBits(br_bits_95p)); + writeInts(patch_list, patch_list_length, getClosestFixedBits(patch_gap_width + patch_width)); +} + +} diff --git a/src/Processors/Formats/Impl/ORC/RLE.h b/src/Processors/Formats/Impl/ORC/RLE.h new file mode 100644 index 0000000000000..c5212bf5fbf6e --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/RLE.h @@ -0,0 +1,151 @@ +#pragma once + +#include + +#include +#include + +/// Run-length encodings of ORC: https://orc.apache.org/specification/ORCv1/#run-length-encoding +/// +/// All decoders fill the positions where `not_null` is non-zero (or all positions if it is +/// nullptr), consuming one value from the stream for each of them; the other positions are set to zero. + +namespace DB::ORC +{ + +/// Byte run-length encoding, used for the bytes and for the booleans (packed into bytes). +class ByteRleDecoder +{ +public: + explicit ByteRleDecoder(std::unique_ptr stream_) : stream(std::move(stream_)) {} + + void next(char * data, size_t num_values, const char * not_null); + void skip(size_t num_values); + void seek(PositionProvider & positions); + +private: + void readHeader(); + + std::unique_ptr stream; + size_t remaining_values = 0; + bool repeating = false; + char value = 0; +}; + +/// Booleans packed into bytes, most significant bit first, encoded with the byte run-length encoding. +class BooleanRleDecoder +{ +public: + explicit BooleanRleDecoder(std::unique_ptr stream_) : byte_decoder(std::move(stream_)) {} + + /// Writes 0 or 1. + void next(char * data, size_t num_values, const char * not_null); + void skip(size_t num_values); + void seek(PositionProvider & positions); + +private: + ByteRleDecoder byte_decoder; + size_t remaining_bits = 0; + UInt8 last_byte = 0; +}; + +/// Integer run-length encodings, version 1 and 2. +class IntRleDecoder +{ +public: + virtual ~IntRleDecoder() = default; + /// The values are converted to the type of the output, which can be narrower than the encoded values. + virtual void next(Int64 * data, size_t num_values, const char * not_null) = 0; + virtual void next(UInt64 * data, size_t num_values, const char * not_null) = 0; + virtual void next(Int32 * data, size_t num_values, const char * not_null) = 0; + virtual void next(Int16 * data, size_t num_values, const char * not_null) = 0; + virtual void skip(size_t num_values) = 0; + virtual void seek(PositionProvider & positions) = 0; +}; + +enum class RleVersion : UInt8 +{ + V1, + V2, +}; + +std::unique_ptr createIntRleDecoder(std::unique_ptr stream, bool is_signed, RleVersion version); + +/// Reads a base 128 varint, as used by the DECIMAL columns. +UInt64 readVarint(InputByteStream & stream); +Int64 readSignedVarint(InputByteStream & stream); + + +class ByteRleEncoder +{ +public: + explicit ByteRleEncoder(OutputByteStream & stream_) : stream(stream_) {} + + void write(char value); + void recordPosition(std::vector & positions) const; + /// Writes out the buffered values. + void flush(); + +private: + void writeValues(); + + static constexpr size_t MIN_REPEAT = 3; + static constexpr size_t MAX_LITERAL_SIZE = 128; + static constexpr size_t MAX_REPEAT_SIZE = 127 + MIN_REPEAT; + + OutputByteStream & stream; + char literals[MAX_LITERAL_SIZE]{}; + size_t num_literals = 0; + size_t tail_run_length = 0; + bool repeat = false; +}; + +class BooleanRleEncoder +{ +public: + explicit BooleanRleEncoder(OutputByteStream & stream_) : byte_encoder(stream_) {} + + void write(bool value); + void recordPosition(std::vector & positions) const; + void flush(); + +private: + ByteRleEncoder byte_encoder; + UInt8 current = 0; + size_t bits_in_current = 0; +}; + +/// Integer run-length encoding, version 2. +class IntRleEncoderV2 +{ +public: + IntRleEncoderV2(OutputByteStream & stream_, bool is_signed_) : stream(stream_), is_signed(is_signed_) {} + + void write(Int64 value) + { + literals[num_literals++] = value; + if (num_literals == MAX_LITERAL_SIZE) + flush(); + } + + void recordPosition(std::vector & positions) const; + /// Writes out the buffered values. + void flush(); + + static constexpr size_t MAX_LITERAL_SIZE = 512; + +private: + void encodeLiterals(const Int64 * values, size_t count); + void encodeRepeat(Int64 value, size_t count); + void writeDirect(const Int64 * values, size_t count); + void writeVarint(UInt64 value); + void writeSignedVarint(Int64 value) { writeVarint((static_cast(value) << 1) ^ static_cast(value >> 63)); } + void writeInts(const Int64 * values, size_t count, UInt32 bit_width); + + OutputByteStream & stream; + const bool is_signed; + Int64 literals[MAX_LITERAL_SIZE]{}; + size_t num_literals = 0; +}; + +} diff --git a/src/Processors/Formats/Impl/ORC/Reader.cpp b/src/Processors/Formats/Impl/ORC/Reader.cpp new file mode 100644 index 0000000000000..76c6a432d1b06 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Reader.cpp @@ -0,0 +1,842 @@ +#include + +#include +#include +#include +#include + +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int INCORRECT_DATA; + extern const int NOT_IMPLEMENTED; +} +} + +namespace DB::ORC +{ + +std::future InputStream::readAsync(void * buf, UInt64 length, UInt64 offset) +{ + read(buf, length, offset); + std::promise promise; + promise.set_value(); + return promise.get_future(); +} + + +Reader::Reader(std::unique_ptr stream_, const ReaderOptions & options_) + : stream(std::move(stream_)), options(options_) +{ + readTail(); +} + +Reader::~Reader() +{ + /// The background reads write into the buffers of the cache. + for (auto & range : cache) + if (!range.ready && range.future.valid()) + range.future.wait(); +} + +void Reader::readTail() +{ + const UInt64 file_size = stream->getLength(); + /// The header "ORC" and at least the length of the postscript. + if (file_size < 4) + throw Exception(ErrorCodes::INCORRECT_DATA, "Not an ORC file: {} is too small ({} bytes)", stream->getName(), file_size); + + static constexpr UInt64 DIRECTORY_SIZE_GUESS = 16 * 1024; + UInt64 tail_size = std::min(file_size, DIRECTORY_SIZE_GUESS); + std::string tail(tail_size, '\0'); + stream->read(tail.data(), tail_size, file_size - tail_size); + + postscript_length = static_cast(tail.back()); + if (postscript_length + 1 > tail_size) + throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid postscript length {} in ORC file {}", postscript_length, stream->getName()); + + postscript.parse(std::string_view(tail).substr(tail_size - 1 - postscript_length, postscript_length)); + + if (postscript.magic != "ORC") + { + /// Very old files do not have the magic in the postscript, but they have it at the beginning. + char header[3]; + stream->read(header, 3, 0); + if (memcmp(header, "ORC", 3) != 0) + throw Exception(ErrorCodes::INCORRECT_DATA, "Not an ORC file: {}", stream->getName()); + } + + if (postscript.compression != CompressionKind::NONE) + { + if (postscript.compression_block_size == 0 || postscript.compression_block_size > (1ULL << 30)) + throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid compression block size {} in ORC file {}", postscript.compression_block_size, stream->getName()); + if (postscript.compression == CompressionKind::BROTLI) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "BROTLI compression in ORC files is not supported"); + } + + UInt64 tail_length = 0; + if (__builtin_add_overflow(postscript.footer_length, postscript.metadata_length, &tail_length) + || __builtin_add_overflow(tail_length, postscript_length + 1, &tail_length) + || tail_length > file_size) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Invalid footer length {} and metadata length {} in ORC file {} of size {}", + postscript.footer_length, postscript.metadata_length, stream->getName(), file_size); + + if (tail_length > tail_size) + { + tail.resize(tail_length); + tail_size = tail_length; + stream->read(tail.data(), tail_size, file_size - tail_size); + } + + std::string_view footer_data = std::string_view(tail).substr(tail_size - 1 - postscript_length - postscript.footer_length, postscript.footer_length); + footer.parse(decompress(footer_data, "footer")); + + if (footer.has_encryption) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Encrypted ORC files are not supported"); + + schema = Type::fromProto(footer.types); + + /// Validate the stripes once, so that the sums of their offsets and lengths cannot overflow later. + if (footer.stripes.size() > static_cast(std::numeric_limits::max())) + throw Exception(ErrorCodes::INCORRECT_DATA, "Too many stripes {} in ORC file {}", footer.stripes.size(), stream->getName()); + for (size_t i = 0; i < footer.stripes.size(); ++i) + { + const auto & stripe = footer.stripes[i]; + UInt64 stripe_end = 0; + if (common::addOverflow(stripe.offset, stripe.index_length, stripe_end) + || common::addOverflow(stripe_end, stripe.data_length, stripe_end) + || common::addOverflow(stripe_end, stripe.footer_length, stripe_end) + || stripe_end > file_size) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Malformed stripe information at stripe index {} in ORC file {}: offset={}, indexLength={}, dataLength={}, footerLength={}, fileLength={}", + i, stream->getName(), stripe.offset, stripe.index_length, stripe.data_length, stripe.footer_length, file_size); + } + + std::string_view metadata_data = std::string_view(tail).substr( + tail_size - 1 - postscript_length - postscript.footer_length - postscript.metadata_length, postscript.metadata_length); + if (!metadata_data.empty()) + metadata.emplace().parse(decompress(metadata_data, "metadata"), footer.stripes.size(), footer.types.size()); + else + metadata.emplace(); +} + +const Proto::Metadata & Reader::getMetadata() +{ + return *metadata; +} + +std::string Reader::decompress(std::string_view data, const std::string & name) const +{ + if (postscript.compression == CompressionKind::NONE) + return std::string(data); + + std::string result; + InputByteStream in(data, postscript.compression, postscript.compression_block_size, name); + while (in.nextChunk()) + result.append(in.pos, in.end); + return result; +} + +Proto::StripeFooter Reader::readStripeFooter(size_t stripe_index) +{ + const auto & stripe = getStripe(stripe_index); + UInt64 offset = stripe.offset + stripe.index_length + stripe.data_length; + UInt64 end = 0; + if (__builtin_add_overflow(offset, stripe.footer_length, &end) || end > stream->getLength() || offset < stripe.offset) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Malformed stripe information at stripe index {} in ORC file {}: offset={}, indexLength={}, dataLength={}, footerLength={}, fileLength={}", + stripe_index, stream->getName(), stripe.offset, stripe.index_length, stripe.data_length, stripe.footer_length, stream->getLength()); + + auto slices = readRanges({{offset, stripe.footer_length}}); + Proto::StripeFooter result; + result.parse(decompress(slices[0].data, "stripe footer"), footer.types.size()); + if (result.encryption_variants) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Encrypted ORC files are not supported"); + return result; +} + +std::vector> Reader::getStripeRanges( + size_t stripe_index, const Proto::StripeFooter & stripe_footer, const std::vector & selected) const +{ + const auto & stripe = getStripe(stripe_index); + std::vector> ranges; + UInt64 offset = stripe.offset; + const UInt64 data_end = stripe.offset + stripe.index_length + stripe.data_length; + for (const auto & s : stripe_footer.streams) + { + /// Every stream is checked: the offsets of the following streams depend on its length. + UInt64 stream_end = 0; + if (common::addOverflow(offset, s.length, stream_end) || stream_end > data_end) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Malformed stream at offset {} of length {} in stripe {} of ORC file {}", offset, s.length, stripe_index, stream->getName()); + if (s.kind && s.column < selected.size() && selected[s.column] && s.length) + ranges.emplace_back(offset, s.length); + offset = stream_end; + } + return ranges; +} + +std::vector Reader::readRanges(const std::vector> & ranges) +{ + std::vector result(ranges.size()); + + std::vector order(ranges.size()); + for (size_t i = 0; i < order.size(); ++i) + order[i] = i; + std::sort(order.begin(), order.end(), [&](size_t a, size_t b) { return ranges[a].first < ranges[b].first; }); + + size_t i = 0; + while (i < order.size()) + { + UInt64 begin = ranges[order[i]].first; + UInt64 end = begin + ranges[order[i]].second; + size_t j = i + 1; + while (j < order.size()) + { + UInt64 next_begin = ranges[order[j]].first; + UInt64 next_end = next_begin + ranges[order[j]].second; + UInt64 new_end = std::max(end, next_end); + if (next_begin > end && next_begin - end > options.hole_size_limit) + break; + if (new_end - begin > options.range_size_limit) + break; + end = new_end; + ++j; + } + + std::shared_ptr> holder; + const char * base = nullptr; + UInt64 base_offset = begin; + { + std::lock_guard lock(cache_mutex); + for (auto & cached : cache) + { + if (cached.offset <= begin && end <= cached.offset + cached.length) + { + if (!cached.ready) + { + cached.ready = true; + cached.future.get(); + } + holder = cached.buffer; + base = cached.buffer->data(); + base_offset = cached.offset; + break; + } + } + } + + if (!holder) + { + if (end > stream->getLength()) + throw Exception(ErrorCodes::INCORRECT_DATA, "A range at {} of length {} goes beyond the end of ORC file {}", begin, end - begin, stream->getName()); + auto buffer = std::make_shared>(end - begin); + if (end > begin) + stream->read(buffer->data(), end - begin, begin); + base = buffer->data(); + holder = std::move(buffer); + } + + for (size_t k = i; k < j; ++k) + { + const auto & range = ranges[order[k]]; + result[order[k]] = Slice{holder, std::string_view(base + (range.first - base_offset), range.second)}; + } + i = j; + } + return result; +} + +void Reader::preBuffer(const std::vector & stripes, const std::vector & include_types) +{ + auto selected = selectColumnsByTypeIds(*schema, include_types); + for (UInt32 stripe_index : stripes) + { + const auto & stripe = getStripe(stripe_index); + auto stripe_footer = readStripeFooter(stripe_index); + auto ranges = getStripeRanges(stripe_index, stripe_footer, selected); + ranges.emplace_back(stripe.offset + stripe.index_length + stripe.data_length, stripe.footer_length); + std::sort(ranges.begin(), ranges.end()); + + size_t i = 0; + while (i < ranges.size()) + { + UInt64 begin = ranges[i].first; + UInt64 end = begin + ranges[i].second; + size_t j = i + 1; + while (j < ranges.size()) + { + UInt64 next_begin = ranges[j].first; + UInt64 new_end = std::max(end, next_begin + ranges[j].second); + if ((next_begin > end && next_begin - end > options.hole_size_limit) || new_end - begin > options.range_size_limit) + break; + end = new_end; + ++j; + } + + auto buffer = std::make_shared>(end - begin); + { + /// The entry must be owned by the cache before the background read starts, so that + /// the destructor waits for the read even if an exception is thrown in between. + std::lock_guard lock(cache_mutex); + auto & cached = cache.emplace_back(CachedRange{.offset = begin, .length = end - begin, .buffer = std::move(buffer), .future = {}}); + try + { + cached.future = stream->readAsync(cached.buffer->data(), end - begin, begin); + } + catch (...) + { + /// Nothing has been scheduled, so the entry can be removed safely. + cache.pop_back(); + throw; + } + } + i = j; + } + } +} + +void Reader::releaseBuffer(UInt64 boundary) +{ + std::lock_guard lock(cache_mutex); + for (auto it = cache.begin(); it != cache.end();) + { + if (it->offset + it->length <= boundary) + { + /// The buffer can be destroyed only after the background read finishes. The data has + /// not been used, so an error of reading it does not matter. + if (!it->ready && it->future.valid()) + it->future.wait(); + it = cache.erase(it); + } + else + ++it; + } +} + +std::unique_ptr Reader::createRowReader(const RowReaderOptions & row_options) +{ + return std::make_unique(*this, row_options); +} + + +class RowReader::StripeStreamsImpl final : public StripeStreams +{ +public: + StripeStreamsImpl( + Reader & reader_, + size_t stripe_index_, + Proto::StripeFooter footer_, + const std::vector & selected_columns_, + const DateLUTImpl & writer_timezone_, + const DateLUTImpl & reader_timezone_) + : reader(reader_) + , stripe_index(stripe_index_) + , footer(std::move(footer_)) + , selected_columns(selected_columns_) + , writer_timezone(writer_timezone_) + , reader_timezone(reader_timezone_) + { + const auto & stripe = reader.getStripe(stripe_index); + UInt64 offset = stripe.offset; + const UInt64 data_end = stripe.offset + stripe.index_length + stripe.data_length; + + for (const auto & s : footer.streams) + { + /// Every stream is checked: the offsets of the following streams depend on its length. + UInt64 stream_end = 0; + if (common::addOverflow(offset, s.length, stream_end) || stream_end > data_end) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Malformed stream meta in stripe {} of ORC file: streamOffset={}, streamLength={}, stripeOffset={}, stripeIndexLength={}, stripeDataLength={}", + stripe_index, offset, s.length, stripe.offset, stripe.index_length, stripe.data_length); + if (s.kind && s.column < selected_columns.size() && selected_columns[s.column]) + { + /// Use the first stream if there are several of the same kind, as the reference implementation. + stream_ranges.try_emplace({static_cast(s.column), *s.kind}, offset, s.length); + } + else if (s.kind && s.column >= selected_columns.size()) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Malformed stream meta in stripe {} of ORC file: column {} is out of range, the file has {} columns", + stripe_index, s.column, selected_columns.size()); + offset = stream_end; + } + } + + static bool isIndexStream(UInt32 kind) + { + return kind == static_cast(Proto::StreamKind::ROW_INDEX) + || kind == static_cast(Proto::StreamKind::BLOOM_FILTER) + || kind == static_cast(Proto::StreamKind::BLOOM_FILTER_UTF8); + } + + /// Reads the streams into memory at once, only the index streams or all of them. The streams + /// that are not read are loaded on demand by `getStream`. + void loadStreams(bool only_index) + { + std::vector> ranges; + std::vector> keys; + for (const auto & [key, range] : stream_ranges) + { + if (streams.contains(key) || (only_index && !isIndexStream(key.second))) + continue; + keys.push_back(key); + ranges.push_back(range); + } + + auto new_slices = reader.readRanges(ranges); + for (size_t i = 0; i < keys.size(); ++i) + streams.emplace(keys[i], new_slices[i].data); + slices.insert(slices.end(), std::make_move_iterator(new_slices.begin()), std::make_move_iterator(new_slices.end())); + } + + const Proto::ColumnEncoding & getEncoding(UInt64 column_id) const override + { + if (column_id >= footer.columns.size()) + throw Exception(ErrorCodes::INCORRECT_DATA, + "ORC stripe {} has no encoding for column {}, it has {} column encodings", stripe_index, column_id, footer.columns.size()); + return footer.columns[column_id]; + } + + std::unique_ptr getStream(UInt64 column_id, Proto::StreamKind kind) const override + { + std::pair key{column_id, static_cast(kind)}; + auto name = fmt::format("column {} kind {} in stripe {}", column_id, static_cast(kind), stripe_index); + if (auto it = streams.find(key); it != streams.end()) + return std::make_unique(it->second, reader.getCompression(), reader.getCompressionBlockSize(), std::move(name)); + + auto it = stream_ranges.find(key); + if (it == stream_ranges.end()) + return nullptr; + auto loader = [&file_reader = reader, stream_offset = it->second.first](UInt64 offset, UInt64 length) + { + auto loaded = file_reader.readRanges({{stream_offset + offset, length}}); + return LoadedRange{std::move(loaded[0].holder), loaded[0].data}; + }; + return std::make_unique( + std::move(loader), it->second.second, reader.getCompression(), reader.getCompressionBlockSize(), std::move(name)); + } + + std::optional getStreamData(UInt64 column_id, Proto::StreamKind kind) const + { + auto it = streams.find({column_id, static_cast(kind)}); + if (it == streams.end()) + return {}; + return it->second; + } + + const std::vector & getSelectedColumns() const override { return selected_columns; } + const DateLUTImpl & getWriterTimezone() const override { return writer_timezone; } + const DateLUTImpl & getReaderTimezone() const override { return reader_timezone; } + UInt64 getNumberOfRows() const override { return reader.getStripe(stripe_index).number_of_rows; } + bool isDecimalAsLong() const override + { + const auto & version = reader.getPostScript().version; + return version.size() == 2 && version[0] == 1 && version[1] == 9999; + } + + const Proto::StripeFooter & getFooter() const { return footer; } + +private: + Reader & reader; + size_t stripe_index; + Proto::StripeFooter footer; + const std::vector & selected_columns; + const DateLUTImpl & writer_timezone; + const DateLUTImpl & reader_timezone; + /// The offset in the file and the length of each selected stream. + std::map, std::pair> stream_ranges; + /// The streams read into memory. + std::vector slices; + std::map, std::string_view> streams; +}; + + +RowReader::RowReader(Reader & reader_, const RowReaderOptions & options_) + : reader(reader_) + , options(options_) + , reader_timezone(DateLUT::instance(options.timezone_name)) +{ + const Type & schema = reader.getType(); + if (options.include_types) + selected_columns = selectColumnsByTypeIds(schema, *options.include_types); + else + selected_columns.assign(schema.getMaximumColumnId() + 1, true); + selected_type = Type::buildSelectedType(schema, selected_columns); + + UInt64 range_end = options.range_offset + std::min(options.range_length, std::numeric_limits::max() - options.range_offset); + const auto & stripes = reader.getFooter().stripes; + current_stripe = stripes.size(); + last_stripe = 0; + for (size_t i = 0; i < stripes.size(); ++i) + { + if (stripes[i].offset >= options.range_offset && stripes[i].offset < range_end) + { + current_stripe = std::min(current_stripe, i); + last_stripe = std::max(last_stripe, i + 1); + } + } + if (current_stripe >= last_stripe) + current_stripe = last_stripe = 0; + + /// The search argument can be applied only if there are row indexes. + if (reader.getFooter().row_index_stride == 0) + options.search_argument.reset(); +} + +RowReader::~RowReader() = default; + +std::unique_ptr RowReader::createRowBatch() const +{ + return ORC::createRowBatch(*selected_type, options.enable_lazy_decoding); +} + +bool Reader::hasBadBloomFilters() const +{ + /// Same as in the reference implementation. Only the C++ writer (id 1) of some old releases wrote bad bloom filters. + if (footer.writer.value_or(0) != 1) + return false; + /// `software_version` is written since 1.5.13, 1.6.11 and 1.7.0, and the C++ writer supports bloom filters since 1.6.0, + /// so the files of the C++ writer without `software_version` have bad bloom filters. + if (!footer.software_version) + return true; + /// Strip the suffix of snapshot versions, like 1.6.12-SNAPSHOT. + std::string_view version = *footer.software_version; + version = version.substr(0, version.find('-')); + static constexpr std::string_view bad_versions[] = { + "1.6.0", "1.6.1", "1.6.2", "1.6.3", "1.6.4", "1.6.5", "1.6.6", "1.6.7", "1.6.8", "1.6.9", "1.6.10", "1.6.11", "1.7.0"}; + return std::find(std::begin(bad_versions), std::end(bad_versions), version) != std::end(bad_versions); +} + +const DateLUTImpl & RowReader::getWriterTimezone(const Proto::StripeFooter & stripe_footer) const +{ + if (stripe_footer.writer_timezone) + return DateLUT::instance(*stripe_footer.writer_timezone); + /// The reference implementation uses the local time zone of the reader if the file does not specify it. + return DateLUT::serverTimezoneInstance(); +} + +bool RowReader::evaluateStatistics(const std::vector & stats) const +{ + const auto & leaves = options.search_argument->getLeaves(); + std::vector values(leaves.size(), TruthValue::YES_NO_NULL); + UInt32 writer_version = reader.getPostScript().writer_version; + for (size_t i = 0; i < leaves.size(); ++i) + { + UInt64 column = leaves[i].getColumnId(); + if (column < stats.size()) + values[i] = leaves[i].evaluate(writer_version, stats[column], nullptr); + } + return isNeeded(options.search_argument->evaluate(values)); +} + +UInt64 RowReader::numberOfRowGroups() const +{ + const UInt64 stride = reader.getFooter().row_index_stride; + return rows_in_current_stripe / stride + (rows_in_current_stripe % stride != 0); +} + +void RowReader::loadStripeIndex() +{ + row_indexes.clear(); + bloom_filters.clear(); + /// One extra entry is accepted from lenient writers. + const UInt64 max_entries = std::min(numberOfRowGroups(), std::numeric_limits::max() - 1) + 1; + const bool use_bloom_filters = !reader.hasBadBloomFilters(); + const auto & stripe_footer = stripe_streams->getFooter(); + for (const auto & s : stripe_footer.streams) + { + if (!s.kind || s.column >= selected_columns.size() || !selected_columns[s.column]) + continue; + auto kind = static_cast(*s.kind); + if (kind == Proto::StreamKind::ROW_INDEX) + { + if (row_indexes.contains(s.column)) + continue; + auto data = stripe_streams->getStreamData(s.column, kind); + row_indexes[s.column].parse(reader.decompress(data.value_or(std::string_view{}), "row index"), max_entries); + } + else if (kind == Proto::StreamKind::BLOOM_FILTER_UTF8 && use_bloom_filters) + { + if (bloom_filters.contains(s.column)) + continue; + const auto & encoding = stripe_streams->getEncoding(s.column); + Proto::BloomFilterIndex index; + auto data = stripe_streams->getStreamData(s.column, kind); + index.parse(reader.decompress(data.value_or(std::string_view{}), "bloom filter"), max_entries); + auto & filters = bloom_filters[s.column]; + for (const auto & bf : index.bloom_filter) + { + /// Only the bloom filters with the UTF-8 strings and UTC timestamps can be used. + if (encoding.bloom_encoding.value_or(0) == 1 && bf.num_hash_functions && bf.utf8bitset) + filters.push_back(std::make_shared(bf)); + else + filters.push_back(nullptr); + } + } + } +} + +void RowReader::pickRowGroups() +{ + const UInt64 stride = reader.getFooter().row_index_stride; + const UInt64 groups = numberOfRowGroups(); + const auto & leaves = options.search_argument->getLeaves(); + + /// The number of the row groups comes from the file. Check it against the row indexes, which are already + /// read, before allocating memory for it. + bool has_leaf_index = false; + for (const auto & leaf : leaves) + { + auto it = row_indexes.find(leaf.getColumnId()); + if (it == row_indexes.end()) + continue; + has_leaf_index = true; + if (it->second.entry.size() < groups) + throw Exception(ErrorCodes::INCORRECT_DATA, + "The row index of ORC column {} has {} entries, less than the {} row groups of stripe {}", + it->first, it->second.entry.size(), groups, current_stripe); + } + + if (!has_leaf_index) + { + /// Without the row indexes all row groups are read. + next_skipped_rows.clear(); + return; + } + + next_skipped_rows.assign(groups, 0); + std::vector values(leaves.size(), TruthValue::YES_NO_NULL); + const UInt32 writer_version = reader.getPostScript().writer_version; + const Proto::ColumnStatistics empty_statistics; + + UInt64 next_skipped_group = groups; + for (UInt64 group = groups; group > 0;) + { + --group; + for (size_t i = 0; i < leaves.size(); ++i) + { + UInt64 column = leaves[i].getColumnId(); + auto it = row_indexes.find(column); + if (it == row_indexes.end()) + { + values[i] = TruthValue::YES_NO_NULL; + continue; + } + if (group >= it->second.entry.size()) + throw Exception(ErrorCodes::INCORRECT_DATA, + "The row index of ORC column {} has {} entries, less than the {} row groups of stripe {}", + column, it->second.entry.size(), groups, current_stripe); + const auto & entry = it->second.entry[group]; + const BloomFilter * bloom_filter = nullptr; + if (auto bf = bloom_filters.find(column); bf != bloom_filters.end() && group < bf->second.size()) + bloom_filter = bf->second[group].get(); + values[i] = leaves[i].evaluate(writer_version, entry.statistics ? *entry.statistics : empty_statistics, bloom_filter); + } + + bool needed = isNeeded(options.search_argument->evaluate(values)); + if (!needed) + { + next_skipped_rows[group] = 0; + next_skipped_group = group; + } + else + next_skipped_rows[group] = (next_skipped_group == groups) ? rows_in_current_stripe : next_skipped_group * stride; + } +} + +UInt64 RowReader::advanceToNextRowGroup(UInt64 current_row) const +{ + const UInt64 groups = next_skipped_rows.size(); + if (groups == 0) + return std::min(current_row, rows_in_current_stripe); + + const UInt64 stride = reader.getFooter().row_index_stride; + UInt64 group = current_row / stride; + if (group >= groups) + return rows_in_current_stripe; + if (next_skipped_rows[group] != 0) + return current_row; + while (group < groups && next_skipped_rows[group] == 0) + ++group; + if (group < groups) + return group * stride; + return rows_in_current_stripe; +} + +UInt64 RowReader::computeBatchSize(UInt64 requested, UInt64 current_row) const +{ + UInt64 end_row = rows_in_current_stripe; + const UInt64 groups = next_skipped_rows.size(); + if (groups > 0) + { + const UInt64 stride = reader.getFooter().row_index_stride; + UInt64 group = current_row / stride; + if (group >= groups) + return 0; + UInt64 next_skipped_row = next_skipped_rows[group]; + if (next_skipped_row == 0) + return 0; + end_row = next_skipped_row; + } + return std::min(requested, end_row - current_row); +} + +void RowReader::seekToRowGroup(size_t row_group) +{ + std::unordered_map positions; + for (const auto & [column, index] : row_indexes) + { + if (row_group >= index.entry.size()) + throw Exception(ErrorCodes::INCORRECT_DATA, + "The row index of ORC column {} has {} entries, the row group {} is requested", column, index.entry.size(), row_group); + positions.emplace(column, PositionProvider(index.entry[row_group].positions)); + } + column_reader->seekToRowGroup(positions); +} + +void RowReader::startNextStripe() +{ + column_reader.reset(); + stripe_streams.reset(); + row_indexes.clear(); + bloom_filters.clear(); + next_skipped_rows.clear(); + + const auto & search_argument = options.search_argument; + const auto & footer = reader.getFooter(); + + if (search_argument && !file_statistics_evaluated) + { + file_statistics_result = footer.statistics.empty() || evaluateStatistics(footer.statistics); + file_statistics_evaluated = true; + } + if (search_argument && !file_statistics_result) + { + current_stripe = last_stripe; + return; + } + + do + { + const auto & stripe = footer.stripes[current_stripe]; + rows_in_current_stripe = stripe.number_of_rows; + + bool is_stripe_needed = true; + const auto & stripe_statistics = reader.getMetadata().stripe_stats; + if (search_argument && stripe_statistics.size() == footer.stripes.size()) + { + const auto & stats = stripe_statistics[current_stripe].col_stats; + is_stripe_needed = stats.empty() || evaluateStatistics(stats); + } + + if (is_stripe_needed) + { + auto stripe_footer = reader.readStripeFooter(current_stripe); + const DateLUTImpl & writer_timezone = getWriterTimezone(stripe_footer); + stripe_streams = std::make_unique( + reader, current_stripe, std::move(stripe_footer), selected_columns, writer_timezone, reader_timezone); + if (!search_argument) + stripe_streams->loadStreams(/* only_index */ false); + + if (search_argument) + { + stripe_streams->loadStreams(/* only_index */ true); + loadStripeIndex(); + pickRowGroups(); + bool has_selected = next_skipped_rows.empty(); + for (UInt64 group = current_row_in_stripe / footer.row_index_stride; !has_selected && group < next_skipped_rows.size(); ++group) + has_selected = next_skipped_rows[group] != 0; + if (has_selected) + { + /// If some row groups are skipped, the data is loaded on demand, only around the + /// selected row groups. Otherwise the whole stripe is read at once. + bool has_skipped = std::find(next_skipped_rows.begin(), next_skipped_rows.end(), 0) != next_skipped_rows.end(); + if (!has_skipped) + stripe_streams->loadStreams(/* only_index */ false); + break; + } + is_stripe_needed = false; + stripe_streams.reset(); + } + } + + if (!is_stripe_needed) + { + ++current_stripe; + current_row_in_stripe = 0; + } + } while (search_argument && current_stripe < last_stripe); + + if (current_stripe >= last_stripe) + return; + + column_reader = buildColumnReader(reader.getType(), *stripe_streams); + + if (search_argument) + { + current_row_in_stripe = advanceToNextRowGroup(current_row_in_stripe); + if (current_row_in_stripe > 0) + seekToRowGroup(current_row_in_stripe / footer.row_index_stride); + } +} + +bool RowReader::next(ColumnVectorBatch & batch, size_t max_rows) +{ + if (current_stripe >= last_stripe) + { + batch.numElements = 0; + return false; + } + + if (current_row_in_stripe == 0) + startNextStripe(); + + if (current_stripe >= last_stripe) + { + batch.numElements = 0; + return false; + } + + UInt64 rows_to_read = std::min(max_rows, rows_in_current_stripe - current_row_in_stripe); + if (options.search_argument && rows_to_read > 0) + rows_to_read = computeBatchSize(rows_to_read, current_row_in_stripe); + + batch.numElements = rows_to_read; + if (rows_to_read == 0) + { + current_stripe = last_stripe; + return false; + } + + if (options.enable_lazy_decoding) + column_reader->nextEncoded(batch, rows_to_read, nullptr); + else + column_reader->next(batch, rows_to_read, nullptr); + + current_row_in_stripe += rows_to_read; + + if (options.search_argument) + { + UInt64 next_row = advanceToNextRowGroup(current_row_in_stripe); + if (current_row_in_stripe != next_row) + { + current_row_in_stripe = next_row; + if (current_row_in_stripe < rows_in_current_stripe) + seekToRowGroup(current_row_in_stripe / reader.getFooter().row_index_stride); + } + } + + if (current_row_in_stripe >= rows_in_current_stripe) + { + ++current_stripe; + current_row_in_stripe = 0; + } + return true; +} + +} diff --git a/src/Processors/Formats/Impl/ORC/Reader.h b/src/Processors/Formats/Impl/ORC/Reader.h new file mode 100644 index 0000000000000..981ab8c5c7996 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Reader.h @@ -0,0 +1,188 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +/// The reader of the ORC files: https://orc.apache.org/specification/ORCv1/ + +namespace DB::ORC +{ + +/// Random access to the file. +class InputStream +{ +public: + virtual ~InputStream() = default; + + virtual UInt64 getLength() const = 0; + /// Reads exactly `length` bytes at `offset`. Must be thread-safe if `readAsync` is used. + virtual void read(void * buf, UInt64 length, UInt64 offset) = 0; + /// Reads in the background. The default implementation reads synchronously. + virtual std::future readAsync(void * buf, UInt64 length, UInt64 offset); + virtual const std::string & getName() const = 0; +}; + +struct ReaderOptions +{ + /// Two ranges are read with one request if the gap between them is at most this size... + UInt64 hole_size_limit = 8192; + /// ... and the combined range is at most this size. + UInt64 range_size_limit = 32 * 1024 * 1024; +}; + +struct RowReaderOptions +{ + /// The ids of the columns to read, with the same semantics as in the reference implementation + /// (see `selectColumnsByTypeIds`). If not set, all columns are read. + std::optional> include_types; + /// Read only the stripes that start in `[range_offset, range_offset + range_length)`. + UInt64 range_offset = 0; + UInt64 range_length = std::numeric_limits::max(); + /// For the dictionary-encoded string columns, fill `EncodedStringVectorBatch`. + bool enable_lazy_decoding = false; + /// The time zone to show the values of the TIMESTAMP (not TIMESTAMP_INSTANT) columns in. + std::string timezone_name = "GMT"; + std::shared_ptr search_argument; +}; + +class RowReader; + +class Reader +{ +public: + Reader(std::unique_ptr stream_, const ReaderOptions & options_); + ~Reader(); + + const Type & getType() const { return *schema; } + UInt64 getNumberOfRows() const { return footer.number_of_rows; } + UInt64 getNumberOfStripes() const { return footer.stripes.size(); } + const Proto::StripeInformation & getStripe(size_t index) const { return footer.stripes.at(index); } + static UInt64 getStripeLength(const Proto::StripeInformation & stripe) { return stripe.index_length + stripe.data_length + stripe.footer_length; } + + const Proto::Footer & getFooter() const { return footer; } + const Proto::PostScript & getPostScript() const { return postscript; } + CompressionKind getCompression() const { return postscript.compression; } + /// Whether the bloom filters of the file were written by a version of the C++ writer + /// that hashed the integers incorrectly (ORC-1024), so they must not be used. + bool hasBadBloomFilters() const; + UInt64 getCompressionBlockSize() const { return postscript.compression_block_size; } + + /// The statistics of the whole file, one per column. + const std::vector & getStatistics() const { return footer.statistics; } + /// The statistics of the stripes, read on the first call. Can be empty. + const Proto::Metadata & getMetadata(); + + /// Reads the stripe footer. Not thread-safe. + Proto::StripeFooter readStripeFooter(size_t stripe_index); + + std::unique_ptr createRowReader(const RowReaderOptions & options); + + /// Starts reading the data of the selected columns of the stripes in the background. + void preBuffer(const std::vector & stripes, const std::vector & include_types); + /// Releases the prefetched data that ends before `boundary`. + void releaseBuffer(UInt64 boundary); + + /// A piece of the file in memory. + struct Slice + { + std::shared_ptr> holder; + std::string_view data; + }; + + /// Reads the ranges `(offset, length)` of the file, from the prefetched data if possible. + /// The adjacent ranges are coalesced according to the options. + std::vector readRanges(const std::vector> & ranges); + + /// Decompresses a whole metadata stream (a footer, an index). + std::string decompress(std::string_view data, const std::string & name) const; + +private: + void readTail(); + + struct CachedRange + { + UInt64 offset; + UInt64 length; + std::shared_ptr> buffer; + std::future future; + bool ready = false; + }; + + /// Returns the ranges of the streams of the selected columns in the stripe, and the stripe footer. + std::vector> getStripeRanges(size_t stripe_index, const Proto::StripeFooter & stripe_footer, const std::vector & selected) const; + + std::unique_ptr stream; + ReaderOptions options; + Proto::PostScript postscript; + UInt64 postscript_length = 0; + Proto::Footer footer; + std::optional metadata; + std::unique_ptr schema; + + std::mutex cache_mutex; + std::list cache; +}; + +/// Reads the rows of the selected stripes and columns in batches. +class RowReader +{ +public: + RowReader(Reader & reader_, const RowReaderOptions & options_); + ~RowReader(); + + std::unique_ptr createRowBatch() const; + + /// Fills the batch with the next rows, at most `max_rows`. Returns false at the end. + bool next(ColumnVectorBatch & batch, size_t max_rows); + + /// The type of the file with only the selected columns. The batches have this structure. + const Type & getSelectedType() const { return *selected_type; } + +private: + class StripeStreamsImpl; + + void startNextStripe(); + void loadStripeIndex(); + UInt64 numberOfRowGroups() const; + void pickRowGroups(); + void seekToRowGroup(size_t row_group); + UInt64 advanceToNextRowGroup(UInt64 current_row) const; + UInt64 computeBatchSize(UInt64 requested, UInt64 current_row) const; + bool evaluateStatistics(const std::vector & stats) const; + const DateLUTImpl & getWriterTimezone(const Proto::StripeFooter & stripe_footer) const; + + Reader & reader; + RowReaderOptions options; + std::vector selected_columns; + std::unique_ptr selected_type; + const DateLUTImpl & reader_timezone; + + UInt64 current_stripe = 0; + UInt64 last_stripe = 0; + UInt64 rows_in_current_stripe = 0; + UInt64 current_row_in_stripe = 0; + bool file_statistics_evaluated = false; + bool file_statistics_result = true; + + std::unique_ptr stripe_streams; + std::unique_ptr column_reader; + + /// For the predicate pushdown, by the column id. + std::unordered_map row_indexes; + std::unordered_map>> bloom_filters; + /// For each row group of the current stripe: zero if it is skipped, otherwise the row (in the + /// stripe) of the next skipped row group or the number of rows in the stripe. + std::vector next_skipped_rows; +}; + +} diff --git a/src/Processors/Formats/Impl/ORC/SearchArgument.cpp b/src/Processors/Formats/Impl/ORC/SearchArgument.cpp new file mode 100644 index 0000000000000..e0543230c614a --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/SearchArgument.cpp @@ -0,0 +1,1075 @@ +#include + +#include +#include +#include + +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int INCORRECT_DATA; +} +} + +namespace DB::ORC +{ + +TruthValue operator||(TruthValue left, TruthValue right) +{ + if (right == TruthValue::YES || left == TruthValue::YES) + return TruthValue::YES; + if (right == TruthValue::YES_NULL || left == TruthValue::YES_NULL) + return TruthValue::YES_NULL; + if (right == TruthValue::NO) + return left; + if (left == TruthValue::NO) + return right; + if (left == TruthValue::IS_NULL) + { + if (right == TruthValue::NO_NULL || right == TruthValue::IS_NULL) + return TruthValue::IS_NULL; + return TruthValue::YES_NULL; + } + if (right == TruthValue::IS_NULL) + { + if (left == TruthValue::NO_NULL) + return TruthValue::IS_NULL; + return TruthValue::YES_NULL; + } + if (left == TruthValue::NO_NULL && right == TruthValue::NO_NULL) + return TruthValue::NO_NULL; + return TruthValue::YES_NO_NULL; +} + +TruthValue operator&&(TruthValue left, TruthValue right) +{ + if (right == TruthValue::NO || left == TruthValue::NO) + return TruthValue::NO; + if (right == TruthValue::NO_NULL || left == TruthValue::NO_NULL) + return TruthValue::NO_NULL; + if (right == TruthValue::YES) + return left; + if (left == TruthValue::YES) + return right; + if (left == TruthValue::IS_NULL) + { + if (right == TruthValue::YES_NULL || right == TruthValue::IS_NULL) + return TruthValue::IS_NULL; + return TruthValue::NO_NULL; + } + if (right == TruthValue::IS_NULL) + { + if (left == TruthValue::YES_NULL) + return TruthValue::IS_NULL; + return TruthValue::NO_NULL; + } + if (left == TruthValue::YES_NULL && right == TruthValue::YES_NULL) + return TruthValue::YES_NULL; + return TruthValue::YES_NO_NULL; +} + +TruthValue operator!(TruthValue value) +{ + switch (value) + { + case TruthValue::NO: return TruthValue::YES; + case TruthValue::YES: return TruthValue::NO; + case TruthValue::IS_NULL: + case TruthValue::YES_NO: + case TruthValue::YES_NO_NULL: + return value; + case TruthValue::NO_NULL: return TruthValue::YES_NULL; + case TruthValue::YES_NULL: return TruthValue::NO_NULL; + } + return value; +} + +bool isNeeded(TruthValue value) +{ + switch (value) + { + case TruthValue::NO: + case TruthValue::IS_NULL: + case TruthValue::NO_NULL: + return false; + default: + return true; + } +} + + +bool Literal::Decimal::parse(std::string_view s) +{ + bool negative = false; + size_t i = 0; + if (i < s.size() && (s[i] == '-' || s[i] == '+')) + { + negative = s[i] == '-'; + ++i; + } + Int256 result = 0; + bool has_digits = false; + bool has_point = false; + Int32 result_scale = 0; + for (; i < s.size(); ++i) + { + char c = s[i]; + if (c == '.' && !has_point) + { + has_point = true; + continue; + } + if (c < '0' || c > '9') + return false; + has_digits = true; + result = result * 10 + (c - '0'); + if (has_point) + ++result_scale; + /// More than 38 digits do not fit into the 128-bit value. + if (result > Int256(std::numeric_limits::max())) + return false; + } + if (!has_digits) + return false; + value = static_cast(negative ? -result : result); + scale = result_scale; + return true; +} + +std::string Literal::Decimal::toString() const +{ + bool negative = value < 0; + std::string digits = DB::toString(negative ? -Int256(value) : Int256(value)); + std::string result; + if (scale <= 0) + result = digits; + else + { + if (digits.size() <= static_cast(scale)) + digits = std::string(scale - digits.size() + 1, '0') + digits; + result = digits.substr(0, digits.size() - scale) + "." + digits.substr(digits.size() - scale); + size_t last = result.find_last_not_of('0'); + if (result[last] == '.') + result.resize(last); + else + result.resize(last + 1); + } + return negative ? "-" + result : result; +} + +namespace +{ + +int compareDecimals(const Literal::Decimal & a, const Literal::Decimal & b) +{ + Int256 left = a.value; + Int256 right = b.value; + for (Int32 s = a.scale; s < b.scale; ++s) + left *= 10; + for (Int32 s = b.scale; s < a.scale; ++s) + right *= 10; + return left < right ? -1 : (left == right ? 0 : 1); +} + +} + +Literal::Literal(bool value) : type(PredicateDataType::BOOLEAN), int_value(value) {} +Literal::Literal(Int64 value) : type(PredicateDataType::LONG), int_value(value) {} +Literal::Literal(double value) : type(PredicateDataType::FLOAT), float_value(value) {} +Literal::Literal(const char * data, size_t size) : type(PredicateDataType::STRING), string_value(data, size) {} + +Literal::Literal(PredicateDataType type_, Int64 value) : type(type_), int_value(value) +{ + if (type != PredicateDataType::DATE) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Only DATE literals can be created from a type and an integer"); +} + +Literal::Literal(Int64 second, Int32 nanos) : type(PredicateDataType::TIMESTAMP), timestamp_value{second, nanos} {} + +Literal::Literal(Int128 value, Int32 /*precision*/, Int32 scale) : type(PredicateDataType::DECIMAL), decimal_value{value, scale} {} + + +namespace +{ + +/// Where a point is relative to the range [min, max]. +enum class Location : UInt8 +{ + BEFORE, + MIN, + MIDDLE, + MAX, + AFTER, +}; + +template +Location compareToRange(const T & point, const T & min, const T & max, Less less, Equal equal) +{ + if (less(point, min)) + return Location::BEFORE; + if (equal(point, min)) + return Location::MIN; + if (less(max, point)) + return Location::AFTER; + if (equal(point, max)) + return Location::MAX; + return Location::MIDDLE; +} + +template +TruthValue evaluatePredicateRange( + PredicateLeaf::Operator op, const std::vector & values, const T & min_value, const T & max_value, bool has_null, Less less, Equal equal) +{ + auto loc_of = [&](const T & v) { return compareToRange(v, min_value, max_value, less, equal); }; + bool min_equals_max = equal(min_value, max_value); + + switch (op) + { + case PredicateLeaf::Operator::NULL_SAFE_EQUALS: + { + auto loc = loc_of(values.at(0)); + if (loc == Location::BEFORE || loc == Location::AFTER) + return TruthValue::NO; + return TruthValue::YES_NO; + } + case PredicateLeaf::Operator::EQUALS: + { + auto loc = loc_of(values.at(0)); + if (min_equals_max && loc == Location::MIN) + return has_null ? TruthValue::YES_NULL : TruthValue::YES; + if (loc == Location::BEFORE || loc == Location::AFTER) + return has_null ? TruthValue::NO_NULL : TruthValue::NO; + return has_null ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; + } + case PredicateLeaf::Operator::LESS_THAN: + { + auto loc = loc_of(values.at(0)); + if (loc == Location::AFTER) + return has_null ? TruthValue::YES_NULL : TruthValue::YES; + if (loc == Location::BEFORE || loc == Location::MIN) + return has_null ? TruthValue::NO_NULL : TruthValue::NO; + return has_null ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; + } + case PredicateLeaf::Operator::LESS_THAN_EQUALS: + { + auto loc = loc_of(values.at(0)); + if (loc == Location::AFTER || loc == Location::MAX || (loc == Location::MIN && min_equals_max)) + return has_null ? TruthValue::YES_NULL : TruthValue::YES; + if (loc == Location::BEFORE) + return has_null ? TruthValue::NO_NULL : TruthValue::NO; + return has_null ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; + } + case PredicateLeaf::Operator::IN: + { + if (min_equals_max) + { + for (const auto & value : values) + if (loc_of(value) == Location::MIN) + return has_null ? TruthValue::YES_NULL : TruthValue::YES; + return has_null ? TruthValue::NO_NULL : TruthValue::NO; + } + for (const auto & value : values) + { + auto loc = loc_of(value); + if (loc == Location::MIN || loc == Location::MIDDLE || loc == Location::MAX) + return has_null ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; + } + return has_null ? TruthValue::NO_NULL : TruthValue::NO; + } + case PredicateLeaf::Operator::BETWEEN: + { + if (values.empty()) + return TruthValue::YES_NO; + auto loc = loc_of(values.at(0)); + if (loc == Location::BEFORE || loc == Location::MIN) + { + auto loc2 = loc_of(values.at(1)); + if (loc2 == Location::AFTER || loc2 == Location::MAX) + return has_null ? TruthValue::YES_NULL : TruthValue::YES; + if (loc2 == Location::BEFORE) + return has_null ? TruthValue::NO_NULL : TruthValue::NO; + return has_null ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; + } + if (loc == Location::AFTER) + return has_null ? TruthValue::NO_NULL : TruthValue::NO; + return has_null ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; + } + case PredicateLeaf::Operator::IS_NULL: + return has_null ? TruthValue::YES_NO : TruthValue::NO; + } + return has_null ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; +} + +template +TruthValue evaluateRange(PredicateLeaf::Operator op, const std::vector & values, const T & min_value, const T & max_value, bool has_null) +{ + return evaluatePredicateRange(op, values, min_value, max_value, has_null, + [](const T & a, const T & b) { return a < b; }, + [](const T & a, const T & b) { return a == b; }); +} + +TruthValue evaluateBoolPredicate(PredicateLeaf::Operator op, const std::vector & literals, const Proto::ColumnStatistics & stats) +{ + bool has_null = stats.has_null.value_or(false); + if (!stats.bucket_statistics || stats.bucket_statistics->count.empty()) + return has_null ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; + + UInt64 true_count = stats.bucket_statistics->count[0]; + UInt64 false_count = stats.number_of_values.value_or(0) - true_count; + switch (op) + { + case PredicateLeaf::Operator::IS_NULL: + return has_null ? TruthValue::YES_NO : TruthValue::NO; + case PredicateLeaf::Operator::NULL_SAFE_EQUALS: + { + if (literals.at(0).getBool()) + { + if (true_count == 0) + return TruthValue::NO; + if (false_count == 0) + return TruthValue::YES; + } + else + { + if (false_count == 0) + return TruthValue::NO; + if (true_count == 0) + return TruthValue::YES; + } + return TruthValue::YES_NO; + } + case PredicateLeaf::Operator::EQUALS: + { + if (literals.at(0).getBool()) + { + if (true_count == 0) + return has_null ? TruthValue::NO_NULL : TruthValue::NO; + if (false_count == 0) + return has_null ? TruthValue::YES_NULL : TruthValue::YES; + } + else + { + if (false_count == 0) + return has_null ? TruthValue::NO_NULL : TruthValue::NO; + if (true_count == 0) + return has_null ? TruthValue::YES_NULL : TruthValue::YES; + } + return has_null ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; + } + default: + return has_null ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; + } +} + +template +std::vector literalsTo(const std::vector & literals, Get get) +{ + std::vector result; + for (const auto & literal : literals) + if (!literal.isNull()) + result.push_back(get(literal)); + return result; +} + +/// From the reference implementation: the writer version that fixed the timestamp statistics. +constexpr UInt32 WRITER_VERSION_ORC_135 = 6; + +} + +TruthValue PredicateLeaf::evaluatePredicateMinMax(const Proto::ColumnStatistics & stats) const +{ + TruthValue result = TruthValue::YES_NO_NULL; + bool has_null = stats.has_null.value_or(false); + + switch (type) + { + case PredicateDataType::LONG: + { + if (stats.int_statistics && stats.int_statistics->minimum && stats.int_statistics->maximum) + result = evaluateRange(op, literalsTo(literals, [](const Literal & l) { return l.getLong(); }), + *stats.int_statistics->minimum, *stats.int_statistics->maximum, has_null); + break; + } + case PredicateDataType::FLOAT: + { + if (stats.double_statistics && stats.double_statistics->minimum && stats.double_statistics->maximum) + { + if (!std::isfinite(stats.double_statistics->sum.value_or(0))) + result = has_null ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; + else + result = evaluateRange(op, literalsTo(literals, [](const Literal & l) { return l.getFloat(); }), + *stats.double_statistics->minimum, *stats.double_statistics->maximum, has_null); + } + break; + } + case PredicateDataType::STRING: + { + if (stats.string_statistics && stats.string_statistics->minimum && stats.string_statistics->maximum) + result = evaluateRange(op, literalsTo(literals, [](const Literal & l) { return l.getString(); }), + *stats.string_statistics->minimum, *stats.string_statistics->maximum, has_null); + break; + } + case PredicateDataType::DATE: + { + if (stats.date_statistics && stats.date_statistics->minimum && stats.date_statistics->maximum) + result = evaluateRange(op, literalsTo(literals, [](const Literal & l) { return l.getDate(); }), + static_cast(*stats.date_statistics->minimum), static_cast(*stats.date_statistics->maximum), has_null); + break; + } + case PredicateDataType::TIMESTAMP: + { + const auto & ts = stats.timestamp_statistics; + /// The stored nanoseconds are the sub-millisecond part plus one; malformed ones disable the pruning. + auto valid_nanos = [](const std::optional & nanos) { return !nanos || (*nanos >= 1 && *nanos <= 1'000'000); }; + if (ts && ts->minimum_utc && ts->maximum_utc && valid_nanos(ts->minimum_nanos) && valid_nanos(ts->maximum_nanos)) + { + Int32 min_nanos = ts->minimum_nanos ? static_cast(*ts->minimum_nanos) - 1 : 0; + Int32 max_nanos = ts->maximum_nanos ? static_cast(*ts->maximum_nanos) - 1 : 999999; + /// The milliseconds are rounded down, so the nanoseconds are not negative, as in the literals. + auto to_timestamp = [](Int64 millis, Int32 nanos) + { + Int64 second = millis / 1000; + Int64 rest = millis % 1000; + if (rest < 0) + { + rest += 1000; + --second; + } + return Literal::Timestamp{second, static_cast(rest * 1000000) + nanos}; + }; + Literal::Timestamp min = to_timestamp(*ts->minimum_utc, min_nanos); + Literal::Timestamp max = to_timestamp(*ts->maximum_utc, max_nanos); + result = evaluateRange(op, literalsTo(literals, [](const Literal & l) { return l.getTimestamp(); }), + min, max, has_null); + } + break; + } + case PredicateDataType::DECIMAL: + { + const auto & ds = stats.decimal_statistics; + if (ds && ds->minimum && ds->maximum) + { + Literal::Decimal min; + Literal::Decimal max; + if (min.parse(*ds->minimum) && max.parse(*ds->maximum)) + result = evaluatePredicateRange(op, + literalsTo(literals, [](const Literal & l) { return l.getDecimal(); }), + min, max, has_null, + [](const Literal::Decimal & a, const Literal::Decimal & b) { return compareDecimals(a, b) < 0; }, + [](const Literal::Decimal & a, const Literal::Decimal & b) { return compareDecimals(a, b) == 0; }); + } + break; + } + case PredicateDataType::BOOLEAN: + { + if (stats.bucket_statistics) + result = evaluateBoolPredicate(op, literals, stats); + break; + } + } + + /// Respect a null literal for IN. + if (op == Operator::IN && has_null) + for (const auto & literal : literals) + if (literal.isNull()) + return TruthValue::YES_NO_NULL; + + return result; +} + +namespace +{ + +TruthValue checkInBloomFilter(PredicateDataType type, const Literal & literal, const BloomFilter & bf, bool has_null) +{ + TruthValue result = has_null ? TruthValue::NO_NULL : TruthValue::NO; + if (literal.isNull()) + result = has_null ? TruthValue::YES_NO_NULL : TruthValue::NO; + else + { + bool found = true; + switch (type) + { + case PredicateDataType::LONG: found = bf.testLong(literal.getLong()); break; + case PredicateDataType::FLOAT: found = bf.testDouble(literal.getFloat()); break; + case PredicateDataType::STRING: found = bf.testBytes(literal.getString().data(), literal.getString().size()); break; + case PredicateDataType::DECIMAL: + { + std::string s = literal.getDecimal().toString(); + found = bf.testBytes(s.data(), s.size()); + break; + } + case PredicateDataType::TIMESTAMP: + { + /// The writers round the negative milliseconds differently: the Java one down, the C++ one toward zero. + const auto timestamp = literal.getTimestamp(); + const Int64 millis_down = timestamp.getMillis(); + const bool has_fraction = timestamp.second < 0 && timestamp.nanos % 1000000 != 0; + found = bf.testLong(millis_down) || (has_fraction && bf.testLong(millis_down + 1)); + break; + } + case PredicateDataType::DATE: found = bf.testLong(literal.getDate()); break; + case PredicateDataType::BOOLEAN: found = true; break; + } + if (found) + result = TruthValue::YES_NO_NULL; + } + + if (result == TruthValue::YES_NO_NULL && !has_null) + result = TruthValue::YES_NO; + return result; +} + +} + +TruthValue PredicateLeaf::evaluatePredicateBloomFilter(const BloomFilter & bloom_filter, bool has_null) const +{ + switch (op) + { + case Operator::NULL_SAFE_EQUALS: + /// Null-safe equals does not return the `*_NULL` variants. + return checkInBloomFilter(type, literals.front(), bloom_filter, false); + case Operator::EQUALS: + return checkInBloomFilter(type, literals.front(), bloom_filter, has_null); + case Operator::IN: + for (const auto & literal : literals) + { + TruthValue result = checkInBloomFilter(type, literal, bloom_filter, has_null); + if (result == TruthValue::YES_NO_NULL || result == TruthValue::YES_NO) + return result; + } + return has_null ? TruthValue::NO_NULL : TruthValue::NO; + default: + return has_null ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; + } +} + +TruthValue PredicateLeaf::evaluate(UInt32 writer_version, const Proto::ColumnStatistics & stats, const BloomFilter * bloom_filter) const +{ + /// The files written before ORC-135 store the timestamps in the local time zone of the writer. + if (type == PredicateDataType::TIMESTAMP && writer_version < WRITER_VERSION_ORC_135) + return TruthValue::YES_NO_NULL; + + /// The files written by some writers do not have the `has_null` field. + if (!stats.has_null) + return TruthValue::YES_NO_NULL; + + bool has_null = *stats.has_null; + bool all_null = has_null && stats.number_of_values.value_or(0) == 0; + if (op == Operator::IS_NULL || ((op == Operator::EQUALS || op == Operator::NULL_SAFE_EQUALS) && literals.at(0).isNull())) + return all_null ? TruthValue::YES : (has_null ? TruthValue::YES_NO : TruthValue::NO); + if (all_null) + return TruthValue::IS_NULL; + + TruthValue result = evaluatePredicateMinMax(stats); + if (bloom_filter && result != TruthValue::NO_NULL && result != TruthValue::NO + && (op == Operator::EQUALS || op == Operator::NULL_SAFE_EQUALS || op == Operator::IN)) + return evaluatePredicateBloomFilter(*bloom_filter, has_null); + return result; +} + + +class ExpressionTree +{ +public: + enum class Operator : UInt8 + { + OR, + AND, + NOT, + LEAF, + CONSTANT, + }; + + explicit ExpressionTree(Operator op_) : op(op_) {} + explicit ExpressionTree(TruthValue constant_) : op(Operator::CONSTANT), constant(constant_) {} + explicit ExpressionTree(size_t leaf_) : op(Operator::LEAF), leaf(leaf_) {} + ExpressionTree(Operator op_, std::vector children_) : op(op_), children(std::move(children_)) {} + + TruthValue evaluate(const std::vector & leaves) const + { + switch (op) + { + case Operator::OR: + { + TruthValue result = children.at(0)->evaluate(leaves); + for (size_t i = 1; i < children.size() && !isNeeded(result); ++i) + result = children[i]->evaluate(leaves) || result; + return result; + } + case Operator::AND: + { + TruthValue result = children.at(0)->evaluate(leaves); + for (size_t i = 1; i < children.size() && isNeeded(result); ++i) + result = children[i]->evaluate(leaves) && result; + return result; + } + case Operator::NOT: + return !children.at(0)->evaluate(leaves); + case Operator::LEAF: + return leaves.at(leaf); + case Operator::CONSTANT: + return constant; + } + return TruthValue::YES_NO_NULL; + } + + Operator op; + TruthValue constant = TruthValue::YES_NO_NULL; + size_t leaf = 0; + std::vector children; +}; + + +SearchArgument::SearchArgument(ExpressionTreePtr root_, std::vector leaves_) + : root(std::move(root_)), leaves(std::move(leaves_)) +{ +} + +TruthValue SearchArgument::evaluate(const std::vector & leaf_values) const +{ + return root ? root->evaluate(leaf_values) : TruthValue::YES_NO_NULL; +} + + +namespace +{ + +using Op = ExpressionTree::Operator; + +/// Pushes the negations down to the leaves and removes the double negations. +ExpressionTreePtr pushDownNot(ExpressionTreePtr root) +{ + if (root->op == Op::NOT) + { + ExpressionTreePtr child = root->children.at(0); + switch (child->op) + { + case Op::NOT: + return pushDownNot(child->children.at(0)); + case Op::CONSTANT: + return std::make_shared(!child->constant); + case Op::AND: + case Op::OR: + { + auto result = std::make_shared(child->op == Op::AND ? Op::OR : Op::AND); + for (const auto & kid : child->children) + result->children.push_back(pushDownNot(std::make_shared(Op::NOT, std::vector{kid}))); + return result; + } + case Op::LEAF: + break; + } + } + else + { + for (auto & kid : root->children) + kid = pushDownNot(kid); + } + return root; +} + +/// Removes the MAYBE (YES_NO_NULL) constants: they are dropped from AND and they replace OR. +ExpressionTreePtr foldMaybe(ExpressionTreePtr expr) +{ + for (auto & kid : expr->children) + { + ExpressionTreePtr child = foldMaybe(kid); + if (child->op == Op::CONSTANT && child->constant == TruthValue::YES_NO_NULL) + { + if (expr->op == Op::AND) + kid = nullptr; + else if (expr->op == Op::OR) + return child; + else + throw Exception(ErrorCodes::LOGICAL_ERROR, "Got a maybe as a child of a NOT in ORC search argument"); + } + else + kid = child; + } + + if (!expr->children.empty()) + { + std::erase(expr->children, nullptr); + if (expr->children.empty()) + return std::make_shared(TruthValue::YES_NO_NULL); + } + return expr; +} + +/// Converts multi-level ANDs and ORs into single level ones. +ExpressionTreePtr flatten(ExpressionTreePtr root) +{ + std::vector nodes; + for (auto & kid : root->children) + { + ExpressionTreePtr child = flatten(kid); + if (child->op == root->op && child->op != Op::NOT) + nodes.insert(nodes.end(), child->children.begin(), child->children.end()); + else + nodes.push_back(child); + } + root->children = std::move(nodes); + if ((root->op == Op::OR || root->op == Op::AND) && root->children.size() == 1) + return root->children[0]; + return root; +} + +ExpressionTreePtr copyTree(const ExpressionTreePtr & node) +{ + auto result = std::make_shared(*node); + return result; +} + +void generateAllCombinations( + std::vector & result, const std::vector & and_list, size_t and_index, const std::vector & non_and_list) +{ + const auto & kids = and_list[and_index]->children; + if (result.empty()) + { + for (const auto & kid : kids) + { + auto node = std::make_shared(Op::OR); + for (const auto & non_and : non_and_list) + node->children.push_back(copyTree(non_and)); + node->children.push_back(kid); + result.push_back(node); + } + } + else + { + std::vector work = std::move(result); + result.clear(); + for (const auto & kid : kids) + { + for (const auto & node : work) + { + auto copy = copyTree(node); + copy->children.push_back(kid); + result.push_back(copy); + } + } + } + if (and_index + 1 < and_list.size()) + generateAllCombinations(result, and_list, and_index + 1, non_and_list); +} + +constexpr size_t CNF_COMBINATIONS_THRESHOLD = 256; + +/// Converts the expression into the conjunctive normal form, if it is not too large. +ExpressionTreePtr convertToCNF(ExpressionTreePtr root) +{ + for (auto & kid : root->children) + kid = convertToCNF(kid); + + if (root->op != Op::OR) + return root; + + std::vector non_and_list; + std::vector and_list; + for (const auto & child : root->children) + { + if (child->op == Op::AND) + and_list.push_back(child); + else if (child->op == Op::OR) + non_and_list.insert(non_and_list.end(), child->children.begin(), child->children.end()); + else + non_and_list.push_back(child); + } + + if (and_list.empty()) + return root; + + size_t num_combinations = 1; + for (const auto & node : and_list) + { + num_combinations *= node->children.size(); + if (num_combinations > CNF_COMBINATIONS_THRESHOLD) + return std::make_shared(TruthValue::YES_NO_NULL); + } + + auto result = std::make_shared(Op::AND); + generateAllCombinations(result->children, and_list, 0, non_and_list); + return result; +} + +void collectUsedLeaves(const ExpressionTreePtr & node, std::vector & reorder, size_t & next) +{ + if (node->op == Op::LEAF) + { + if (reorder[node->leaf] == std::numeric_limits::max()) + reorder[node->leaf] = next++; + return; + } + for (const auto & kid : node->children) + collectUsedLeaves(kid, reorder, next); +} + +/// The trees can share nodes after the CNF conversion, so rewrite every leaf node exactly once. +void rewriteLeaves(const ExpressionTreePtr & root, const std::vector & reorder) +{ + std::unordered_set leaves; + std::deque nodes{root.get()}; + while (!nodes.empty()) + { + ExpressionTree * node = nodes.front(); + nodes.pop_front(); + if (node->op == Op::LEAF) + leaves.insert(node); + else + for (const auto & kid : node->children) + nodes.push_back(kid.get()); + } + for (auto * leaf : leaves) + leaf->leaf = reorder[leaf->leaf]; +} + +} + + +SearchArgumentBuilder::SearchArgumentBuilder() +{ + root = std::make_shared(Op::AND); + stack.push_back(root); +} + +SearchArgumentBuilder & SearchArgumentBuilder::start(UInt8 op) +{ + auto node = std::make_shared(static_cast(op)); + stack.back()->children.push_back(node); + stack.push_back(node); + return *this; +} + +SearchArgumentBuilder & SearchArgumentBuilder::startOr() { return start(static_cast(Op::OR)); } +SearchArgumentBuilder & SearchArgumentBuilder::startAnd() { return start(static_cast(Op::AND)); } +SearchArgumentBuilder & SearchArgumentBuilder::startNot() { return start(static_cast(Op::NOT)); } + +SearchArgumentBuilder & SearchArgumentBuilder::end() +{ + auto current = stack.back(); + stack.pop_back(); + if (stack.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unbalanced end() in ORC search argument builder"); + if (current->children.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot create an expression with no children in ORC search argument builder"); + if (current->op == Op::NOT && current->children.size() != 1) + throw Exception(ErrorCodes::LOGICAL_ERROR, "A NOT expression must have exactly one child in ORC search argument builder"); + return *this; +} + +SearchArgumentBuilder & SearchArgumentBuilder::literal(TruthValue value) +{ + stack.back()->children.push_back(std::make_shared(value)); + return *this; +} + +SearchArgumentBuilder & SearchArgumentBuilder::addLeaf(PredicateLeaf leaf) +{ + size_t index = leaves.size(); + leaves.push_back(std::move(leaf)); + stack.back()->children.push_back(std::make_shared(index)); + return *this; +} + +SearchArgumentBuilder & SearchArgumentBuilder::lessThan(UInt64 column_id, PredicateDataType type, Literal value) +{ + return addLeaf(PredicateLeaf(PredicateLeaf::Operator::LESS_THAN, type, column_id, {std::move(value)})); +} + +SearchArgumentBuilder & SearchArgumentBuilder::lessThanEquals(UInt64 column_id, PredicateDataType type, Literal value) +{ + return addLeaf(PredicateLeaf(PredicateLeaf::Operator::LESS_THAN_EQUALS, type, column_id, {std::move(value)})); +} + +SearchArgumentBuilder & SearchArgumentBuilder::equals(UInt64 column_id, PredicateDataType type, Literal value) +{ + return addLeaf(PredicateLeaf(PredicateLeaf::Operator::EQUALS, type, column_id, {std::move(value)})); +} + +SearchArgumentBuilder & SearchArgumentBuilder::nullSafeEquals(UInt64 column_id, PredicateDataType type, Literal value) +{ + return addLeaf(PredicateLeaf(PredicateLeaf::Operator::NULL_SAFE_EQUALS, type, column_id, {std::move(value)})); +} + +SearchArgumentBuilder & SearchArgumentBuilder::in(UInt64 column_id, PredicateDataType type, const std::vector & values) +{ + if (values.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot create an IN expression with no arguments in ORC search argument builder"); + return addLeaf(PredicateLeaf(PredicateLeaf::Operator::IN, type, column_id, values)); +} + +SearchArgumentBuilder & SearchArgumentBuilder::isNull(UInt64 column_id, PredicateDataType type) +{ + return addLeaf(PredicateLeaf(PredicateLeaf::Operator::IS_NULL, type, column_id, {})); +} + +SearchArgumentBuilder & SearchArgumentBuilder::between(UInt64 column_id, PredicateDataType type, Literal lower, Literal upper) +{ + return addLeaf(PredicateLeaf(PredicateLeaf::Operator::BETWEEN, type, column_id, {std::move(lower), std::move(upper)})); +} + +std::unique_ptr SearchArgumentBuilder::build() +{ + if (stack.size() != 1) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Failed to end {} operations in ORC search argument builder", stack.size() - 1); + if (root->children.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty ORC search argument"); + + root = pushDownNot(root); + root = foldMaybe(root); + root = flatten(root); + root = convertToCNF(root); + root = flatten(root); + + std::vector reorder(leaves.size(), std::numeric_limits::max()); + size_t num_used = 0; + collectUsedLeaves(root, reorder, num_used); + rewriteLeaves(root, reorder); + + std::vector used_leaves; + used_leaves.reserve(num_used); + std::vector by_new_index(num_used); + for (size_t i = 0; i < leaves.size(); ++i) + if (reorder[i] != std::numeric_limits::max()) + by_new_index[reorder[i]] = &leaves[i]; + for (const auto * leaf : by_new_index) + used_leaves.push_back(*leaf); + + return std::make_unique(root, std::move(used_leaves)); +} + + +namespace +{ + +constexpr UInt32 MURMUR3_DEFAULT_SEED = 104729; + +inline UInt64 rotl64(UInt64 x, int r) +{ + return (x << r) | (x >> (64 - r)); +} + +inline UInt64 fmix64(UInt64 value) +{ + value ^= (value >> 33); + value *= 0xff51afd7ed558ccdULL; + value ^= (value >> 33); + value *= 0xc4ceb9fe1a85ec53ULL; + value ^= (value >> 33); + return value; +} + +/// The 64-bit variant of Murmur3 used by ORC (the lower half of the 128-bit hash with one state). +UInt64 murmur3Hash64(const UInt8 * data, size_t size) +{ + UInt64 h = MURMUR3_DEFAULT_SEED; + const UInt64 c1 = 0x87c37b91114253d5ULL; + const UInt64 c2 = 0x4cf5ad432745937fULL; + size_t blocks = size >> 3; + for (size_t i = 0; i < blocks; ++i) + { + UInt64 k = 0; + memcpy(&k, data + i * 8, 8); + if constexpr (std::endian::native == std::endian::big) + k = std::byteswap(k); + k *= c1; + k = rotl64(k, 31); + k *= c2; + h ^= k; + h = rotl64(h, 27); + h = h * 5 + 0x52dce729; + } + + UInt64 k = 0; + size_t idx = blocks << 3; + size_t tail = size - idx; + for (size_t i = tail; i > 0; --i) + k ^= static_cast(data[idx + i - 1]) << ((i - 1) * 8); + if (tail) + { + k *= c1; + k = rotl64(k, 31); + k *= c2; + h ^= k; + } + + h ^= static_cast(size); + return fmix64(h); +} + +/// Thomas Wang's 64-bit integer hash, as in the reference implementation. +Int64 getLongHash(Int64 key) +{ + UInt64 k = static_cast(key); + k = (~k) + (k << 21); + k = k ^ static_cast(static_cast(k) >> 24); + k = (k + (k << 3)) + (k << 8); + k = k ^ static_cast(static_cast(k) >> 14); + k = (k + (k << 2)) + (k << 4); + k = k ^ static_cast(static_cast(k) >> 28); + k = k + (k << 31); + return static_cast(k); +} + +} + +BloomFilter::BloomFilter(const Proto::BloomFilter & proto) : num_hash_functions(proto.num_hash_functions) +{ + /// The writers use a few hash functions (the optimal number for the false positive probability), + /// a huge number from a corrupt file would make every probe extremely slow. + static constexpr UInt32 MAX_HASH_FUNCTIONS = 256; + if (num_hash_functions == 0 || num_hash_functions > MAX_HASH_FUNCTIONS) + throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid number of hash functions {} of a bloom filter in ORC file, must be from 1 to {}", + num_hash_functions, MAX_HASH_FUNCTIONS); + const std::string & bitset = proto.utf8bitset.value(); + if (bitset.empty() || bitset.size() % 8 != 0) + throw Exception(ErrorCodes::INCORRECT_DATA, "The size of a bloom filter in ORC file must be a positive multiple of 8 bytes"); + bits.resize(bitset.size() / 8); + memcpy(bits.data(), bitset.data(), bitset.size()); + if constexpr (std::endian::native == std::endian::big) + for (auto & word : bits) + word = std::byteswap(word); +} + +bool BloomFilter::testHash(Int64 hash) const +{ + Int32 hash1 = static_cast(hash & 0xffffffff); + Int32 hash2 = static_cast(static_cast(hash) >> 32); + UInt64 num_bits = bits.size() * 64; + for (UInt32 i = 1; i <= num_hash_functions; ++i) + { + Int32 combined = static_cast(static_cast(hash1) + i * static_cast(hash2)); + if (combined < 0) + combined = ~combined; + UInt64 pos = static_cast(combined) % num_bits; + if (!(bits[pos >> 6] & (1ULL << (pos % 64)))) + return false; + } + return true; +} + +bool BloomFilter::testLong(Int64 value) const +{ + return testHash(getLongHash(value)); +} + +bool BloomFilter::testDouble(double value) const +{ + return testLong(std::bit_cast(value)); +} + +bool BloomFilter::testBytes(const char * data, size_t size) const +{ + return testHash(static_cast(murmur3Hash64(reinterpret_cast(data), size))); +} + +} diff --git a/src/Processors/Formats/Impl/ORC/SearchArgument.h b/src/Processors/Formats/Impl/ORC/SearchArgument.h new file mode 100644 index 0000000000000..da19904275e75 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/SearchArgument.h @@ -0,0 +1,198 @@ +#pragma once + +#include + +#include +#include +#include + +/// Predicates for skipping stripes and row groups by the column statistics and the bloom filters. +/// The semantics (the truth values, the normalization of the expression, the evaluation of each +/// predicate against the statistics) are the same as in the reference implementation. + +namespace DB::ORC +{ + +/// The possible outcomes of a predicate over a set of rows: whether any row can satisfy it (YES), +/// fail it (NO), or evaluate it to NULL. +enum class TruthValue : UInt8 +{ + YES, + NO, + IS_NULL, + YES_NULL, + NO_NULL, + YES_NO, + YES_NO_NULL, +}; + +TruthValue operator||(TruthValue left, TruthValue right); +TruthValue operator&&(TruthValue left, TruthValue right); +TruthValue operator!(TruthValue value); +/// Whether the rows have to be read. +bool isNeeded(TruthValue value); + +enum class PredicateDataType : UInt8 +{ + LONG, + FLOAT, + STRING, + DATE, + DECIMAL, + TIMESTAMP, + BOOLEAN, +}; + +class Literal +{ +public: + struct Timestamp + { + Int64 second = 0; + Int32 nanos = 0; + + /// Rounded down, as the nanoseconds are not negative. + Int64 getMillis() const { return second * 1000 + nanos / 1000000; } + auto operator<=>(const Timestamp &) const = default; + }; + + struct Decimal + { + Int128 value = 0; + Int32 scale = 0; + + /// Parses a decimal number like `-123.45`. Returns false if it is malformed. + bool parse(std::string_view s); + /// Without the trailing zeros in the fraction, as in the bloom filters. + std::string toString() const; + }; + + explicit Literal(bool value); + explicit Literal(Int64 value); + explicit Literal(double value); + Literal(const char * data, size_t size); + /// Only for DATE: the number of days since the epoch. + Literal(PredicateDataType type, Int64 value); + /// TIMESTAMP: the seconds since the epoch in UTC and the nanoseconds. + Literal(Int64 second, Int32 nanos); + Literal(Int128 value, Int32 precision, Int32 scale); + + PredicateDataType getType() const { return type; } + bool isNull() const { return is_null; } + + bool getBool() const { return int_value != 0; } + Int64 getLong() const { return int_value; } + Int64 getDate() const { return int_value; } + double getFloat() const { return float_value; } + const std::string & getString() const { return string_value; } + Timestamp getTimestamp() const { return timestamp_value; } + Decimal getDecimal() const { return decimal_value; } + +private: + PredicateDataType type; + bool is_null = false; + Int64 int_value = 0; + double float_value = 0; + std::string string_value; + Timestamp timestamp_value; + Decimal decimal_value; +}; + +class BloomFilter; + +class PredicateLeaf +{ +public: + enum class Operator : UInt8 + { + EQUALS, + NULL_SAFE_EQUALS, + LESS_THAN, + LESS_THAN_EQUALS, + IN, + BETWEEN, + IS_NULL, + }; + + PredicateLeaf(Operator op_, PredicateDataType type_, UInt64 column_id_, std::vector literals_) + : op(op_), type(type_), column_id(column_id_), literals(std::move(literals_)) {} + + UInt64 getColumnId() const { return column_id; } + + /// `writer_version` is from the postscript of the file. `bloom_filter` can be nullptr. + TruthValue evaluate(UInt32 writer_version, const Proto::ColumnStatistics & stats, const BloomFilter * bloom_filter) const; + +private: + TruthValue evaluatePredicateMinMax(const Proto::ColumnStatistics & stats) const; + TruthValue evaluatePredicateBloomFilter(const BloomFilter & bloom_filter, bool has_null) const; + + Operator op; + PredicateDataType type; + UInt64 column_id; + std::vector literals; +}; + +class ExpressionTree; +using ExpressionTreePtr = std::shared_ptr; + +class SearchArgument +{ +public: + SearchArgument(ExpressionTreePtr root_, std::vector leaves_); + + const std::vector & getLeaves() const { return leaves; } + TruthValue evaluate(const std::vector & leaf_values) const; + +private: + ExpressionTreePtr root; + std::vector leaves; +}; + +class SearchArgumentBuilder +{ +public: + SearchArgumentBuilder(); + + SearchArgumentBuilder & startOr(); + SearchArgumentBuilder & startAnd(); + SearchArgumentBuilder & startNot(); + SearchArgumentBuilder & end(); + + SearchArgumentBuilder & literal(TruthValue value); + SearchArgumentBuilder & lessThan(UInt64 column_id, PredicateDataType type, Literal value); + SearchArgumentBuilder & lessThanEquals(UInt64 column_id, PredicateDataType type, Literal value); + SearchArgumentBuilder & equals(UInt64 column_id, PredicateDataType type, Literal value); + SearchArgumentBuilder & nullSafeEquals(UInt64 column_id, PredicateDataType type, Literal value); + SearchArgumentBuilder & in(UInt64 column_id, PredicateDataType type, const std::vector & values); + SearchArgumentBuilder & isNull(UInt64 column_id, PredicateDataType type); + SearchArgumentBuilder & between(UInt64 column_id, PredicateDataType type, Literal lower, Literal upper); + + std::unique_ptr build(); + +private: + SearchArgumentBuilder & start(UInt8 op); + SearchArgumentBuilder & addLeaf(PredicateLeaf leaf); + + std::vector stack; + ExpressionTreePtr root; + std::vector leaves; +}; + +/// A bloom filter of a row group (or a stripe), from the BLOOM_FILTER_UTF8 stream. +class BloomFilter +{ +public: + explicit BloomFilter(const Proto::BloomFilter & proto); + + bool testLong(Int64 value) const; + bool testDouble(double value) const; + bool testBytes(const char * data, size_t size) const; + +private: + bool testHash(Int64 hash) const; + + UInt32 num_hash_functions; + std::vector bits; +}; + +} diff --git a/src/Processors/Formats/Impl/ORC/Statistics.cpp b/src/Processors/Formats/Impl/ORC/Statistics.cpp new file mode 100644 index 0000000000000..0b1b4c86cd9b3 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Statistics.cpp @@ -0,0 +1,462 @@ +#include +#include + +#include + +#include + + +namespace DB::ORC +{ + +namespace +{ + +int compareDecimals(Int128 a, Int32 a_scale, Int128 b, Int32 b_scale) +{ + Int256 left = a; + Int256 right = b; + for (Int32 s = a_scale; s < b_scale; ++s) + left *= 10; + for (Int32 s = b_scale; s < a_scale; ++s) + right *= 10; + return left < right ? -1 : (left == right ? 0 : 1); +} + +std::string decimalToString(Int128 value, Int32 scale) +{ + return Literal::Decimal{value, scale}.toString(); +} + +} + +void StatisticsBuilder::updateInteger(Int64 value) +{ + if (!has_minimum) + { + has_minimum = true; + int_minimum = int_maximum = value; + } + else + { + int_minimum = std::min(int_minimum, value); + int_maximum = std::max(int_maximum, value); + } + if (has_int_sum && __builtin_add_overflow(int_sum, value, &int_sum)) + has_int_sum = false; +} + +void StatisticsBuilder::updateDouble(double value) +{ + if (!has_minimum) + { + has_minimum = true; + double_minimum = double_maximum = value; + } + else + { + double_minimum = std::min(double_minimum, value); + double_maximum = std::max(double_maximum, value); + } + double_sum += value; +} + +void StatisticsBuilder::updateString(std::string_view value) +{ + if (!has_minimum) + { + has_minimum = true; + string_minimum = value; + string_maximum = value; + } + else + { + if (value < std::string_view(string_minimum)) + string_minimum = value; + if (value > std::string_view(string_maximum)) + string_maximum = value; + } + total_length += value.size(); +} + +void StatisticsBuilder::updateBinary(size_t length) +{ + total_length += length; +} + +void StatisticsBuilder::updateBoolean(bool value) +{ + if (value) + ++true_count; +} + +void StatisticsBuilder::updateDate(Int32 value) +{ + updateInteger(value); +} + +void StatisticsBuilder::updateDecimal(Int128 value, Int32 scale) +{ + if (!has_minimum) + { + has_minimum = true; + decimal_minimum = decimal_maximum = value; + decimal_minimum_scale = decimal_maximum_scale = scale; + } + else + { + if (compareDecimals(value, scale, decimal_minimum, decimal_minimum_scale) < 0) + { + decimal_minimum = value; + decimal_minimum_scale = scale; + } + if (compareDecimals(value, scale, decimal_maximum, decimal_maximum_scale) > 0) + { + decimal_maximum = value; + decimal_maximum_scale = scale; + } + } + + if (has_decimal_sum) + { + Int256 sum = decimal_sum; + Int256 addend = value; + Int32 result_scale = std::max(decimal_sum_scale, scale); + for (Int32 s = decimal_sum_scale; s < result_scale; ++s) + sum *= 10; + for (Int32 s = scale; s < result_scale; ++s) + addend *= 10; + sum += addend; + if (sum > Int256(std::numeric_limits::max()) || sum < Int256(std::numeric_limits::min())) + has_decimal_sum = false; + else + { + decimal_sum = static_cast(sum); + decimal_sum_scale = result_scale; + } + } +} + +void StatisticsBuilder::updateTimestamp(Int64 millis, Int32 nanos) +{ + if (!has_minimum) + { + has_minimum = true; + int_minimum = int_maximum = millis; + minimum_nanos = maximum_nanos = nanos; + return; + } + if (millis <= int_minimum) + { + if (millis < int_minimum || nanos < minimum_nanos) + minimum_nanos = nanos; + int_minimum = millis; + } + if (millis >= int_maximum) + { + if (millis > int_maximum || nanos > maximum_nanos) + maximum_nanos = nanos; + int_maximum = millis; + } +} + +void StatisticsBuilder::updateCollection(UInt64 children) +{ + Int64 value = static_cast(children); + if (!has_minimum) + { + has_minimum = true; + int_minimum = int_maximum = value; + } + else + { + int_minimum = std::min(int_minimum, value); + int_maximum = std::max(int_maximum, value); + } + total_children += children; +} + +void StatisticsBuilder::merge(const StatisticsBuilder & other) +{ + number_of_values += other.number_of_values; + has_null |= other.has_null; + + switch (kind) + { + case BYTE: + case SHORT: + case INT: + case LONG: + case DATE: + if (other.has_minimum) + { + if (!has_minimum) + { + int_minimum = other.int_minimum; + int_maximum = other.int_maximum; + } + else + { + int_minimum = std::min(int_minimum, other.int_minimum); + int_maximum = std::max(int_maximum, other.int_maximum); + } + has_minimum = true; + } + has_int_sum = has_int_sum && other.has_int_sum; + if (has_int_sum && __builtin_add_overflow(int_sum, other.int_sum, &int_sum)) + has_int_sum = false; + break; + case FLOAT: + case DOUBLE: + if (other.has_minimum) + { + if (!has_minimum) + { + double_minimum = other.double_minimum; + double_maximum = other.double_maximum; + } + else + { + double_minimum = std::min(double_minimum, other.double_minimum); + double_maximum = std::max(double_maximum, other.double_maximum); + } + has_minimum = true; + } + double_sum += other.double_sum; + break; + case STRING: + case CHAR: + case VARCHAR: + if (other.has_minimum) + { + if (!has_minimum) + { + string_minimum = other.string_minimum; + string_maximum = other.string_maximum; + } + else + { + if (other.string_minimum < string_minimum) + string_minimum = other.string_minimum; + if (other.string_maximum > string_maximum) + string_maximum = other.string_maximum; + } + has_minimum = true; + } + total_length += other.total_length; + break; + case BINARY: + total_length += other.total_length; + break; + case BOOLEAN: + true_count += other.true_count; + break; + case DECIMAL: + if (other.has_minimum) + { + if (!has_minimum) + { + decimal_minimum = other.decimal_minimum; + decimal_minimum_scale = other.decimal_minimum_scale; + decimal_maximum = other.decimal_maximum; + decimal_maximum_scale = other.decimal_maximum_scale; + } + else + { + if (compareDecimals(other.decimal_minimum, other.decimal_minimum_scale, decimal_minimum, decimal_minimum_scale) < 0) + { + decimal_minimum = other.decimal_minimum; + decimal_minimum_scale = other.decimal_minimum_scale; + } + if (compareDecimals(other.decimal_maximum, other.decimal_maximum_scale, decimal_maximum, decimal_maximum_scale) > 0) + { + decimal_maximum = other.decimal_maximum; + decimal_maximum_scale = other.decimal_maximum_scale; + } + } + has_minimum = true; + } + has_decimal_sum = has_decimal_sum && other.has_decimal_sum; + if (has_decimal_sum) + { + /// Reuse the update of the sum without touching the minimum and the maximum. + bool saved_has_minimum = has_minimum; + Int128 saved_min = decimal_minimum; + Int32 saved_min_scale = decimal_minimum_scale; + Int128 saved_max = decimal_maximum; + Int32 saved_max_scale = decimal_maximum_scale; + updateDecimal(other.decimal_sum, other.decimal_sum_scale); + has_minimum = saved_has_minimum; + decimal_minimum = saved_min; + decimal_minimum_scale = saved_min_scale; + decimal_maximum = saved_max; + decimal_maximum_scale = saved_max_scale; + } + break; + case TIMESTAMP: + case TIMESTAMP_INSTANT: + if (other.has_minimum) + { + if (!has_minimum) + { + int_minimum = other.int_minimum; + int_maximum = other.int_maximum; + minimum_nanos = other.minimum_nanos; + maximum_nanos = other.maximum_nanos; + has_minimum = true; + } + else + { + if (other.int_minimum <= int_minimum) + { + if (other.int_minimum < int_minimum || other.minimum_nanos < minimum_nanos) + minimum_nanos = other.minimum_nanos; + int_minimum = other.int_minimum; + } + if (other.int_maximum >= int_maximum) + { + if (other.int_maximum > int_maximum || other.maximum_nanos > maximum_nanos) + maximum_nanos = other.maximum_nanos; + int_maximum = other.int_maximum; + } + } + } + break; + case LIST: + case MAP: + if (other.has_minimum) + { + if (!has_minimum) + { + int_minimum = other.int_minimum; + int_maximum = other.int_maximum; + } + else + { + int_minimum = std::min(int_minimum, other.int_minimum); + int_maximum = std::max(int_maximum, other.int_maximum); + } + has_minimum = true; + } + total_children += other.total_children; + break; + default: + break; + } +} + +void StatisticsBuilder::reset() +{ + *this = StatisticsBuilder(kind); +} + +Proto::ColumnStatistics StatisticsBuilder::toProto() const +{ + Proto::ColumnStatistics res; + res.has_null = has_null; + res.number_of_values = number_of_values; + + switch (kind) + { + case BYTE: + case SHORT: + case INT: + case LONG: + { + auto & stats = res.int_statistics.emplace(); + if (has_minimum) + { + stats.minimum = int_minimum; + stats.maximum = int_maximum; + } + if (has_int_sum) + stats.sum = int_sum; + break; + } + case DATE: + { + auto & stats = res.date_statistics.emplace(); + if (has_minimum) + { + stats.minimum = static_cast(int_minimum); + stats.maximum = static_cast(int_maximum); + } + break; + } + case FLOAT: + case DOUBLE: + { + auto & stats = res.double_statistics.emplace(); + if (has_minimum) + { + stats.minimum = double_minimum; + stats.maximum = double_maximum; + } + stats.sum = double_sum; + break; + } + case STRING: + case CHAR: + case VARCHAR: + { + auto & stats = res.string_statistics.emplace(); + if (has_minimum) + { + stats.minimum = string_minimum; + stats.maximum = string_maximum; + } + stats.sum = static_cast(total_length); + break; + } + case BINARY: + res.binary_statistics.emplace().sum = static_cast(total_length); + break; + case BOOLEAN: + res.bucket_statistics.emplace().count.push_back(true_count); + break; + case DECIMAL: + { + auto & stats = res.decimal_statistics.emplace(); + if (has_minimum) + { + stats.minimum = decimalToString(decimal_minimum, decimal_minimum_scale); + stats.maximum = decimalToString(decimal_maximum, decimal_maximum_scale); + } + if (has_decimal_sum) + stats.sum = decimalToString(decimal_sum, decimal_sum_scale); + break; + } + case TIMESTAMP: + case TIMESTAMP_INSTANT: + { + auto & stats = res.timestamp_statistics.emplace(); + if (has_minimum) + { + stats.minimum_utc = int_minimum; + stats.maximum_utc = int_maximum; + if (minimum_nanos != 0) + stats.minimum_nanos = static_cast(minimum_nanos) + 1; + if (maximum_nanos != 999999) + stats.maximum_nanos = static_cast(maximum_nanos) + 1; + } + break; + } + case LIST: + case MAP: + { + auto & stats = res.collection_statistics.emplace(); + if (has_minimum) + { + stats.min_children = static_cast(int_minimum); + stats.max_children = static_cast(int_maximum); + } + stats.total_children = total_children; + break; + } + default: + break; + } + return res; +} + +} diff --git a/src/Processors/Formats/Impl/ORC/Statistics.h b/src/Processors/Formats/Impl/ORC/Statistics.h new file mode 100644 index 0000000000000..c00c00e07b944 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Statistics.h @@ -0,0 +1,71 @@ +#pragma once + +#include + +#include + +namespace DB::ORC +{ + +/// Accumulates the statistics of a column for a row group, a stripe or the whole file, +/// in the same way as the reference implementation. +class StatisticsBuilder +{ +public: + explicit StatisticsBuilder(TypeKind kind_) : kind(kind_) {} + + void increase(UInt64 count) { number_of_values += count; } + void setHasNull() { has_null = true; } + + void updateInteger(Int64 value); + void updateDouble(double value); + void updateString(std::string_view value); + void updateBinary(size_t length); + void updateBoolean(bool value); + void updateDate(Int32 value); + void updateDecimal(Int128 value, Int32 scale); + /// `millis` since the epoch in UTC, `nanos` is the sub-millisecond part. + void updateTimestamp(Int64 millis, Int32 nanos); + void updateCollection(UInt64 children); + + void merge(const StatisticsBuilder & other); + void reset(); + + Proto::ColumnStatistics toProto() const; + +private: + TypeKind kind; + UInt64 number_of_values = 0; + bool has_null = false; + bool has_minimum = false; + + Int64 int_minimum = 0; + Int64 int_maximum = 0; + Int64 int_sum = 0; + bool has_int_sum = true; + + double double_minimum = 0; + double double_maximum = 0; + double double_sum = 0; + + std::string string_minimum; + std::string string_maximum; + UInt64 total_length = 0; + + UInt64 true_count = 0; + + Int128 decimal_minimum = 0; + Int32 decimal_minimum_scale = 0; + Int128 decimal_maximum = 0; + Int32 decimal_maximum_scale = 0; + Int128 decimal_sum = 0; + Int32 decimal_sum_scale = 0; + bool has_decimal_sum = true; + + Int32 minimum_nanos = 0; + Int32 maximum_nanos = 999999; + + UInt64 total_children = 0; +}; + +} diff --git a/src/Processors/Formats/Impl/ORC/Stream.cpp b/src/Processors/Formats/Impl/ORC/Stream.cpp new file mode 100644 index 0000000000000..275e75915dca2 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Stream.cpp @@ -0,0 +1,248 @@ +#include + +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int INCORRECT_DATA; +} +} + +namespace DB::ORC +{ + +UInt64 PositionProvider::next() +{ + if (index >= positions.size()) + throw Exception(ErrorCodes::INCORRECT_DATA, "Not enough positions in a row index entry of ORC file"); + return positions[index++]; +} + + +InputByteStream::InputByteStream(std::string_view data_, CompressionKind compression_, size_t block_size_, std::string name_) + : data(data_), length(data_.size()), compression(compression_), block_size(block_size_), name(std::move(name_)) +{ +} + +InputByteStream::InputByteStream(RangeLoader loader_, UInt64 length_, CompressionKind compression_, size_t block_size_, std::string name_) + : length(length_), loader(std::move(loader_)), compression(compression_), block_size(block_size_), name(std::move(name_)) +{ +} + +std::string_view InputByteStream::getRange(UInt64 offset, UInt64 size) +{ + if (!loader) + return data.substr(offset, size); + + if (offset < window_offset || offset + size > window_offset + data.size()) + { + UInt64 load_size = std::min(length - offset, std::max(size, LAZY_READ_SIZE)); + auto range = loader(offset, load_size); + if (range.data.size() != load_size) + throw Exception(ErrorCodes::INCORRECT_DATA, "Loaded {} bytes instead of {} from ORC stream {}", range.data.size(), load_size, name); + window_holder = std::move(range.holder); + data = range.data; + window_offset = offset; + } + return data.substr(offset - window_offset, size); +} + +bool InputByteStream::nextChunk() +{ + while (next_chunk_offset < length) + { + if (compression == CompressionKind::NONE) + { + UInt64 size = length - next_chunk_offset; + if (loader) + size = std::min(size, LAZY_READ_SIZE); + pos = getRange(next_chunk_offset, size).data(); + end = pos + size; + next_chunk_offset += size; + return true; + } + + if (length - next_chunk_offset < COMPRESSION_HEADER_SIZE) + throw Exception(ErrorCodes::INCORRECT_DATA, "Truncated compression chunk header in ORC stream {}", name); + + const UInt8 * header = reinterpret_cast(getRange(next_chunk_offset, COMPRESSION_HEADER_SIZE).data()); + UInt32 header_value = header[0] | (header[1] << 8) | (header[2] << 16); + bool is_original = header_value & 1; + size_t chunk_length = header_value >> 1; + size_t chunk_offset = next_chunk_offset + COMPRESSION_HEADER_SIZE; + + if (chunk_length > length - chunk_offset) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Compression chunk of size {} at offset {} goes beyond the end of ORC stream {} of size {}", + chunk_length, next_chunk_offset, name, length); + + next_chunk_offset = chunk_offset + chunk_length; + const char * chunk = getRange(chunk_offset, chunk_length).data(); + + if (is_original) + { + pos = chunk; + end = chunk + chunk_length; + } + else + { + /// The chunk length is at most 8 MiB, so the bound does not overflow. + size_t capacity = std::min(block_size, maxDecompressedSize(compression, chunk, chunk_length)); + if (decompressed.size() < capacity) + decompressed.resize(capacity); + size_t size = decompressChunk(compression, chunk, chunk_length, decompressed.data(), capacity); + pos = decompressed.data(); + end = decompressed.data() + size; + } + + if (pos != end) + return true; + } + + pos = end = nullptr; + return false; +} + +void InputByteStream::readBytes(char * to, size_t n) +{ + while (n) + { + if (pos == end && !nextChunk()) + throwUnexpectedEnd(); + size_t count = std::min(n, static_cast(end - pos)); + memcpy(to, pos, count); + pos += count; + to += count; + n -= count; + } +} + +void InputByteStream::skipBytes(size_t n) +{ + while (n) + { + if (pos == end && !nextChunk()) + throwUnexpectedEnd(); + size_t count = std::min(n, static_cast(end - pos)); + pos += count; + n -= count; + } +} + +void InputByteStream::seek(PositionProvider & positions) +{ + if (compression == CompressionKind::NONE) + { + UInt64 offset = positions.next(); + if (offset > length) + throw Exception(ErrorCodes::INCORRECT_DATA, "Seek position {} is beyond the end of ORC stream {} of size {}", offset, name, length); + next_chunk_offset = offset; + pos = end = nullptr; + nextChunk(); + return; + } + + UInt64 chunk_offset = positions.next(); + UInt64 uncompressed_offset = positions.next(); + if (chunk_offset > length) + throw Exception(ErrorCodes::INCORRECT_DATA, "Seek position {} is beyond the end of ORC stream {} of size {}", chunk_offset, name, length); + + next_chunk_offset = chunk_offset; + pos = end = nullptr; + if (!nextChunk()) + { + if (uncompressed_offset != 0) + throw Exception(ErrorCodes::INCORRECT_DATA, "Seek position {} is beyond the end of ORC stream {}", uncompressed_offset, name); + return; + } + + if (uncompressed_offset > static_cast(end - pos)) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Seek position {} is beyond the end of the compression chunk of size {} in ORC stream {}", + uncompressed_offset, end - pos, name); + pos += uncompressed_offset; +} + +void InputByteStream::throwUnexpectedEnd() const +{ + throw Exception(ErrorCodes::INCORRECT_DATA, "Unexpected end of ORC stream {}", name); +} + + +OutputByteStream::OutputByteStream(Compressor * compressor_, size_t block_size_) + : compressor(compressor_), block_size(block_size_) +{ + buffer.reserve(std::min(block_size, 64 * 1024)); +} + +void OutputByteStream::write(const char * data, size_t size) +{ + while (size) + { + if (buffer.size() == block_size) + flushChunk(); + size_t count = std::min(size, block_size - buffer.size()); + buffer.append(data, count); + data += count; + size -= count; + } +} + +void OutputByteStream::recordPosition(std::vector & positions) const +{ + if (compressor) + { + positions.push_back(output.size()); + positions.push_back(buffer.size()); + } + else + positions.push_back(output.size() + buffer.size()); +} + +void OutputByteStream::flushChunk() +{ + if (buffer.empty()) + return; + + if (!compressor) + { + output.append(buffer); + buffer.clear(); + return; + } + + size_t header_offset = output.size(); + output.append(COMPRESSION_HEADER_SIZE, '\0'); + size_t compressed_size = compressor->compress(buffer.data(), buffer.size(), output); + + UInt32 header_value = 0; + if (compressed_size) + header_value = static_cast(compressed_size << 1); + else + { + output.append(buffer); + header_value = static_cast((buffer.size() << 1) | 1); + } + + output[header_offset] = static_cast(header_value & 0xFF); + output[header_offset + 1] = static_cast((header_value >> 8) & 0xFF); + output[header_offset + 2] = static_cast((header_value >> 16) & 0xFF); + buffer.clear(); +} + +size_t OutputByteStream::finish() +{ + flushChunk(); + return output.size(); +} + +void OutputByteStream::reset() +{ + buffer.clear(); + output.clear(); +} + +} diff --git a/src/Processors/Formats/Impl/ORC/Stream.h b/src/Processors/Formats/Impl/ORC/Stream.h new file mode 100644 index 0000000000000..44d209d38321e --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Stream.h @@ -0,0 +1,140 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +namespace DB::ORC +{ + +/// A sequence of positions from a row index entry, consumed by the streams and the decoders of a +/// column in their order. +class PositionProvider +{ +public: + explicit PositionProvider(const std::vector & positions_) : positions(positions_) {} + UInt64 next(); + +private: + const std::vector & positions; + size_t index = 0; +}; + +/// A piece of a stream loaded from the file. `data` is valid while `holder` is alive. +struct LoadedRange +{ + std::shared_ptr holder; + std::string_view data; +}; + +/// Loads the range `(offset, length)` of a stream from the file. +using RangeLoader = std::function; + +/// A stream of an ORC column, such as PRESENT or DATA, decompressed chunk by chunk. The decoders +/// read from the buffer `[pos, end)` of the current chunk directly. +class InputByteStream +{ +public: + /// The whole stream in memory. `data` must outlive the stream. + InputByteStream(std::string_view data_, CompressionKind compression_, size_t block_size_, std::string name_); + /// The stream of `length_` bytes is loaded from the file on demand, by `loader_`, a window of + /// at least `LAZY_READ_SIZE` bytes at a time. Used when only some row groups are read. + InputByteStream(RangeLoader loader_, UInt64 length_, CompressionKind compression_, size_t block_size_, std::string name_); + + static constexpr size_t LAZY_READ_SIZE = 1024 * 1024; + + /// Loads the next chunk. Returns false at the end of the stream. + bool nextChunk(); + + bool eof() + { + return pos == end && !nextChunk(); + } + + UInt8 readByte() + { + if (pos == end && !nextChunk()) + throwUnexpectedEnd(); + return static_cast(*pos++); + } + + void readBytes(char * to, size_t n); + void skipBytes(size_t n); + + /// Moves to the position from the row index: one value (the byte offset) for an uncompressed + /// stream, two values (the offset of the compressed chunk and the offset in its decompressed + /// data) for a compressed one. + void seek(PositionProvider & positions); + + [[noreturn]] void throwUnexpectedEnd() const; + + const std::string & getName() const { return name; } + + const char * pos = nullptr; + const char * end = nullptr; + +private: + /// Returns the range `[offset, offset + length)` of the stream. It is valid until the next call. + std::string_view getRange(UInt64 offset, UInt64 length); + + /// The stream in memory, or the currently loaded window of a lazily loaded stream. + std::string_view data; + UInt64 length; + RangeLoader loader; + /// For a lazily loaded stream: the offset of `data` in the stream, and its owner. + UInt64 window_offset = 0; + std::shared_ptr window_holder; + + CompressionKind compression; + size_t block_size; + std::string name; + + /// The offset of the next chunk to read in `data`. + size_t next_chunk_offset = 0; + PODArray decompressed; +}; + +/// Accumulates a stream of an ORC column, compressing it chunk by chunk. +class OutputByteStream +{ +public: + /// `compressor` can be nullptr, which means no compression. + OutputByteStream(Compressor * compressor_, size_t block_size_); + + void writeByte(UInt8 byte) + { + if (buffer.size() == block_size) + flushChunk(); + buffer.push_back(static_cast(byte)); + } + + void write(const char * data, size_t size); + + /// Appends the current position to `positions`, in the format that `InputByteStream::seek` expects. + void recordPosition(std::vector & positions) const; + + /// Finishes the stream and returns its size in bytes. + size_t finish(); + + const std::string & getOutput() const { return output; } + + /// Clears the stream to start a new stripe. + void reset(); + + size_t estimateSize() const { return output.size() + buffer.size(); } + +private: + void flushChunk(); + + Compressor * compressor; + size_t block_size; + std::string buffer; + std::string output; +}; + +} diff --git a/src/Processors/Formats/Impl/ORC/Type.cpp b/src/Processors/Formats/Impl/ORC/Type.cpp new file mode 100644 index 0000000000000..b716b8c9ccb1e --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Type.cpp @@ -0,0 +1,380 @@ +#include + +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int INCORRECT_DATA; + extern const int UNKNOWN_TYPE; + extern const int LOGICAL_ERROR; +} +} + +namespace DB::ORC +{ + +namespace +{ + +bool isUnquotedFieldName(const std::string & name) +{ + for (char c : name) + if (!isalnum(static_cast(c)) && c != '_') + return false; + return true; +} + +std::string algorithmToString(UInt32 algorithm) +{ + switch (algorithm) + { + case 0: return "spherical"; + case 1: return "vincenty"; + case 2: return "thomas"; + case 3: return "andoyer"; + case 4: return "karney"; + default: return std::to_string(algorithm); + } +} + +/// The same limit as in the reference implementation. +constexpr size_t MAX_TYPE_NESTING_DEPTH = 2000; + +std::unique_ptr convertType(const std::vector & types, UInt64 index, UInt64 & next_id, size_t depth) +{ + checkStackSize(); + + if (depth > MAX_TYPE_NESTING_DEPTH) + throw Exception(ErrorCodes::INCORRECT_DATA, "ORC file footer is corrupt: type nesting depth exceeds the limit of {}", MAX_TYPE_NESTING_DEPTH); + + /// The types are listed in pre-order and the column id of each type is its index in the list. + if (index != next_id || index >= types.size()) + throw Exception(ErrorCodes::INCORRECT_DATA, + "ORC file footer is corrupt: the types are not a tree listed in pre-order (type {} is referenced at position {})", + index, next_id); + ++next_id; + + const auto & proto = types[index]; + if (proto.kind > GEOGRAPHY) + throw Exception(ErrorCodes::UNKNOWN_TYPE, "Unknown ORC type kind {}", proto.kind); + + auto kind = static_cast(proto.kind); + auto result = std::make_unique(kind); + + switch (kind) + { + case LIST: + if (proto.subtypes.size() != 1) + throw Exception(ErrorCodes::INCORRECT_DATA, "Illegal ORC LIST type that doesn't contain one subtype"); + break; + case MAP: + if (proto.subtypes.size() != 2) + throw Exception(ErrorCodes::INCORRECT_DATA, "Illegal ORC MAP type that doesn't contain two subtypes"); + break; + case UNION: + if (proto.subtypes.empty()) + throw Exception(ErrorCodes::INCORRECT_DATA, "Illegal ORC UNION type that doesn't contain any subtypes"); + /// The tags of a union are bytes. + if (proto.subtypes.size() > 256) + throw Exception(ErrorCodes::INCORRECT_DATA, "Illegal ORC UNION type with {} subtypes, more than 256", proto.subtypes.size()); + break; + case STRUCT: + if (proto.subtypes.size() > proto.field_names.size()) + throw Exception(ErrorCodes::INCORRECT_DATA, "Illegal ORC STRUCT type that contains less field names than subtypes"); + break; + default: + if (!proto.subtypes.empty()) + throw Exception(ErrorCodes::INCORRECT_DATA, "Illegal ORC type {} with subtypes", proto.kind); + break; + } + + if (kind == CHAR || kind == VARCHAR) + result->setMaximumLength(proto.maximum_length.value_or(0)); + if (kind == DECIMAL) + result->setPrecisionAndScale(proto.precision.value_or(0), proto.scale.value_or(0)); + if (kind == GEOMETRY || kind == GEOGRAPHY) + result->setGeospatial(proto.crs.value_or("OGC:CRS84"), proto.algorithm.value_or(0)); + + for (size_t i = 0; i < proto.subtypes.size(); ++i) + { + auto child = convertType(types, proto.subtypes[i], next_id, depth + 1); + if (kind == STRUCT) + result->addStructField(proto.field_names[i], std::move(child)); + else + result->addChild(std::move(child)); + } + + for (const auto & attribute : proto.attributes) + result->setAttribute(attribute.key, attribute.value); + + return result; +} + +void selectChildren(std::vector & selected, const Type & type) +{ + for (UInt64 c = type.getColumnId(); c <= type.getMaximumColumnId(); ++c) + selected[c] = true; +} + +bool selectParents(std::vector & selected, const Type & type) +{ + size_t id = type.getColumnId(); + bool result = selected[id]; + size_t num_selected_subtypes = 0; + for (UInt64 c = 0; c < type.getSubtypeCount(); ++c) + { + if (selectParents(selected, *type.getSubtype(c))) + { + result = true; + ++num_selected_subtypes; + } + } + selected[id] = result; + + /// The branches of a union are either all selected or all not selected. + if (type.getKind() == UNION && result && num_selected_subtypes > 0 && num_selected_subtypes < type.getSubtypeCount()) + for (UInt64 c = 0; c < type.getSubtypeCount(); ++c) + selectChildren(selected, *type.getSubtype(c)); + + return result; +} + +} + + +std::string Type::toString() const +{ + switch (kind) + { + case BOOLEAN: return "boolean"; + case BYTE: return "tinyint"; + case SHORT: return "smallint"; + case INT: return "int"; + case LONG: return "bigint"; + case FLOAT: return "float"; + case DOUBLE: return "double"; + case STRING: return "string"; + case BINARY: return "binary"; + case TIMESTAMP: return "timestamp"; + case TIMESTAMP_INSTANT: return "timestamp with local time zone"; + case LIST: return "array<" + (subtypes.empty() ? std::string("void") : subtypes[0]->toString()) + ">"; + case MAP: + return "map<" + (subtypes.empty() ? std::string("void") : subtypes[0]->toString()) + "," + + (subtypes.size() < 2 ? std::string("void") : subtypes[1]->toString()) + ">"; + case STRUCT: + { + std::string result = "struct<"; + for (size_t i = 0; i < subtypes.size(); ++i) + { + if (i != 0) + result += ","; + if (isUnquotedFieldName(field_names[i])) + result += field_names[i]; + else + { + result += "`"; + for (char c : field_names[i]) + { + if (c == '`') + result += "``"; + else + result += c; + } + result += "`"; + } + result += ":"; + result += subtypes[i]->toString(); + } + result += ">"; + return result; + } + case UNION: + { + std::string result = "uniontype<"; + for (size_t i = 0; i < subtypes.size(); ++i) + { + if (i != 0) + result += ","; + result += subtypes[i]->toString(); + } + result += ">"; + return result; + } + case DECIMAL: return "decimal(" + std::to_string(precision) + "," + std::to_string(scale) + ")"; + case DATE: return "date"; + case VARCHAR: return "varchar(" + std::to_string(maximum_length) + ")"; + case CHAR: return "char(" + std::to_string(maximum_length) + ")"; + case GEOMETRY: return "geometry(" + crs + ")"; + case GEOGRAPHY: return "geography(" + crs + "," + algorithmToString(algorithm) + ")"; + } + return "unknown"; +} + +Type * Type::addStructField(const std::string & name, std::unique_ptr field) +{ + field_names.push_back(name); + return addChild(std::move(field)); +} + +Type * Type::addChild(std::unique_ptr child) +{ + subtypes.push_back(std::move(child)); + return subtypes.back().get(); +} + +UInt64 Type::assignIds(UInt64 start) +{ + column_id = start; + UInt64 next = start + 1; + for (auto & subtype : subtypes) + next = subtype->assignIds(next); + maximum_column_id = next - 1; + return next; +} + +void Type::toProto(std::vector & out) const +{ + size_t index = out.size(); + if (index != column_id) + throw Exception(ErrorCodes::LOGICAL_ERROR, "ORC type ids are not assigned in pre-order"); + out.emplace_back(); + { + auto & proto = out.back(); + proto.kind = kind; + for (const auto & subtype : subtypes) + proto.subtypes.push_back(subtype->getColumnId()); + proto.field_names = field_names; + if (kind == CHAR || kind == VARCHAR) + proto.maximum_length = static_cast(maximum_length); + if (kind == DECIMAL) + { + proto.precision = static_cast(precision); + proto.scale = static_cast(scale); + } + for (const auto & [key, value] : attributes) + proto.attributes.push_back({key, value}); + } + /// `out` may be reallocated by the recursive calls, so do not keep the reference. + for (const auto & subtype : subtypes) + subtype->toProto(out); +} + +std::unique_ptr Type::createPrimitiveType(TypeKind kind) +{ + return std::make_unique(kind); +} + +std::unique_ptr Type::createCharType(TypeKind kind, UInt64 maximum_length) +{ + auto result = std::make_unique(kind); + result->maximum_length = maximum_length; + return result; +} + +std::unique_ptr Type::createDecimalType(UInt64 precision, UInt64 scale) +{ + auto result = std::make_unique(DECIMAL); + result->precision = precision; + result->scale = scale; + return result; +} + +std::unique_ptr Type::createStructType() +{ + return std::make_unique(STRUCT); +} + +std::unique_ptr Type::createListType(std::unique_ptr element) +{ + auto result = std::make_unique(LIST); + result->addChild(std::move(element)); + return result; +} + +std::unique_ptr Type::createMapType(std::unique_ptr key, std::unique_ptr value) +{ + auto result = std::make_unique(MAP); + result->addChild(std::move(key)); + result->addChild(std::move(value)); + return result; +} + +std::unique_ptr Type::createUnionType() +{ + return std::make_unique(UNION); +} + +std::unique_ptr Type::fromProto(const std::vector & types) +{ + if (types.empty()) + throw Exception(ErrorCodes::INCORRECT_DATA, "ORC file footer is corrupt: it has no types"); + UInt64 next_id = 0; + auto root = convertType(types, 0, next_id, 0); + if (next_id != types.size()) + throw Exception(ErrorCodes::INCORRECT_DATA, + "ORC file footer is corrupt: {} of {} types are not reachable from the root type", types.size() - next_id, types.size()); + root->assignIds(0); + return root; +} + +std::unique_ptr Type::buildSelectedType(const Type & type, const std::vector & selected) +{ + if (!selected.at(type.column_id)) + return nullptr; + + auto result = std::make_unique(type.kind); + result->maximum_length = type.maximum_length; + result->precision = type.precision; + result->scale = type.scale; + result->crs = type.crs; + result->algorithm = type.algorithm; + result->attributes = type.attributes; + result->column_id = type.column_id; + result->maximum_column_id = type.maximum_column_id; + + for (size_t i = 0; i < type.subtypes.size(); ++i) + { + auto child = buildSelectedType(*type.subtypes[i], selected); + if (!child) + continue; + if (type.kind == STRUCT) + result->addStructField(type.field_names[i], std::move(child)); + else + result->addChild(std::move(child)); + } + return result; +} + +std::vector selectColumnsByTypeIds(const Type & root, const std::vector & type_ids) +{ + std::vector selected(root.getMaximumColumnId() + 1, false); + + /// Find the types by ids with a walk over the tree. + std::vector by_id(selected.size()); + std::vector stack{&root}; + while (!stack.empty()) + { + const Type * type = stack.back(); + stack.pop_back(); + by_id[type->getColumnId()] = type; + for (UInt64 i = 0; i < type->getSubtypeCount(); ++i) + stack.push_back(type->getSubtype(i)); + } + + for (UInt64 id : type_ids) + { + if (id >= selected.size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid ORC type id {} selected out of {}", id, selected.size()); + selectChildren(selected, *by_id[id]); + } + + selectParents(selected, root); + selected[0] = true; + return selected; +} + +} diff --git a/src/Processors/Formats/Impl/ORC/Type.h b/src/Processors/Formats/Impl/ORC/Type.h new file mode 100644 index 0000000000000..9a30f4455076c --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Type.h @@ -0,0 +1,114 @@ +#pragma once + +#include + +#include +#include +#include +#include + +namespace DB::ORC +{ + +enum TypeKind : UInt32 +{ + BOOLEAN = 0, + BYTE = 1, + SHORT = 2, + INT = 3, + LONG = 4, + FLOAT = 5, + DOUBLE = 6, + STRING = 7, + BINARY = 8, + TIMESTAMP = 9, + LIST = 10, + MAP = 11, + STRUCT = 12, + UNION = 13, + DECIMAL = 14, + DATE = 15, + VARCHAR = 16, + CHAR = 17, + TIMESTAMP_INSTANT = 18, + GEOMETRY = 19, + GEOGRAPHY = 20, +}; + +/// A node of the type tree of an ORC file. Every node is a column of the file; the column ids are +/// assigned in pre-order, so the subtree of a column `c` occupies the ids `[c, c.getMaximumColumnId()]`. +class Type +{ +public: + explicit Type(TypeKind kind_) : kind(kind_) {} + + TypeKind getKind() const { return kind; } + UInt64 getColumnId() const { return column_id; } + UInt64 getMaximumColumnId() const { return maximum_column_id; } + UInt64 getSubtypeCount() const { return subtypes.size(); } + const Type * getSubtype(UInt64 i) const { return subtypes.at(i).get(); } + /// Only for STRUCT. + const std::string & getFieldName(UInt64 i) const { return field_names.at(i); } + /// Only for CHAR and VARCHAR. + UInt64 getMaximumLength() const { return maximum_length; } + /// Only for DECIMAL. + UInt64 getPrecision() const { return precision; } + UInt64 getScale() const { return scale; } + + bool hasAttributeKey(const std::string & key) const { return attributes.contains(key); } + const std::string & getAttributeValue(const std::string & key) const { return attributes.at(key); } + void setAttribute(const std::string & key, const std::string & value) { attributes[key] = value; } + const std::map & getAttributes() const { return attributes; } + + /// The type name in the same format as in the reference implementation (Hive type names). + std::string toString() const; + + void setMaximumLength(UInt64 value) { maximum_length = value; } + void setPrecisionAndScale(UInt64 precision_, UInt64 scale_) { precision = precision_; scale = scale_; } + void setGeospatial(std::string crs_, UInt32 algorithm_) { crs = std::move(crs_); algorithm = algorithm_; } + + Type * addStructField(const std::string & name, std::unique_ptr field); + Type * addChild(std::unique_ptr child); + + /// Assigns the column ids in pre-order, starting from `start`. Returns the next free id. + UInt64 assignIds(UInt64 start); + + /// Appends the types of this subtree to `out` in pre-order. The ids must be assigned. + void toProto(std::vector & out) const; + + static std::unique_ptr createPrimitiveType(TypeKind kind); + static std::unique_ptr createCharType(TypeKind kind, UInt64 maximum_length); + static std::unique_ptr createDecimalType(UInt64 precision, UInt64 scale); + static std::unique_ptr createStructType(); + static std::unique_ptr createListType(std::unique_ptr element); + static std::unique_ptr createMapType(std::unique_ptr key, std::unique_ptr value); + static std::unique_ptr createUnionType(); + + /// Builds the type tree from the types in the file footer. Throws `INCORRECT_DATA` if they do + /// not form a tree with the ids in pre-order. + static std::unique_ptr fromProto(const std::vector & types); + + /// Builds a copy of the tree with only the selected columns (by id). The parent of every + /// selected column must also be selected. Returns nullptr if the root is not selected. + static std::unique_ptr buildSelectedType(const Type & type, const std::vector & selected); + +private: + TypeKind kind; + UInt64 column_id = 0; + UInt64 maximum_column_id = 0; + std::vector> subtypes; + std::vector field_names; + UInt64 maximum_length = 0; + UInt64 precision = 0; + UInt64 scale = 0; + std::string crs; + UInt32 algorithm = 0; + std::map attributes; +}; + +/// The column selection with the same semantics as in the reference implementation: +/// selecting a column selects its whole subtree, and then all parents of the selected columns +/// are selected. If only some branches of a UNION are selected, all its branches are selected. +std::vector selectColumnsByTypeIds(const Type & root, const std::vector & type_ids); + +} diff --git a/src/Processors/Formats/Impl/ORC/Vector.cpp b/src/Processors/Formats/Impl/ORC/Vector.cpp new file mode 100644 index 0000000000000..ce7b5cdac8de7 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Vector.cpp @@ -0,0 +1,143 @@ +#include + +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; + extern const int INCORRECT_DATA; +} +} + +namespace DB::ORC +{ + +ColumnPtr decodeDictionary(const ColumnString & dictionary, const Int64 * index, const char * not_null, size_t num_values) +{ + const size_t dictionary_size = dictionary.size(); + const auto & dictionary_offsets = dictionary.getOffsets(); + const auto & dictionary_chars = dictionary.getChars(); + + auto result = ColumnString::create(); + auto & chars = result->getChars(); + auto & offsets = result->getOffsets(); + offsets.resize(num_values); + + size_t total = 0; + for (size_t i = 0; i < num_values; ++i) + { + if (!not_null || not_null[i]) + { + Int64 entry = index[i]; + if (entry < 0 || static_cast(entry) >= dictionary_size) + throw Exception(ErrorCodes::INCORRECT_DATA, "ORC dictionary index {} at row {} is out of range [0, {})", entry, i, dictionary_size); + total += dictionary_offsets[entry] - dictionary_offsets[entry - 1]; + } + offsets[i] = total; + } + + chars.resize(total); + UInt8 * out = chars.data(); + for (size_t i = 0; i < num_values; ++i) + { + if (!not_null || not_null[i]) + { + size_t begin = dictionary_offsets[index[i] - 1]; + size_t size = dictionary_offsets[index[i]] - begin; + memcpy(out, dictionary_chars.data() + begin, size); + out += size; + } + } + return result; +} + +ColumnPtr getStringColumn(const StringVectorBatch & batch) +{ + if (!batch.isEncoded) + return batch.column; + + const auto & encoded = typeid_cast(batch); + return decodeDictionary( + assert_cast(*encoded.dictionary->column), + encoded.index.data(), + batch.hasNulls ? batch.notNull.data() : nullptr, + batch.numElements); +} + +std::unique_ptr createRowBatch(const Type & type, bool encoded) +{ + checkStackSize(); + + switch (type.getKind()) + { + case BOOLEAN: + return std::make_unique(); + case BYTE: + return std::make_unique(); + case SHORT: + return std::make_unique(); + case INT: + return std::make_unique(); + case LONG: + case DATE: + return std::make_unique(); + case FLOAT: + return std::make_unique(); + case DOUBLE: + return std::make_unique(); + case STRING: + case BINARY: + case CHAR: + case VARCHAR: + case GEOMETRY: + case GEOGRAPHY: + if (encoded) + return std::make_unique(); + return std::make_unique(); + case TIMESTAMP: + case TIMESTAMP_INSTANT: + return std::make_unique(); + case STRUCT: + { + auto result = std::make_unique(); + for (UInt64 i = 0; i < type.getSubtypeCount(); ++i) + result->addField(createRowBatch(*type.getSubtype(i), encoded)); + return result; + } + case LIST: + { + auto result = std::make_unique(); + if (type.getSubtypeCount()) + result->elements = createRowBatch(*type.getSubtype(0), encoded); + return result; + } + case MAP: + { + auto result = std::make_unique(); + if (type.getSubtypeCount() > 0) + result->keys = createRowBatch(*type.getSubtype(0), encoded); + if (type.getSubtypeCount() > 1) + result->elements = createRowBatch(*type.getSubtype(1), encoded); + return result; + } + case UNION: + { + auto result = std::make_unique(); + for (UInt64 i = 0; i < type.getSubtypeCount(); ++i) + result->addChild(createRowBatch(*type.getSubtype(i), encoded)); + return result; + } + case DECIMAL: + if (type.getPrecision() == 0 || type.getPrecision() > 18) + return std::make_unique(); + return std::make_unique(); + } + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "ORC type {} is not supported", type.toString()); +} + +} diff --git a/src/Processors/Formats/Impl/ORC/Vector.h b/src/Processors/Formats/Impl/ORC/Vector.h new file mode 100644 index 0000000000000..1e44a80a61d11 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Vector.h @@ -0,0 +1,174 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include + +/// Batches of column values filled by the reader. The values of the primitive types are decoded +/// directly into ClickHouse columns, which the conversion to the requested types takes without +/// copying. The structure follows the reference implementation: a batch has the null mask of its +/// values, and the batches of the compound types have the batches of their children. + +namespace DB::ORC +{ + +struct ColumnVectorBatch +{ + ColumnVectorBatch() = default; + virtual ~ColumnVectorBatch() = default; + + ColumnVectorBatch(const ColumnVectorBatch &) = delete; + ColumnVectorBatch & operator=(const ColumnVectorBatch &) = delete; + + /// Prepares the batch for `num_values` values. + void reset(size_t num_values) + { + numElements = num_values; + notNull.resize(num_values); + } + + size_t numElements = 0; + /// Non-zero for the non-null values, valid only if `hasNulls`. + PaddedPODArray notNull; + bool hasNulls = false; + /// For string batches read with lazy dictionary decoding. + bool isEncoded = false; +}; + +/// The values of a primitive type in a ClickHouse column (`ColumnVector` of `T`). The null values are zeros. +template +struct NumericVectorBatch : public ColumnVectorBatch +{ + using Container = PaddedPODArray; + + /// Creates a new column for the next values: the previous column can be still referenced. + Container & prepareColumn(size_t num_values) + { + auto new_column = ColumnVector::create(num_values); + auto & data = new_column->getData(); + column = std::move(new_column); + return data; + } + + const Container & getData() const { return assert_cast &>(*column).getData(); } + + ColumnPtr column; +}; + +/// BOOLEAN: 0 or 1. +using BoolVectorBatch = NumericVectorBatch; +using ByteVectorBatch = NumericVectorBatch; +using ShortVectorBatch = NumericVectorBatch; +using IntVectorBatch = NumericVectorBatch; +/// LONG and DATE (days since the epoch, not narrowed, so that the range can be checked). +using LongVectorBatch = NumericVectorBatch; +using FloatVectorBatch = NumericVectorBatch; +using DoubleVectorBatch = NumericVectorBatch; + +/// The values in a `ColumnString`. The null values are empty strings. +struct StringVectorBatch : public ColumnVectorBatch +{ + ColumnPtr column; +}; + +struct StringDictionary +{ + /// The entries of the dictionary. + ColumnPtr column; +}; + +/// A string batch with the dictionary indices, without decoding the values. `column` is not set. +struct EncodedStringVectorBatch : public StringVectorBatch +{ + std::shared_ptr dictionary; + PaddedPODArray index; +}; + +/// Builds a `ColumnString` from the dictionary entries by the indices (the null values are empty strings). +/// Throws `INCORRECT_DATA` if an index is out of range. +ColumnPtr decodeDictionary(const ColumnString & dictionary, const Int64 * index, const char * not_null, size_t num_values); + +/// Returns the values of a string batch as a `ColumnString`, decoding the dictionary for an encoded batch. +ColumnPtr getStringColumn(const StringVectorBatch & batch); + +struct StructVectorBatch : public ColumnVectorBatch +{ + void addField(std::unique_ptr field) + { + fields.push_back(field.get()); + owned_fields.push_back(std::move(field)); + } + + std::vector fields; + +private: + std::vector> owned_fields; +}; + +/// LIST and MAP: the end offsets of the values in the children, as in `ColumnArray`. +struct ListVectorBatch : public ColumnVectorBatch +{ + /// `ColumnArray::ColumnOffsets`. + ColumnPtr offsets; + std::unique_ptr elements; +}; + +struct MapVectorBatch : public ColumnVectorBatch +{ + /// `ColumnArray::ColumnOffsets`. + ColumnPtr offsets; + std::unique_ptr keys; + std::unique_ptr elements; +}; + +struct UnionVectorBatch : public ColumnVectorBatch +{ + void addChild(std::unique_ptr child) + { + children.push_back(child.get()); + owned_children.push_back(std::move(child)); + } + + /// The branch of each value. + PaddedPODArray tags; + /// The position of each value in its branch. + PaddedPODArray offsets; + std::vector children; + +private: + std::vector> owned_children; +}; + +/// For DECIMAL with the precision at most 18. +struct Decimal64VectorBatch : public ColumnVectorBatch +{ + Int32 precision = 0; + Int32 scale = 0; + PaddedPODArray values; +}; + +/// For DECIMAL with the precision above 18 or unspecified (zero). +struct Decimal128VectorBatch : public ColumnVectorBatch +{ + Int32 precision = 0; + Int32 scale = 0; + PaddedPODArray values; +}; + +struct TimestampVectorBatch : public ColumnVectorBatch +{ + /// Seconds since the epoch in UTC. + PaddedPODArray data; + PaddedPODArray nanoseconds; +}; + +/// Creates a batch tree matching the type. With `encoded`, the string columns get +/// `EncodedStringVectorBatch`, which the reader fills without decoding the dictionaries. +std::unique_ptr createRowBatch(const Type & type, bool encoded = false); + +} diff --git a/src/Processors/Formats/Impl/ORC/Writer.cpp b/src/Processors/Formats/Impl/ORC/Writer.cpp new file mode 100644 index 0000000000000..78bd4299eb5a1 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Writer.cpp @@ -0,0 +1,1492 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int LOGICAL_ERROR; + extern const int NOT_IMPLEMENTED; + extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE; +} +} + +namespace DB::ORC +{ + +struct WriterContext +{ + Compressor * compressor = nullptr; + size_t block_size = 0; + bool enable_index = false; + double dictionary_key_size_threshold = 0; + const DateLUTImpl * timezone = nullptr; + bool timezone_is_utc = true; + /// 2015-01-01 00:00:00 in the time zone of the writer, as seconds since the epoch. + Int64 epoch_offset = 0; +}; + +namespace +{ + +struct StreamData +{ + Proto::StreamKind kind; + UInt32 column; + std::string data; +}; + +std::string compressWhole(const WriterContext & context, const std::string & data) +{ + OutputByteStream stream(context.compressor, context.block_size); + stream.write(data.data(), data.size()); + stream.finish(); + return stream.getOutput(); +} + +} + +class ColumnWriter +{ +public: + ColumnWriter(const Type & type, WriterContext & context_) + : column_id(type.getColumnId()) + , context(context_) + , present_stream(context.compressor, context.block_size) + , present_encoder(present_stream) + , index_statistics(type.getKind()) + , stripe_statistics(type.getKind()) + , file_statistics(type.getKind()) + { + } + + virtual ~ColumnWriter() = default; + + /// Must be called by the most derived constructor, after the streams are created. + void init() + { + if (context.enable_index) + recordPositions(); + } + + /// Adds the values `[offset, offset + num_values)` of `column`, which is not `Nullable`. + /// `null_map` (1 for a null) and `parent_null_map` (1 for a null in the parent struct, for which + /// no value is written at all) are for these values only, so they start at the first value. + virtual void add(const IColumn & /*column*/, const UInt8 * null_map, size_t /*offset*/, size_t num_values, const UInt8 * parent_null_map) + { + addPresent(null_map, num_values, parent_null_map); + } + + /// Writes the PRESENT stream for the values. + void addPresent(const UInt8 * null_map, size_t num_values, const UInt8 * parent_null_map) + { + for (size_t i = 0; i < num_values; ++i) + { + if (parent_null_map && parent_null_map[i]) + continue; + present_encoder.write(!null_map || !null_map[i]); + } + if (null_map) + { + for (size_t i = 0; i < num_values && !has_null_value; ++i) + if (null_map[i]) + has_null_value = true; + } + } + + virtual void createRowIndexEntry() + { + entries.push_back(Entry{std::move(current_present_positions), std::move(current_data_positions), index_statistics.toProto()}); + current_present_positions.clear(); + current_data_positions.clear(); + stripe_statistics.merge(index_statistics); + index_statistics.reset(); + recordPositions(); + } + + virtual void mergeRowGroupStatsIntoStripeStats() + { + stripe_statistics.merge(index_statistics); + index_statistics.reset(); + } + + /// Encodes the deferred data of the stripe. + virtual void finalizeStripe() {} + + virtual void writeIndex(std::vector & streams) + { + Proto::RowIndex index; + for (const auto & entry : entries) + { + auto & proto_entry = index.entry.emplace_back(); + /// The positions of the PRESENT stream are written only if the stream is written. + if (has_null_value) + proto_entry.positions = entry.present_positions; + proto_entry.positions.insert(proto_entry.positions.end(), entry.data_positions.begin(), entry.data_positions.end()); + proto_entry.statistics = entry.statistics; + } + streams.push_back({Proto::StreamKind::ROW_INDEX, static_cast(column_id), compressWhole(context, index.serialize())}); + } + + virtual void flush(std::vector & streams) + { + present_encoder.flush(); + present_stream.finish(); + if (has_null_value) + streams.push_back({Proto::StreamKind::PRESENT, static_cast(column_id), present_stream.getOutput()}); + } + + virtual void getColumnEncodings(std::vector & encodings) const + { + encodings.push_back({Proto::ColumnEncodingKind::DIRECT, 0, {}}); + } + + virtual void getStripeStatistics(std::vector & stats) const + { + stats.push_back(stripe_statistics.toProto()); + } + + virtual void getFileStatistics(std::vector & stats) const + { + stats.push_back(file_statistics.toProto()); + } + + virtual void mergeStripeStatsIntoFileStats() + { + file_statistics.merge(stripe_statistics); + stripe_statistics.reset(); + } + + virtual size_t estimateSize() const + { + return present_stream.estimateSize(); + } + + /// Starts a new stripe. + virtual void reset() + { + present_stream.reset(); + has_null_value = false; + entries.clear(); + current_present_positions.clear(); + current_data_positions.clear(); + resetStreams(); + if (context.enable_index) + recordPositions(); + } + +protected: + /// Appends the positions of the data streams, in the order the reader seeks them. + virtual void recordDataPositions(std::vector & positions) = 0; + /// Clears the data streams for a new stripe. + virtual void resetStreams() = 0; + + /// The number of the non-null values, also updates the statistics with the count. + size_t countValues(const UInt8 * null_map, size_t num_values) + { + size_t count = num_values; + if (null_map) + { + count = 0; + for (size_t i = 0; i < num_values; ++i) + count += !null_map[i]; + } + index_statistics.increase(count); + if (count < num_values) + index_statistics.setHasNull(); + return count; + } + + struct Entry + { + std::vector present_positions; + std::vector data_positions; + Proto::ColumnStatistics statistics; + }; + + const UInt64 column_id; + WriterContext & context; + OutputByteStream present_stream; + BooleanRleEncoder present_encoder; + bool has_null_value = false; + StatisticsBuilder index_statistics; + StatisticsBuilder stripe_statistics; + StatisticsBuilder file_statistics; + std::vector entries; + +private: + void recordPositions() + { + present_encoder.recordPosition(current_present_positions); + recordDataPositions(current_data_positions); + } + + std::vector current_present_positions; + std::vector current_data_positions; +}; + +namespace +{ + +std::unique_ptr buildColumnWriter(const Type & type, const DataTypePtr & data_type, WriterContext & context); + +/// Adds the values of a column that can be `Nullable`. The values that are null in the parent +/// (`outer_null_map`, for the fields of a struct) are null as well. +void addColumn( + ColumnWriter & writer, const IColumn & column, const UInt8 * outer_null_map, size_t offset, size_t num_values, const UInt8 * parent_null_map) +{ + if (const auto * nullable = typeid_cast(&column)) + { + const UInt8 * null_map = nullable->getNullMapData().data() + offset; + if (!outer_null_map) + { + writer.add(nullable->getNestedColumn(), null_map, offset, num_values, parent_null_map); + return; + } + PaddedPODArray merged(num_values); + for (size_t i = 0; i < num_values; ++i) + merged[i] = null_map[i] | outer_null_map[i]; + writer.add(nullable->getNestedColumn(), merged.data(), offset, num_values, parent_null_map); + return; + } + writer.add(column, outer_null_map, offset, num_values, parent_null_map); +} + +/// The raw values of a column with fixed-size values. +template +const T * getRawValues(const IColumn & column) +{ + auto raw = column.getRawData(); + if (column.sizeOfValueIfFixed() != sizeof(T)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected column {} with values of size {} for ORC (expected {})", + column.getName(), column.sizeOfValueIfFixed(), sizeof(T)); + return reinterpret_cast(raw.data()); +} + +/// The value of the integer column (of any width up to 8 bytes) as a signed number of the same width, +/// as ORC does not have unsigned types. +Int64 readSignedValue(const char * data, size_t width, size_t row) +{ + switch (width) + { + case 1: return unalignedLoad(data + row); + case 2: return unalignedLoad(data + row * 2); + case 4: return unalignedLoad(data + row * 4); + default: return unalignedLoad(data + row * 8); + } +} + +/// The writers that have children call the same methods for them. +class CompoundColumnWriter : public ColumnWriter +{ +public: + using ColumnWriter::ColumnWriter; + + void createRowIndexEntry() override + { + ColumnWriter::createRowIndexEntry(); + for (auto & child : children) + child->createRowIndexEntry(); + } + + void mergeRowGroupStatsIntoStripeStats() override + { + ColumnWriter::mergeRowGroupStatsIntoStripeStats(); + for (auto & child : children) + child->mergeRowGroupStatsIntoStripeStats(); + } + + void finalizeStripe() override + { + for (auto & child : children) + child->finalizeStripe(); + } + + void writeIndex(std::vector & streams) override + { + ColumnWriter::writeIndex(streams); + for (auto & child : children) + child->writeIndex(streams); + } + + void flush(std::vector & streams) override + { + ColumnWriter::flush(streams); + flushOwnStreams(streams); + for (auto & child : children) + child->flush(streams); + } + + void getColumnEncodings(std::vector & encodings) const override + { + getOwnColumnEncoding(encodings); + for (const auto & child : children) + child->getColumnEncodings(encodings); + } + + void getStripeStatistics(std::vector & stats) const override + { + ColumnWriter::getStripeStatistics(stats); + for (const auto & child : children) + child->getStripeStatistics(stats); + } + + void getFileStatistics(std::vector & stats) const override + { + ColumnWriter::getFileStatistics(stats); + for (const auto & child : children) + child->getFileStatistics(stats); + } + + void mergeStripeStatsIntoFileStats() override + { + ColumnWriter::mergeStripeStatsIntoFileStats(); + for (auto & child : children) + child->mergeStripeStatsIntoFileStats(); + } + + size_t estimateSize() const override + { + size_t size = ColumnWriter::estimateSize() + estimateOwnSize(); + for (const auto & child : children) + size += child->estimateSize(); + return size; + } + + void reset() override + { + ColumnWriter::reset(); + for (auto & child : children) + (*child).reset(); + } + +protected: + virtual void flushOwnStreams(std::vector &) {} + virtual void getOwnColumnEncoding(std::vector & encodings) const + { + encodings.push_back({Proto::ColumnEncodingKind::DIRECT, 0, {}}); + } + virtual size_t estimateOwnSize() const { return 0; } + + std::vector> children; +}; + +class StructColumnWriter final : public CompoundColumnWriter +{ +public: + StructColumnWriter(const Type & type, const DataTypes & field_types, WriterContext & context_) + : CompoundColumnWriter(type, context_) + { + if (field_types.size() != type.getSubtypeCount()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "ORC struct {} has {} fields, the ClickHouse type has {}", + type.toString(), type.getSubtypeCount(), field_types.size()); + for (UInt64 i = 0; i < type.getSubtypeCount(); ++i) + children.push_back(buildColumnWriter(*type.getSubtype(i), field_types[i], context)); + init(); + } + + void add(const IColumn & column, const UInt8 * null_map, size_t offset, size_t num_values, const UInt8 * parent_null_map) override + { + const auto & tuple = assert_cast(column); + ColumnWriter::add(column, null_map, offset, num_values, parent_null_map); + /// The fields of the null values are null, and they are not written. + for (size_t i = 0; i < children.size(); ++i) + addColumn(*children[i], tuple.getColumn(i), null_map, offset, num_values, null_map); + countValues(null_map, num_values); + } + + /// Adds the rows of the root struct, which are the columns of the file. + void addRoot(const Columns & columns, size_t offset, size_t num_values) + { + if (columns.size() != children.size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected {} columns for ORC, got {}", children.size(), columns.size()); + addPresent(nullptr, num_values, nullptr); + for (size_t i = 0; i < children.size(); ++i) + addColumn(*children[i], *columns[i], nullptr, offset, num_values, nullptr); + countValues(nullptr, num_values); + } + +protected: + void recordDataPositions(std::vector &) override {} + void resetStreams() override {} +}; + +template +class CollectionColumnWriter final : public CompoundColumnWriter +{ +public: + CollectionColumnWriter(const Type & type, const DataTypes & child_types, WriterContext & context_) + : CompoundColumnWriter(type, context_) + , length_stream(context.compressor, context.block_size) + , length_encoder(length_stream, false) + { + if (child_types.size() != type.getSubtypeCount()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "ORC type {} does not match the ClickHouse type", type.toString()); + for (UInt64 i = 0; i < type.getSubtypeCount(); ++i) + children.push_back(buildColumnWriter(*type.getSubtype(i), child_types[i], context)); + init(); + } + + void add(const IColumn & column, const UInt8 * null_map, size_t offset, size_t num_values, const UInt8 * parent_null_map) override + { + const ColumnArray & array = is_map ? assert_cast(column).getNestedColumn() : assert_cast(column); + /// The offsets of `ColumnArray` are the end offsets, and the one before the first is zero. + const auto & offsets = array.getOffsets(); + + ColumnWriter::add(column, null_map, offset, num_values, parent_null_map); + + UInt64 element_offset = offsets[static_cast(offset) - 1]; + UInt64 total_elements = offsets[offset + num_values - 1] - element_offset; + + if constexpr (is_map) + { + const auto & key_value = assert_cast(array.getData()); + addColumn(*children[0], key_value.getColumn(0), nullptr, element_offset, total_elements, nullptr); + addColumn(*children[1], key_value.getColumn(1), nullptr, element_offset, total_elements, nullptr); + } + else + { + addColumn(*children[0], array.getData(), nullptr, element_offset, total_elements, nullptr); + } + + size_t count = 0; + for (size_t i = 0; i < num_values; ++i) + { + if (null_map && null_map[i]) + continue; + size_t row = offset + i; + UInt64 length = offsets[row] - offsets[static_cast(row) - 1]; + length_encoder.write(static_cast(length)); + index_statistics.updateCollection(length); + ++count; + } + index_statistics.increase(count); + if (count < num_values) + index_statistics.setHasNull(); + } + +protected: + void recordDataPositions(std::vector & positions) override + { + length_encoder.recordPosition(positions); + } + + void resetStreams() override + { + length_stream.reset(); + } + + void flushOwnStreams(std::vector & streams) override + { + length_encoder.flush(); + length_stream.finish(); + streams.push_back({Proto::StreamKind::LENGTH, static_cast(column_id), length_stream.getOutput()}); + } + + void getOwnColumnEncoding(std::vector & encodings) const override + { + encodings.push_back({Proto::ColumnEncodingKind::DIRECT_V2, 0, {}}); + } + + size_t estimateOwnSize() const override { return length_stream.estimateSize(); } + +private: + OutputByteStream length_stream; + IntRleEncoderV2 length_encoder; +}; + +class UnionColumnWriter final : public CompoundColumnWriter +{ +public: + /// The branches of the union are in the same order as the variants of the `Variant` type, + /// so the global discriminator of a value is its tag. + UnionColumnWriter(const Type & type, const DataTypes & variant_types, WriterContext & context_) + : CompoundColumnWriter(type, context_) + , tag_stream(context.compressor, context.block_size) + , tag_encoder(tag_stream) + { + if (variant_types.size() != type.getSubtypeCount()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "ORC union {} does not match the ClickHouse type", type.toString()); + for (UInt64 i = 0; i < type.getSubtypeCount(); ++i) + children.push_back(buildColumnWriter(*type.getSubtype(i), variant_types[i], context)); + init(); + } + + void add(const IColumn & column, const UInt8 * null_map, size_t offset, size_t num_values, const UInt8 * parent_null_map) override + { + const auto & variant = assert_cast(column); + + /// A `NULL` of `Variant` is a null value of the union. + PaddedPODArray union_null_map(num_values); + bool has_nulls = false; + for (size_t i = 0; i < num_values; ++i) + { + union_null_map[i] = (null_map && null_map[i]) || variant.globalDiscriminatorAt(offset + i) == ColumnVariant::NULL_DISCRIMINATOR; + has_nulls |= union_null_map[i]; + } + const UInt8 * union_nulls = has_nulls ? union_null_map.data() : nullptr; + + ColumnWriter::add(column, union_nulls, offset, num_values, parent_null_map); + + std::vector child_offset(children.size(), -1); + std::vector child_length(children.size(), 0); + size_t count = 0; + for (size_t i = 0; i < num_values; ++i) + { + if (union_nulls && union_nulls[i]) + continue; + size_t row = offset + i; + auto tag = variant.globalDiscriminatorAt(row); + if (tag >= children.size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid union tag {} for union with {} children", static_cast(tag), children.size()); + if (child_offset[tag] == -1) + child_offset[tag] = static_cast(variant.offsetAt(row)); + ++child_length[tag]; + tag_encoder.write(static_cast(tag)); + ++count; + } + + /// The values of each branch are consecutive in its column. + for (size_t i = 0; i < children.size(); ++i) + if (child_length[i]) + addColumn(*children[i], variant.getVariantByGlobalDiscriminator(i), nullptr, static_cast(child_offset[i]), child_length[i], nullptr); + + index_statistics.increase(count); + if (count < num_values) + index_statistics.setHasNull(); + } + +protected: + void recordDataPositions(std::vector & positions) override + { + tag_encoder.recordPosition(positions); + } + + void resetStreams() override + { + tag_stream.reset(); + } + + void flushOwnStreams(std::vector & streams) override + { + tag_encoder.flush(); + tag_stream.finish(); + streams.push_back({Proto::StreamKind::DATA, static_cast(column_id), tag_stream.getOutput()}); + } + + size_t estimateOwnSize() const override { return tag_stream.estimateSize(); } + +private: + OutputByteStream tag_stream; + ByteRleEncoder tag_encoder; +}; + + +/// The writers of the primitive columns. +class PrimitiveColumnWriter : public ColumnWriter +{ +public: + using ColumnWriter::ColumnWriter; + + void getColumnEncodings(std::vector & encodings) const override + { + encodings.push_back({encoding_kind, dictionary_size, {}}); + } + +protected: + Proto::ColumnEncodingKind encoding_kind = Proto::ColumnEncodingKind::DIRECT; + UInt32 dictionary_size = 0; +}; + +class BooleanColumnWriter final : public PrimitiveColumnWriter +{ +public: + BooleanColumnWriter(const Type & type, WriterContext & context_) + : PrimitiveColumnWriter(type, context_) + , data_stream(context.compressor, context.block_size) + , encoder(data_stream) + { + init(); + } + + void add(const IColumn & column, const UInt8 * null_map, size_t offset, size_t num_values, const UInt8 * parent_null_map) override + { + ColumnWriter::add(column, null_map, offset, num_values, parent_null_map); + const UInt8 * data = getRawValues(column) + offset; + for (size_t i = 0; i < num_values; ++i) + { + if (null_map && null_map[i]) + continue; + encoder.write(data[i] != 0); + index_statistics.updateBoolean(data[i] != 0); + } + countValues(null_map, num_values); + } + + void flush(std::vector & streams) override + { + ColumnWriter::flush(streams); + encoder.flush(); + data_stream.finish(); + streams.push_back({Proto::StreamKind::DATA, static_cast(column_id), data_stream.getOutput()}); + } + + size_t estimateSize() const override { return ColumnWriter::estimateSize() + data_stream.estimateSize(); } + +protected: + void recordDataPositions(std::vector & positions) override { encoder.recordPosition(positions); } + void resetStreams() override { data_stream.reset(); } + +private: + OutputByteStream data_stream; + BooleanRleEncoder encoder; +}; + +class ByteColumnWriter final : public PrimitiveColumnWriter +{ +public: + ByteColumnWriter(const Type & type, WriterContext & context_) + : PrimitiveColumnWriter(type, context_) + , data_stream(context.compressor, context.block_size) + , encoder(data_stream) + { + init(); + } + + void add(const IColumn & column, const UInt8 * null_map, size_t offset, size_t num_values, const UInt8 * parent_null_map) override + { + ColumnWriter::add(column, null_map, offset, num_values, parent_null_map); + /// The unsigned values are written as signed ones of the same width. + const Int8 * data = getRawValues(column) + offset; + for (size_t i = 0; i < num_values; ++i) + { + if (null_map && null_map[i]) + continue; + encoder.write(static_cast(data[i])); + index_statistics.updateInteger(data[i]); + } + countValues(null_map, num_values); + } + + void flush(std::vector & streams) override + { + ColumnWriter::flush(streams); + encoder.flush(); + data_stream.finish(); + streams.push_back({Proto::StreamKind::DATA, static_cast(column_id), data_stream.getOutput()}); + } + + size_t estimateSize() const override { return ColumnWriter::estimateSize() + data_stream.estimateSize(); } + +protected: + void recordDataPositions(std::vector & positions) override { encoder.recordPosition(positions); } + void resetStreams() override { data_stream.reset(); } + +private: + OutputByteStream data_stream; + ByteRleEncoder encoder; +}; + +/// SHORT, INT, LONG and DATE. +class IntegerColumnWriter final : public PrimitiveColumnWriter +{ +public: + IntegerColumnWriter(const Type & type, WriterContext & context_) + : PrimitiveColumnWriter(type, context_) + , data_stream(context.compressor, context.block_size) + , encoder(data_stream, true) + , is_date(type.getKind() == DATE) + { + encoding_kind = Proto::ColumnEncodingKind::DIRECT_V2; + init(); + } + + void add(const IColumn & column, const UInt8 * null_map, size_t offset, size_t num_values, const UInt8 * parent_null_map) override + { + ColumnWriter::add(column, null_map, offset, num_values, parent_null_map); + /// The unsigned values are written as signed ones of the same width (except `Date`). + const size_t width = column.sizeOfValueIfFixed(); + if (width != 2 && width != 4 && width != 8) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected column {} for ORC integer", column.getName()); + const char * data = column.getRawData().data(); + for (size_t i = 0; i < num_values; ++i) + { + if (null_map && null_map[i]) + continue; + /// `Date` is the unsigned number of days (up to 2149), which must not wrap around + /// to the negative ones after 2059. `Date32` is signed, as the other integers. + Int64 value = (is_date && width == 2) + ? unalignedLoad(data + (offset + i) * sizeof(UInt16)) + : readSignedValue(data, width, offset + i); + encoder.write(value); + if (is_date) + index_statistics.updateDate(static_cast(value)); + else + index_statistics.updateInteger(value); + } + countValues(null_map, num_values); + } + + void flush(std::vector & streams) override + { + ColumnWriter::flush(streams); + encoder.flush(); + data_stream.finish(); + streams.push_back({Proto::StreamKind::DATA, static_cast(column_id), data_stream.getOutput()}); + } + + size_t estimateSize() const override { return ColumnWriter::estimateSize() + data_stream.estimateSize(); } + +protected: + void recordDataPositions(std::vector & positions) override { encoder.recordPosition(positions); } + void resetStreams() override { data_stream.reset(); } + +private: + OutputByteStream data_stream; + IntRleEncoderV2 encoder; + const bool is_date; +}; + +template +class FloatingColumnWriter final : public PrimitiveColumnWriter +{ +public: + FloatingColumnWriter(const Type & type, WriterContext & context_) + : PrimitiveColumnWriter(type, context_) + , data_stream(context.compressor, context.block_size) + { + init(); + } + + void add(const IColumn & column, const UInt8 * null_map, size_t offset, size_t num_values, const UInt8 * parent_null_map) override + { + ColumnWriter::add(column, null_map, offset, num_values, parent_null_map); + const ItemType * data = getRawValues(column) + offset; + if (!null_map && std::endian::native == std::endian::little) + { + data_stream.write(reinterpret_cast(data), num_values * sizeof(ItemType)); + for (size_t i = 0; i < num_values; ++i) + index_statistics.updateDouble(static_cast(data[i])); + } + else + { + for (size_t i = 0; i < num_values; ++i) + { + if (null_map && null_map[i]) + continue; + ItemType value = data[i]; + if constexpr (std::endian::native == std::endian::big) + value = std::bit_cast(std::byteswap(std::bit_cast>(value))); + data_stream.write(reinterpret_cast(&value), sizeof(value)); + index_statistics.updateDouble(static_cast(data[i])); + } + } + countValues(null_map, num_values); + } + + void flush(std::vector & streams) override + { + ColumnWriter::flush(streams); + data_stream.finish(); + streams.push_back({Proto::StreamKind::DATA, static_cast(column_id), data_stream.getOutput()}); + } + + size_t estimateSize() const override { return ColumnWriter::estimateSize() + data_stream.estimateSize(); } + +protected: + void recordDataPositions(std::vector & positions) override { data_stream.recordPosition(positions); } + void resetStreams() override { data_stream.reset(); } + +private: + OutputByteStream data_stream; +}; + +/// STRING, VARCHAR, CHAR and BINARY. The values of a stripe are kept in memory and encoded at the +/// end of the stripe, either directly or with a dictionary. +class StringColumnWriter final : public PrimitiveColumnWriter +{ +public: + StringColumnWriter(const Type & type, WriterContext & context_) + : PrimitiveColumnWriter(type, context_) + , is_binary(type.getKind() == BINARY) + , data_stream(context.compressor, context.block_size) + , length_stream(context.compressor, context.block_size) + , dictionary_data_stream(context.compressor, context.block_size) + , length_encoder(length_stream, false) + , index_encoder(data_stream, false) + { + /// The binary columns never use dictionaries, as in the reference implementation. + if (is_binary || context.dictionary_key_size_threshold <= 0) + use_dictionary = false; + encoding_kind = Proto::ColumnEncodingKind::DIRECT_V2; + init(); + } + + void add(const IColumn & column, const UInt8 * null_map, size_t offset, size_t num_values, const UInt8 * parent_null_map) override + { + ColumnWriter::add(column, null_map, offset, num_values, parent_null_map); + /// `String`, or the raw bytes of the values of fixed size (`FixedString`, `IPv6`, big integers). + auto add_value = [&](std::string_view value) + { + blob.append(value); + value_ends.push_back(blob.size()); + if (is_binary) + index_statistics.updateBinary(value.size()); + else + index_statistics.updateString(value); + }; + + if (const auto * strings = typeid_cast(&column)) + { + const auto & offsets = strings->getOffsets(); + const auto * chars = reinterpret_cast(strings->getChars().data()); + for (size_t i = 0; i < num_values; ++i) + { + if (null_map && null_map[i]) + continue; + size_t row = offset + i; + add_value(std::string_view(chars + offsets[row - 1], offsets[row] - offsets[row - 1])); + } + } + else + { + for (size_t i = 0; i < num_values; ++i) + { + if (null_map && null_map[i]) + continue; + add_value(column.getDataAt(offset + i)); + } + } + countValues(null_map, num_values); + } + + void createRowIndexEntry() override + { + if (!use_dictionary) + decideDictionary(); + ColumnWriter::createRowIndexEntry(); + } + + void finalizeStripe() override + { + if (!use_dictionary) + decideDictionary(); + + /// The start of the group without an entry is not needed. + group_starts.resize(entries.size()); + + if (*use_dictionary) + encodeWithDictionary(); + else + encodeDirect(); + } + + void flush(std::vector & streams) override + { + ColumnWriter::flush(streams); + if (*use_dictionary) + { + index_encoder.flush(); + data_stream.finish(); + length_encoder.flush(); + length_stream.finish(); + dictionary_data_stream.finish(); + streams.push_back({Proto::StreamKind::DATA, static_cast(column_id), data_stream.getOutput()}); + streams.push_back({Proto::StreamKind::DICTIONARY_DATA, static_cast(column_id), dictionary_data_stream.getOutput()}); + streams.push_back({Proto::StreamKind::LENGTH, static_cast(column_id), length_stream.getOutput()}); + } + else + { + data_stream.finish(); + length_encoder.flush(); + length_stream.finish(); + streams.push_back({Proto::StreamKind::DATA, static_cast(column_id), data_stream.getOutput()}); + streams.push_back({Proto::StreamKind::LENGTH, static_cast(column_id), length_stream.getOutput()}); + } + } + + size_t estimateSize() const override + { + return ColumnWriter::estimateSize() + blob.size() + value_ends.size() * sizeof(UInt64); + } + +protected: + void recordDataPositions(std::vector &) override + { + /// The positions are known only after the encoding, at the end of the stripe. + group_starts.push_back(value_ends.size()); + } + + void resetStreams() override + { + data_stream.reset(); + length_stream.reset(); + dictionary_data_stream.reset(); + blob.clear(); + value_ends.clear(); + group_starts.clear(); + dictionary_size = 0; + } + +private: + std::string_view getValue(size_t i) const + { + size_t begin = i ? value_ends[i - 1] : 0; + return std::string_view(blob.data() + begin, value_ends[i] - begin); + } + + /// The decision is made once for the file, by the values of the first row group (or the first + /// stripe without the row index), as in the reference implementation. + void decideDictionary() + { + std::unordered_set distinct; + for (size_t i = 0; i < value_ends.size(); ++i) + distinct.insert(getValue(i)); + /// Compare as doubles: the threshold is user-provided and the product may not fit in `size_t`. + use_dictionary = static_cast(distinct.size()) <= static_cast(value_ends.size()) * context.dictionary_key_size_threshold; + } + + void encodeDirect() + { + encoding_kind = Proto::ColumnEncodingKind::DIRECT_V2; + size_t group = 0; + for (size_t i = 0; i <= value_ends.size(); ++i) + { + while (group < group_starts.size() && group_starts[group] == i) + { + auto & positions = entries[group].data_positions; + data_stream.recordPosition(positions); + length_encoder.recordPosition(positions); + ++group; + } + if (i == value_ends.size()) + break; + auto value = getValue(i); + data_stream.write(value.data(), value.size()); + length_encoder.write(static_cast(value.size())); + } + } + + void encodeWithDictionary() + { + encoding_kind = Proto::ColumnEncodingKind::DICTIONARY_V2; + + /// The dictionary is sorted, as in the reference implementation. + std::unordered_map index_of; + std::vector keys; + for (size_t i = 0; i < value_ends.size(); ++i) + { + auto value = getValue(i); + if (index_of.emplace(value, 0).second) + keys.push_back(value); + } + std::sort(keys.begin(), keys.end()); + for (size_t i = 0; i < keys.size(); ++i) + index_of[keys[i]] = static_cast(i); + dictionary_size = static_cast(keys.size()); + + size_t group = 0; + for (size_t i = 0; i <= value_ends.size(); ++i) + { + while (group < group_starts.size() && group_starts[group] == i) + { + index_encoder.recordPosition(entries[group].data_positions); + ++group; + } + if (i == value_ends.size()) + break; + index_encoder.write(index_of[getValue(i)]); + } + + for (const auto & key : keys) + { + dictionary_data_stream.write(key.data(), key.size()); + length_encoder.write(static_cast(key.size())); + } + } + + const bool is_binary; + std::optional use_dictionary; + OutputByteStream data_stream; + OutputByteStream length_stream; + OutputByteStream dictionary_data_stream; + IntRleEncoderV2 length_encoder; + IntRleEncoderV2 index_encoder; + + std::string blob; + std::vector value_ends; + /// The index of the first value of each row group. + std::vector group_starts; +}; + +class TimestampColumnWriter final : public PrimitiveColumnWriter +{ +public: + /// `scale` is of `DateTime64`, or nullopt for `DateTime` (`UInt32` seconds). + TimestampColumnWriter(const Type & type, std::optional scale_, WriterContext & context_) + : PrimitiveColumnWriter(type, context_) + , scale(scale_) + , seconds_stream(context.compressor, context.block_size) + , nanos_stream(context.compressor, context.block_size) + , seconds_encoder(seconds_stream, true) + , nanos_encoder(nanos_stream, false) + , is_utc(type.getKind() == TIMESTAMP_INSTANT || context.timezone_is_utc) + , epoch_offset(type.getKind() == TIMESTAMP_INSTANT ? 1420070400 : context.epoch_offset) + { + encoding_kind = Proto::ColumnEncodingKind::DIRECT_V2; + init(); + } + + void add(const IColumn & column, const UInt8 * null_map, size_t offset, size_t num_values, const UInt8 * parent_null_map) override + { + ColumnWriter::add(column, null_map, offset, num_values, parent_null_map); + const char * data = column.getRawData().data(); + const Int64 multiplier = scale ? common::exp10_i64(static_cast(*scale)) : 1; + const Int64 nanos_multiplier = scale ? common::exp10_i64(9 - static_cast(*scale)) : 0; + for (size_t i = 0; i < num_values; ++i) + { + if (null_map && null_map[i]) + continue; + Int64 secs = 0; + Int64 nano = 0; + if (scale) + { + /// For the values before the epoch, the nanoseconds are negative (as in the Arrow ORC adapter). + Int64 value = unalignedLoad(data + (offset + i) * sizeof(Int64)); + secs = value / multiplier; + nano = (value % multiplier) * nanos_multiplier; + } + else + { + secs = unalignedLoad(data + (offset + i) * sizeof(UInt32)); + nano = 0; + } + + /// The statistics have the milliseconds rounded down and the rest of the nanoseconds, which is not negative. + Int64 nano_millis = nano / 1000000; + Int64 nano_rest = nano % 1000000; + if (nano_rest < 0) + { + nano_rest += 1000000; + --nano_millis; + } + Int64 utc_seconds = secs; + Int64 millis = 0; + Int64 encoded_seconds = 0; + if ((!is_utc && common::addOverflow(secs, static_cast(context.timezone->timezoneOffset(secs)), utc_seconds)) + || common::mulOverflow(utc_seconds, Int64{1000}, millis) + || common::addOverflow(millis, nano_millis, millis) + || common::subOverflow(secs + (secs < 0 && nano > 999999), epoch_offset, encoded_seconds)) + throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, + "Timestamp (seconds={}, nanoseconds={}) in ORC column {} is out of the range supported by the ORC format", secs, nano, column_id); + index_statistics.updateTimestamp(millis, static_cast(nano_rest)); + + seconds_encoder.write(encoded_seconds); + nanos_encoder.write(formatNanos(nano)); + } + countValues(null_map, num_values); + } + + void flush(std::vector & streams) override + { + ColumnWriter::flush(streams); + seconds_encoder.flush(); + seconds_stream.finish(); + nanos_encoder.flush(); + nanos_stream.finish(); + streams.push_back({Proto::StreamKind::DATA, static_cast(column_id), seconds_stream.getOutput()}); + streams.push_back({Proto::StreamKind::SECONDARY, static_cast(column_id), nanos_stream.getOutput()}); + } + + size_t estimateSize() const override + { + return ColumnWriter::estimateSize() + seconds_stream.estimateSize() + nanos_stream.estimateSize(); + } + +protected: + void recordDataPositions(std::vector & positions) override + { + seconds_encoder.recordPosition(positions); + nanos_encoder.recordPosition(positions); + } + + void resetStreams() override + { + seconds_stream.reset(); + nanos_stream.reset(); + } + +private: + /// The trailing zeros are removed and their number is kept in the lowest 3 bits. + static Int64 formatNanos(Int64 nanos) + { + if (nanos == 0) + return 0; + if (nanos % 100 != 0) + return nanos << 3; + nanos /= 100; + Int64 trailing_zeros = 1; + while (nanos % 10 == 0 && trailing_zeros < 7) + { + nanos /= 10; + ++trailing_zeros; + } + return (nanos << 3) | trailing_zeros; + } + + const std::optional scale; + OutputByteStream seconds_stream; + OutputByteStream nanos_stream; + IntRleEncoderV2 seconds_encoder; + IntRleEncoderV2 nanos_encoder; + const bool is_utc; + const Int64 epoch_offset; +}; + +class DecimalColumnWriter final : public PrimitiveColumnWriter +{ +public: + DecimalColumnWriter(const Type & type, WriterContext & context_) + : PrimitiveColumnWriter(type, context_) + , scale(static_cast(type.getScale())) + , value_stream(context.compressor, context.block_size) + , scale_stream(context.compressor, context.block_size) + , scale_encoder(scale_stream, true) + { + encoding_kind = Proto::ColumnEncodingKind::DIRECT_V2; + init(); + } + + void add(const IColumn & column, const UInt8 * null_map, size_t offset, size_t num_values, const UInt8 * parent_null_map) override + { + ColumnWriter::add(column, null_map, offset, num_values, parent_null_map); + /// `Decimal32`, `Decimal64` or `Decimal128`. + const size_t width = column.sizeOfValueIfFixed(); + const char * data = column.getRawData().data(); + for (size_t i = 0; i < num_values; ++i) + { + if (null_map && null_map[i]) + continue; + size_t row = offset + i; + Int128 value; + if (width == 16) + value = unalignedLoad(data + row * 16); + else if (width == 8) + value = unalignedLoad(data + row * 8); + else if (width == 4) + value = unalignedLoad(data + row * 4); + else + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected column {} for ORC decimal", column.getName()); + + /// Zigzag-encoded base 128 varint. + UInt128 encoded = (static_cast(value) << 1) ^ static_cast(value >> 127); + while (encoded >= 0x80) + { + char byte = static_cast(static_cast(encoded & 0x7F) | 0x80); + value_stream.write(&byte, 1); + encoded >>= 7; + } + char last = static_cast(static_cast(encoded)); + value_stream.write(&last, 1); + + scale_encoder.write(scale); + index_statistics.updateDecimal(value, scale); + } + countValues(null_map, num_values); + } + + void flush(std::vector & streams) override + { + ColumnWriter::flush(streams); + value_stream.finish(); + scale_encoder.flush(); + scale_stream.finish(); + streams.push_back({Proto::StreamKind::DATA, static_cast(column_id), value_stream.getOutput()}); + streams.push_back({Proto::StreamKind::SECONDARY, static_cast(column_id), scale_stream.getOutput()}); + } + + size_t estimateSize() const override + { + return ColumnWriter::estimateSize() + value_stream.estimateSize() + scale_stream.estimateSize(); + } + +protected: + void recordDataPositions(std::vector & positions) override + { + value_stream.recordPosition(positions); + scale_encoder.recordPosition(positions); + } + + void resetStreams() override + { + value_stream.reset(); + scale_stream.reset(); + } + +private: + const Int32 scale; + OutputByteStream value_stream; + OutputByteStream scale_stream; + IntRleEncoderV2 scale_encoder; +}; + +std::unique_ptr buildColumnWriter(const Type & type, const DataTypePtr & data_type_with_nulls, WriterContext & context) +{ + checkStackSize(); + + /// The nulls are handled by `addColumn`. + DataTypePtr data_type = removeNullable(data_type_with_nulls); + + switch (type.getKind()) + { + case BOOLEAN: + return std::make_unique(type, context); + case BYTE: + return std::make_unique(type, context); + case SHORT: + case INT: + case LONG: + case DATE: + return std::make_unique(type, context); + case FLOAT: + return std::make_unique>(type, context); + case DOUBLE: + return std::make_unique>(type, context); + case STRING: + case BINARY: + case CHAR: + case VARCHAR: + return std::make_unique(type, context); + case TIMESTAMP: + case TIMESTAMP_INSTANT: + { + std::optional scale; + if (const auto * datetime64 = typeid_cast(data_type.get())) + scale = datetime64->getScale(); + return std::make_unique(type, scale, context); + } + case DECIMAL: + return std::make_unique(type, context); + case STRUCT: + return std::make_unique(type, assert_cast(*data_type).getElements(), context); + case LIST: + return std::make_unique>( + type, DataTypes{assert_cast(*data_type).getNestedType()}, context); + case MAP: + { + const auto & map_type = assert_cast(*data_type); + return std::make_unique>(type, DataTypes{map_type.getKeyType(), map_type.getValueType()}, context); + } + case UNION: + return std::make_unique(type, assert_cast(*data_type).getVariants(), context); + case GEOMETRY: + case GEOGRAPHY: + break; + } + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Writing ORC type {} is not supported", type.toString()); +} + +} + + +Writer::Writer(const Type & schema_, const DataTypes & column_types, OutputStream & out_, const WriterOptions & options_) + : out(out_), options(options_) +{ + if (options.compression_block_size == 0 || options.compression_block_size >= (1ULL << 23)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "The ORC compression block size must be positive and less than 8 MiB, got {}", options.compression_block_size); + + schema = Type::buildSelectedType(schema_, std::vector(schema_.getMaximumColumnId() + 1, true)); + schema->assignIds(0); + + compressor = createCompressor(options.compression); + + context = std::make_unique(); + context->compressor = compressor.get(); + context->block_size = options.compression_block_size; + context->enable_index = options.row_index_stride > 0; + context->dictionary_key_size_threshold = options.dictionary_key_size_threshold; + context->timezone = &DateLUT::instance(options.timezone_name); + context->timezone_is_utc = options.timezone_name == "GMT" || options.timezone_name == "UTC"; + static constexpr Int64 UTC_2015_01_01 = 1420070400; + context->epoch_offset = UTC_2015_01_01 - context->timezone->timezoneOffset(UTC_2015_01_01); + + if (schema->getKind() != STRUCT) + throw Exception(ErrorCodes::LOGICAL_ERROR, "The root type of ORC file must be a struct"); + column_writer = std::make_unique(*schema, column_types, *context); + + write("ORC"); + footer.header_length = current_offset; + footer.row_index_stride = static_cast(options.row_index_stride); + /// The identifier of the C++ writer of the reference implementation, which this writer follows. + footer.writer = 1; + footer.software_version = "ClickHouse"; + schema->toProto(footer.types); +} + +Writer::~Writer() = default; + +void Writer::write(const std::string & data) +{ + out.write(data.data(), data.size()); + current_offset += data.size(); +} + +void Writer::add(const Columns & columns, size_t num_rows) +{ + auto & root = typeid_cast(*column_writer); + + if (context->enable_index) + { + size_t position = 0; + while (position < num_rows) + { + size_t chunk = std::min(num_rows - position, options.row_index_stride - index_rows); + root.addRoot(columns, position, chunk); + position += chunk; + index_rows += chunk; + stripe_rows += chunk; + if (index_rows >= options.row_index_stride) + { + column_writer->createRowIndexEntry(); + index_rows = 0; + } + } + } + else + { + stripe_rows += num_rows; + root.addRoot(columns, 0, num_rows); + } + + if (column_writer->estimateSize() >= options.stripe_size) + writeStripe(); +} + +void Writer::writeStripe() +{ + if (context->enable_index && index_rows != 0) + { + column_writer->createRowIndexEntry(); + index_rows = 0; + } + else + column_writer->mergeRowGroupStatsIntoStripeStats(); + + column_writer->finalizeStripe(); + + std::vector index_streams; + std::vector data_streams; + if (context->enable_index) + column_writer->writeIndex(index_streams); + column_writer->flush(data_streams); + + Proto::StripeFooter stripe_footer; + UInt64 index_length = 0; + UInt64 data_length = 0; + for (const auto & s : index_streams) + { + stripe_footer.streams.push_back({static_cast(s.kind), s.column, s.data.size()}); + index_length += s.data.size(); + } + for (const auto & s : data_streams) + { + stripe_footer.streams.push_back({static_cast(s.kind), s.column, s.data.size()}); + data_length += s.data.size(); + } + column_writer->getColumnEncodings(stripe_footer.columns); + stripe_footer.writer_timezone = options.timezone_name; + + auto & stripe_statistics = metadata.stripe_stats.emplace_back(); + column_writer->getStripeStatistics(stripe_statistics.col_stats); + column_writer->mergeStripeStatsIntoFileStats(); + + Proto::StripeInformation stripe; + stripe.offset = current_offset; + + for (const auto & s : index_streams) + write(s.data); + for (const auto & s : data_streams) + write(s.data); + + std::string footer_data = compressWhole(*context, stripe_footer.serialize()); + write(footer_data); + + stripe.index_length = index_length; + stripe.data_length = data_length; + stripe.footer_length = footer_data.size(); + stripe.number_of_rows = stripe_rows; + footer.stripes.push_back(stripe); + + total_rows += stripe_rows; + stripe_rows = 0; + index_rows = 0; + (*column_writer).reset(); +} + +void Writer::close() +{ + if (stripe_rows > 0) + writeStripe(); + + Proto::PostScript postscript; + + /// The content length covers only the stripes, not the metadata section with the stripe statistics. + footer.content_length = current_offset - footer.header_length; + + std::string metadata_data = compressWhole(*context, metadata.serialize()); + write(metadata_data); + postscript.metadata_length = metadata_data.size(); + + footer.number_of_rows = total_rows; + footer.statistics.clear(); + column_writer->getFileStatistics(footer.statistics); + std::string footer_data = compressWhole(*context, footer.serialize()); + write(footer_data); + + postscript.footer_length = footer_data.size(); + postscript.compression = options.compression; + postscript.compression_block_size = options.compression_block_size; + postscript.version = {0, 12}; + /// ORC-135: the timestamp statistics are in UTC. + postscript.writer_version = 6; + postscript.magic = "ORC"; + std::string postscript_data = postscript.serialize(); + if (postscript_data.size() > 255) + throw Exception(ErrorCodes::LOGICAL_ERROR, "ORC postscript is too large: {} bytes", postscript_data.size()); + write(postscript_data); + write(std::string(1, static_cast(postscript_data.size()))); +} + +} diff --git a/src/Processors/Formats/Impl/ORC/Writer.h b/src/Processors/Formats/Impl/ORC/Writer.h new file mode 100644 index 0000000000000..111c20ac0fab1 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Writer.h @@ -0,0 +1,78 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include + +class DateLUTImpl; + +namespace DB::ORC +{ + +class OutputStream +{ +public: + virtual ~OutputStream() = default; + virtual void write(const void * buf, size_t length) = 0; +}; + +struct WriterOptions +{ + CompressionKind compression = CompressionKind::ZSTD; + UInt64 compression_block_size = 256 * 1024; + /// The number of rows between the row index entries; 0 means no row index. + UInt64 row_index_stride = 10000; + /// Use the dictionary encoding for a string column if the number of distinct values in the + /// first row group is at most this fraction of the number of values. 0 disables the dictionaries. + double dictionary_key_size_threshold = 0.0; + /// The time zone of the writer, which is written into the stripe footers. + std::string timezone_name = "GMT"; + /// A stripe is written when the estimated size of its data reaches this size. + UInt64 stripe_size = 64 * 1024 * 1024; +}; + +class ColumnWriter; +struct WriterContext; + +class Writer +{ +public: + /// `column_types` are the ClickHouse types (without `LowCardinality`) of the columns of the file, + /// which are the fields of `schema` (a struct). The values are converted as follows: + /// the unsigned integers are written as the signed ones of the same width (but `Date` as is); + /// `DateTime` and `DateTime64` as TIMESTAMP; `FixedString`, `IPv6` and the big integers + /// as their raw bytes; `Variant` as UNION with the branches in the order of its variants. + Writer(const Type & schema_, const DataTypes & column_types, OutputStream & out_, const WriterOptions & options_); + ~Writer(); + + /// Adds the rows of the columns (in the order of the fields of the schema). + void add(const Columns & columns, size_t num_rows); + + /// Writes the remaining data and the file footer. + void close(); + +private: + void writeStripe(); + void write(const std::string & data); + + std::unique_ptr schema; + OutputStream & out; + WriterOptions options; + std::unique_ptr compressor; + std::unique_ptr context; + std::unique_ptr column_writer; + + Proto::Footer footer; + Proto::Metadata metadata; + UInt64 current_offset = 0; + UInt64 stripe_rows = 0; + UInt64 index_rows = 0; + UInt64 total_rows = 0; +}; + +} diff --git a/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp index 7c2fa8963137d..896189c0a1fad 100644 --- a/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp @@ -45,24 +45,24 @@ namespace ErrorCodes namespace { -orc::CompressionKind getORCCompression(FormatSettings::ORCCompression method) +ORC::CompressionKind getORCCompression(FormatSettings::ORCCompression method) { if (method == FormatSettings::ORCCompression::NONE) - return orc::CompressionKind::CompressionKind_NONE; + return ORC::CompressionKind::NONE; #if USE_SNAPPY if (method == FormatSettings::ORCCompression::SNAPPY) - return orc::CompressionKind::CompressionKind_SNAPPY; + return ORC::CompressionKind::SNAPPY; #endif if (method == FormatSettings::ORCCompression::ZSTD) - return orc::CompressionKind::CompressionKind_ZSTD; + return ORC::CompressionKind::ZSTD; if (method == FormatSettings::ORCCompression::LZ4) - return orc::CompressionKind::CompressionKind_LZ4; + return ORC::CompressionKind::LZ4; if (method == FormatSettings::ORCCompression::ZLIB) - return orc::CompressionKind::CompressionKind_ZLIB; + return ORC::CompressionKind::ZLIB; throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported compression method"); } @@ -71,23 +71,23 @@ orc::CompressionKind getORCCompression(FormatSettings::ORCCompression method) /// them back as the same ClickHouse type: `BINARY` and `STRING` both read as `String`, and a nested /// `UNION` reads as a `Variant`, which sorts its branches, while ORC keeps them positional. /// A field name is length-prefixed so a name containing `:` or `,` cannot forge another key. -String orcTypeDedupKey(const orc::Type & type) +String orcTypeDedupKey(const ORC::Type & type) { const auto kind = type.getKind(); - if (kind == orc::TypeKind::BINARY || kind == orc::TypeKind::STRING) + if (kind == ORC::TypeKind::BINARY || kind == ORC::TypeKind::STRING) return "string"; - if (kind == orc::TypeKind::DECIMAL) + if (kind == ORC::TypeKind::DECIMAL) return fmt::format("decimal({},{})", type.getPrecision(), type.getScale()); std::vector children; for (size_t i = 0; i < type.getSubtypeCount(); ++i) { auto child = orcTypeDedupKey(*type.getSubtype(i)); - if (kind == orc::TypeKind::STRUCT) + if (kind == ORC::TypeKind::STRUCT) child = fmt::format("{}:{}:{}", type.getFieldName(i).size(), type.getFieldName(i), child); children.push_back(std::move(child)); } - if (kind == orc::TypeKind::UNION) + if (kind == ORC::TypeKind::UNION) std::sort(children.begin(), children.end()); /// Every remaining kind the writer emits is a parameterless primitive mapping to a distinct /// ClickHouse type, so for those the kind alone is the key and there are no children. @@ -98,17 +98,6 @@ String orcTypeDedupKey(const orc::Type & type) ORCOutputStream::ORCOutputStream(WriteBuffer & out_) : out(out_) {} -uint64_t ORCOutputStream::getLength() const -{ - return out.count(); -} - -uint64_t ORCOutputStream::getNaturalWriteSize() const -{ - out.nextIfAtEnd(); - return out.available(); -} - void ORCOutputStream::write(const void* buf, size_t length) { out.write(static_cast(buf), length); @@ -124,7 +113,7 @@ ORCBlockOutputFormat::ORCBlockOutputFormat(WriteBuffer & out_, SharedHeader head data_types.push_back(recursiveRemoveLowCardinality(type)); } -std::unique_ptr ORCBlockOutputFormat::getORCType(const DataTypePtr & type, const String & column_path) +std::unique_ptr ORCBlockOutputFormat::getORCType(const DataTypePtr & type, const String & column_path) { /// Nullable(T) maps to the same ORC node as T; only the Iceberg `required` flag differs. /// A complex container (list/map/struct) is never wrapped in Nullable, so its Iceberg @@ -135,63 +124,63 @@ std::unique_ptr ORCBlockOutputFormat::getORCType(const DataTypePtr & required = !column_mapper->isIcebergOptionalPath(column_path); const DataTypePtr unwrapped = removeNullable(type); - std::unique_ptr result; + std::unique_ptr result; switch (unwrapped->getTypeId()) { case TypeIndex::UInt8: { if (isBool(unwrapped)) - result = orc::createPrimitiveType(orc::TypeKind::BOOLEAN); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::BOOLEAN); else - result = orc::createPrimitiveType(orc::TypeKind::BYTE); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::BYTE); break; } case TypeIndex::Enum8: [[fallthrough]]; case TypeIndex::Int8: { - result = orc::createPrimitiveType(orc::TypeKind::BYTE); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::BYTE); break; } case TypeIndex::Enum16: [[fallthrough]]; case TypeIndex::UInt16: [[fallthrough]]; case TypeIndex::Int16: { - result = orc::createPrimitiveType(orc::TypeKind::SHORT); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::SHORT); break; } case TypeIndex::UInt32: [[fallthrough]]; case TypeIndex::IPv4: [[fallthrough]]; case TypeIndex::Int32: { - result = orc::createPrimitiveType(orc::TypeKind::INT); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::INT); break; } case TypeIndex::UInt64: [[fallthrough]]; case TypeIndex::Int64: { - result = orc::createPrimitiveType(orc::TypeKind::LONG); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::LONG); break; } case TypeIndex::Float32: { - result = orc::createPrimitiveType(orc::TypeKind::FLOAT); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::FLOAT); break; } case TypeIndex::Float64: { - result = orc::createPrimitiveType(orc::TypeKind::DOUBLE); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::DOUBLE); break; } case TypeIndex::Date32: [[fallthrough]]; case TypeIndex::Date: { - result = orc::createPrimitiveType(orc::TypeKind::DATE); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::DATE); break; } case TypeIndex::DateTime: [[fallthrough]]; case TypeIndex::DateTime64: { - result = orc::createPrimitiveType(orc::TypeKind::TIMESTAMP); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::TIMESTAMP); break; } case TypeIndex::Int128: [[fallthrough]]; @@ -199,7 +188,7 @@ std::unique_ptr ORCBlockOutputFormat::getORCType(const DataTypePtr & case TypeIndex::Int256: [[fallthrough]]; case TypeIndex::UInt256: [[fallthrough]]; case TypeIndex::Decimal256: - result = orc::createPrimitiveType(orc::TypeKind::BINARY); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::BINARY); break; case TypeIndex::String: { @@ -208,7 +197,7 @@ std::unique_ptr ORCBlockOutputFormat::getORCType(const DataTypePtr & bool force_string = format_settings.orc.output_string_as_string; if (column_mapper && column_mapper->hasIcebergStringInfo()) force_string = column_mapper->isIcebergStringPath(column_path); - result = orc::createPrimitiveType(force_string ? orc::TypeKind::STRING : orc::TypeKind::BINARY); + result = ORC::Type::createPrimitiveType(force_string ? ORC::TypeKind::STRING : ORC::TypeKind::BINARY); break; } case TypeIndex::FixedString: @@ -216,38 +205,38 @@ std::unique_ptr ORCBlockOutputFormat::getORCType(const DataTypePtr & /// Iceberg `fixed[L]` (FixedString) must stay ORC `binary`, so for Iceberg writes /// force `binary` regardless of the setting; keep the setting-driven choice otherwise. if (format_settings.orc.output_string_as_string && !column_mapper) - result = orc::createPrimitiveType(orc::TypeKind::STRING); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::STRING); else - result = orc::createPrimitiveType(orc::TypeKind::BINARY); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::BINARY); break; } case TypeIndex::IPv6: { - result = orc::createPrimitiveType(orc::TypeKind::BINARY); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::BINARY); break; } case TypeIndex::Array: { const auto * array_type = assert_cast(unwrapped.get()); - result = orc::createListType(getORCType(array_type->getNestedType(), Nested::concatenateName(column_path, "element"))); + result = ORC::Type::createListType(getORCType(array_type->getNestedType(), Nested::concatenateName(column_path, "element"))); break; } case TypeIndex::Decimal32: { const auto * decimal_type = assert_cast *>(unwrapped.get()); - result = orc::createDecimalType(decimal_type->getPrecision(), decimal_type->getScale()); + result = ORC::Type::createDecimalType(decimal_type->getPrecision(), decimal_type->getScale()); break; } case TypeIndex::Decimal64: { const auto * decimal_type = assert_cast *>(unwrapped.get()); - result = orc::createDecimalType(decimal_type->getPrecision(), decimal_type->getScale()); + result = ORC::Type::createDecimalType(decimal_type->getPrecision(), decimal_type->getScale()); break; } case TypeIndex::Decimal128: { const auto * decimal_type = assert_cast *>(unwrapped.get()); - result = orc::createDecimalType(decimal_type->getPrecision(), decimal_type->getScale()); + result = ORC::Type::createDecimalType(decimal_type->getPrecision(), decimal_type->getScale()); break; } case TypeIndex::Tuple: @@ -255,7 +244,7 @@ std::unique_ptr ORCBlockOutputFormat::getORCType(const DataTypePtr & const auto * tuple_type = assert_cast(unwrapped.get()); const auto & nested_names = tuple_type->getElementNames(); const auto & nested_types = tuple_type->getElements(); - auto struct_type = orc::createStructType(); + auto struct_type = ORC::Type::createStructType(); for (size_t i = 0; i < nested_types.size(); ++i) struct_type->addStructField(nested_names[i], getORCType(nested_types[i], Nested::concatenateName(column_path, nested_names[i]))); result = std::move(struct_type); @@ -264,7 +253,7 @@ std::unique_ptr ORCBlockOutputFormat::getORCType(const DataTypePtr & case TypeIndex::Map: { const auto * map_type = assert_cast(unwrapped.get()); - result = orc::createMapType( + result = ORC::Type::createMapType( getORCType(map_type->getKeyType(), Nested::concatenateName(column_path, "key")), getORCType(map_type->getValueType(), Nested::concatenateName(column_path, "value")) ); @@ -273,7 +262,7 @@ std::unique_ptr ORCBlockOutputFormat::getORCType(const DataTypePtr & case TypeIndex::Variant: { const auto & variant_type = assert_cast(*unwrapped); - auto union_type = orc::createUnionType(); + auto union_type = ORC::Type::createUnionType(); /// ORC keeps one physical stream per union branch and identifies a branch only by its /// ORC type, so two variants that map to the same ORC type (e.g. `Int32` and `UInt32`, /// both ORC `int`) would produce a union with duplicate branches, which the reader @@ -293,7 +282,7 @@ std::unique_ptr ORCBlockOutputFormat::getORCType(const DataTypePtr & "Type {} is not supported for ORC output format: variant {} is written as ORC type '{}' and variant {} as " "'{}', which are read back as the same type, and ORC unions with duplicate branch types cannot be read back", unwrapped->getName(), it->second.first, it->second.second, nested_type->getName(), child_type->toString()); - union_type->addUnionChild(std::move(child_type)); + union_type->addChild(std::move(child_type)); } result = std::move(union_type); break; @@ -318,406 +307,18 @@ std::unique_ptr ORCBlockOutputFormat::getORCType(const DataTypePtr & return result; } -template -void ORCBlockOutputFormat::writeNumbers( - orc::ColumnVectorBatch & orc_column, - const IColumn & column, - const PaddedPODArray * /*null_bytemap*/, - ConvertFunc convert) -{ - NumberVectorBatch & number_orc_column = dynamic_cast(orc_column); - const auto & number_column = assert_cast &>(column); - - number_orc_column.data.resize(number_column.size()); - for (size_t i = 0; i != number_column.size(); ++i) - number_orc_column.data[i] = convert(number_column.getElement(i)); -} - -template -void ORCBlockOutputFormat::writeDecimals( - orc::ColumnVectorBatch & orc_column, - const IColumn & column, - DataTypePtr & type, - const PaddedPODArray * /*null_bytemap*/, - ConvertFunc convert) -{ - DecimalVectorBatch & decimal_orc_column = dynamic_cast(orc_column); - const auto & decimal_column = assert_cast &>(column); - const auto * decimal_type = assert_cast *>(type.get()); - decimal_orc_column.precision = decimal_type->getPrecision(); - decimal_orc_column.scale = decimal_type->getScale(); - decimal_orc_column.resize(decimal_column.size()); - - decimal_orc_column.values.resize(decimal_column.size()); - for (size_t i = 0; i != decimal_column.size(); ++i) - decimal_orc_column.values[i] = convert(decimal_column.getElement(i).value); -} - -template -void ORCBlockOutputFormat::writeStrings( - orc::ColumnVectorBatch & orc_column, - const IColumn & column, - const PaddedPODArray * null_bytemap) -{ - orc::StringVectorBatch & string_orc_column = dynamic_cast(orc_column); - const auto & string_column = assert_cast(column); - - string_orc_column.data.resize(string_column.size()); - string_orc_column.length.resize(string_column.size()); - for (size_t i = 0; i != string_column.size(); ++i) - { - const std::string_view string = string_column.getDataAt(i); - string_orc_column.data[i] = (null_bytemap && (*null_bytemap)[i]) ? nullptr : const_cast(string.data()); - string_orc_column.length[i] = string.size(); - } -} - -template -void ORCBlockOutputFormat::writeDateTimes( - orc::ColumnVectorBatch & orc_column, - const IColumn & column, - const PaddedPODArray * /*null_bytemap*/, - GetSecondsFunc get_seconds, - GetNanosecondsFunc get_nanoseconds) -{ - orc::TimestampVectorBatch & timestamp_orc_column = dynamic_cast(orc_column); - const auto & timestamp_column = assert_cast(column); - - timestamp_orc_column.data.resize(timestamp_column.size()); - timestamp_orc_column.nanoseconds.resize(timestamp_column.size()); - for (size_t i = 0; i != timestamp_column.size(); ++i) - { - timestamp_orc_column.data[i] = static_cast(get_seconds(timestamp_column.getElement(i))); - timestamp_orc_column.nanoseconds[i] = static_cast(get_nanoseconds(timestamp_column.getElement(i))); - } -} - -void ORCBlockOutputFormat::writeColumn( - orc::ColumnVectorBatch & orc_column, - const IColumn & column, - DataTypePtr & type, - const PaddedPODArray * null_bytemap) -{ - size_t rows = column.size(); - orc_column.resize(rows); - orc_column.numElements = rows; - - /// Calculate orc_column.hasNulls - if (null_bytemap) - orc_column.hasNulls = !memoryIsZero(null_bytemap->data(), 0, null_bytemap->size()); - else - orc_column.hasNulls = false; - - /// Fill orc_column.notNull - if (orc_column.hasNulls) - { - for (size_t i = 0; i < rows; ++i) - orc_column.notNull[i] = !(*null_bytemap)[i]; - } - else - { - for (size_t i = 0; i < rows; ++i) - orc_column.notNull[i] = 1; - } - - /// ORC doesn't have unsigned types, so cast everything to signed and sign-extend to Int64 to - /// make the ORC library calculate min and max correctly. - switch (type->getTypeId()) - { - case TypeIndex::Enum8: [[fallthrough]]; - case TypeIndex::Int8: - { - /// Note: Explicit cast to avoid clang-tidy error: 'signed char' to 'long' conversion; consider casting to 'unsigned char' first. - writeNumbers(orc_column, column, null_bytemap, [](const Int8 & value){ return Int64(Int8(value)); }); - break; - } - case TypeIndex::UInt8: - { - writeNumbers(orc_column, column, null_bytemap, [](const UInt8 & value){ return Int64(Int8(value)); }); - break; - } - case TypeIndex::Enum16: [[fallthrough]]; - case TypeIndex::Int16: - { - writeNumbers(orc_column, column, null_bytemap, [](const Int16 & value){ return value; }); - break; - } - case TypeIndex::Date: [[fallthrough]]; - case TypeIndex::UInt16: - { - writeNumbers(orc_column, column, null_bytemap, [](const UInt16 & value){ return Int64(Int16(value)); }); - break; - } - case TypeIndex::Date32: [[fallthrough]]; - case TypeIndex::Int32: - { - writeNumbers(orc_column, column, null_bytemap, [](const Int32 & value){ return value; }); - break; - } - case TypeIndex::UInt32: - { - writeNumbers(orc_column, column, null_bytemap, [](const UInt32 & value){ return Int64(Int32(value)); }); - break; - } - case TypeIndex::IPv4: - { - writeNumbers(orc_column, column, null_bytemap, [](const IPv4 & value){ return Int64(Int32(value.toUnderType())); }); - break; - } - case TypeIndex::Int64: - { - writeNumbers(orc_column, column, null_bytemap, [](const Int64 & value){ return value; }); - break; - } - case TypeIndex::UInt64: - { - writeNumbers(orc_column, column, null_bytemap, [](const UInt64 & value){ return value; }); - break; - } - case TypeIndex::Int128: - { - writeStrings(orc_column, column, null_bytemap); - break; - } - case TypeIndex::UInt128: - { - writeStrings(orc_column, column, null_bytemap); - break; - } - case TypeIndex::Int256: - { - writeStrings(orc_column, column, null_bytemap); - break; - } - case TypeIndex::UInt256: - { - writeStrings(orc_column, column, null_bytemap); - break; - } - case TypeIndex::Float32: - { - writeNumbers(orc_column, column, null_bytemap, [](const Float32 & value){ return static_cast(value); }); - break; - } - case TypeIndex::Float64: - { - writeNumbers(orc_column, column, null_bytemap, [](const Float64 & value){ return value; }); - break; - } - case TypeIndex::FixedString: - { - writeStrings(orc_column, column, null_bytemap); - break; - } - case TypeIndex::String: - { - writeStrings(orc_column, column, null_bytemap); - break; - } - case TypeIndex::IPv6: - { - writeStrings(orc_column, column, null_bytemap); - break; - } - case TypeIndex::DateTime: - { - writeDateTimes( - orc_column, - column, null_bytemap, - [](UInt32 value){ return value; }, - [](UInt32){ return 0; }); - break; - } - case TypeIndex::DateTime64: - { - const auto * timestamp_type = assert_cast(type.get()); - UInt32 scale = timestamp_type->getScale(); - writeDateTimes( - orc_column, - column, - null_bytemap, - [scale](Int64 value) { return value / Int64(std::pow(10, scale)); }, - [scale](Int64 value) { return (value % Int64(std::pow(10, scale))) * Int64(std::pow(10, 9 - scale)); }); - break; - } - case TypeIndex::Decimal32: { - writeDecimals( - orc_column, - column, - type, - null_bytemap, - [](Int32 value){ return value; }); - break; - } - case TypeIndex::Decimal64: - { - writeDecimals( - orc_column, - column, - type, - null_bytemap, - [](Int64 value){ return value; }); - break; - } - case TypeIndex::Decimal128: - { - writeDecimals( - orc_column, - column, - type, - null_bytemap, - [](Int128 value){ return orc::Int128(static_cast(value >> 64), static_cast((value << 64) >> 64)); }); - break; - } - case TypeIndex::Decimal256: - { - writeStrings>(orc_column, column, null_bytemap); - break; - } - case TypeIndex::Nullable: - { - chassert(!null_bytemap); - const auto & nullable_column = assert_cast(column); - const PaddedPODArray & new_null_bytemap = assert_cast &>(*nullable_column.getNullMapColumnPtr()).getData(); - auto nested_type = removeNullable(type); - writeColumn(orc_column, nullable_column.getNestedColumn(), nested_type, &new_null_bytemap); - break; - } - case TypeIndex::Array: - { - orc::ListVectorBatch & list_orc_column = dynamic_cast(orc_column); - const auto & list_column = assert_cast(column); - auto nested_type = assert_cast(*type).getNestedType(); - const ColumnArray::Offsets & offsets = list_column.getOffsets(); - - size_t column_size = list_column.size(); - list_orc_column.offsets.resize(column_size + 1); - - /// The length of list i in ListVectorBatch is offsets[i+1] - offsets[i]. - list_orc_column.offsets[0] = 0; - for (size_t i = 0; i != column_size; ++i) - list_orc_column.offsets[i + 1] = offsets[i]; - - orc::ColumnVectorBatch & nested_orc_column = *list_orc_column.elements; - writeColumn(nested_orc_column, list_column.getData(), nested_type, nullptr); - break; - } - case TypeIndex::Tuple: - { - orc::StructVectorBatch & struct_orc_column = dynamic_cast(orc_column); - const auto & tuple_column = assert_cast(column); - auto nested_types = assert_cast(type.get())->getElements(); - for (size_t i = 0; i != tuple_column.tupleSize(); ++i) - { - if (null_bytemap && nested_types[i]->isNullable()) - { - /// When both the struct and the element are nullable, we need to merge the two null bitmaps: - /// a child value is null if either the struct row is null OR the element itself is null. - const auto & nullable_col = assert_cast(tuple_column.getColumn(i)); - const auto & element_null_map = nullable_col.getNullMapData(); - PaddedPODArray merged_null_map(element_null_map.size()); - for (size_t j = 0; j < element_null_map.size(); ++j) - merged_null_map[j] = element_null_map[j] | (*null_bytemap)[j]; - - auto nested_type = removeNullable(nested_types[i]); - writeColumn(*struct_orc_column.fields[i], nullable_col.getNestedColumn(), nested_type, &merged_null_map); - } - else - { - /// Propagate the struct-level null_bytemap to children so the ORC library correctly handles - /// null struct rows (child values at null positions must also be marked null). - writeColumn(*struct_orc_column.fields[i], tuple_column.getColumn(i), nested_types[i], null_bytemap); - } - } - break; - } - case TypeIndex::Map: - { - orc::MapVectorBatch & map_orc_column = dynamic_cast(orc_column); - const auto & list_column = assert_cast(column).getNestedColumn(); - const auto & map_type = assert_cast(*type); - const ColumnArray::Offsets & offsets = list_column.getOffsets(); - - size_t column_size = list_column.size(); - - map_orc_column.offsets.resize(column_size + 1); - /// The length of list i in ListVectorBatch is offsets[i+1] - offsets[i]. - map_orc_column.offsets[0] = 0; - for (size_t i = 0; i != column_size; ++i) - map_orc_column.offsets[i + 1] = offsets[i]; - - const auto nested_columns = assert_cast(list_column.getDataPtr().get())->getColumns(); - - orc::ColumnVectorBatch & keys_orc_column = *map_orc_column.keys; - auto key_type = map_type.getKeyType(); - writeColumn(keys_orc_column, *nested_columns[0], key_type, nullptr); - - orc::ColumnVectorBatch & values_orc_column = *map_orc_column.elements; - auto value_type = map_type.getValueType(); - writeColumn(values_orc_column, *nested_columns[1], value_type, nullptr); - break; - } - case TypeIndex::Variant: - { - auto & union_orc_column = dynamic_cast(orc_column); - const auto & variant_column = assert_cast(column); - const auto & variant_types = assert_cast(*type).getVariants(); - - /// The ORC union branches are created in the same order as the Variant's variants (see - /// getORCType), so a global discriminator is exactly the ORC tag. A Variant NULL row (the - /// NULL discriminator) is written as a NULL union row. - bool has_nulls = orc_column.hasNulls; - for (size_t i = 0; i < rows; ++i) - { - auto global_discr = variant_column.globalDiscriminatorAt(i); - if (global_discr == ColumnVariant::NULL_DISCRIMINATOR || (null_bytemap && (*null_bytemap)[i])) - { - union_orc_column.notNull[i] = 0; - union_orc_column.tags[i] = 0; - union_orc_column.offsets[i] = 0; - has_nulls = true; - } - else - { - union_orc_column.tags[i] = static_cast(global_discr); - union_orc_column.offsets[i] = variant_column.offsetAt(i); - } - } - union_orc_column.hasNulls = has_nulls; - - /// Each branch column already contains exactly the rows routed to it (in offset order). - for (size_t i = 0; i < variant_types.size(); ++i) - { - auto nested_type = variant_types[i]; - writeColumn(*union_orc_column.children[i], variant_column.getVariantByGlobalDiscriminator(i), nested_type, nullptr); - } - break; - } - default: - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Type {} is not supported for ORC output format", type->getName()); - } -} - void ORCBlockOutputFormat::consume(Chunk chunk) { if (!writer) prepareWriter(); - size_t columns_num = chunk.getNumColumns(); size_t rows_num = chunk.getNumRows(); - - std::unique_ptr batch = writer->createRowBatch(chunk.getNumRows()); - orc::StructVectorBatch & root = dynamic_cast(*batch); - auto columns = chunk.detachColumns(); for (auto & column : columns) column = recursiveRemoveLowCardinality(column); - for (size_t i = 0; i != columns_num; ++i) - writeColumn(*root.fields[i], *columns[i], data_types[i], nullptr); - - root.numElements = rows_num; - writer->add(*batch); + /// The values are encoded directly from the columns. + writer->add(columns, rows_num); } void ORCBlockOutputFormat::finalizeImpl() @@ -736,20 +337,19 @@ void ORCBlockOutputFormat::resetFormatterImpl() void ORCBlockOutputFormat::prepareWriter() { const Block & header = getPort(PortKind::Main).getHeader(); - schema = orc::createStructType(); - options.setMemoryPool(&getORCMemoryPool()); - options.setCompression(getORCCompression(format_settings.orc.output_compression_method)); - options.setRowIndexStride(format_settings.orc.output_row_index_stride); - options.setDictionaryKeySizeThreshold(format_settings.orc.output_dictionary_key_size_threshold); - options.setCompressionBlockSize(format_settings.orc.output_compression_block_size); - options.setTimezoneName(format_settings.orc.writer_time_zone_name); + schema = ORC::Type::createStructType(); + options.compression = getORCCompression(format_settings.orc.output_compression_method); + options.row_index_stride = format_settings.orc.output_row_index_stride; + options.dictionary_key_size_threshold = format_settings.orc.output_dictionary_key_size_threshold; + options.compression_block_size = format_settings.orc.output_compression_block_size; + options.timezone_name = format_settings.orc.writer_time_zone_name; size_t columns_count = header.columns(); for (size_t i = 0; i != columns_count; ++i) { const auto & column_name = header.safeGetByPosition(i).name; schema->addStructField(column_name, getORCType(recursiveRemoveLowCardinality(data_types[i]), column_name)); } - writer = orc::createWriter(*schema, &output_stream, options); + writer = std::make_unique(*schema, data_types, output_stream, options); } void registerOutputFormatORC(FormatFactory & factory); diff --git a/src/Processors/Formats/Impl/ORCBlockOutputFormat.h b/src/Processors/Formats/Impl/ORCBlockOutputFormat.h index 10832fbbcbb6b..c7cac88d52cc8 100644 --- a/src/Processors/Formats/Impl/ORCBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/ORCBlockOutputFormat.h @@ -9,7 +9,7 @@ #include #include #include -#include +#include namespace DB @@ -21,22 +21,15 @@ using DataTypes = VectorWithMemoryTracking; class WriteBuffer; -/// orc::Writer writes only in orc::OutputStream -class ORCOutputStream : public orc::OutputStream +class ORCOutputStream : public ORC::OutputStream { public: explicit ORCOutputStream(WriteBuffer & out_); - uint64_t getLength() const override; - uint64_t getNaturalWriteSize() const override; void write(const void * buf, size_t length) override; - void close() override {} - const std::string& getName() const override { return name; } - private: WriteBuffer & out; - std::string name = "ORCOutputStream"; }; @@ -54,37 +47,16 @@ class ORCBlockOutputFormat final : public IOutputFormat /// For Iceberg writes, attaches the field-id `iceberg.id` type attribute so readers project /// by id. `column_path` is the dotted name (`t.x`, `arr.element`) used to look the id up. - std::unique_ptr getORCType(const DataTypePtr & type, const String & column_path); - - /// ConvertFunc is needed for type UInt8, because firstly UInt8 (char8_t) must be - /// converted to unsigned char (bugprone-signed-char-misuse in clang). - template - void writeNumbers(orc::ColumnVectorBatch & orc_column, const IColumn & column, const PaddedPODArray * null_bytemap, ConvertFunc convert); - - /// ConvertFunc is needed to convert ClickHouse Int128 to ORC Int128. - template - void writeDecimals(orc::ColumnVectorBatch & orc_column, const IColumn & column, DataTypePtr & type, - const PaddedPODArray * null_bytemap, ConvertFunc convert); - - template - void writeStrings(orc::ColumnVectorBatch & orc_column, const IColumn & column, const PaddedPODArray * null_bytemap); - - /// ORC column TimestampVectorBatch stores only seconds and nanoseconds, - /// GetSecondsFunc and GetNanosecondsFunc are needed to extract them from DateTime type. - template - void writeDateTimes(orc::ColumnVectorBatch & orc_column, const IColumn & column, const PaddedPODArray * null_bytemap, - GetSecondsFunc get_seconds, GetNanosecondsFunc get_nanoseconds); - - void writeColumn(orc::ColumnVectorBatch & orc_column, const IColumn & column, DataTypePtr & type, const PaddedPODArray * null_bytemap); + std::unique_ptr getORCType(const DataTypePtr & type, const String & column_path); void prepareWriter(); const FormatSettings format_settings; ORCOutputStream output_stream; DataTypes data_types; - std::unique_ptr writer; - std::unique_ptr schema; - orc::WriterOptions options; + std::unique_ptr writer; + std::unique_ptr schema; + ORC::WriterOptions options; /// Non-null only for Iceberg writes; maps dotted column name -> Iceberg field-id. ColumnMapperPtr column_mapper; }; diff --git a/src/Processors/tests/gtest_write_orc_iceberg_required.cpp b/src/Processors/tests/gtest_write_orc_iceberg_required.cpp index c8917a248a924..2355d618525fd 100644 --- a/src/Processors/tests/gtest_write_orc_iceberg_required.cpp +++ b/src/Processors/tests/gtest_write_orc_iceberg_required.cpp @@ -17,7 +17,8 @@ # include # include -# include +# include +# include # include # include @@ -122,24 +123,24 @@ void writeOneEmptyBlock(const Block & header, ColumnMapperPtr mapper, const Stri /// Walk the ORC footer type tree, recording iceberg.required for every path (dotted convention: /// t.field, arr.element, m.key, m.value) that carries the attribute. -void collectRequired(const orc::Type & type, const String & path, std::map & out) +void collectRequired(const ORC::Type & type, const String & path, std::map & out) { if (type.hasAttributeKey("iceberg.required")) out[path] = type.getAttributeValue("iceberg.required"); switch (type.getKind()) { - case orc::TypeKind::STRUCT: + case ORC::TypeKind::STRUCT: for (uint64_t i = 0; i < type.getSubtypeCount(); ++i) { const String child = path.empty() ? type.getFieldName(i) : path + "." + type.getFieldName(i); collectRequired(*type.getSubtype(i), child, out); } break; - case orc::TypeKind::LIST: + case ORC::TypeKind::LIST: collectRequired(*type.getSubtype(0), path + ".element", out); break; - case orc::TypeKind::MAP: + case ORC::TypeKind::MAP: collectRequired(*type.getSubtype(0), path + ".key", out); collectRequired(*type.getSubtype(1), path + ".value", out); break; @@ -150,8 +151,9 @@ void collectRequired(const orc::Type & type, const String & path, std::map readIcebergRequired(const String & path) { - orc::ReaderOptions options; - auto reader = orc::createReader(orc::readLocalFile(path), options); + ReadBufferFromFile in(path); + std::atomic is_cancelled{0}; + auto reader = createORCReader(asORCInputStreamLoadIntoMemory(in, is_cancelled), ORC::ReaderOptions{}); std::map out; // The root is the file struct; descend into its top-level columns with an empty prefix. collectRequired(reader->getType(), "", out); diff --git a/src/Storages/Hive/HiveFile.cpp b/src/Storages/Hive/HiveFile.cpp index 378bb0840ef73..15d92e6ad1155 100644 --- a/src/Storages/Hive/HiveFile.cpp +++ b/src/Storages/Hive/HiveFile.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include @@ -34,23 +33,16 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -template -Range createRangeFromOrcStatistics(const StatisticsType * stats) +template +Range createRangeFromOrcStatistics(const std::optional & minimum, const std::optional & maximum) { /// Null values or NaN/Inf values of double type. - if (stats->hasMinimum() && stats->hasMaximum()) - { - return Range(FieldType(stats->getMinimum()), true, FieldType(stats->getMaximum()), true); - } - if (stats->hasMinimum()) - { - return Range::createLeftBounded(FieldType(stats->getMinimum()), true); - } - if (stats->hasMaximum()) - { - return Range::createRightBounded(FieldType(stats->getMaximum()), true); - } - + if (minimum && maximum) + return Range(FieldType(*minimum), true, FieldType(*maximum), true); + if (minimum) + return Range::createLeftBounded(FieldType(*minimum), true); + if (maximum) + return Range::createRightBounded(FieldType(*maximum), true); return Range::createWholeUniverseWithoutNull(); } @@ -109,48 +101,40 @@ void IHiveFile::loadSplitMinMaxIndexes() split_minmax_idxes_loaded = true; } -Range HiveORCFile::buildRange(const orc::ColumnStatistics * col_stats) +Range HiveORCFile::buildRange(const ORC::Proto::ColumnStatistics * col_stats) { - if (!col_stats || col_stats->hasNull()) + /// A missing `has_null` means that the column may contain nulls. + if (!col_stats || col_stats->has_null.value_or(true)) return Range::createWholeUniverseWithoutNull(); - if (const auto * int_stats = dynamic_cast(col_stats)) - { - return createRangeFromOrcStatistics(int_stats); - } - if (const auto * double_stats = dynamic_cast(col_stats)) - { - return createRangeFromOrcStatistics(double_stats); - } - if (const auto * string_stats = dynamic_cast(col_stats)) + if (const auto & int_stats = col_stats->int_statistics) + return createRangeFromOrcStatistics(int_stats->minimum, int_stats->maximum); + if (const auto & double_stats = col_stats->double_statistics) + return createRangeFromOrcStatistics(double_stats->minimum, double_stats->maximum); + if (const auto & string_stats = col_stats->string_statistics) + return createRangeFromOrcStatistics(string_stats->minimum, string_stats->maximum); + if (const auto & bool_stats = col_stats->bucket_statistics) { - return createRangeFromOrcStatistics(string_stats); - } - if (const auto * bool_stats = dynamic_cast(col_stats)) - { - auto false_cnt = bool_stats->getFalseCount(); - auto true_cnt = bool_stats->getTrueCount(); + if (bool_stats->count.empty()) + return Range::createWholeUniverseWithoutNull(); + UInt64 true_cnt = bool_stats->count[0]; + UInt64 false_cnt = col_stats->number_of_values.value_or(0) - true_cnt; if (false_cnt && true_cnt) - { return Range(UInt8(0), true, UInt8(1), true); - } if (false_cnt) - { return Range::createLeftBounded(UInt8(0), true); - } if (true_cnt) - { return Range::createRightBounded(UInt8(1), true); - } - } - else if (const auto * timestamp_stats = dynamic_cast(col_stats)) - { - return createRangeFromOrcStatistics(timestamp_stats); + return Range::createWholeUniverseWithoutNull(); } - else if (const auto * date_stats = dynamic_cast(col_stats)) + if (const auto & timestamp_stats = col_stats->timestamp_statistics) { - return createRangeFromOrcStatistics(date_stats); + /// The deprecated `minimum` and `maximum` are in the local time zone of the writer, which is + /// unknown for the statistics of the whole file, so only the UTC bounds are usable. + return createRangeFromOrcStatistics(timestamp_stats->minimum_utc, timestamp_stats->maximum_utc); } + if (const auto & date_stats = col_stats->date_statistics) + return createRangeFromOrcStatistics(date_stats->minimum, date_stats->maximum); return Range::createWholeUniverseWithoutNull(); } @@ -159,14 +143,12 @@ void HiveORCFile::prepareReader() in = std::make_unique(namenode_url, path, getContext()->getGlobalContext()->getConfigRef(), getContext()->getReadSettings()); auto format_settings = getFormatSettings(getContext()); std::atomic is_stopped{0}; - orc::ReaderOptions options; - options.setMemoryPool(getORCMemoryPool()); - reader = createORCReader(asORCInputStream(*in, format_settings, /*use_prefetch=*/false, is_stopped), options); + reader = createORCReader(asORCInputStream(*in, format_settings, /*use_prefetch=*/false, is_stopped), ORC::ReaderOptions{}); } void HiveORCFile::prepareColumnMapping() { - const orc::Type & type = reader->getType(); + const ORC::Type & type = reader->getType(); size_t count = type.getSubtypeCount(); for (size_t pos = 0; pos < count; pos++) { @@ -183,7 +165,7 @@ bool HiveORCFile::useFileMinMaxIndex() const } -std::unique_ptr HiveORCFile::buildMinMaxIndex(const orc::Statistics * statistics) +std::unique_ptr HiveORCFile::buildMinMaxIndex(const std::vector * statistics) { if (!statistics) return nullptr; @@ -206,7 +188,7 @@ std::unique_ptr HiveORCFile::buildMinMaxIndex(c { size_t pos = it->second; /// Attention: column statistics start from 1. 0 has special purpose. - const orc::ColumnStatistics * col_stats = statistics->getColumnStatistics(static_cast(pos + 1)); + const ORC::Proto::ColumnStatistics * col_stats = pos + 1 < statistics->size() ? &(*statistics)[pos + 1] : nullptr; idx->hyperrectangle[i] = buildRange(col_stats); } ++i; @@ -223,8 +205,7 @@ void HiveORCFile::loadFileMinMaxIndexImpl() prepareColumnMapping(); } - auto statistics = reader->getStatistics(); - file_minmax_idx = buildMinMaxIndex(statistics.get()); + file_minmax_idx = buildMinMaxIndex(&reader->getStatistics()); } bool HiveORCFile::useSplitMinMaxIndex() const @@ -243,7 +224,8 @@ void HiveORCFile::loadSplitMinMaxIndexesImpl() auto * raw_reader = reader.get(); auto stripe_num = raw_reader->getNumberOfStripes(); - auto stripe_stats_num = raw_reader->getNumberOfStripeStatistics(); + const auto & stripe_statistics = raw_reader->getMetadata().stripe_stats; + auto stripe_stats_num = stripe_statistics.size(); if (stripe_num != stripe_stats_num) throw Exception(ErrorCodes::BAD_ARGUMENTS, "orc file:{} has different strip num {} and strip statistics num {}", path, stripe_num, stripe_stats_num); @@ -251,8 +233,7 @@ void HiveORCFile::loadSplitMinMaxIndexesImpl() split_minmax_idxes.resize(stripe_num); for (size_t i = 0; i < stripe_num; ++i) { - auto stripe_stats = raw_reader->getStripeStatistics(i); - split_minmax_idxes[i] = buildMinMaxIndex(stripe_stats.get()); + split_minmax_idxes[i] = buildMinMaxIndex(&stripe_statistics[i].col_stats); } } diff --git a/src/Storages/Hive/HiveFile.h b/src/Storages/Hive/HiveFile.h index 61625c2c76c63..4b82ef293b6d8 100644 --- a/src/Storages/Hive/HiveFile.h +++ b/src/Storages/Hive/HiveFile.h @@ -8,7 +8,7 @@ #include #include -#include +#include #include #include @@ -225,18 +225,18 @@ class HiveORCFile : public IHiveFile bool useSplitMinMaxIndex() const override; private: - static Range buildRange(const orc::ColumnStatistics * col_stats); + static Range buildRange(const ORC::Proto::ColumnStatistics * col_stats); void loadFileMinMaxIndexImpl() override; void loadSplitMinMaxIndexesImpl() override; - std::unique_ptr buildMinMaxIndex(const orc::Statistics * statistics); + std::unique_ptr buildMinMaxIndex(const std::vector * statistics); void prepareReader(); void prepareColumnMapping(); std::optional getRowsImpl() override; std::unique_ptr in; - std::unique_ptr reader; + std::unique_ptr reader; std::map orc_column_positions; }; diff --git a/src/configure_config.cmake b/src/configure_config.cmake index fd24de2f1ba87..5cbe9efaaf1b2 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -160,7 +160,8 @@ if (TARGET ch_contrib::parquet) set(USE_PARQUET 1) set(USE_ARROW 1) endif() -if (TARGET _orc) +# The ORC reader and writer are implemented in ClickHouse and need only the compression libraries. +if (TARGET ch_contrib::snappy) set(USE_ORC 1) endif() if (TARGET ch_contrib::liburing) diff --git a/tests/queries/0_stateless/01308_orc_output_format_arrays.reference b/tests/queries/0_stateless/01308_orc_output_format_arrays.reference index d771371b26442..0eb78aef36483 100644 --- a/tests/queries/0_stateless/01308_orc_output_format_arrays.reference +++ b/tests/queries/0_stateless/01308_orc_output_format_arrays.reference @@ -1 +1 @@ -0bedc9eaf0b07ce14f59ffd25e1a6df6 - +75b5b0cbd2734c37f4f02bd726c4ea60 - diff --git a/tests/queries/0_stateless/02426_orc_bug.reference b/tests/queries/0_stateless/02426_orc_bug.reference index 4f76072e70d1c..d5f93b43aa8dd 100644 --- a/tests/queries/0_stateless/02426_orc_bug.reference +++ b/tests/queries/0_stateless/02426_orc_bug.reference @@ -1 +1 @@ -64a50db9526e0e3aa8c969939922658d - +f8d66b83061a9b0ce960c0b9ce32a64c - diff --git a/tests/queries/0_stateless/04337_orc_schema_inference_deep_recursion.sh b/tests/queries/0_stateless/04337_orc_schema_inference_deep_recursion.sh index 6dfb8038211b1..1639da5845ab2 100755 --- a/tests/queries/0_stateless/04337_orc_schema_inference_deep_recursion.sh +++ b/tests/queries/0_stateless/04337_orc_schema_inference_deep_recursion.sh @@ -7,10 +7,9 @@ # grind (and could overflow the native stack), uncancellable. It must now be rejected as # TOO_DEEP_RECURSION once the nesting exceeds max_parser_depth. # -# 2. The Apache ORC C++ library (convertType) builds its own orc::Type tree by recursing over the -# file footer before any ClickHouse code runs. A pathologically deep file used to overflow the -# native stack there (crashing the process / triggering the OOM killer). The library now bounds -# the nesting depth and rejects such a file instead of crashing. +# 2. The ORC reader builds its type tree by recursing over the file footer before the schema +# inference runs. A pathologically deep file must not overflow the native stack there: the +# reader bounds the nesting depth and rejects such a file. CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -55,18 +54,24 @@ orc.write_table(pa.table({'x': pa.array([], type=t)}), '$2') " } -# (1) A moderate file (200 levels) that the ORC library builds fine, with max_parser_depth lowered +# (1) A moderate file (200 levels) that the ORC reader builds fine, with max_parser_depth lowered # below the nesting so parseORCType rejects it deterministically and build-independently. gen_orc 200 "${TMP_DIR}/moderate.orc" $CLICKHOUSE_LOCAL --query "DESC TABLE file('${TMP_DIR}/moderate.orc', ORC) SETTINGS max_parser_depth = 100" 2>&1 \ | expect_contains parser_depth_limit TOO_DEEP_RECURSION -# (2) A file nested past the ORC library's own limit. Even with max_parser_depth raised so high that -# parseORCType would accept it, the library must reject it (without crashing). The library throws +# (2) A file nested past the ORC reader's own limit. Even with max_parser_depth raised so high that +# parseORCType would accept it, the reader must reject it (without crashing). The reader throws # before recursing past its limit, so this is safe to run. gen_orc 3000 "${TMP_DIR}/deep.orc" -$CLICKHOUSE_LOCAL --query "DESC TABLE file('${TMP_DIR}/deep.orc', ORC) SETTINGS max_parser_depth = 100000" 2>&1 \ - | expect_contains orc_library_depth_limit "nesting depth exceeds the limit" +# Under TSan the stack size check allows only 5% of the stack, so it can reject the file before the depth limit. +out=$($CLICKHOUSE_LOCAL --query "DESC TABLE file('${TMP_DIR}/deep.orc', ORC) SETTINGS max_parser_depth = 100000" 2>&1) +if printf '%s\n' "$out" | grep -qF -e "nesting depth exceeds the limit" -e "TOO_DEEP_RECURSION"; then + echo "orc_library_depth_limit: nesting depth exceeds the limit" +else + echo "orc_library_depth_limit: expected 'nesting depth exceeds the limit', got:" + printf '%s\n' "$out" | head -3 +fi # (3) max_parser_depth=0 means unlimited (matching the SQL parser), so a moderately nested file must # still be inferred normally rather than rejected by the explicit limit. diff --git a/tests/queries/0_stateless/04707_tuple_subcolumn_pushdown_orc.sh b/tests/queries/0_stateless/04707_tuple_subcolumn_pushdown_orc.sh index d9f7e0d839f97..e96ecaf629530 100755 --- a/tests/queries/0_stateless/04707_tuple_subcolumn_pushdown_orc.sh +++ b/tests/queries/0_stateless/04707_tuple_subcolumn_pushdown_orc.sh @@ -22,7 +22,7 @@ ${CLICKHOUSE_CLIENT} --query " FROM numbers(20000) SETTINGS engine_file_truncate_on_insert = 1, output_format_orc_row_index_stride = 2000" -# rows_read is the pruning oracle: liborc skips row index strides whose statistics exclude the +# rows_read is the pruning oracle: the ORC reader skips row index strides whose statistics exclude the # predicate, so a pruned read reports fewer rows than the file holds. run_and_report() { local label="$1" diff --git a/tests/queries/0_stateless/04856_orc_corrupt_stripe_footer.reference b/tests/queries/0_stateless/04856_orc_corrupt_stripe_footer.reference index 0072ff8938135..7bed391e68f39 100644 --- a/tests/queries/0_stateless/04856_orc_corrupt_stripe_footer.reference +++ b/tests/queries/0_stateless/04856_orc_corrupt_stripe_footer.reference @@ -1,5 +1,4 @@ INCORRECT_DATA -Read past EOF in DecompressionStream::readBuffer 0 0 1 1 2 2 diff --git a/tests/queries/0_stateless/04856_orc_corrupt_stripe_footer.sh b/tests/queries/0_stateless/04856_orc_corrupt_stripe_footer.sh index b9b83b3b2e369..57a8727696be6 100755 --- a/tests/queries/0_stateless/04856_orc_corrupt_stripe_footer.sh +++ b/tests/queries/0_stateless/04856_orc_corrupt_stripe_footer.sh @@ -5,11 +5,8 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -# The stripe footer of this blob is truncated, so ORC's decompression stream reports the -# failure by throwing from inside the protobuf parser. The parse must surface as a regular -# INCORRECT_DATA exception; a protobuf has-bits assertion instead means the message was left -# inconsistent while the exception unwound. Only debug and sanitizer builds check that -# assertion, so the discriminating arm of this test is those builds. +# The stripe footer of this blob is truncated, so reading it must fail with a regular +# INCORRECT_DATA exception. BLOB='4f52431100000a061204080150005500000a280a06000000000000121e08014a1818b8fef997d7d63820b8fef997d7d63828c1fc1530c1fc1550000f0000780039604d293e0b00006e0007dda36300f00a0608061000180b0a0608061001182d0a0608011001180a0a060805100118081204080010001204080210001a03474d545100000a260a04080150000a1e08014a1818b8fef997d7d63820b8fef997d7d63828c1fc1530c1fc155000b500000803107e1a0a08031038181220342801220f080c1201011a0263302000280030002208080920002800300030013a04080150003a1e08014a1818b8fef997d7d63820b8fef997d7d63828c1fc1530c1fc15500040904e48016200085d1005188080102202000c282b300682f403034f524317' # The fixture must stay pure even-length lowercase hex: unhex's result for a non-hex character is @@ -21,8 +18,7 @@ if [[ ! $BLOB =~ ^[0-9a-f]+$ ]] || (( ${#BLOB} % 2 )); then fi $CLICKHOUSE_LOCAL -q "SELECT c0 FROM format(ORC, 'c0 DateTime64(3)', unhex('$BLOB'));" 2>&1 \ - | grep -oE "Code: [0-9]+.*Read past EOF in DecompressionStream::readBuffer|INCORRECT_DATA|Has bits mismatch|Check failure stack trace" \ - | sed -E 's/^Code: [0-9]+.*Read past EOF in DecompressionStream::readBuffer$/Read past EOF in DecompressionStream::readBuffer/' \ + | grep -oE "INCORRECT_DATA" \ | sort -u # Well-formed ORC must still read back unchanged. diff --git a/tests/queries/0_stateless/05036_orc_unimplemented_codec_no_abort.reference b/tests/queries/0_stateless/05036_orc_unimplemented_codec_no_abort.reference new file mode 100644 index 0000000000000..1f714f4b2082b --- /dev/null +++ b/tests/queries/0_stateless/05036_orc_unimplemented_codec_no_abort.reference @@ -0,0 +1,14 @@ +BROTLI compression in ORC files is not supported +BROTLI compression in ORC files is not supported +BROTLI compression in ORC files is not supported +BROTLI compression in ORC files is not supported +BROTLI compression in ORC files is not supported +BROTLI compression in ORC files is not supported +CANNOT_EXTRACT_TABLE_STRUCTURE +INCORRECT_DATA +0 +CANNOT_EXTRACT_TABLE_STRUCTURE +INCORRECT_DATA +0 +1 +2 diff --git a/tests/queries/0_stateless/05036_orc_unimplemented_codec_no_abort.sh b/tests/queries/0_stateless/05036_orc_unimplemented_codec_no_abort.sh new file mode 100755 index 0000000000000..dd54288e64f9c --- /dev/null +++ b/tests/queries/0_stateless/05036_orc_unimplemented_codec_no_abort.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Truncating this fixture to 289 bytes leaves byte 6 in the postscript's compression field. Codec 6 +# is BROTLI, which the ORC format declares but this reader does not implement. The reader rejects it +# with a NOT_IMPLEMENTED exception; it must never be a logical error, which aborts in debug and +# sanitizer builds. +DATA_FILE="$CLICKHOUSE_TMP/05036_orc_codec6_${CLICKHOUSE_DATABASE}.orc" +head -c 289 "$CUR_DIR"/data_orc/orc_nested_union_type.orc > "$DATA_FILE" + +# Both the schema-inference path and the explicit-structure path reach the reader, so both are +# asserted here. Match on the error text, never on the exit status: ClickHouse exits with the error +# code truncated to a byte, so 636 and 124 are the same outcome. +for query in \ + "SELECT * FROM file('$DATA_FILE', ORC) FORMAT Null" \ + "SELECT * FROM file('$DATA_FILE', ORC, 'x Int32') FORMAT Null" \ + "SELECT count() FROM file('$DATA_FILE', ORC, 'x Int32')" \ + "DESCRIBE file('$DATA_FILE', ORC)" \ + "SELECT * FROM format(ORC, 'x Int32', (SELECT * FROM file('$DATA_FILE', RawBLOB))) FORMAT Null" +do + # An abort prints nothing at all, so classify explicitly rather than letting the arm go silent: + # a missing line is much harder to read in a diff than a named verdict. + OUT=$($CLICKHOUSE_LOCAL --query "$query" 2>&1) + if [ -z "$OUT" ]; then + echo "no output at all" + else + echo "$OUT" | grep -o -m1 'BROTLI compression in ORC files is not supported\|Logical error' || echo "unexpected: $OUT" + fi +done + +# The message must name the codec, and must not call the file corrupt: a well-formed +# Brotli-compressed ORC file takes this same path. +$CLICKHOUSE_LOCAL --query "SELECT * FROM file('$DATA_FILE', ORC, 'x Int32') FORMAT Null" 2>&1 \ + | grep -o -m1 'BROTLI compression in ORC files is not supported\|CORRUPTED\|INCORRECT_DATA' + +# Neighbouring truncations are genuinely corrupt and must keep reporting corruption rather than the +# codec gap. Schema inference wraps every non-retryable cause in one outer code, so that arm alone +# cannot tell the two apart; the explicit-structure arm bypasses the wrapper and surfaces the inner +# classification, and counting the codec error there pins them apart. +for length in 200 400 +do + NEIGHBOUR="$CLICKHOUSE_TMP/05036_orc_${length}_${CLICKHOUSE_DATABASE}.orc" + head -c $length "$CUR_DIR"/data_orc/orc_nested_union_type.orc > "$NEIGHBOUR" + $CLICKHOUSE_LOCAL --query "SELECT * FROM file('$NEIGHBOUR', ORC) FORMAT Null" 2>&1 \ + | grep -o -m1 'CANNOT_EXTRACT_TABLE_STRUCTURE\|Logical error' + EXPLICIT=$($CLICKHOUSE_LOCAL --query "SELECT * FROM file('$NEIGHBOUR', ORC, 'x Int32') FORMAT Null" 2>&1) + echo "$EXPLICIT" | grep -o -m1 'INCORRECT_DATA\|Logical error' || echo "unexpected: $EXPLICIT" + echo "$EXPLICIT" | grep -c 'BROTLI compression' + rm "$NEIGHBOUR" +done + +# Well-formed ORC must still read back unchanged. +$CLICKHOUSE_LOCAL --query "SELECT i FROM file('$CUR_DIR/data_orc/orc_union_type.orc', ORC) ORDER BY i" + +rm "$DATA_FILE" diff --git a/tests/queries/0_stateless/05317_orc_corrupt_stripe_footer_stream_column.reference b/tests/queries/0_stateless/05317_orc_corrupt_stripe_footer_stream_column.reference new file mode 100644 index 0000000000000..88b9566a6b12d --- /dev/null +++ b/tests/queries/0_stateless/05317_orc_corrupt_stripe_footer_stream_column.reference @@ -0,0 +1,3 @@ +column 20 is out of range, the file has 5 columns +column 20 is out of range, the file has 5 columns +100 4950 diff --git a/tests/queries/0_stateless/05317_orc_corrupt_stripe_footer_stream_column.sh b/tests/queries/0_stateless/05317_orc_corrupt_stripe_footer_stream_column.sh new file mode 100755 index 0000000000000..27b41467be726 --- /dev/null +++ b/tests/queries/0_stateless/05317_orc_corrupt_stripe_footer_stream_column.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: Depends on S3 + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A 569-byte ORC file (id Int64, name String, v Float64, part Int32) with one row, whose stripe footer declares a +# row index stream for column 20, but the file has 5 columns. Two readers indexed the selected-column mask with +# that id and aborted the server; both must report a malformed file instead: +# - remote reads prebuffer stripes (`ORC::Reader::preBuffer`), so the file is read through s3; +# - predicate pushdown loads the row indexes (`ORC::RowReader::loadStripeIndex`), also for local files. The +# predicate must keep the stripe (`id >= 0`; the row has id 1), or the stripe statistics skip it first. +ORC_B64=$(tr -d '\n' <<'B64' +T1JDEQAACgYSBAgBUAArAAAKEwoDAAAAEgwIARIGCAIQAhgCUAAzAAAKFwoFAAAAAAASDggBIggKAWESAWEYAlAAUwAACicKAgAA +EiEIARobCQAAAAAAAPg/EQAAAAAAAPg/GQAAAAAAAPg/UAArAAAKEwoDAAAAEgwIARIGCAAQABgAUAAHAABCAIADAABhBwAAQACA +EQAAAAAAAAAA+D8HAABAAACuAAAotS/9IGR1AgCEAwoGCAYQABgLCgYIBhABGBgCGBwDGCwUARABGAYCGAQKBggCEAIDARAEGAYS +AggAEgIIAgASAggCCAB7/hAam+nkirydJ0AbQIZdZQm6AAAotS/9IF+lAgBEBApdCgQIAVAACg4IARIGCAIQAhgCUABYBgoQCAEi +CAoBYRIBYQoKIwgBGhsJAPg/EQD4PxkA+D9QAFgLABAAGABQAFgGBQBKyAIaAUJwd1jtfAJ4AQAotS/9IOydBQBCyiYskLc5//// +///w6acgxiIELbemIDUZs4IjVzcRUBwT/pE0KZF9o02pZZNSphSQJ3okw0BbBghYiEDLlRuOvFiuU8aKKePElHHiyPYGVW7/bcVn +e6TQcm1Ni83htzOUa/ujDT2c/5Sg/4wO/hPqp6rnKrCqiyUKQ5//PDC42JMILQmsez1JVU0+1/o9wp3ZDoEFo2mF6HyRLSE6DwgA +SsgCGgFCcHdas32g06DT47p5Bgi/ARAFGICAECICAAwoYDAJgvQDA09SQxg= +B64 +) +FILE="05317_${CLICKHOUSE_DATABASE}.orc" + +$CLICKHOUSE_CLIENT --s3_truncate_on_insert 1 \ + --query "INSERT INTO FUNCTION s3(s3_conn, filename='$FILE', format='RawBLOB') SELECT base64Decode('$ORC_B64')" + +# Stripes are prebuffered only with `remote_filesystem_read_prefetch`, which CI randomizes. +$CLICKHOUSE_CLIENT --remote_filesystem_read_prefetch 1 \ + --query "SELECT * FROM s3(s3_conn, filename='$FILE', format='ORC') FORMAT Null" 2>&1 \ + | grep -o -m1 'column 20 is out of range, the file has 5 columns' + +# Predicate pushdown (on by default) on a local file, which is never prebuffered. +LOCAL_FILE="05317_${CLICKHOUSE_DATABASE}.orc" +$CLICKHOUSE_CLIENT --engine_file_truncate_on_insert 1 \ + --query "INSERT INTO FUNCTION file('$LOCAL_FILE', 'RawBLOB') SELECT base64Decode('$ORC_B64')" +$CLICKHOUSE_CLIENT --input_format_orc_filter_push_down 1 \ + --query "SELECT * FROM file('$LOCAL_FILE', 'ORC') WHERE id >= 0 FORMAT Null" 2>&1 \ + | grep -o -m1 'column 20 is out of range, the file has 5 columns' +rm -f "${USER_FILES_PATH:?}/$LOCAL_FILE" + +# The server is still alive, and a valid ORC file is still read through the same path. +VALID="05317_valid_${CLICKHOUSE_DATABASE}.orc" +$CLICKHOUSE_CLIENT --s3_truncate_on_insert 1 \ + --query "INSERT INTO FUNCTION s3(s3_conn, filename='$VALID', format='ORC') SELECT number AS id, toString(number) AS s FROM numbers(100)" +$CLICKHOUSE_CLIENT --query "SELECT count(), sum(id) FROM s3(s3_conn, filename='$VALID', format='ORC')" diff --git a/tests/queries/0_stateless/05407_orc_timezone_traversal_abort.reference b/tests/queries/0_stateless/05407_orc_timezone_traversal_abort.reference new file mode 100644 index 0000000000000..fcc653eab3e4a --- /dev/null +++ b/tests/queries/0_stateless/05407_orc_timezone_traversal_abort.reference @@ -0,0 +1,5 @@ +writer: time zone is rejected +reader: time zone is rejected +reader file: time zone is rejected +--- stateless server is still alive --- +1 diff --git a/tests/queries/0_stateless/05407_orc_timezone_traversal_abort.sh b/tests/queries/0_stateless/05407_orc_timezone_traversal_abort.sh new file mode 100755 index 0000000000000..a94f164952f4b --- /dev/null +++ b/tests/queries/0_stateless/05407_orc_timezone_traversal_abort.sh @@ -0,0 +1,134 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh +set -e + +# A malformed ORC timezone name used to reach an empty TZif file outside TZDIR and abort +# inside libc++. The time zones are now resolved only from the embedded time zone database, +# so such a name must be rejected without touching the file system. Keep each attempt in a child clickhouse-local process and disable core dumps. +ulimit -c 0 + +WORK_DIR="${CLICKHOUSE_TMP}/orc_timezone_traversal_${CLICKHOUSE_TEST_UNIQUE_NAME}" +rm -rf "${WORK_DIR}" +mkdir -p "${WORK_DIR}" +trap 'rm -rf "${WORK_DIR}"' EXIT + +regression_status=0 +python3 - "${WORK_DIR}" <<'PY' || regression_status=$? +import os +import pathlib +import shlex +import shutil +import subprocess +import sys + +work_dir = pathlib.Path(sys.argv[1]) +tz_dir = work_dir / "zoneinfo" / "a" / "b" +tz_dir.mkdir(parents=True) + +# Use real, minimal zone files so the valid setup and the ORC fixture can be loaded while +# TZDIR points at this private directory. The traversal resolves to this test's empty file. +system_zoneinfo = pathlib.Path("/usr/share/zoneinfo") +for zone in ("UTC", "America/Vancouver"): + source = system_zoneinfo / zone + if not source.is_file(): + raise SystemExit(f"required time zone file not found: {source}") + target = tz_dir / zone + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, target) + +(work_dir / "outside").write_bytes(b"") +if (tz_dir / "../../../outside").resolve() != (work_dir / "outside").resolve(): + raise SystemExit("private timezone traversal fixture is misconfigured") +(work_dir / "dev").mkdir() +(work_dir / "dev" / "null").write_bytes(b"") +if (tz_dir / "../../../dev/null").resolve() != (work_dir / "dev" / "null").resolve(): + raise SystemExit("private ORC metadata traversal fixture is misconfigured") + +env = os.environ.copy() +env["TZDIR"] = str(tz_dir) +env["TZ"] = "UTC" +local_command = shlex.split(env["CLICKHOUSE_LOCAL"]) + + +def run_local(path, *args, query=None, input_data=None): + path.mkdir(parents=True, exist_ok=True) + command = local_command + ["--path", str(path), *args] + if query is not None: + command += ["--query", query] + return subprocess.run( + command, + input=input_data, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + +safe_query = ( + "SELECT toDateTime(0, 'America/Vancouver') AS t FORMAT ORC " + "SETTINGS output_format_orc_writer_time_zone_name='America/Vancouver', " + "output_format_orc_compression_method='none'" +) +fixture = run_local(work_dir / "safe", query=safe_query) +if fixture.returncode != 0 or not fixture.stdout.startswith(b"ORC"): + diagnostic = (fixture.stderr + fixture.stdout).decode(errors="replace") + raise SystemExit(f"could not create valid ORC fixture: {diagnostic[:1000]}") + +# Preserve the ORC file's byte lengths while replacing its writer timezone metadata +# with the path from the customer reproducer. This exercises a file supplied by a +# reader, without requiring a malicious query setting. +safe_name = b"America/Vancouver" +unsafe_name = b"../../../dev/null" +if len(safe_name) != len(unsafe_name) or fixture.stdout.count(safe_name) != 1: + raise SystemExit("valid ORC fixture has unexpected timezone metadata") +malicious_file = fixture.stdout.replace(safe_name, unsafe_name, 1) + +malicious_writer_query = ( + "SELECT toDateTime(0, 'America/Vancouver') AS t FORMAT ORC " + "SETTINGS output_format_orc_writer_time_zone_name='../../../outside', " + "output_format_orc_compression_method='none'" +) +writer = run_local(work_dir / "writer", query=malicious_writer_query) +reader = run_local( + work_dir / "reader", + "--input-format=ORC", + "--input_format_orc_reader_time_zone_name=../../../outside", + query="SELECT * FROM table", + input_data=fixture.stdout, +) +reader_file = run_local( + work_dir / "reader_file", + "--input-format=ORC", + query="SELECT * FROM table", + input_data=malicious_file, +) + +failed = False +for label, result in (("writer", writer), ("reader", reader), ("reader file", reader_file)): + if result.returncode < 0: + print(f"{label}: terminated by signal {-result.returncode}") + failed = True + elif result.returncode == 0: + print(f"{label}: malformed timezone unexpectedly succeeded") + failed = True + elif b"is not supported. Use a name from `system.time_zones`" in result.stdout + result.stderr: + print(f"{label}: time zone is rejected") + else: + diagnostic = (result.stderr + result.stdout).decode(errors="replace") + first_line = next((line for line in diagnostic.splitlines() if line), "no diagnostic") + print(f"{label}: unexpected error: {first_line[:300]}") + failed = True + +if failed: + raise SystemExit(1) +PY + +echo '--- stateless server is still alive ---' +${CLICKHOUSE_CLIENT} --query "SELECT 1" +exit "${regression_status}" diff --git a/tests/queries/0_stateless/05447_orc_native_reader_writer.reference b/tests/queries/0_stateless/05447_orc_native_reader_writer.reference new file mode 100644 index 0000000000000..a46fd04699683 --- /dev/null +++ b/tests/queries/0_stateless/05447_orc_native_reader_writer.reference @@ -0,0 +1,14 @@ +1900-06-01 00:01:41.000007000 -2195942298999993 +1900-06-01 00:01:41.000000000 -2195942299000000 +1960-06-01 00:01:41.000007000 -302486298999993 +2000-06-01 00:01:41.000007000 959817701000007 +none 25000 609103159500350648 +zstd 25000 609103159500350648 +lz4 25000 609103159500350648 +snappy 25000 609103159500350648 +zlib 25000 609103159500350648 +expected 25000 609103159500350648 +1001 4500 5500 +1923 +['a','b','a','b'] +['a','b','a','b'] Array(LowCardinality(String)) diff --git a/tests/queries/0_stateless/05447_orc_native_reader_writer.sh b/tests/queries/0_stateless/05447_orc_native_reader_writer.sh new file mode 100755 index 0000000000000..bee99fd058401 --- /dev/null +++ b/tests/queries/0_stateless/05447_orc_native_reader_writer.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The Arrow ORC adapter stores negative nanoseconds for the timestamps before the epoch, +# and the reader must decode them as the reference implementation does. +$CLICKHOUSE_LOCAL -q "SELECT t, toUnixTimestamp64Micro(t) FROM file('$CUR_DIR/data_orc/timestamps_before_epoch_arrow.orc', ORC) SETTINGS session_timezone = 'UTC'" + +DATA_FILE="$CLICKHOUSE_TMP/05447_${CLICKHOUSE_DATABASE}.orc" + +# Dictionary encoding of strings (with the row index and several row groups), a non-GMT writer time +# zone, and the values that need all integer encodings: repeats, deltas, outliers. +QUERY="SELECT number AS n, + toString(number % 13) AS low_cardinality, + if(number % 5 = 0, NULL, concat('value_', toString(number))) AS high_cardinality, + toDateTime64(1700000000 + number * 37, 3, 'UTC') AS ts, + if(number % 1000 = 0, 1000000000000, number % 7) AS outliers, + intDiv(number, 100) AS runs, + toDecimal128(number, 10) * -3 AS dec +FROM numbers(25000)" + +for compression in none zstd lz4 snappy zlib +do + $CLICKHOUSE_LOCAL -q "$QUERY FORMAT ORC SETTINGS output_format_orc_compression_method = '$compression', + output_format_orc_dictionary_key_size_threshold = 0.5, output_format_orc_row_index_stride = 1000, + output_format_orc_writer_time_zone_name = 'America/Los_Angeles'" > "$DATA_FILE" + $CLICKHOUSE_LOCAL -q "SELECT '$compression', count(), sum(cityHash64(*)) FROM file('$DATA_FILE', ORC, + 'n UInt64, low_cardinality String, high_cardinality Nullable(String), ts DateTime64(3, \'UTC\'), outliers UInt64, runs UInt64, dec Decimal(38, 10)') + SETTINGS input_format_orc_reader_time_zone_name = 'America/Los_Angeles'" +done +$CLICKHOUSE_LOCAL -q "SELECT 'expected', count(), sum(cityHash64(*)) FROM ($QUERY)" + +# Filter pushdown skips the row groups by the statistics in the row index. +$CLICKHOUSE_LOCAL -q "SELECT count(), min(n), max(n) FROM file('$DATA_FILE', ORC) WHERE n BETWEEN 4500 AND 5500 SETTINGS input_format_orc_filter_push_down = 1" +$CLICKHOUSE_LOCAL -q "SELECT count() FROM file('$DATA_FILE', ORC) WHERE low_cardinality = '12' SETTINGS input_format_orc_filter_push_down = 1" + +# The dictionary of a string column nested in an array can have more entries than the stripe has rows. +$CLICKHOUSE_LOCAL -q "SELECT ['a', 'b', 'a', 'b'] AS arr FORMAT ORC SETTINGS output_format_orc_dictionary_key_size_threshold = 0.5" > "$DATA_FILE" +$CLICKHOUSE_LOCAL -q "SELECT arr FROM file('$DATA_FILE', ORC, 'arr Array(String)')" +$CLICKHOUSE_LOCAL -q "SELECT arr, toTypeName(arr) FROM file('$DATA_FILE', ORC, 'arr Array(LowCardinality(String))')" + +rm "$DATA_FILE" diff --git a/tests/queries/0_stateless/05448_orc_date_after_2059.reference b/tests/queries/0_stateless/05448_orc_date_after_2059.reference new file mode 100644 index 0000000000000..62cea6b447e40 --- /dev/null +++ b/tests/queries/0_stateless/05448_orc_date_after_2059.reference @@ -0,0 +1,12 @@ +0 1970-01-01 Nullable(Date32) +1 2059-09-17 Nullable(Date32) +2 2059-09-18 Nullable(Date32) +3 2100-01-01 Nullable(Date32) +4 2149-06-06 Nullable(Date32) +0 1970-01-01 +1 2059-09-17 +2 2059-09-18 +3 2100-01-01 +4 2149-06-06 +3 +4 diff --git a/tests/queries/0_stateless/05448_orc_date_after_2059.sh b/tests/queries/0_stateless/05448_orc_date_after_2059.sh new file mode 100755 index 0000000000000..8cc480ff4b24c --- /dev/null +++ b/tests/queries/0_stateless/05448_orc_date_after_2059.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `Date` is the unsigned number of days, and the dates after 2059-09-18 (more than 32767 days) used +# to be written to ORC as negative numbers, which read back as the dates before 1970. +DATA_FILE="$CLICKHOUSE_TMP/05448_${CLICKHOUSE_DATABASE}.orc" + +$CLICKHOUSE_LOCAL -q "SELECT number AS n, toDate(d) AS d FROM values('number UInt64, d String', (0, '1970-01-01'), (1, '2059-09-17'), (2, '2059-09-18'), (3, '2100-01-01'), (4, '2149-06-06')) FORMAT ORC" > "$DATA_FILE" + +$CLICKHOUSE_LOCAL -q "SELECT n, d, toTypeName(d) FROM file('$DATA_FILE', ORC) ORDER BY n" +$CLICKHOUSE_LOCAL -q "SELECT n, d FROM file('$DATA_FILE', ORC, 'n UInt64, d Date') ORDER BY n" + +# The statistics are correct as well, so the filter pushdown keeps the rows. +$CLICKHOUSE_LOCAL -q "SELECT n FROM file('$DATA_FILE', ORC, 'n UInt64, d Date') WHERE d > '2060-01-01' ORDER BY n SETTINGS input_format_orc_filter_push_down = 1" + +rm "$DATA_FILE" diff --git a/tests/queries/0_stateless/05453_orc_bad_bloom_filter.reference b/tests/queries/0_stateless/05453_orc_bad_bloom_filter.reference new file mode 100644 index 0000000000000..a4a9c43f01b26 --- /dev/null +++ b/tests/queries/0_stateless/05453_orc_bad_bloom_filter.reference @@ -0,0 +1,8 @@ +1.6.0 0 18000000000 Mike +1.6.0 0 1 +1.6.0 1 18000000000 Mike +1.6.0 1 1 +1.6.11 0 18000000000 Mike +1.6.11 0 1 +1.6.11 1 18000000000 Mike +1.6.11 1 1 diff --git a/tests/queries/0_stateless/05453_orc_bad_bloom_filter.sh b/tests/queries/0_stateless/05453_orc_bad_bloom_filter.sh new file mode 100755 index 0000000000000..30badff1fa0dc --- /dev/null +++ b/tests/queries/0_stateless/05453_orc_bad_bloom_filter.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + + +# The bloom filters written by the C++ writer of ORC 1.6.0 - 1.6.11 and 1.7.0 use incorrect hashing (ORC-1024) +# and must be ignored, otherwise the filter pushdown skips the matching rows. +for version in 1.6.0 1.6.11 +do + for push_down in 0 1 + do + $CLICKHOUSE_LOCAL -q " + SELECT '$version', $push_down, id, name + FROM file('$CUR_DIR/data_orc/bad_bloom_filter_$version.orc') + WHERE id = 18000000000 + SETTINGS input_format_orc_filter_push_down = $push_down" + $CLICKHOUSE_LOCAL -q " + SELECT '$version', $push_down, count() + FROM file('$CUR_DIR/data_orc/bad_bloom_filter_$version.orc') + WHERE id IN (18000000000, 18000000001) + SETTINGS input_format_orc_filter_push_down = $push_down" + done +done diff --git a/tests/queries/0_stateless/05473_orc_footer_content_length.reference b/tests/queries/0_stateless/05473_orc_footer_content_length.reference new file mode 100644 index 0000000000000..ae5399ce4997a --- /dev/null +++ b/tests/queries/0_stateless/05473_orc_footer_content_length.reference @@ -0,0 +1,2 @@ +1 True True +1 True True diff --git a/tests/queries/0_stateless/05473_orc_footer_content_length.sh b/tests/queries/0_stateless/05473_orc_footer_content_length.sh new file mode 100755 index 0000000000000..78921273f950c --- /dev/null +++ b/tests/queries/0_stateless/05473_orc_footer_content_length.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest. +# The `content_length` in the ORC footer covers only the stripes, not the metadata section with the stripe statistics. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +check() +{ + $CLICKHOUSE_LOCAL --query "$1" | python3 -c ' +import sys + +data = sys.stdin.buffer.read() + +def varint(buf, pos): + result = shift = 0 + while True: + byte = buf[pos] + pos += 1 + result |= (byte & 0x7f) << shift + shift += 7 + if byte < 0x80: + return result, pos + +def fields(buf): + pos = 0 + while pos < len(buf): + key, pos = varint(buf, pos) + if key & 7 == 0: + value, pos = varint(buf, pos) + elif key & 7 == 2: + length, pos = varint(buf, pos) + value = buf[pos:pos + length] + pos += length + else: + raise Exception("unexpected wire type") + yield key >> 3, value + +postscript_length = data[-1] +postscript = dict(fields(data[-1 - postscript_length:-1])) +footer_length = postscript[1] +footer = list(fields(data[-1 - postscript_length - footer_length:-1 - postscript_length])) +content_length = next(v for f, v in footer if f == 2) +stripes = [dict(fields(v)) for f, v in footer if f == 3] +stripes_length = sum(s.get(2, 0) + s.get(3, 0) + s.get(4, 0) for s in stripes) +print(len(stripes), content_length == stripes_length, postscript[5] > 0) +' +} + +check "SELECT number FROM numbers(3) FORMAT ORC SETTINGS output_format_orc_compression_method = 'none'" +check "SELECT number, toString(number) FROM numbers(100000) FORMAT ORC SETTINGS output_format_orc_compression_method = 'none'" diff --git a/tests/queries/0_stateless/05474_orc_decimal64_v2.reference b/tests/queries/0_stateless/05474_orc_decimal64_v2.reference new file mode 100644 index 0000000000000..754cd5dae7684 --- /dev/null +++ b/tests/queries/0_stateless/05474_orc_decimal64_v2.reference @@ -0,0 +1,20 @@ +a Nullable(Int64) +b Nullable(Decimal(12, 0)) +c Nullable(Decimal(20, 2)) +d Nullable(Decimal(12, 2)) +e Nullable(Decimal(2, 2)) +17292380420 24 36164.16 0.03 0.01 +17292380421 38 63351.7 0.08 0.01 +17292380421 28 42673.96 0.09 0.06 +17292380421 40 76677.6 0.05 0.04 +17292380421 2 2096.48 0.07 0.07 +17292380421 42 45284.82 0.07 0.05 +17292380421 10 18572.9 0.01 0.08 +17292380422 12 14836.8 0.09 0.06 +17292380422 41 82152.52 0.07 0.02 +17292380422 38 47240.84 0.1 0 +38 0.08 +40 0.05 +42 0.07 +41 0.07 +38 0.1 diff --git a/tests/queries/0_stateless/05474_orc_decimal64_v2.sh b/tests/queries/0_stateless/05474_orc_decimal64_v2.sh new file mode 100755 index 0000000000000..94e31bf561544 --- /dev/null +++ b/tests/queries/0_stateless/05474_orc_decimal64_v2.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `decimal64_v2.orc` from the examples of apache/orc has the format version 1.9999 (`UNSTABLE_PRE_2_0`), +# where the DECIMAL columns with the precision up to 18 are signed RLEv2 integers without the SECONDARY stream. +$CLICKHOUSE_LOCAL -q "DESCRIBE file('$CUR_DIR/data_orc/decimal64_v2.orc', ORC)" +$CLICKHOUSE_LOCAL -q "SELECT * FROM file('$CUR_DIR/data_orc/decimal64_v2.orc', ORC)" +$CLICKHOUSE_LOCAL -q "SELECT b, d FROM file('$CUR_DIR/data_orc/decimal64_v2.orc', ORC) WHERE b > 30 SETTINGS input_format_orc_filter_push_down = 1" diff --git a/tests/queries/0_stateless/05507_orc_malformed_timestamps.reference b/tests/queries/0_stateless/05507_orc_malformed_timestamps.reference new file mode 100644 index 0000000000000..573a680fbe868 --- /dev/null +++ b/tests/queries/0_stateless/05507_orc_malformed_timestamps.reference @@ -0,0 +1,16 @@ +nanoseconds local push_down=0 lazy=0: (INCORRECT_DATA) +nanoseconds local push_down=0 lazy=1: (INCORRECT_DATA) +nanoseconds local push_down=1 lazy=0: (INCORRECT_DATA) +nanoseconds local push_down=1 lazy=1: (INCORRECT_DATA) +nanoseconds remote push_down=0 lazy=0: (INCORRECT_DATA) +nanoseconds remote push_down=0 lazy=1: (INCORRECT_DATA) +nanoseconds remote push_down=1 lazy=0: (INCORRECT_DATA) +nanoseconds remote push_down=1 lazy=1: (INCORRECT_DATA) +statistics local push_down=0 lazy=0: 3 +statistics local push_down=0 lazy=1: 3 +statistics local push_down=1 lazy=0: 3 +statistics local push_down=1 lazy=1: 3 +statistics remote push_down=0 lazy=0: 3 +statistics remote push_down=0 lazy=1: 3 +statistics remote push_down=1 lazy=0: 3 +statistics remote push_down=1 lazy=1: 3 diff --git a/tests/queries/0_stateless/05507_orc_malformed_timestamps.sh b/tests/queries/0_stateless/05507_orc_malformed_timestamps.sh new file mode 100755 index 0000000000000..53eae4b38eb09 --- /dev/null +++ b/tests/queries/0_stateless/05507_orc_malformed_timestamps.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest, the test uses S3. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh +# shellcheck source=./orc_malformed.lib +. "$CUR_DIR"/orc_malformed.lib + +DATA="$CUR_DIR/data_orc" + +# The nanoseconds of a timestamp with the 7 removed trailing zeros overflow `Int64`. +orc_check nanoseconds "$DATA/malformed_timestamp_nanoseconds.orc" 't Nullable(DateTime64(9))' "SELECT count(), max(t) FROM SOURCE" \ + 'Invalid timestamp in ORC column 1: seconds=[0-9]+, encoded nanoseconds=-?[0-9]+' + +# The minimum nanoseconds in the statistics is 0x80000000: such statistics must not be used by the filter pushdown. +orc_check statistics "$DATA/malformed_timestamp_statistics_nanoseconds.orc" 't DateTime64(9)' \ + "SELECT count() FROM SOURCE WHERE t > toDateTime64('2000-01-01 00:00:00', 9, 'UTC')" diff --git a/tests/queries/0_stateless/05508_orc_writer_out_of_range_values.reference b/tests/queries/0_stateless/05508_orc_writer_out_of_range_values.reference new file mode 100644 index 0000000000000..17ac0b58ed17d --- /dev/null +++ b/tests/queries/0_stateless/05508_orc_writer_out_of_range_values.reference @@ -0,0 +1,7 @@ +1e308 ['0','1','2','0','1','2','0','1','2','0'] +-1e308 ['0','1','2','0','1','2','0','1','2','0'] +(VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE) +(VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE) +(VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE) +9223372036854775807 +-9223372036854775808 diff --git a/tests/queries/0_stateless/05508_orc_writer_out_of_range_values.sh b/tests/queries/0_stateless/05508_orc_writer_out_of_range_values.sh new file mode 100755 index 0000000000000..6da24b0909d78 --- /dev/null +++ b/tests/queries/0_stateless/05508_orc_writer_out_of_range_values.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A dictionary threshold that does not fit in `size_t` when multiplied by the number of values. +for threshold in 1e308 -1e308; do + $CLICKHOUSE_LOCAL -q "SELECT toString(number % 3) AS s FROM numbers(10) FORMAT ORC SETTINGS output_format_orc_dictionary_key_size_threshold = $threshold" \ + | $CLICKHOUSE_LOCAL --input-format ORC -q "SELECT '$threshold', groupArray(s) FROM table" +done + +# The extreme `DateTime64` values that cannot be stored in ORC with their statistics in milliseconds. +for value in 'reinterpret(toInt64(9223372036854775807), '"'"'DateTime64(0)'"'"')' \ + 'reinterpret(toInt64(-9223372036854775808), '"'"'DateTime64(0)'"'"')' \ + 'reinterpret(toInt64(9223372036854775807), '"'"'DateTime64(1)'"'"')'; do + $CLICKHOUSE_LOCAL -q "SELECT $value AS t FORMAT ORC" 2>&1 >/dev/null | grep -oE '\(VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE\)' | head -n1 | grep . || echo "$value: no error" +done + +# The extreme values with the nanoseconds can be stored. +for value in 9223372036854775807 -9223372036854775808; do + $CLICKHOUSE_LOCAL -q "SELECT reinterpret(toInt64($value), 'DateTime64(9, \'UTC\')') AS t FORMAT ORC" \ + | $CLICKHOUSE_LOCAL --input-format ORC -q "SELECT reinterpret(t, 'Int64') FROM table" +done diff --git a/tests/queries/0_stateless/05509_orc_malformed_dictionaries.reference b/tests/queries/0_stateless/05509_orc_malformed_dictionaries.reference new file mode 100644 index 0000000000000..2d165fb8a6c0b --- /dev/null +++ b/tests/queries/0_stateless/05509_orc_malformed_dictionaries.reference @@ -0,0 +1,16 @@ +empty local push_down=0 lazy=0: (INCORRECT_DATA) +empty local push_down=0 lazy=1: (INCORRECT_DATA) +empty local push_down=1 lazy=0: (INCORRECT_DATA) +empty local push_down=1 lazy=1: (INCORRECT_DATA) +empty remote push_down=0 lazy=0: (INCORRECT_DATA) +empty remote push_down=0 lazy=1: (INCORRECT_DATA) +empty remote push_down=1 lazy=0: (INCORRECT_DATA) +empty remote push_down=1 lazy=1: (INCORRECT_DATA) +nulls local push_down=0 lazy=0: 5 5 +nulls local push_down=0 lazy=1: 5 5 +nulls local push_down=1 lazy=0: 5 5 +nulls local push_down=1 lazy=1: 5 5 +nulls remote push_down=0 lazy=0: 5 5 +nulls remote push_down=0 lazy=1: 5 5 +nulls remote push_down=1 lazy=0: 5 5 +nulls remote push_down=1 lazy=1: 5 5 diff --git a/tests/queries/0_stateless/05509_orc_malformed_dictionaries.sh b/tests/queries/0_stateless/05509_orc_malformed_dictionaries.sh new file mode 100755 index 0000000000000..437d071ce5a5a --- /dev/null +++ b/tests/queries/0_stateless/05509_orc_malformed_dictionaries.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest, the test uses S3. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh +# shellcheck source=./orc_malformed.lib +. "$CUR_DIR"/orc_malformed.lib + +DATA="$CUR_DIR/data_orc" + +# A dictionary without entries referenced by the non-null values. +orc_check empty "$DATA/malformed_dictionary_empty.orc" 's Nullable(String)' "SELECT count(), countIf(s IS NULL) FROM SOURCE" \ + 'ORC dictionary index 0 at row 0 is out of range \[0, 0\)' + +# A valid empty dictionary of a column with only nulls. +FILE="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}_nulls.orc" +trap 'rm -f "$FILE"' EXIT +$CLICKHOUSE_LOCAL -q "SELECT CAST(NULL, 'Nullable(String)') AS s FROM numbers(5) INTO OUTFILE '$FILE' TRUNCATE FORMAT ORC SETTINGS output_format_orc_dictionary_key_size_threshold = 1" +orc_check nulls "$FILE" 's Nullable(String)' "SELECT count(), countIf(s IS NULL) FROM SOURCE" diff --git a/tests/queries/0_stateless/05510_orc_allocation_amplification.reference b/tests/queries/0_stateless/05510_orc_allocation_amplification.reference new file mode 100644 index 0000000000000..1e94a8329c54f --- /dev/null +++ b/tests/queries/0_stateless/05510_orc_allocation_amplification.reference @@ -0,0 +1,32 @@ +compression_block local push_down=0 lazy=0: 1 10000 +compression_block local push_down=0 lazy=1: 1 10000 +compression_block local push_down=1 lazy=0: 1 10000 +compression_block local push_down=1 lazy=1: 1 10000 +compression_block remote push_down=0 lazy=0: 1 10000 +compression_block remote push_down=0 lazy=1: 1 10000 +compression_block remote push_down=1 lazy=0: 1 10000 +compression_block remote push_down=1 lazy=1: 1 10000 +dictionary_size local push_down=0 lazy=0: (INCORRECT_DATA) +dictionary_size local push_down=0 lazy=1: (INCORRECT_DATA) +dictionary_size local push_down=1 lazy=0: (INCORRECT_DATA) +dictionary_size local push_down=1 lazy=1: (INCORRECT_DATA) +dictionary_size remote push_down=0 lazy=0: (INCORRECT_DATA) +dictionary_size remote push_down=0 lazy=1: (INCORRECT_DATA) +dictionary_size remote push_down=1 lazy=0: (INCORRECT_DATA) +dictionary_size remote push_down=1 lazy=1: (INCORRECT_DATA) +snappy_chunk_size local push_down=0 lazy=0: (INCORRECT_DATA) +snappy_chunk_size local push_down=0 lazy=1: (INCORRECT_DATA) +snappy_chunk_size local push_down=1 lazy=0: (INCORRECT_DATA) +snappy_chunk_size local push_down=1 lazy=1: (INCORRECT_DATA) +snappy_chunk_size remote push_down=0 lazy=0: (INCORRECT_DATA) +snappy_chunk_size remote push_down=0 lazy=1: (INCORRECT_DATA) +snappy_chunk_size remote push_down=1 lazy=0: (INCORRECT_DATA) +snappy_chunk_size remote push_down=1 lazy=1: (INCORRECT_DATA) +zstd_chunk_size local push_down=0 lazy=0: (INCORRECT_DATA) +zstd_chunk_size local push_down=0 lazy=1: (INCORRECT_DATA) +zstd_chunk_size local push_down=1 lazy=0: (INCORRECT_DATA) +zstd_chunk_size local push_down=1 lazy=1: (INCORRECT_DATA) +zstd_chunk_size remote push_down=0 lazy=0: (INCORRECT_DATA) +zstd_chunk_size remote push_down=0 lazy=1: (INCORRECT_DATA) +zstd_chunk_size remote push_down=1 lazy=0: (INCORRECT_DATA) +zstd_chunk_size remote push_down=1 lazy=1: (INCORRECT_DATA) diff --git a/tests/queries/0_stateless/05510_orc_allocation_amplification.sh b/tests/queries/0_stateless/05510_orc_allocation_amplification.sh new file mode 100755 index 0000000000000..bc5685ba31872 --- /dev/null +++ b/tests/queries/0_stateless/05510_orc_allocation_amplification.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest, the test uses S3. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh +# shellcheck source=./orc_malformed.lib +. "$CUR_DIR"/orc_malformed.lib + +DATA="$CUR_DIR/data_orc" + +# The memory must be allocated by the size of the data, not by the sizes declared in the file. +LIMIT='max_memory_usage = 100000000' + +# The declared compression block size is 1 GiB, the chunks are small. The schema inference reads the compressed footer. +orc_check compression_block "$DATA/malformed_compression_block_1g.orc" '' "SELECT count(), sum(length(s)) FROM SOURCE" '' "$LIMIT" + +# The declared dictionary size is 2^32 - 1, the stream has a few entries. +orc_check dictionary_size "$DATA/malformed_dictionary_size_max.orc" 's Nullable(String)' "SELECT count(), sum(length(s)) FROM SOURCE" \ + 'Unexpected end of ORC stream column 1 kind 2' "$LIMIT" + +# The sizes stored in the snappy and zstd chunks are 1 GiB, the chunks have a few bytes. +orc_check snappy_chunk_size "$DATA/malformed_snappy_chunk_size.orc" 'x Int64' "SELECT count() FROM SOURCE" \ + 'snappy-compressed chunk of ORC file decompresses to 1073741824 bytes' "$LIMIT" +orc_check zstd_chunk_size "$DATA/malformed_zstd_chunk_size.orc" 'x Int64' "SELECT count() FROM SOURCE" \ + 'Malformed zstd-compressed data in ORC file' "$LIMIT" diff --git a/tests/queries/0_stateless/05511_orc_malformed_stripes_and_streams.reference b/tests/queries/0_stateless/05511_orc_malformed_stripes_and_streams.reference new file mode 100644 index 0000000000000..3c50f86ded0e7 --- /dev/null +++ b/tests/queries/0_stateless/05511_orc_malformed_stripes_and_streams.reference @@ -0,0 +1,16 @@ +stripe local push_down=0 lazy=0: (INCORRECT_DATA) +stripe local push_down=0 lazy=1: (INCORRECT_DATA) +stripe local push_down=1 lazy=0: (INCORRECT_DATA) +stripe local push_down=1 lazy=1: (INCORRECT_DATA) +stripe remote push_down=0 lazy=0: (INCORRECT_DATA) +stripe remote push_down=0 lazy=1: (INCORRECT_DATA) +stripe remote push_down=1 lazy=0: (INCORRECT_DATA) +stripe remote push_down=1 lazy=1: (INCORRECT_DATA) +stream local push_down=0 lazy=0: (INCORRECT_DATA) +stream local push_down=0 lazy=1: (INCORRECT_DATA) +stream local push_down=1 lazy=0: (INCORRECT_DATA) +stream local push_down=1 lazy=1: (INCORRECT_DATA) +stream remote push_down=0 lazy=0: (INCORRECT_DATA) +stream remote push_down=0 lazy=1: (INCORRECT_DATA) +stream remote push_down=1 lazy=0: (INCORRECT_DATA) +stream remote push_down=1 lazy=1: (INCORRECT_DATA) diff --git a/tests/queries/0_stateless/05511_orc_malformed_stripes_and_streams.sh b/tests/queries/0_stateless/05511_orc_malformed_stripes_and_streams.sh new file mode 100755 index 0000000000000..2d3b50f7f31eb --- /dev/null +++ b/tests/queries/0_stateless/05511_orc_malformed_stripes_and_streams.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest, the test uses S3. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh +# shellcheck source=./orc_malformed.lib +. "$CUR_DIR"/orc_malformed.lib + +DATA="$CUR_DIR/data_orc" + +# The sum of the offset and the lengths of the stripe overflows `UInt64`. +orc_check stripe "$DATA/malformed_stripe_offset_overflow.orc" 'a UInt64, b String' "SELECT count(), sum(length(b)) FROM SOURCE" \ + 'Malformed stripe information at stripe index 0' + +# The length of a stream of a column that is not read is 2^64 - 1, the offsets of the next streams wrap around. +orc_check stream "$DATA/malformed_stream_offset_overflow.orc" 'b String' "SELECT count(), sum(length(b)) FROM SOURCE" \ + 'Malformed stream.*11.*18446744073709551615' diff --git a/tests/queries/0_stateless/05512_orc_malformed_row_groups.reference b/tests/queries/0_stateless/05512_orc_malformed_row_groups.reference new file mode 100644 index 0000000000000..1fafd3d526a4f --- /dev/null +++ b/tests/queries/0_stateless/05512_orc_malformed_row_groups.reference @@ -0,0 +1,32 @@ +row_groups local push_down=1 lazy=0: (INCORRECT_DATA) +row_groups local push_down=1 lazy=1: (INCORRECT_DATA) +row_groups remote push_down=1 lazy=0: (INCORRECT_DATA) +row_groups remote push_down=1 lazy=1: (INCORRECT_DATA) +row_groups local push_down=0 lazy=0: (INCORRECT_DATA) +row_groups local push_down=0 lazy=1: (INCORRECT_DATA) +row_groups remote push_down=0 lazy=0: (INCORRECT_DATA) +row_groups remote push_down=0 lazy=1: (INCORRECT_DATA) +patched_base local push_down=0 lazy=0: (INCORRECT_DATA) +patched_base local push_down=0 lazy=1: (INCORRECT_DATA) +patched_base local push_down=1 lazy=0: (INCORRECT_DATA) +patched_base local push_down=1 lazy=1: (INCORRECT_DATA) +patched_base remote push_down=0 lazy=0: (INCORRECT_DATA) +patched_base remote push_down=0 lazy=1: (INCORRECT_DATA) +patched_base remote push_down=1 lazy=0: (INCORRECT_DATA) +patched_base remote push_down=1 lazy=1: (INCORRECT_DATA) +row_index_seek local push_down=0 lazy=0: 1 +row_index_seek local push_down=0 lazy=1: 1 +row_index_seek local push_down=1 lazy=0: (INCORRECT_DATA) +row_index_seek local push_down=1 lazy=1: (INCORRECT_DATA) +row_index_seek remote push_down=0 lazy=0: 1 +row_index_seek remote push_down=0 lazy=1: 1 +row_index_seek remote push_down=1 lazy=0: (INCORRECT_DATA) +row_index_seek remote push_down=1 lazy=1: (INCORRECT_DATA) +row_index_control local push_down=0 lazy=0: 1 +row_index_control local push_down=0 lazy=1: 1 +row_index_control local push_down=1 lazy=0: 1 +row_index_control local push_down=1 lazy=1: 1 +row_index_control remote push_down=0 lazy=0: 1 +row_index_control remote push_down=0 lazy=1: 1 +row_index_control remote push_down=1 lazy=0: 1 +row_index_control remote push_down=1 lazy=1: 1 diff --git a/tests/queries/0_stateless/05512_orc_malformed_row_groups.sh b/tests/queries/0_stateless/05512_orc_malformed_row_groups.sh new file mode 100755 index 0000000000000..163a644dea9f4 --- /dev/null +++ b/tests/queries/0_stateless/05512_orc_malformed_row_groups.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest, the test uses S3. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh +# shellcheck source=./orc_malformed.lib +. "$CUR_DIR"/orc_malformed.lib + +DATA="$CUR_DIR/data_orc" + +# The stripe has 2^64 - 1 rows: rounding up the number of row groups overflows. +ORC_PUSH_DOWN_VALUES=1 orc_check row_groups "$DATA/malformed_row_group_count_overflow.orc" 'x Int64' "SELECT x FROM SOURCE WHERE x = 1" \ + 'The row index of ORC column 1 has 2 entries, less than the [0-9]+ row groups of stripe 0' +ORC_PUSH_DOWN_VALUES=0 orc_check row_groups "$DATA/malformed_row_group_count_overflow.orc" 'x Int64' "SELECT x FROM SOURCE WHERE x = 1" \ + 'Unexpected end of ORC stream column 1 kind 1' + +# A PATCHED_BASE run whose last patch is a continuation (gap 255, patch 0). +orc_check patched_base "$DATA/malformed_patched_base_terminal_continuation.orc" 'x Int64' "SELECT count(), sum(x) FROM SOURCE" \ + 'Corrupt PATCHED_BASE encoded data in ORC stream column 1 kind 1 in stripe 0 \(unterminated gap\)' + +# A row index position beyond the end of the stream, and the same file with the valid position. +orc_check row_index_seek "$DATA/malformed_row_index_seek_past_stream.orc" 'x Int64' "SELECT x FROM SOURCE WHERE x = 1" \ + 'Seek position 127 is beyond the end of ORC stream column 1 kind 1 in stripe 0 of size 3' +orc_check row_index_control "$DATA/row_index_seek_control.orc" 'x Int64' "SELECT x FROM SOURCE WHERE x = 1" diff --git a/tests/queries/0_stateless/05513_orc_metadata_repeated_field_limits.reference b/tests/queries/0_stateless/05513_orc_metadata_repeated_field_limits.reference new file mode 100644 index 0000000000000..bac3e0b9f016c --- /dev/null +++ b/tests/queries/0_stateless/05513_orc_metadata_repeated_field_limits.reference @@ -0,0 +1,16 @@ +footer_statistics local push_down=0 lazy=0: (INCORRECT_DATA) +footer_statistics local push_down=0 lazy=1: (INCORRECT_DATA) +footer_statistics local push_down=1 lazy=0: (INCORRECT_DATA) +footer_statistics local push_down=1 lazy=1: (INCORRECT_DATA) +footer_statistics remote push_down=0 lazy=0: (INCORRECT_DATA) +footer_statistics remote push_down=0 lazy=1: (INCORRECT_DATA) +footer_statistics remote push_down=1 lazy=0: (INCORRECT_DATA) +footer_statistics remote push_down=1 lazy=1: (INCORRECT_DATA) +stripe_statistics local push_down=0 lazy=0: (INCORRECT_DATA) +stripe_statistics local push_down=0 lazy=1: (INCORRECT_DATA) +stripe_statistics local push_down=1 lazy=0: (INCORRECT_DATA) +stripe_statistics local push_down=1 lazy=1: (INCORRECT_DATA) +stripe_statistics remote push_down=0 lazy=0: (INCORRECT_DATA) +stripe_statistics remote push_down=0 lazy=1: (INCORRECT_DATA) +stripe_statistics remote push_down=1 lazy=0: (INCORRECT_DATA) +stripe_statistics remote push_down=1 lazy=1: (INCORRECT_DATA) diff --git a/tests/queries/0_stateless/05513_orc_metadata_repeated_field_limits.sh b/tests/queries/0_stateless/05513_orc_metadata_repeated_field_limits.sh new file mode 100755 index 0000000000000..9c0fb713fb91f --- /dev/null +++ b/tests/queries/0_stateless/05513_orc_metadata_repeated_field_limits.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest, the test uses S3. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh +# shellcheck source=./orc_malformed.lib +. "$CUR_DIR"/orc_malformed.lib + +DATA="$CUR_DIR/data_orc" + +# The number of the repeated objects in the metadata is limited by the numbers of the columns and the stripes. +orc_check footer_statistics "$DATA/malformed_footer_too_many_statistics.orc" 'a UInt64, b String' "SELECT count(), sum(length(b)) FROM SOURCE" \ + 'Too many column statistics in ORC footer: 13, the file has 3 columns' +orc_check stripe_statistics "$DATA/malformed_metadata_too_many_stripes.orc" 'a UInt64, b String' "SELECT count(), sum(length(b)) FROM SOURCE" \ + 'Too many stripe statistics in ORC metadata, at most 1 expected' diff --git a/tests/queries/0_stateless/05514_orc_timestamp_statistics_before_epoch.reference b/tests/queries/0_stateless/05514_orc_timestamp_statistics_before_epoch.reference new file mode 100644 index 0000000000000..7f41d79fbe442 --- /dev/null +++ b/tests/queries/0_stateless/05514_orc_timestamp_statistics_before_epoch.reference @@ -0,0 +1,21 @@ +t < fromUnixTimestamp64Nano(-500000001, 'UTC') +500 +500 0 499 +500 0 499 +t <= fromUnixTimestamp64Nano(-500000001, 'UTC') +600 +501 0 500 +501 0 500 +t = fromUnixTimestamp64Nano(-950000001, 'UTC') +100 +1 50 50 +1 50 50 +t = fromUnixTimestamp64Nano(-999500000, 'UTC') +1 +1 1 1 +1 1 1 +-2000000000 1 +-1 1 +-999500000 1 +-1500000 1 +1000000000 1 diff --git a/tests/queries/0_stateless/05514_orc_timestamp_statistics_before_epoch.sh b/tests/queries/0_stateless/05514_orc_timestamp_statistics_before_epoch.sh new file mode 100755 index 0000000000000..21a4b8bb4428a --- /dev/null +++ b/tests/queries/0_stateless/05514_orc_timestamp_statistics_before_epoch.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +FILE="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}.orc" +trap 'rm -f "$FILE"' EXIT + +# Prints the number of rows that are not pruned, and the result with and without the filter pushdown. +function check() +{ + local condition=$1 + echo "$condition" + $CLICKHOUSE_LOCAL -q " + SELECT count() FROM file('$FILE', 'ORC', 'number UInt64, t DateTime64(9, \'UTC\')') WHERE indexHint($condition); + SELECT count(), min(number), max(number) FROM file('$FILE', 'ORC', 'number UInt64, t DateTime64(9, \'UTC\')') WHERE $condition; + SELECT count(), min(number), max(number) FROM file('$FILE', 'ORC', 'number UInt64, t DateTime64(9, \'UTC\')') WHERE $condition + SETTINGS input_format_orc_filter_push_down = 0" +} + +# The timestamps before the epoch with a negative sub-millisecond part, from -1.000000001 to -0.001000001 seconds, +# in row groups of 100 rows. Their statistics must allow the pruning. +$CLICKHOUSE_LOCAL -q " + SELECT number, fromUnixTimestamp64Nano(toInt64(number) * 1000000 - 1000000001, 'UTC') AS t FROM numbers(1000) + INTO OUTFILE '$FILE' TRUNCATE FORMAT ORC SETTINGS output_format_orc_row_index_stride = 100" +check "t < fromUnixTimestamp64Nano(-500000001, 'UTC')" +check "t <= fromUnixTimestamp64Nano(-500000001, 'UTC')" +check "t = fromUnixTimestamp64Nano(-950000001, 'UTC')" + +# A timestamp in the millisecond after -1 second, in its own row group: the pruning must keep it. +$CLICKHOUSE_LOCAL -q " + SELECT number, fromUnixTimestamp64Nano([-2000000000, -999500000, -500000000][number + 1], 'UTC') AS t FROM numbers(3) + INTO OUTFILE '$FILE' TRUNCATE FORMAT ORC SETTINGS output_format_orc_row_index_stride = 1" +check "t = fromUnixTimestamp64Nano(-999500000, 'UTC')" + +# A file from PyArrow (the C++ writer of ORC) with a bloom filter, which rounds the negative milliseconds toward zero, +# and one row in each row group: the bloom filter must keep the matching rows. +for value in -2000000000 -1 -999500000 -1500000 1000000000; do + $CLICKHOUSE_LOCAL -q " + SELECT $value, count() FROM file('$CUR_DIR/data_orc/timestamps_before_epoch_pyarrow_bloom_filter.orc', 'ORC', 't Nullable(DateTime64(9))') + WHERE t = fromUnixTimestamp64Nano($value)" +done diff --git a/tests/queries/0_stateless/05515_orc_nullable_floats.reference b/tests/queries/0_stateless/05515_orc_nullable_floats.reference new file mode 100644 index 0000000000000..4aae050a32fc2 --- /dev/null +++ b/tests/queries/0_stateless/05515_orc_nullable_floats.reference @@ -0,0 +1,2 @@ +1 +1086 1410 5000 10000 0 diff --git a/tests/queries/0_stateless/05515_orc_nullable_floats.sh b/tests/queries/0_stateless/05515_orc_nullable_floats.sh new file mode 100755 index 0000000000000..c7eba0810a432 --- /dev/null +++ b/tests/queries/0_stateless/05515_orc_nullable_floats.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +FILE="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}.orc" +trap 'rm -f "$FILE"' EXIT + +# Nullable floats with different patterns of nulls, and special values, at the top level and nested. +QUERY=" + SELECT + number, + multiIf(k = 0, NULL, k = 1, nan, k = 2, inf, k = 3, -inf, k = 4, -0., k = 5, toFloat32(1e-42), toFloat32(number) / 7)::Nullable(Float32) AS f32, + multiIf(k2 = 0, NULL, k2 = 1, nan, k2 = 2, -inf, k2 = 3, 5e-324, k2 = 4, -0., number / 3.)::Nullable(Float64) AS f64, + if(intDiv(number, 1000) % 2 = 0, NULL, number / 2.)::Nullable(Float64) AS runs, + NULL::Nullable(Float32) AS all_null, + (number * 1.5)::Nullable(Float64) AS no_null, + arrayMap(x -> if(x % 3 = 0, NULL, toFloat32(x) / 4), range(number % 5))::Array(Nullable(Float32)) AS arr + FROM (SELECT number, cityHash64(number) % 9 AS k, cityHash64(number, 1) % 7 AS k2 FROM numbers(10000))" +STRUCTURE='number UInt64, f32 Nullable(Float32), f64 Nullable(Float64), runs Nullable(Float64), all_null Nullable(Float32), no_null Nullable(Float64), arr Array(Nullable(Float32))' +DIGEST="sum(cityHash64(number, isNull(f32), reinterpretAsUInt32(ifNull(f32, 0)), isNull(f64), reinterpretAsUInt64(ifNull(f64, 0)), + isNull(runs), ifNull(runs, 0), isNull(all_null), no_null, arrayMap(x -> (isNull(x), reinterpretAsUInt32(ifNull(x, 0))), arr)))" + +$CLICKHOUSE_LOCAL -q "$QUERY INTO OUTFILE '$FILE' TRUNCATE FORMAT ORC" +$CLICKHOUSE_LOCAL -q " + SELECT (SELECT $DIGEST FROM ($QUERY)) = (SELECT $DIGEST FROM file('$FILE', 'ORC', '$STRUCTURE')); + SELECT countIf(f32 IS NULL), countIf(f64 IS NULL), countIf(runs IS NULL), countIf(all_null IS NULL), countIf(no_null IS NULL) FROM file('$FILE', 'ORC', '$STRUCTURE')" diff --git a/tests/queries/0_stateless/data_orc/bad_bloom_filter_1.6.0.orc b/tests/queries/0_stateless/data_orc/bad_bloom_filter_1.6.0.orc new file mode 100644 index 0000000000000000000000000000000000000000..6c52f4f0f98b40dec581f1b02ec3936d34d62796 GIT binary patch literal 709 zcmeYda+YOa;Nsz8VE_Ul77o?`1}g@J$3ah$(i<2Q4f(`X9foPp;g3RC==BExo1XeDT)j`@d*0UouL)T2hk0z`R(Z zLXP?4@vapcj<8#u@|yho1lyA79E~@e53qDH@+2HEcDVcTjX@hD4~ygm1wIL$ga_Wt zzJC_DaqeWa0J?nA^G7KSjEahUVXAU^VaMm3c@m;{OoekM0~>=D(Badqxf%=vTtb6b zHg4E<%CX3zKUbOK)={TKV@@6Y*h6V$&s+aIGd{jSwsL{0WmAA^^S*OhsY*>Z94N8MFvLc#_M{U zPrgjEZ*TIGPvvkpz~A`dxL36L%3Q1DZO2aANS!XVi~Tn%;2*O-D+9y-A7Da(f#LVT zLkEs9u=FG}9eptIV>|;`oST8cjU`yn0T@?~Iho0+PWefRNxqrcsZ$sjOivl|H5l+X zUsQAGG`hGcLZ?7l&5W5; zG3liD8Q;)^3ke1Z0goOeG;Y43sKXqlEyj0^flV7Dfr$jfbTo%{ws&7OR5!K^?J)Fo mQfoMv*fgJAnSmomfKj5Mfklalfk(r{fUW5ZGqZn?vp4|E0T8hO literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_orc/bad_bloom_filter_1.6.11.orc b/tests/queries/0_stateless/data_orc/bad_bloom_filter_1.6.11.orc new file mode 100644 index 0000000000000000000000000000000000000000..fc3ffef8f18c17dc95f532108f94846c6cb9b894 GIT binary patch literal 717 zcmeYda+YOa;Nsz8VE_Ul77o?`1}g@J$3ah$(i<2Q4f(`X9foPp;g3RC==BExo1XeDT)j`@d*0UouL)T2hk0z`R(Z zLXP?4@vapcj<8#u@|yho1lyA79E~@e53qDH@+2HEcDVcTjX@hD4~ygm1wIL$ga_Wt zzJC_DaqeWa0J?nA^G7KSjEahUVXAU^VaMm3c@m;{OoekM0~>=D(Badqxf%=vTtb6b zHg4E<%CX3zKUbOK)={TKV@@6Y*h6V$&s+aIGd{jSwsL{0WmAA^^S*OhsY*>Z94N8MFvLc#_M{U zPrgjEZ*TIGPvvkpz~A`dxL36L%3Q1DZO2aANS!XVi~Tn%;2*O-D+9y-A7Da(f#LVT zLkEs9u=FG}9eptIV>|;`oST8cjU`yn0T@?~Iho0+PWefRNxqrcsZ$sjOivl|H5l+X zUsQAGG`hGcLZ?7l&5W5; zG3liD8Q;)^3ke1Z0goOeG;Y43sKXqlEyj0^flV7Dfr$jfbTo%{ws&7OR5!K^?J)Fo vQfoMv*fgI#S!njcc{ArNRAk^N5nz;PXkbxdV&Ks*F<@)@!p!U+JFZE-iyIBTGB(h%cBfX4BGjAJ&?Gn8U6&@(d& zz$y=vGO&;Wsxk>+P+(wKo$%nn**QlR@+j>TaZurBkYr%+<23NG2w({jkqGi}XJ9z6 zYbW!qRlXhHI#@oOiG6Td^2pN_EKS$M7W4_Nc>Rj~#nl5W{0t0koK8+IN-kjSYyv`D ze0)qod?H{j6B7>yAC~|R3qu>wgGrAQgcBw_c*b{mZuY+kFmEzQ>GXvtHD{W zWs;4rS`L~_X0e}<#`Zdy^-@fN9e2o_(5Co~&Su7rnHMtH82$lGnRVvG(L)DC4;)ZE zeNa{9)WwY{H(#^4HOd*NetdLvKd-FANsdH4L0}-t>A5j0stGG=rYRQ(FVL``E)@{c z*M0ZIflEH;k2rN4K6K*36_4{*Ji0C)IdSR0m4Neim{y!?@a9?3$G>FuO^5RVT}K_x zpW#^}ug521v|$3%saXw27$-3@JWde%aZ=^z*@HR-4S|as4*E8Gdp0$@cTAnwIkPpi z%P_FZ(6h@hvdhr5%P_RFv)Q$?x7ok>cmtzPw}gS?Pk~t{8%{Qy?rLq;ZaC;Qt=ZdS z%Hrgco38WrFxe(}Jv!Q5FKyK@N$^mZl1z-7O-v7)laHEP&>6R9%Nzphv`rVRkP+WK zv2p3ld7Z5S?X43RuADe;;?|1sEk78U&P>8TXu+-id_MC9Fa-<` z?lW*QFhnuTafo1Gc%1Y&K{#Q;gJ(<^dxYv@a%6g#dW4FAv=5MebWNDr~+4;VnvT^!awPNw&jholK~7@| z!qQ)f-`+hkxUucGmEX^Ib+-3k9pAk34oO3dhvpw=li&Ht=fd(`*2q-Zz@MuUHur+;QX7&$q76$-W;u-w_ literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_orc/malformed_dictionary_size_max.orc b/tests/queries/0_stateless/data_orc/malformed_dictionary_size_max.orc new file mode 100644 index 0000000000000000000000000000000000000000..27d3a7a5694c836382a434b145463145bc7e4969 GIT binary patch literal 174 zcmeYdaz2pEz)<19(7?sU!6?8e!2+b2fixqKW&+a8LQEVCLL5M`|3JVm#q91I!X?hd z!jTfdz{SszqQuF?SRuq%A#s8+fB~q=Sc(fsGDvVJXltY>ad7YmF)~Uq7ArAvup6XU xfwftoYja@mU`*n2&dE&9_Q)?SPUUb18qm-npv1($qakU)*7Sv$*+0lx8~}Ex9uxoo literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_orc/malformed_footer_too_many_statistics.orc b/tests/queries/0_stateless/data_orc/malformed_footer_too_many_statistics.orc new file mode 100644 index 0000000000000000000000000000000000000000..00ab37f164451f7d41fdd9918d3a60e6eb3fb9e1 GIT binary patch literal 353 zcmb7!55@c<-_od@Ma)J`I4HxfN+mPlz%p40|nh@xK=W-?pJwH%K)jX%p}<3F&Fh@rNx z?t8Y(IS+(X;JUy~Nt{KsDqAPB`oye0GaG+COmQR4^X~C!`ttU={aZS&{tbFtXANNv_gqUE3}AonBeEZn8f9rlbM|DkzZPz$`J=lJ`D{5 WN=ytq8jF|>*qXjDGy4ZQO8@|~r#gH9 literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_orc/malformed_patched_base_terminal_continuation.orc b/tests/queries/0_stateless/data_orc/malformed_patched_base_terminal_continuation.orc new file mode 100644 index 0000000000000000000000000000000000000000..43552b05c31d975806e7b28bf0348ac911b93320 GIT binary patch literal 83 zcmeYda_(bbf5@QqpMi^wgHeD{fZBafx-+D4Gk42F|jIl^(saf&@Ul}_i@Q96kobdi?g;4J`i%|p7O zHQHy4J^a)g0`Qj6YX%1aWcQxocSeo^^{~g%Xx3tINijrGbQqKXl}3-r5QS%EZ#Tnm$hsO_t_nvG%Tof%Rfa2K6BY3f#1KM=U?J&`Kze_YKPs*0 zx=Eu0j}P9!do!~|CB`MJ=O9i9z@@C6VX*Am!Mha&!ECV3S+H<8^mY#W1gIGH9|rxw zXpDOxF>EY&jzr&%B;N@{i#~~@xF<_qJ7Nf;voiJcyp((SJL0nbs+X1h!K@W3q*{Nxeoi)D+c$m>ztpwO3DRD- MFyFsQ%@$SX1Y@lyc>n+a literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_orc/malformed_timestamp_nanoseconds.orc b/tests/queries/0_stateless/data_orc/malformed_timestamp_nanoseconds.orc new file mode 100644 index 0000000000000000000000000000000000000000..ab57ba54f6cb5f35ce2f2225cedead3e3c24d9e6 GIT binary patch literal 238 zcmeYdaz4P>@V{v>Ba?moe+amFYzD)Ut4a(EH!q|}F!G&RDZuEwo9hH)g+~tq6u9SsRejxb{KXpyLT!00RfJK%^8GkYtdsQIOT(QsUs?5n^PNVk}W&;@~vk yvI3i7h1U!R1`oz0F6W%gPb;1ObElwvGVV&dR5V73CA xVnx6dhY5Zjj7eP1Iho1X9{HuksT|=zmo_vAC^0ecXlNL)HGN@b_78Fv2LL-3L?-|M literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_orc/malformed_zstd_chunk_size.orc b/tests/queries/0_stateless/data_orc/malformed_zstd_chunk_size.orc new file mode 100644 index 0000000000000000000000000000000000000000..4f30ed6c7f1acfc4f1587ecd851ca828b38d2f80 GIT binary patch literal 38 scmeYda+Y9V(AcW~cL4(fg99T20|&nVt3(43uxKzeePL$y4{{a;0H`qt$^ZZW literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_orc/row_index_seek_control.orc b/tests/queries/0_stateless/data_orc/row_index_seek_control.orc new file mode 100644 index 0000000000000000000000000000000000000000..ede347aa81181c970ec5d83161d8595eef07a424 GIT binary patch literal 213 zcmY+7F$=;#426?iIYPMMI7E+5r66?dD4oO#c9E9SA0Uc@C^(4VSp3cY0MA1Nhvb8o z_g>n*7VQ$o4t{h(0M0Nv&0xVpxP3JIYGlb%2P@1(vknFuiM|L#i(WZm2t%pbYzLW0 z!oe?wDK^G5E4xc2?1fj^yr7)fIK&hmIdM`_6*JXeW{C~%rM{dF=jM999eEv|kE6{A OOffC>^QBbV*RwZ`5EX0y literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_orc/timestamps_before_epoch_arrow.orc b/tests/queries/0_stateless/data_orc/timestamps_before_epoch_arrow.orc new file mode 100644 index 0000000000000000000000000000000000000000..d6e0144685e7e91cbc4e2f76aba78db27c503b4f GIT binary patch literal 294 zcmeYda^_+aV&Py3VBnJHVqpLRA#n~CFF}d-S0A;$tXG(DYv0WG<_0&-0vIZo8Q1fF zHU<%<3}+*pj2X=Co-wUs{tp7J@2o(YK>&?!#l^uMlK_Jhv%7By$R%JG;c^KFv%n-qDJ~$TDPf^tsKKJd$H61S$SB2FqQIcR zV8Ecn!NCb(8L(J^&9cH}mcs-;55^=`BRwGGhyc2?p@Bt-iGfE$(SWV#3p2BSkh3@d Dw(m#M literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_orc/timestamps_before_epoch_pyarrow_bloom_filter.orc b/tests/queries/0_stateless/data_orc/timestamps_before_epoch_pyarrow_bloom_filter.orc new file mode 100644 index 0000000000000000000000000000000000000000..2afd747824ec0515a92db943df384c0a15d9573e GIT binary patch literal 517 zcmah_F-XHu5WUM!;u%X*Gw3%hG-L`zN_4CaLP4z{m7=SIgM;WMjv~5taS^)MxuUCE zXQx6PTq+J-6>$+1LF@hef*^>uyn8P%clTelR(5R9BwT}kal$cy%S#ikkZk6IWVM&3+yxPgX3uK-?@cBI7ut#aQ69Ap@RG z2HVjWORMWDLb5o9G+8=Y{U{Qd6XB0B)D1Zo%tuz3rj&>1 [] [] +function orc_check() +{ + local label=$1 file=$2 structure=$3 query=$4 expected_error=${5:-} extra_settings=${6:-} + local name + name=$(basename "$file") + $CLICKHOUSE_CLIENT -q "INSERT INTO FUNCTION s3('${ORC_S3_URL}/${name}', 'test', 'testtest', 'RawBLOB') SETTINGS s3_truncate_on_insert = 1 FORMAT RawBLOB" < "$file" + + local source push_down lazy table client output status + for source in local remote; do + for push_down in $ORC_PUSH_DOWN_VALUES; do + for lazy in 0 1; do + if [ "$source" = local ]; then + table="file('${file}', 'ORC'${structure:+, '${structure}'})" + client=$CLICKHOUSE_LOCAL + else + table="s3('${ORC_S3_URL}/${name}', 'test', 'testtest', 'ORC'${structure:+, '${structure}'})" + client=$CLICKHOUSE_CLIENT + fi + output=$($client -q "${query//SOURCE/$table} SETTINGS input_format_orc_filter_push_down = ${push_down}, input_format_orc_dictionary_as_low_cardinality = ${lazy}${extra_settings:+, $extra_settings}" 2>&1) + status=$? + echo "${label} ${source} push_down=${push_down} lazy=${lazy}: $(orc_result "$output" "$status" "$expected_error")" + done + done + done +} From 55b740ffb006244ad3af2540e03f0da0cd82a48e Mon Sep 17 00:00:00 2001 From: Alexey Milovidov Date: Sun, 8 Nov 2026 14:47:12 +0000 Subject: [PATCH 129/219] Accept the 26.8 time zone error message in `05407_orc_timezone_traversal_abort` On 26.8, `DateLUTImpl` rejects unknown time zone names with `Cannot load time zone`, while master has a different message. The behavior (rejection without abort) is the same. CI report: https://s3.amazonaws.com/clickhouse-test-reports/json.html?PR=129779&sha=7934808df3fb52d5bd596bfdd7c47cdc0c7bfc1f&name_0=BackportPR&name_1=Stateless%20tests%20%28amd_asan_ubsan%2C%20distributed%20plan%2C%20parallel%29 Co-Authored-By: Claude Opus 5.5 --- .../0_stateless/05407_orc_timezone_traversal_abort.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/queries/0_stateless/05407_orc_timezone_traversal_abort.sh b/tests/queries/0_stateless/05407_orc_timezone_traversal_abort.sh index a94f164952f4b..f8e04add8d562 100755 --- a/tests/queries/0_stateless/05407_orc_timezone_traversal_abort.sh +++ b/tests/queries/0_stateless/05407_orc_timezone_traversal_abort.sh @@ -117,7 +117,10 @@ for label, result in (("writer", writer), ("reader", reader), ("reader file", re elif result.returncode == 0: print(f"{label}: malformed timezone unexpectedly succeeded") failed = True - elif b"is not supported. Use a name from `system.time_zones`" in result.stdout + result.stderr: + elif any( + message in result.stdout + result.stderr + for message in (b"is not supported. Use a name from `system.time_zones`", b"Cannot load time zone ../../../") + ): print(f"{label}: time zone is rejected") else: diagnostic = (result.stderr + result.stdout).decode(errors="replace") From a50389d9da87a11c65125356e4a8014482d306a3 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 8 Nov 2026 16:17:17 +0000 Subject: [PATCH 130/219] Backport #130045 to 26.8: Fix counter overflow in `groupArraySample` merge --- .../AggregateFunctionGroupArray.cpp | 31 +++++++++++++++---- ...up_array_sample_counter_overflow.reference | 3 ++ ...67_group_array_sample_counter_overflow.sql | 28 +++++++++++++++++ 3 files changed, 56 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/05567_group_array_sample_counter_overflow.reference create mode 100644 tests/queries/0_stateless/05567_group_array_sample_counter_overflow.sql diff --git a/src/AggregateFunctions/AggregateFunctionGroupArray.cpp b/src/AggregateFunctions/AggregateFunctionGroupArray.cpp index 89c46451c9272..51b83e5fd0733 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupArray.cpp +++ b/src/AggregateFunctions/AggregateFunctionGroupArray.cpp @@ -24,6 +24,8 @@ #include +#include + #include constexpr size_t AGGREGATE_FUNCTION_GROUP_ARRAY_MAX_ELEMENT_SIZE = 0xFFFFFF; @@ -90,6 +92,17 @@ struct GroupArraySamplerData size_t total_values = 0; pcg32_fast rng; + /// The counter may come from a deserialized state, so it can be arbitrarily large, + /// and wrapping it around would make `genRandom` divide by zero. + void addTotalValues(UInt64 count) + { + UInt64 new_total_values = 0; + if (common::addOverflow(static_cast(total_values), count, new_total_values)) + throw Exception(ErrorCodes::INCORRECT_DATA, + "The number of values in groupArraySample state overflows: {} + {}", total_values, count); + total_values = new_total_values; + } + UInt64 genRandom(size_t lim) { chassert(lim != 0); @@ -160,7 +173,7 @@ class GroupArrayNumericImpl final void insertWithSampler(Data & a, const T & v, Arena * arena) const { - ++a.total_values; + a.addTotalValues(1); if (a.value.size() < max_elems) a.value.push_back(v, arena); else @@ -183,7 +196,10 @@ class GroupArrayNumericImpl final const auto & row_value = assert_cast &>(*columns[0]).getData()[row_num]; auto & cur_elems = this->data(place); - ++cur_elems.total_values; + if constexpr (Trait::sampler == Sampler::NONE) + ++cur_elems.total_values; + else + cur_elems.addTotalValues(1); if constexpr (Trait::sampler == Sampler::NONE) { @@ -275,8 +291,8 @@ class GroupArrayNumericImpl final } else { + cur_elems.addTotalValues(rhs_elems.total_values); cur_elems.randomShuffle(); - cur_elems.total_values += rhs_elems.total_values; for (size_t i = 0; i < max_elems; ++i) { UInt64 rnd = cur_elems.genRandom(cur_elems.total_values); @@ -525,7 +541,7 @@ class GroupArrayGeneralImpl final void insertWithSampler(Data & a, const Node * v, Arena * arena) const { - ++a.total_values; + a.addTotalValues(1); if (a.value.size() < max_elems) a.value.push_back(v->clone(arena), arena); else @@ -547,7 +563,10 @@ class GroupArrayGeneralImpl final { auto & cur_elems = data(place); - ++cur_elems.total_values; + if constexpr (Trait::sampler == Sampler::NONE) + ++cur_elems.total_values; + else + cur_elems.addTotalValues(1); if constexpr (Trait::sampler == Sampler::NONE) { @@ -642,8 +661,8 @@ class GroupArrayGeneralImpl final } else { + cur_elems.addTotalValues(rhs_elems.total_values); cur_elems.randomShuffle(); - cur_elems.total_values += rhs_elems.total_values; for (size_t i = 0; i < max_elems; ++i) { UInt64 rnd = cur_elems.genRandom(cur_elems.total_values); diff --git a/tests/queries/0_stateless/05567_group_array_sample_counter_overflow.reference b/tests/queries/0_stateless/05567_group_array_sample_counter_overflow.reference new file mode 100644 index 0000000000000..ec1c393fef3a5 --- /dev/null +++ b/tests/queries/0_stateless/05567_group_array_sample_counter_overflow.reference @@ -0,0 +1,3 @@ +1 +1 +still alive diff --git a/tests/queries/0_stateless/05567_group_array_sample_counter_overflow.sql b/tests/queries/0_stateless/05567_group_array_sample_counter_overflow.sql new file mode 100644 index 0000000000000..36af866963144 --- /dev/null +++ b/tests/queries/0_stateless/05567_group_array_sample_counter_overflow.sql @@ -0,0 +1,28 @@ +-- A deserialized `groupArraySample` state may carry an arbitrary `total_values` counter. +-- Merging must not wrap it around to zero, which led to a division by zero. + +-- Valid states merge fine. +SELECT length(arrayReduce('groupArraySampleMerge(1, 1)', [ + CAST(unhex('0101410100000000000000173633363431333632323338343637393330303520302033'), 'AggregateFunction(groupArraySample(1, 1), String)'), + CAST(unhex('0101420100000000000000173633363431333632323338343637393330303520302033'), 'AggregateFunction(groupArraySample(1, 1), String)')])); +SELECT length(arrayReduce('groupArraySampleMerge(1, 1)', [ + CAST(unhex('01410100000000000000173633363431333632323338343637393330303520302033'), 'AggregateFunction(groupArraySample(1, 1), UInt8)'), + CAST(unhex('01420100000000000000173633363431333632323338343637393330303520302033'), 'AggregateFunction(groupArraySample(1, 1), UInt8)')])); + +-- total_values = UINT64_MAX, merged with a state with one value. +SELECT arrayReduce('groupArraySampleMerge(1, 1)', [ + CAST(unhex('010141FFFFFFFFFFFFFFFF173633363431333632323338343637393330303520302033'), 'AggregateFunction(groupArraySample(1, 1), String)'), + CAST(unhex('0101420100000000000000173633363431333632323338343637393330303520302033'), 'AggregateFunction(groupArraySample(1, 1), String)')]); -- { serverError INCORRECT_DATA } +SELECT arrayReduce('groupArraySampleMerge(1, 1)', [ + CAST(unhex('0141FFFFFFFFFFFFFFFF173633363431333632323338343637393330303520302033'), 'AggregateFunction(groupArraySample(1, 1), UInt8)'), + CAST(unhex('01420100000000000000173633363431333632323338343637393330303520302033'), 'AggregateFunction(groupArraySample(1, 1), UInt8)')]); -- { serverError INCORRECT_DATA } + +-- Both states have more values than the sample size, and their counters sum to 2^64. +SELECT arrayReduce('groupArraySampleMerge(1, 1)', [ + CAST(unhex('0101410000000000000080173633363431333632323338343637393330303520302033'), 'AggregateFunction(groupArraySample(1, 1), String)'), + CAST(unhex('0101420000000000000080173633363431333632323338343637393330303520302033'), 'AggregateFunction(groupArraySample(1, 1), String)')]); -- { serverError INCORRECT_DATA } +SELECT arrayReduce('groupArraySampleMerge(1, 1)', [ + CAST(unhex('01410000000000000080173633363431333632323338343637393330303520302033'), 'AggregateFunction(groupArraySample(1, 1), UInt8)'), + CAST(unhex('01420000000000000080173633363431333632323338343637393330303520302033'), 'AggregateFunction(groupArraySample(1, 1), UInt8)')]); -- { serverError INCORRECT_DATA } + +SELECT 'still alive'; From 53eebcefcde15211f749896250a6ee7084ed5feb Mon Sep 17 00:00:00 2001 From: Azat Khuzhin Date: Sun, 8 Nov 2026 18:52:52 +0100 Subject: [PATCH 131/219] Backport #127478 to 26.8: Bump jemalloc for the fiber-safe TLS rework (upstream solution) This reverts initial implementation of fiber aware TLS for ClickHouse/jemalloc#7, which was slightly more optimal, but maintainers voted for a simpler solution. And replaces it with a simpler one - ClickHouse/jemalloc#11 Backport note: the jemalloc submodule pointer is left at the one already on the release branch (from the backport of #129308), which already contains ClickHouse/jemalloc#11 and the revert of #7. Without these header changes the new `JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS` gate is not defined and no fiber-safe TLS fix is compiled in at all. Original pull-request https://github.com/ClickHouse/ClickHouse/pull/127478 (cherry picked from commit 8b81289f3121cdeb7c7fedada772e99e6a7e6bcd) Co-Authored-By: Claude Fable 5.1 --- .../jemalloc/internal/jemalloc_internal_defs.h.in | 9 +++++---- .../jemalloc/internal/jemalloc_internal_defs.h.in | 9 +++++---- .../jemalloc/internal/jemalloc_internal_defs.h.in | 9 +++++---- .../jemalloc/internal/jemalloc_internal_defs.h.in | 9 +++++---- .../jemalloc/internal/jemalloc_internal_defs.h.in | 9 +++++---- .../jemalloc/internal/jemalloc_internal_defs.h.in | 9 +++++---- .../jemalloc/internal/jemalloc_internal_defs.h.in | 8 ++++++++ .../jemalloc/internal/jemalloc_internal_defs.h.in | 9 +++++---- .../jemalloc/internal/jemalloc_internal_defs.h.in | 9 +++++---- .../jemalloc/internal/jemalloc_internal_defs.h.in | 9 +++++---- .../jemalloc/internal/jemalloc_internal_defs.h.in | 9 +++++---- .../jemalloc/internal/jemalloc_internal_defs.h.in | 9 +++++---- .../jemalloc/internal/jemalloc_internal_defs.h.in | 9 +++++---- 13 files changed, 68 insertions(+), 48 deletions(-) diff --git a/contrib/jemalloc-cmake/include_darwin_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_darwin_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in index 3f3b872e850f5..e71ff144fb27d 100644 --- a/contrib/jemalloc-cmake/include_darwin_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_darwin_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -163,11 +163,12 @@ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) /* - * Defined when the TSD thread-locals use the initial-exec (static) TLS model, - * i.e. live at a fixed offset from the thread pointer. Gates the fast path of - * JEMALLOC_TLS_ADDR (see tsd_internals.h). + * Route TSD thread-local address computation through a noinline + * optimization-barrier accessor so the compiler cannot cache it across a + * user-space context switch (fibers migrating between OS threads). See + * tsd_tls_addr.h. */ -#define JEMALLOC_TLS_MODEL_INITIAL_EXEC +#define JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables diff --git a/contrib/jemalloc-cmake/include_darwin_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_darwin_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in index e00bb848e7b09..a8cd1ce994698 100644 --- a/contrib/jemalloc-cmake/include_darwin_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_darwin_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -163,11 +163,12 @@ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) /* - * Defined when the TSD thread-locals use the initial-exec (static) TLS model, - * i.e. live at a fixed offset from the thread pointer. Gates the fast path of - * JEMALLOC_TLS_ADDR (see tsd_internals.h). + * Route TSD thread-local address computation through a noinline + * optimization-barrier accessor so the compiler cannot cache it across a + * user-space context switch (fibers migrating between OS threads). See + * tsd_tls_addr.h. */ -#define JEMALLOC_TLS_MODEL_INITIAL_EXEC +#define JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables diff --git a/contrib/jemalloc-cmake/include_freebsd_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_freebsd_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in index 83b88f0fd9d46..9086baf5c2fe8 100644 --- a/contrib/jemalloc-cmake/include_freebsd_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_freebsd_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -142,11 +142,12 @@ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) /* - * Defined when the TSD thread-locals use the initial-exec (static) TLS model, - * i.e. live at a fixed offset from the thread pointer. Gates the fast path of - * JEMALLOC_TLS_ADDR (see tsd_internals.h). + * Route TSD thread-local address computation through a noinline + * optimization-barrier accessor so the compiler cannot cache it across a + * user-space context switch (fibers migrating between OS threads). See + * tsd_tls_addr.h. */ -#define JEMALLOC_TLS_MODEL_INITIAL_EXEC +#define JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables diff --git a/contrib/jemalloc-cmake/include_freebsd_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_freebsd_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in index 86f1f3ceca3d0..56d2ce53f4567 100644 --- a/contrib/jemalloc-cmake/include_freebsd_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_freebsd_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in @@ -140,11 +140,12 @@ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) /* - * Defined when the TSD thread-locals use the initial-exec (static) TLS model, - * i.e. live at a fixed offset from the thread pointer. Gates the fast path of - * JEMALLOC_TLS_ADDR (see tsd_internals.h). + * Route TSD thread-local address computation through a noinline + * optimization-barrier accessor so the compiler cannot cache it across a + * user-space context switch (fibers migrating between OS threads). See + * tsd_tls_addr.h. */ -#define JEMALLOC_TLS_MODEL_INITIAL_EXEC +#define JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables diff --git a/contrib/jemalloc-cmake/include_freebsd_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_freebsd_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in index 672f828f3ca45..229cc9593830b 100644 --- a/contrib/jemalloc-cmake/include_freebsd_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_freebsd_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -142,11 +142,12 @@ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) /* - * Defined when the TSD thread-locals use the initial-exec (static) TLS model, - * i.e. live at a fixed offset from the thread pointer. Gates the fast path of - * JEMALLOC_TLS_ADDR (see tsd_internals.h). + * Route TSD thread-local address computation through a noinline + * optimization-barrier accessor so the compiler cannot cache it across a + * user-space context switch (fibers migrating between OS threads). See + * tsd_tls_addr.h. */ -#define JEMALLOC_TLS_MODEL_INITIAL_EXEC +#define JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in index 9a98fbd3810a0..d8ba561bd2075 100644 --- a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -140,11 +140,12 @@ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) /* - * Defined when the TSD thread-locals use the initial-exec (static) TLS model, - * i.e. live at a fixed offset from the thread pointer. Gates the fast path of - * JEMALLOC_TLS_ADDR (see tsd_internals.h). + * Route TSD thread-local address computation through a noinline + * optimization-barrier accessor so the compiler cannot cache it across a + * user-space context switch (fibers migrating between OS threads). See + * tsd_tls_addr.h. */ -#define JEMALLOC_TLS_MODEL_INITIAL_EXEC +#define JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables diff --git a/contrib/jemalloc-cmake/include_linux_aarch64_musl/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_aarch64_musl/jemalloc/internal/jemalloc_internal_defs.h.in index 0ac9fb9c4526d..a19d65dccf5fe 100644 --- a/contrib/jemalloc-cmake/include_linux_aarch64_musl/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_aarch64_musl/jemalloc/internal/jemalloc_internal_defs.h.in @@ -142,6 +142,14 @@ /* Non-empty if the tls_model attribute is supported. */ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) +/* + * Route TSD thread-local address computation through a noinline + * optimization-barrier accessor so the compiler cannot cache it across a + * user-space context switch (fibers migrating between OS threads). See + * tsd_tls_addr.h. + */ +#define JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS + /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables * inline functions. diff --git a/contrib/jemalloc-cmake/include_linux_e2k/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_e2k/jemalloc/internal/jemalloc_internal_defs.h.in index 94f65fa0ce076..971ab7bd66d0c 100644 --- a/contrib/jemalloc-cmake/include_linux_e2k/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_e2k/jemalloc/internal/jemalloc_internal_defs.h.in @@ -127,11 +127,12 @@ #undef JEMALLOC_TLS_MODEL /* - * Defined when the TSD thread-locals use the initial-exec (static) TLS model, - * i.e. live at a fixed offset from the thread pointer. Gates the fast path of - * JEMALLOC_TLS_ADDR (see tsd_internals.h). + * Route TSD thread-local address computation through a noinline + * optimization-barrier accessor so the compiler cannot cache it across a + * user-space context switch (fibers migrating between OS threads). See + * tsd_tls_addr.h. */ -#undef JEMALLOC_TLS_MODEL_INITIAL_EXEC +#define JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables diff --git a/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in index 9f6bce289868e..39a69c4a449d8 100644 --- a/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in @@ -140,11 +140,12 @@ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) /* - * Defined when the TSD thread-locals use the initial-exec (static) TLS model, - * i.e. live at a fixed offset from the thread pointer. Gates the fast path of - * JEMALLOC_TLS_ADDR (see tsd_internals.h). + * Route TSD thread-local address computation through a noinline + * optimization-barrier accessor so the compiler cannot cache it across a + * user-space context switch (fibers migrating between OS threads). See + * tsd_tls_addr.h. */ -#define JEMALLOC_TLS_MODEL_INITIAL_EXEC +#define JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables diff --git a/contrib/jemalloc-cmake/include_linux_riscv64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_riscv64/jemalloc/internal/jemalloc_internal_defs.h.in index 9e495d40d94aa..64c97e6d9340d 100644 --- a/contrib/jemalloc-cmake/include_linux_riscv64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_riscv64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -140,11 +140,12 @@ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) /* - * Defined when the TSD thread-locals use the initial-exec (static) TLS model, - * i.e. live at a fixed offset from the thread pointer. Gates the fast path of - * JEMALLOC_TLS_ADDR (see tsd_internals.h). + * Route TSD thread-local address computation through a noinline + * optimization-barrier accessor so the compiler cannot cache it across a + * user-space context switch (fibers migrating between OS threads). See + * tsd_tls_addr.h. */ -#define JEMALLOC_TLS_MODEL_INITIAL_EXEC +#define JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables diff --git a/contrib/jemalloc-cmake/include_linux_s390x/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_s390x/jemalloc/internal/jemalloc_internal_defs.h.in index 8373a973f759e..867b247302e27 100644 --- a/contrib/jemalloc-cmake/include_linux_s390x/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_s390x/jemalloc/internal/jemalloc_internal_defs.h.in @@ -140,11 +140,12 @@ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) /* - * Defined when the TSD thread-locals use the initial-exec (static) TLS model, - * i.e. live at a fixed offset from the thread pointer. Gates the fast path of - * JEMALLOC_TLS_ADDR (see tsd_internals.h). + * Route TSD thread-local address computation through a noinline + * optimization-barrier accessor so the compiler cannot cache it across a + * user-space context switch (fibers migrating between OS threads). See + * tsd_tls_addr.h. */ -#define JEMALLOC_TLS_MODEL_INITIAL_EXEC +#define JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in index b1267a4dfa56d..e8983ec778d03 100644 --- a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -140,11 +140,12 @@ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) /* - * Defined when the TSD thread-locals use the initial-exec (static) TLS model, - * i.e. live at a fixed offset from the thread pointer. Gates the fast path of - * JEMALLOC_TLS_ADDR (see tsd_internals.h). + * Route TSD thread-local address computation through a noinline + * optimization-barrier accessor so the compiler cannot cache it across a + * user-space context switch (fibers migrating between OS threads). See + * tsd_tls_addr.h. */ -#define JEMALLOC_TLS_MODEL_INITIAL_EXEC +#define JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables diff --git a/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in index 70e304e1629ef..894d9b6d63a21 100644 --- a/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in @@ -141,11 +141,12 @@ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) /* - * Defined when the TSD thread-locals use the initial-exec (static) TLS model, - * i.e. live at a fixed offset from the thread pointer. Gates the fast path of - * JEMALLOC_TLS_ADDR (see tsd_internals.h). + * Route TSD thread-local address computation through a noinline + * optimization-barrier accessor so the compiler cannot cache it across a + * user-space context switch (fibers migrating between OS threads). See + * tsd_tls_addr.h. */ -#define JEMALLOC_TLS_MODEL_INITIAL_EXEC +#define JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables From b6f1b01f6fd6ebc139b2d4a240ae733c347c72c3 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 8 Nov 2026 20:00:44 +0000 Subject: [PATCH 132/219] Backport #125901 to 26.8: Start executor threads for CPU slots granted after the request --- src/Processors/Executors/ExecutorTasks.cpp | 105 ++++++-- src/Processors/Executors/ExecutorTasks.h | 42 ++- src/Processors/Executors/PipelineExecutor.cpp | 99 +++---- src/Processors/Executors/PipelineExecutor.h | 20 +- ...gtest_pipeline_executor_late_cpu_slots.cpp | 241 ++++++++++++++++++ .../test.py | 69 +++++ .../test_scheduler_cpu_preemptive/test.py | 77 ++++++ .../small_block_contention_workload.xml | 13 + 8 files changed, 555 insertions(+), 111 deletions(-) create mode 100644 src/Processors/tests/gtest_pipeline_executor_late_cpu_slots.cpp create mode 100644 tests/performance/small_block_contention_workload.xml diff --git a/src/Processors/Executors/ExecutorTasks.cpp b/src/Processors/Executors/ExecutorTasks.cpp index fd87df1f1043c..20c5b596a820a 100644 --- a/src/Processors/Executors/ExecutorTasks.cpp +++ b/src/Processors/Executors/ExecutorTasks.cpp @@ -141,12 +141,16 @@ void ExecutorTasks::tryGetTask(ExecutionThreadContext & context) /// Enqueue thread into stack of waiting threads. threads_queue.push(context.thread_number); + + /// No queued or asynchronous tasks are left for the requested threads. + if (async_task_queue.empty() && threads_to_spawn.load(std::memory_order_relaxed)) + threads_to_spawn.store(0, std::memory_order_relaxed); } context.wait(finished); } -size_t ExecutorTasks::pushTasks(Queue & queue, Queue & async_queue, ExecutionThreadContext & context) +void ExecutorTasks::pushTasks(Queue & queue, Queue & async_queue, ExecutionThreadContext & context) { /// Take local task from queue if has one. if (!queue.empty() && !has_fast_tasks.load(std::memory_order_relaxed) @@ -162,13 +166,13 @@ size_t ExecutorTasks::pushTasks(Queue & queue, Queue & async_queue, ExecutionThr context.setTask(nullptr); } - if (!queue.empty() || !async_queue.empty()) + if (queue.empty() && async_queue.empty()) + return; // No new tasks -- no need for new threads + + SlotAllocationPtr slots; { std::unique_lock lock(mutex); - /// Measure the parallelism of the push: how many threads could run the new tasks - /// concurrently. Used by `PipelineExecutor` to size the slot-allocation ceiling via - /// `ISlotAllocation::setMax` on the pipeline's `cpu_slots` handle. const size_t pushed = queue.size() + async_queue.size(); #if defined(OS_LINUX) || defined(OS_DARWIN) @@ -186,25 +190,57 @@ size_t ExecutorTasks::pushTasks(Queue & queue, Queue & async_queue, ExecutionThr queue.pop(); } - /// Capture state under the lock — tryWakeUp may unlock it internally to wake a - /// thread, after which `total_slots` could change. - const size_t idle_before = threads_queue.size(); - const size_t total_slots_now = total_slots; - const size_t num_threads_now = num_threads; // const after init but read for clarity + /// A thread for every pushed task that idle threads can't cover, within the pipeline width. + /// Computed before `tryWakeUpAnyOtherThreadWithTasks`, which may unlock the mutex. + const size_t uncovered = pushed - std::min(pushed, threads_queue.size()); + const size_t requested = std::min(uncovered, widthLeft()); + addThreadsToSpawn(requested); + if (growDesiredThreads(requested)) + slots = cpu_slots; /// Wake up at least one thread that will wake up other threads if required tryWakeUpAnyOtherThreadWithTasks(context, lock); + } + + if (slots) + applyDesiredThreads(*slots); +} + +void ExecutorTasks::initDesiredThreads(bool lazy_allocation, size_t initial_parallel) +{ + /// The eager allocation already asks for all the threads. + if (!lazy_allocation) + { + desired_threads.store(num_threads, std::memory_order_relaxed); + return; + } - /// Return the number of additional pipeline threads the caller should target: pushed - /// tasks that idle threads can't cover, clamped to remaining pipeline-width headroom - /// (`num_threads - total_slots`). `PipelineExecutor` adds this delta to the - /// `cpu_slots->setMax(...)` target so the slot allocator grows in step with demand. - const size_t coverable = std::min(pushed, idle_before); - const size_t uncovered = pushed - coverable; - const size_t headroom = (num_threads_now > total_slots_now) ? (num_threads_now - total_slots_now) : 0; - return std::min(uncovered, headroom); + /// The initial tasks are not pushed by `pushTasks`, so ask for threads to run them in parallel here. + SlotAllocationPtr slots; + { + std::lock_guard lock(mutex); + if (initial_parallel > 1 && growDesiredThreads(initial_parallel - 1)) + slots = cpu_slots; } - return 0; // No new tasks -- no need for new threads + if (slots) + applyDesiredThreads(*slots); +} + +bool ExecutorTasks::growDesiredThreads(size_t threads) +{ + const size_t current = desired_threads.load(std::memory_order_relaxed); + const size_t target = std::min(num_threads, current + threads); + if (target <= current) + return false; + desired_threads.store(target, std::memory_order_relaxed); + return true; +} + +void ExecutorTasks::applyDesiredThreads(ISlotAllocation & slots) +{ + /// Each caller sets the latest target, so the last `setMax` is never below it. + std::lock_guard lock(set_max_mutex); + slots.setMax(desired_threads.load(std::memory_order_relaxed)); } void ExecutorTasks::init(size_t num_threads_, size_t use_threads_, const SlotAllocationPtr & cpu_slots_, bool profile_processors, bool trace_processors, const StepWallClockRegistry * step_to_wall_clock_registry, ReadProgressCallback * callback) @@ -278,14 +314,45 @@ size_t ExecutorTasks::upscale(size_t slot_id) ++total_slots; use_threads = std::max(use_threads, slot_id + 1); + /// Concurrent spawners may acquire more slots than requested; such slots are granted anyway, so they get a thread. + if (const size_t requested = threads_to_spawn.load(std::memory_order_relaxed)) + threads_to_spawn.store(requested - 1, std::memory_order_relaxed); + + if (num_threads <= total_slots) + { + if (threads_to_spawn.load(std::memory_order_relaxed)) + threads_to_spawn.store(0, std::memory_order_relaxed); + return 0; + } + // Too many idle threads can already cover incoming work — don't spawn more. if (threads_queue.size() > TOO_MANY_IDLE_THRESHOLD) return 0; // Remaining pipeline-width capacity. Callers spawn one at a time and re-call upscale. + return num_threads - total_slots; +} + +size_t ExecutorTasks::widthLeft() const +{ return num_threads > total_slots ? num_threads - total_slots : 0; } +void ExecutorTasks::addThreadsToSpawn(size_t threads) +{ + /// Requests that wait for a CPU slot are not dropped, but they can't exceed the pipeline width. + const size_t requested = threads_to_spawn.load(std::memory_order_relaxed); + const size_t new_requested = std::min(requested + threads, widthLeft()); + if (new_requested != requested) + threads_to_spawn.store(new_requested, std::memory_order_relaxed); +} + +void ExecutorTasks::requestThreads(size_t threads) +{ + std::lock_guard lock(mutex); + addThreadsToSpawn(threads); +} + void ExecutorTasks::downscale(size_t slot_id) { std::lock_guard lock(mutex); diff --git a/src/Processors/Executors/ExecutorTasks.h b/src/Processors/Executors/ExecutorTasks.h index 2e677cf53b0b8..19886cea89a90 100644 --- a/src/Processors/Executors/ExecutorTasks.h +++ b/src/Processors/Executors/ExecutorTasks.h @@ -58,9 +58,29 @@ class ExecutorTasks /// CPU slots for each thread. SlotAllocationPtr cpu_slots; + /// Requested threads (for queued tasks or in place of downscaled ones) that wait for a CPU slot. + /// Changed under `mutex`, read without it after every task, so it has its own cache line. + alignas(64) std::atomic threads_to_spawn = 0; + /// Threshold found by rolling dice. const static size_t TOO_MANY_IDLE_THRESHOLD = 4; + /// Pipeline width not taken by started threads. Requires `mutex`. + size_t widthLeft() const; + /// Add to `threads_to_spawn`, up to `widthLeft`. Requires `mutex`. + void addThreadsToSpawn(size_t threads); + + /// Current `setMax` target of the CPU slots. Only grows. Changed under `mutex`. + std::atomic desired_threads = 1; + + /// Orders `setMax` calls. Not `mutex`: the allocation takes its own locks, and it calls `resume` under them. + std::mutex set_max_mutex; + + /// Raise the `setMax` target by `threads`, up to `num_threads`. Returns whether it changed. Requires `mutex`. + bool growDesiredThreads(size_t threads); + /// Pass the current target to `setMax`. Must not be called under `mutex`. + void applyDesiredThreads(ISlotAllocation & slots); + public: using Stack = std::stack; /// This queue can grow a lot and lead to OOM. That is why we use non-default @@ -92,9 +112,13 @@ class ExecutorTasks // Local task optimization: the first regular task could be placed directly into thread to be executed next. // For async tasks proessor->schedule() is called. // If non-local tasks were added, wake up one thread to process them. - // Returns the number of additional threads the caller should try to spawn (0 if idle - // threads can cover the push, or no new tasks were added). - size_t pushTasks(Queue & queue, Queue & async_queue, ExecutionThreadContext & context); + // Requests a thread for every new task that idle threads can't cover, within the pipeline width not taken + // by started threads: adds them to `threads_to_spawn` and raises the `setMax` target of the CPU slots by as much. + void pushTasks(Queue & queue, Queue & async_queue, ExecutionThreadContext & context); + + /// Set the initial `setMax` target: the pipeline width for the eager allocation, + /// enough threads for the `initial_parallel` initial tasks for the lazy one. + void initDesiredThreads(bool lazy_allocation, size_t initial_parallel); void init(size_t num_threads_, size_t use_threads_, const SlotAllocationPtr & cpu_slots_, bool profile_processors, bool trace_processors, const StepWallClockRegistry * step_to_wall_clock_registry, ReadProgressCallback * callback); @@ -108,14 +132,22 @@ class ExecutorTasks void freeCPU(); /// Upscale to include slot_id. Updates use_threads to max(use_threads, slot_id + 1). + /// The new thread is one of `threads_to_spawn`. /// Returns the number of additional threads the caller may try to spawn next (0 means - /// "stop spawning": either pipeline width is saturated or enough idle threads can absorb - /// the work). Callers should still spawn one at a time and re-call upscale after each. + /// "stop spawning": either pipeline width is saturated, and the rest of `threads_to_spawn` is dropped, + /// or enough idle threads can absorb the work). Callers should still spawn one at a time and + /// re-call upscale after each. size_t upscale(size_t slot_id); /// Downscale by removing slot_id from active slots. Updates use_threads to highest active slot + 1 void downscale(size_t slot_id); + /// Request `threads` more threads (e.g. for the initial tasks), clamped to the pipeline width. + void requestThreads(size_t threads); + + /// Whether requested threads wait for a CPU slot. Lock-free, so it is checked after every task. + bool hasThreadsToSpawn() const { return threads_to_spawn.load(std::memory_order_relaxed) > 0; } + /// Temporarily release slot_id without downscale. Later either downscale() or resume() is called. void preempt(size_t slot_id); diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index e23ef59b1d300..528122b0b95eb 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -303,10 +303,8 @@ void PipelineExecutor::finalizeExecution() { single_thread_cpu_slot.reset(); tasks.freeCPU(); - { - std::lock_guard lock(spawn_mutex); - cpu_slots.reset(); - } + /// No thread spawns anymore: the pool is waited for, or there is no pool. + cpu_slots.reset(); checkTimeLimit(); @@ -398,7 +396,6 @@ void PipelineExecutor::executeStepImpl(size_t thread_num, WorkloadResources && r #endif /// Try to execute neighbour processor. - size_t spawn_count = 0; { Queue queue; Queue async_queue; @@ -410,7 +407,17 @@ void PipelineExecutor::executeStepImpl(size_t thread_num, WorkloadResources && r /// Push other tasks to global queue. if (status == ExecutingGraph::UpdateNodeStatus::Done) - spawn_count = tasks.pushTasks(queue, async_queue, context); + { + try + { + tasks.pushTasks(queue, async_queue, context); + } + catch (...) + { + cancel(ExecutionStatus::Exception); + throw; + } + } } #ifndef NDEBUG @@ -418,50 +425,21 @@ void PipelineExecutor::executeStepImpl(size_t thread_num, WorkloadResources && r #endif - /// Upscaling: `pushTasks` tells us how many additional threads would usefully - /// parallelize the tasks it just enqueued. Publish our spawn_count into the shared - /// `pending_demand` counter; the thread that actually holds `spawn_mutex` drains it - /// so no demand is lost when multiple threads push concurrently. - if (pool && spawn_count > 0) - pending_demand.fetch_add(spawn_count, std::memory_order_relaxed); - - // Drain pending_demand under `spawn_mutex`. A thread that fails `try_lock` - // must not exit while `pending_demand > 0` -- otherwise its add could race - // with the owner's final `exchange(0)` and be left stranded after the owner - // unlocks. The outer loop holds the thread until either it drains the demand - // itself or observes the counter return to zero (meaning the owner caught it). - while (pool && pending_demand.load(std::memory_order_acquire) > 0) + /// Upscaling: threads requested by `pushTasks` wait in `tasks` until their CPU slots are granted, + /// so a slot granted later (e.g. released by another query) still gets a thread. + if (pool && tasks.hasThreadsToSpawn()) { - if (!spawn_mutex.try_lock()) - { - std::this_thread::yield(); - continue; - } try { - std::lock_guard lock(spawn_mutex, std::adopt_lock); - while (true) - { - size_t accumulated = pending_demand.exchange(0, std::memory_order_acq_rel); - if (accumulated == 0) - break; - size_t target = std::min(max_pipeline_threads, desired_threads + accumulated); - if (target > desired_threads) - { - desired_threads = target; - cpu_slots->setMax(target); - } - } spawnThreads({}); } catch (...) { - /// spawnThreads can throw an exception, for example CANNOT_SCHEDULE_TASK. + /// spawnThreads can throw an exception, for example CANNOT_SCHEDULE_TASK or RESOURCE_ACCESS_DENIED. /// We should cancel execution properly before rethrow. cancel(ExecutionStatus::Exception); throw; } - break; } /// Preemption and downscaling. @@ -474,6 +452,8 @@ void PipelineExecutor::executeStepImpl(size_t thread_num, WorkloadResources && r if (!resources.renewCPULease()) { tasks.downscale(resources.getCPULeaseSlotId()); + /// The slot may be granted again later: start a thread for it then. + tasks.requestThreads(1); yield = true; break; // Downscaling. Unable to renew the lease - thread should stop (but could be rerun later). } @@ -620,21 +600,13 @@ void PipelineExecutor::initializeExecution(size_t num_threads, bool concurrency_ is_execution_initialized = true; tryUpdateExecutionStatus(ExecutionStatus::NotStarted, ExecutionStatus::Executing); - /// Capture the ceiling so the upscaling block can cap `setMax` at this value. - max_pipeline_threads = num_threads; - /// Read the flag ONCE. A config reload mid-initialization would otherwise split a /// single query between lazy and eager strategies (e.g. allocate lazy with max=1 but - /// then initialize desired_threads=num_threads, stranding subsequent setMax as a no-op). + /// then start from the full `setMax` target, stranding subsequent setMax as a no-op). const bool lazy_allocation = ConcurrencyControl::instance().getLazyAllocation(); cpu_slots = allocateCPU(num_threads, concurrency_control, lazy_allocation); - /// If rollback flag is off, we used eager allocate(1, num_threads) and will not call - /// setMax in the upscaling block (nothing to grow). Initialize desired_threads - /// to the full ceiling so the growth check in the block becomes a no-op. - desired_threads = lazy_allocation ? 1 : num_threads; - Queue queue; Queue async_queue; graph->initializeExecution(queue, async_queue); @@ -645,22 +617,12 @@ void PipelineExecutor::initializeExecution(size_t num_threads, bool concurrency_ tasks.init(num_threads, 1, cpu_slots, profile_processors, trace_processors, step_wall_clock_registry, read_progress_callback.get()); const size_t initial_parallel = tasks.fill(queue, async_queue); - /// Initial queued parallelism never routes through `pushTasks`, so size setMax here to - /// cover it. For multi-source pipelines (e.g. UNION ALL of N subqueries) this prevents - /// the pipeline from being stuck single-threaded if the master's first task happens not - /// to push more work. Only matters on the lazy path; the eager path already has the full - /// ceiling. `initial_parallel` is the total target — the master thread is one of its - /// consumers, no +1 needed. - if (lazy_allocation && initial_parallel > 1) - { - std::lock_guard lock(spawn_mutex); - const size_t target = std::min(max_pipeline_threads, initial_parallel); - if (target > desired_threads) - { - desired_threads = target; - cpu_slots->setMax(target); - } - } + /// For multi-source pipelines (e.g. UNION ALL of N subqueries) this prevents the pipeline from being + /// stuck single-threaded if the master's first task happens not to push more work. + tasks.initDesiredThreads(lazy_allocation, initial_parallel); + + /// The eager path spawns a thread for every granted slot, as many as the pipeline width allows. + tasks.requestThreads(lazy_allocation ? std::max(initial_parallel, 1) : num_threads); if (num_threads > 1) pool = std::make_unique(CurrentMetrics::QueryPipelineExecutorThreads, CurrentMetrics::QueryPipelineExecutorThreadsActive, CurrentMetrics::QueryPipelineExecutorThreadsScheduled, num_threads); @@ -668,7 +630,7 @@ void PipelineExecutor::initializeExecution(size_t num_threads, bool concurrency_ void PipelineExecutor::spawnThreads(AcquiredSlotPtr slot) { - while (cpu_slots) + while (true) { if (!slot) slot = cpu_slots->tryAcquire(); @@ -714,11 +676,8 @@ void PipelineExecutor::executeImpl(size_t num_threads, bool concurrency_control) { if (num_threads > 1) { - { - std::lock_guard lock(spawn_mutex); - // Start at least one thread, could block to acquire the first CPU slot - spawnThreads(cpu_slots->acquire()); - } + // Start at least one thread, could block to acquire the first CPU slot + spawnThreads(cpu_slots->acquire()); tasks.processAsyncTasks(); pool->wait(); } diff --git a/src/Processors/Executors/PipelineExecutor.h b/src/Processors/Executors/PipelineExecutor.h index efcde90f48cdf..b1dc4741a2072 100644 --- a/src/Processors/Executors/PipelineExecutor.h +++ b/src/Processors/Executors/PipelineExecutor.h @@ -85,22 +85,6 @@ class PipelineExecutor SlotAllocationPtr cpu_slots; AcquiredSlotPtr single_thread_cpu_slot; // cpu slot for single-thread mode to work using executeStep() std::unique_ptr pool; - std::mutex spawn_mutex; - - /// Pipeline's max thread count (captured from execute(num_threads)). - size_t max_pipeline_threads = 1; - - /// Current setMax target. Written by initializeExecution before any worker thread exists, - /// then read and updated only under `spawn_mutex` from the upscaling block. Non-atomic - /// because all mutations are serialized by spawn_mutex and initial write happens-before - /// any spawned worker via the ThreadPool barrier in initializeExecution. - size_t desired_threads = 1; - - /// Accumulates spawn_count from pushTasks calls that couldn't acquire `spawn_mutex` - /// (another thread was already handling the spawn). The thread that owns the mutex - /// drains this counter before deciding the new setMax target, so no demand is ever - /// silently dropped. - std::atomic pending_demand{0}; /// Flag that checks that initializeExecution was called. bool is_execution_initialized = false; @@ -138,7 +122,9 @@ class PipelineExecutor // Methods for CPU scheduling SlotAllocationPtr allocateCPU(size_t num_threads, bool concurrency_control, bool lazy_allocation); - void spawnThreads(AcquiredSlotPtr slot) TSA_REQUIRES(spawn_mutex); + /// Start a thread for `slot` (if any) and for every other granted slot, until `tasks` has no capacity left. + /// Lock-free: any thread calls it after a task while `tasks` has requested threads. + void spawnThreads(AcquiredSlotPtr slot); /// If execution_status == from, change it to desired. bool tryUpdateExecutionStatus(ExecutionStatus expected, ExecutionStatus desired); diff --git a/src/Processors/tests/gtest_pipeline_executor_late_cpu_slots.cpp b/src/Processors/tests/gtest_pipeline_executor_late_cpu_slots.cpp new file mode 100644 index 0000000000000..a50e9e78439d1 --- /dev/null +++ b/src/Processors/tests/gtest_pipeline_executor_late_cpu_slots.cpp @@ -0,0 +1,241 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +using namespace DB; + +namespace +{ + +constexpr size_t num_streams = 4; +constexpr auto timeout = std::chrono::seconds(60); + +struct LateSlotsState +{ + std::mutex mutex; + std::condition_variable cv; + + bool first_step_started = false; + bool slots_released = false; + + /// Threads that ran a step after the release. + std::set threads; + bool done = false; + bool timed_out = false; +}; + +/// The first step of the pipeline waits until the test releases the CPU slots held by another allocation. +/// Every later step waits until `num_streams` threads are in it. A thread is started for a granted slot only +/// at a task boundary of a running thread, so without one the only running thread waits here for the timeout. +class LateSlotsSource : public ISource +{ +public: + LateSlotsSource(SharedHeader header, LateSlotsState & state_) + : ISource(std::move(header)) + , state(state_) + { + } + + String getName() const override { return "LateSlotsSource"; } + +protected: + Chunk generate() override + { + std::unique_lock lock(state.mutex); + if (!state.first_step_started) + { + state.first_step_started = true; + state.cv.notify_all(); + state.cv.wait_for(lock, timeout, [&] { return state.slots_released; }); + return Chunk(Columns{ColumnUInt8::create(1, static_cast(1))}, 1); + } + + if (!state.done) + { + state.threads.insert(std::this_thread::get_id()); + if (state.threads.size() >= num_streams) + state.done = true; + else if (!state.cv.wait_for(lock, timeout, [&] { return state.done; })) + state.done = state.timed_out = true; + state.cv.notify_all(); + } + return {}; + } + +private: + LateSlotsState & state; +}; + +void checkThreadsForLateSlots(bool lazy_allocation) +{ + auto & concurrency_control = ConcurrencyControl::instance(); + const String prev_scheduler = concurrency_control.getScheduler(); + const bool prev_lazy_allocation = concurrency_control.getLazyAllocation(); + SCOPE_EXIT({ + concurrency_control.setMaxConcurrency(UnlimitedSlots); + concurrency_control.setScheduler(prev_scheduler); + concurrency_control.setLazyAllocation(prev_lazy_allocation); + }); + concurrency_control.setScheduler("fair_round_robin"); + concurrency_control.setLazyAllocation(lazy_allocation); + concurrency_control.setMaxConcurrency(num_streams); + + /// Another query holds all the competing slots, so the pipeline starts with its master thread only. + SlotAllocationPtr other_query = concurrency_control.allocate(0, num_streams); + + LateSlotsState state; + SharedHeader header = std::make_shared(Block{ColumnWithTypeAndName(ColumnUInt8::create(), std::make_shared(), "x")}); + auto processors = std::make_shared(); + for (size_t i = 0; i < num_streams; ++i) + { + auto source = std::make_shared(header, state); + auto sink = std::make_shared(source->getPort().getSharedHeader()); + connect(source->getPort(), sink->getPort()); + processors->emplace_back(std::move(source)); + processors->emplace_back(std::move(sink)); + } + + QueryStatusPtr element; + PipelineExecutor executor(processors, element); + std::thread query([&] { executor.execute(num_streams, /* concurrency_control */ true); }); + + { + std::unique_lock lock(state.mutex); + EXPECT_TRUE(state.cv.wait_for(lock, timeout, [&] { return state.first_step_started; })) << "The pipeline did not start"; + } + + /// The other query finishes: its slots are granted to the pipeline, which is in the middle of execution. + other_query.reset(); + { + std::lock_guard lock(state.mutex); + state.slots_released = true; + } + state.cv.notify_all(); + + query.join(); + + std::lock_guard lock(state.mutex); + EXPECT_FALSE(state.timed_out); + EXPECT_EQ(state.threads.size(), num_streams); +} + +} + +/// A CPU slot granted to a running pipeline gets a thread at the next task boundary, +/// even if no step pushes tasks to the shared queue. +TEST(PipelineExecutor, ThreadsForSlotsGrantedAfterStart) +{ + checkThreadsForLateSlots(/* lazy_allocation */ true); +} + +TEST(PipelineExecutor, ThreadsForSlotsGrantedAfterStartEagerAllocation) +{ + checkThreadsForLateSlots(/* lazy_allocation */ false); +} + +/// A thread requested in place of a downscaled one is kept until a slot is granted and the thread is started. +TEST(PipelineExecutor, ThreadRequestedAfterDownscale) +{ + ExecutorTasks tasks; + tasks.init(num_streams, 1, std::make_shared(num_streams), false, false, nullptr, nullptr); + tasks.upscale(0); + tasks.upscale(1); + EXPECT_FALSE(tasks.hasThreadsToSpawn()); + + tasks.preempt(1); + tasks.downscale(1); + EXPECT_FALSE(tasks.hasThreadsToSpawn()); + tasks.requestThreads(1); + EXPECT_TRUE(tasks.hasThreadsToSpawn()); + + tasks.upscale(1); + EXPECT_FALSE(tasks.hasThreadsToSpawn()); + tasks.finish(); +} + +/// A thread that finds no queued task to run drops the threads requested for queued tasks: +/// none are left for them. +TEST(PipelineExecutor, RequestedThreadsDroppedWhenThreadGoesIdle) +{ + ExecutorTasks tasks; + tasks.init(num_streams, 1, std::make_shared(num_streams), false, false, nullptr, nullptr); + tasks.upscale(0); + tasks.upscale(1); + tasks.requestThreads(2); + EXPECT_TRUE(tasks.hasThreadsToSpawn()); + + /// One of the two running threads waits for a task; the queues are empty. + std::thread idle([&] { tasks.tryGetTask(tasks.getThreadContext(1)); }); + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (tasks.hasThreadsToSpawn() && std::chrono::steady_clock::now() < deadline) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + EXPECT_FALSE(tasks.hasThreadsToSpawn()); + + tasks.finish(); + idle.join(); +} + +#if defined(OS_LINUX) || defined(OS_DARWIN) +namespace +{ + +/// Waits for a pipe that is never written to. +class PendingAsyncProcessor : public IProcessor +{ +public: + explicit PendingAsyncProcessor(int fd_) : fd(fd_) {} + String getName() const override { return "PendingAsyncProcessor"; } + Status prepare() override { return Status::Async; } + int schedule() override { return fd; } + +private: + int fd; +}; + +} + +/// Requested threads are kept while asynchronous tasks are pending: they can run the tasks once these are ready. +TEST(PipelineExecutor, RequestedThreadsKeptWhileAsyncTasksPending) +{ + int fds[2]; + ASSERT_EQ(pipe(fds), 0); + SCOPE_EXIT({ + close(fds[0]); + close(fds[1]); + }); + Processors processors{std::make_shared(fds[0])}; + ExecutingGraph::Node node(processors.begin(), 0); + + ExecutorTasks tasks; + tasks.init(num_streams, 1, std::make_shared(num_streams), false, false, nullptr, nullptr); + tasks.upscale(0); + tasks.upscale(1); + + ExecutorTasks::Queue queue; + ExecutorTasks::Queue async_queue; + async_queue.push(&node); + tasks.pushTasks(queue, async_queue, tasks.getThreadContext(0)); + EXPECT_TRUE(tasks.hasThreadsToSpawn()); + + /// One of the two running threads waits for a task; only the asynchronous one is pending. + std::thread idle([&] { tasks.tryGetTask(tasks.getThreadContext(1)); }); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + EXPECT_TRUE(tasks.hasThreadsToSpawn()); + + tasks.finish(); + idle.join(); +} +#endif diff --git a/tests/integration/test_concurrent_threads_soft_limit/test.py b/tests/integration/test_concurrent_threads_soft_limit/test.py index b06f90664a4cd..3d39a8c2e9d7a 100644 --- a/tests/integration/test_concurrent_threads_soft_limit/test.py +++ b/tests/integration/test_concurrent_threads_soft_limit/test.py @@ -333,3 +333,72 @@ def limit_reached(): assert count < 6 node4.query("KILL QUERY WHERE query_id = 'background_query' SYNC") background_thread.join() + + +# A query that starts while the limit is reached gets only its non-competing slot. When the other +# query finishes, the released slots are granted to it and it must start using them, even if its +# pipeline never pushes new tasks to the executor's global queue (each stream is an independent +# chain whose steps are slow, as in object storage scans). +def test_concurrent_threads_soft_limit_upscale_after_release(started_cluster): + background_id = "background_query_upscale" + foreground_id = "test_concurrent_threads_soft_limit_5" + errors = {} + + def run(name, query, query_id): + try: + node4.query(query, query_id=query_id) + except QueryRuntimeException as e: + errors[name] = e + + def wait_for(condition, what, timeout=60): + deadline = time.monotonic() + timeout + while not condition(): + assert time.monotonic() < deadline, f"Timed out waiting for {what}, errors: {errors}" + time.sleep(0.1) + + def foreground_event(profile_event): + value = node4.query( + f"SELECT ProfileEvents['{profile_event}'] FROM system.processes WHERE query_id = '{foreground_id}'" + ).strip() + return int(value) if value else None + + background = threading.Thread( + target=run, + args=("background", "SELECT count(*) FROM numbers_mt(1e11) SETTINGS max_threads=100", background_id), + ) + # 8 streams; every step of a stream sleeps 1 s, so a thread that runs only local tasks pushes to the + # global queue only after more than 20 s. The query runs until it is killed. + foreground = threading.Thread( + target=run, + args=( + "foreground", + "SELECT count() FROM numbers_mt(1e9) WHERE NOT ignore(sleepEachRow(0.02)) " + "SETTINGS max_threads=8, max_block_size=50", + foreground_id, + ), + ) + try: + background.start() + wait_for( + lambda: int(node4.query("SELECT sum(length(thread_ids)) FROM system.processes").strip() or 0) >= 11, + "the background query to take all slots", + ) + + foreground.start() + wait_for(lambda: foreground_event("ConcurrencyControlSlotsDelayed"), "the foreground query to wait for slots") + + # Release all competing slots. The foreground query has to start threads for them at the next + # step (~1 s), long before its threads push to the global queue on their own. + node4.query(f"KILL QUERY WHERE query_id = '{background_id}' SYNC") + wait_for( + lambda: (foreground_event("ConcurrencyControlSlotsAcquired") or 0) >= 4, + "the foreground query to use the released slots", + timeout=8, + ) + finally: + try: + node4.query(f"KILL QUERY WHERE query_id IN ('{background_id}', '{foreground_id}') SYNC") + finally: + for thread in (background, foreground): + if thread.ident is not None: + thread.join() diff --git a/tests/integration/test_scheduler_cpu_preemptive/test.py b/tests/integration/test_scheduler_cpu_preemptive/test.py index 6f647f8c9b24e..ec986552e3a18 100644 --- a/tests/integration/test_scheduler_cpu_preemptive/test.py +++ b/tests/integration/test_scheduler_cpu_preemptive/test.py @@ -41,6 +41,8 @@ def clear_workloads_and_resources(): drop workload if exists production; drop workload if exists development; drop workload if exists admin; + drop workload if exists victim; + drop workload if exists competitor; drop workload if exists all; drop resource if exists cpu; """ @@ -628,3 +630,78 @@ def test_create_workload_under_load(): development2.stop() assert development2.get_errors() == 0, "Errors occurred in development2 workload" assert production2.get_errors() == 0, "Errors occurred in production2 workload" + + +# A worker whose preempted CPU lease is not granted back in time is downscaled. When the competing query +# finishes and the capacity is back, the same query starts a thread for it again, without waiting for its +# threads to push to the executor's global queue (each step here takes ~1 s of CPU, and a thread that runs +# only local tasks pushes after 128 of them). +@pytest.mark.parametrize( + "with_custom_config", + [{"node": {"cpu_slot_preemption_timeout_ms": 100}}], + indirect=True, +) +def test_worker_replaced_after_downscale(with_custom_config): + if node.is_built_with_address_sanitizer() or node.is_built_with_thread_sanitizer() or node.is_built_with_llvm_coverage(): + pytest.skip("doesn't fit in timeouts due to heavy workload") + + node.query( + """ + create resource cpu (master thread, worker thread); + create workload all settings max_concurrent_threads=4; + create workload victim in all; + create workload competitor in all; + """ + ) + + victim_id = "test_worker_replaced_after_downscale_victim" + competitor_id = "test_worker_replaced_after_downscale_competitor" + # CPU-bound steps of ~1 s each with little memory; the query runs until it is killed. + hashes = "number" + for _ in range(150): + hashes = f"cityHash64({hashes})" + heavy = f"select count() from numbers_mt(1000000000000) where not ignore({hashes}) settings max_block_size=5000000, max_threads=4" + errors = {} + + def run(name, query_id, workload): + try: + node.query(f"{heavy}, workload='{workload}'", query_id=query_id) + except QueryRuntimeException as e: + errors[name] = e + + def wait_for(condition, what, timeout=60): + deadline = time.monotonic() + timeout + while not condition(): + assert time.monotonic() < deadline, f"Timed out waiting for {what}, errors: {errors}" + time.sleep(0.1) + + def victim_event(profile_event): + value = node.query( + f"select ProfileEvents['{profile_event}'] from system.processes where query_id = '{victim_id}'" + ).strip() + return int(value) if value else 0 + + victim = threading.Thread(target=run, args=("victim", victim_id, "victim")) + competitor = threading.Thread(target=run, args=("competitor", competitor_id, "competitor")) + try: + victim.start() + wait_for(lambda: victim_event("ConcurrencyControlSlotsAcquired") >= 4, "the victim to take all slots") + + # The competitor takes its share, so the victim's workers are preempted and downscaled after the timeout. + competitor.start() + wait_for(lambda: victim_event("ConcurrencyControlDownscales") >= 1, "the victim to be downscaled") + + upscales = victim_event("ConcurrencyControlUpscales") + node.query(f"kill query where query_id = '{competitor_id}' sync") + wait_for( + lambda: victim_event("ConcurrencyControlUpscales") > upscales, + "the victim to start a thread for the released capacity", + timeout=15, + ) + finally: + try: + node.query(f"kill query where query_id in ('{victim_id}', '{competitor_id}') sync") + finally: + for thread in (victim, competitor): + if thread.ident is not None: + thread.join() diff --git a/tests/performance/small_block_contention_workload.xml b/tests/performance/small_block_contention_workload.xml new file mode 100644 index 0000000000000..089e89add6c49 --- /dev/null +++ b/tests/performance/small_block_contention_workload.xml @@ -0,0 +1,13 @@ + + + CREATE RESOURCE IF NOT EXISTS perf_small_block_cpu (WORKER THREAD) + CREATE WORKLOAD IF NOT EXISTS perf_small_block_all + CREATE WORKLOAD IF NOT EXISTS perf_small_block IN perf_small_block_all SETTINGS max_concurrent_threads = 4 + + select sum(number) from numbers_mt(200000) settings max_threads=100, max_block_size = 1, workload = 'perf_small_block' format Null + + DROP WORKLOAD IF EXISTS perf_small_block + DROP WORKLOAD IF EXISTS perf_small_block_all + DROP RESOURCE IF EXISTS perf_small_block_cpu + From a0be1cce7c38133215d729f656178160104c3eff Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 9 Nov 2026 02:54:09 +0000 Subject: [PATCH 133/219] Backport #130211 to 26.8: Fix a segfault in `Pretty*` and `JSONColumns` output when a memory limit is hit while the result is accumulated --- src/Processors/Chunk.cpp | 12 +++++------- ...ting_result_memory_limit_in_local.reference | 6 ++++++ ...ccumulating_result_memory_limit_in_local.sh | 18 ++++++++++++++++++ 3 files changed, 29 insertions(+), 7 deletions(-) create mode 100644 tests/queries/0_stateless/05615_formats_accumulating_result_memory_limit_in_local.reference create mode 100755 tests/queries/0_stateless/05615_formats_accumulating_result_memory_limit_in_local.sh diff --git a/src/Processors/Chunk.cpp b/src/Processors/Chunk.cpp index bd5e8027801c2..18e2314e0b259 100644 --- a/src/Processors/Chunk.cpp +++ b/src/Processors/Chunk.cpp @@ -70,13 +70,11 @@ void Chunk::checkNumRowsIsConsistent() MutableColumns Chunk::mutateColumns() { - size_t num_columns = columns.size(); - MutableColumns mutable_columns(num_columns); - for (size_t i = 0; i < num_columns; ++i) - mutable_columns[i] = IColumn::mutate(std::move(columns[i])); - - columns.clear(); - num_rows = 0; + MutableColumns mutable_columns(columns.size()); + /// Detach first: `IColumn::mutate` consumes the column even when it throws, and a chunk never holds a null column. + Columns detached_columns = detachColumns(); + for (size_t i = 0; i < detached_columns.size(); ++i) + mutable_columns[i] = IColumn::mutate(std::move(detached_columns[i])); return mutable_columns; } diff --git a/tests/queries/0_stateless/05615_formats_accumulating_result_memory_limit_in_local.reference b/tests/queries/0_stateless/05615_formats_accumulating_result_memory_limit_in_local.reference new file mode 100644 index 0000000000000..d0fcfd538e173 --- /dev/null +++ b/tests/queries/0_stateless/05615_formats_accumulating_result_memory_limit_in_local.reference @@ -0,0 +1,6 @@ +MEMORY_LIMIT_EXCEEDED +PrettyCompactMonoBlock 241 +MEMORY_LIMIT_EXCEEDED +PrettyCompact 241 +MEMORY_LIMIT_EXCEEDED +JSONColumns 241 diff --git a/tests/queries/0_stateless/05615_formats_accumulating_result_memory_limit_in_local.sh b/tests/queries/0_stateless/05615_formats_accumulating_result_memory_limit_in_local.sh new file mode 100755 index 0000000000000..551f08b3f82cb --- /dev/null +++ b/tests/queries/0_stateless/05615_formats_accumulating_result_memory_limit_in_local.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Every stored block is larger than the memory limit, so the format that accumulates the result fails to copy it +# when the next block arrives: the query must fail with an error, not crash. +for format in PrettyCompactMonoBlock PrettyCompact JSONColumns; do + $CLICKHOUSE_LOCAL --query " + CREATE TABLE t (x UInt64) ENGINE = Memory; + INSERT INTO t SELECT number FROM numbers(5000000) SETTINGS max_block_size = 5000000, max_insert_block_size = 5000000; + INSERT INTO t SELECT number FROM numbers(5000000) SETTINGS max_block_size = 5000000, max_insert_block_size = 5000000; + SELECT * FROM t FORMAT $format SETTINGS max_threads = 1, max_memory_usage = '20Mi', + output_format_pretty_squash_consecutive_ms = 1000000, output_format_pretty_squash_max_wait_ms = 1000000; + " 2>&1 >/dev/null | grep -o -m1 'MEMORY_LIMIT_EXCEEDED' | head -n1 + echo "$format ${PIPESTATUS[0]}" +done From 26d13df66811556423ba3bdbf345d2db1e66ade8 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 6 Nov 2026 20:15:15 +0000 Subject: [PATCH 134/219] Backport #129467 to 26.8: Build amd_compat and arm_v80compat on release and backport branches --- .github/workflows/backport_branches.yml | 72 +++++++++++++++- .github/workflows/release_branches.yml | 106 ++++++++++++++++++++++++ ci/workflows/backport_branches.py | 2 +- ci/workflows/release_branches.py | 2 +- 4 files changed, 179 insertions(+), 3 deletions(-) diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml index 5087d25f38766..35dd0cae11954 100644 --- a/.github/workflows/backport_branches.yml +++ b/.github/workflows/backport_branches.yml @@ -367,6 +367,76 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_darwin)' --workflow "BackportPR" --ci --timestamp + build_arm_v80compat: + runs-on: [self-hosted, arm-large] + needs: [config_workflow, dockers_build_amd, dockers_build_arm] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV92ODBjb21wYXQp') }} + name: "Build (arm_v80compat)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_v80compat)' --workflow "BackportPR" --ci --timestamp + + build_amd_compat: + runs-on: [self-hosted, arm-large] + needs: [config_workflow, dockers_build_amd, dockers_build_arm] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9jb21wYXQp') }} + name: "Build (amd_compat)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_compat)' --workflow "BackportPR" --ci --timestamp + docker_server_image: runs-on: [self-hosted, style-checker] needs: [build_amd_release, build_arm_release, config_workflow, dockers_build_amd, dockers_build_arm] @@ -1384,7 +1454,7 @@ jobs: finish_workflow: runs-on: [self-hosted, style-checker-aarch64] - needs: [build_amd_asan_ubsan, build_amd_darwin, build_amd_debug, build_amd_release, build_amd_tsan, build_arm_darwin, build_arm_release, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_3_3, stateless_tests_amd_asan_ubsan_distributed_plan_parallel, stress_test_amd_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] + needs: [build_amd_asan_ubsan, build_amd_compat, build_amd_darwin, build_amd_debug, build_amd_release, build_amd_tsan, build_arm_darwin, build_arm_release, build_arm_v80compat, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_3_3, stateless_tests_amd_asan_ubsan_distributed_plan_parallel, stress_test_amd_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] if: ${{ always() }} name: "Finish Workflow" outputs: diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 1f6cc30dbf57a..e09c72f617696 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -540,6 +540,76 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_darwin)' --workflow "ReleaseBranchCI" --ci --timestamp + build_arm_v80compat: + runs-on: [self-hosted, arm-large] + needs: [config_workflow, dockers_build_amd, dockers_build_arm] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV92ODBjb21wYXQp') }} + name: "Build (arm_v80compat)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_v80compat)' --workflow "ReleaseBranchCI" --ci --timestamp + + build_amd_compat: + runs-on: [self-hosted, arm-large] + needs: [config_workflow, dockers_build_amd, dockers_build_arm] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9jb21wYXQp') }} + name: "Build (amd_compat)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_compat)' --workflow "ReleaseBranchCI" --ci --timestamp + sign_macos_binary_amd_darwin: runs-on: [self-hosted, release-runner] needs: [build_amd_darwin, config_workflow, dockers_build_amd, dockers_build_arm] @@ -2114,3 +2184,39 @@ jobs: run: | . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Stress test (arm_msan)' --workflow "ReleaseBranchCI" --ci --timestamp + + + finish_workflow: + runs-on: [self-hosted, style-checker-aarch64] + needs: [build_amd_asan_ubsan, build_amd_compat, build_amd_darwin, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_darwin, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_arm_v80compat, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_1_8, integration_tests_amd_asan_ubsan_db_disk_2_8, integration_tests_amd_asan_ubsan_db_disk_3_8, integration_tests_amd_asan_ubsan_db_disk_4_8, integration_tests_amd_asan_ubsan_db_disk_5_8, integration_tests_amd_asan_ubsan_db_disk_6_8, integration_tests_amd_asan_ubsan_db_disk_7_8, integration_tests_amd_asan_ubsan_db_disk_8_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, sign_macos_binary_amd_darwin, sign_macos_binary_arm_darwin, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_3_3, stateless_tests_amd_asan_ubsan_distributed_plan_parallel, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, unit_tests_asan_ubsan, unit_tests_msan, unit_tests_tsan] + if: ${{ always() }} + name: "Finish Workflow" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Finish Workflow' --workflow "ReleaseBranchCI" --ci --timestamp diff --git a/ci/workflows/backport_branches.py b/ci/workflows/backport_branches.py index 92a5c210d533b..5bde2757897b3 100644 --- a/ci/workflows/backport_branches.py +++ b/ci/workflows/backport_branches.py @@ -18,7 +18,7 @@ *[ job for job in JobConfigs.special_build_jobs - if any(t in job.name for t in ("darwin",)) + if any(t in job.name for t in ("darwin", "compat")) ], JobConfigs.docker_server, JobConfigs.docker_keeper, diff --git a/ci/workflows/release_branches.py b/ci/workflows/release_branches.py index bfff879cc774a..66711c40744a0 100644 --- a/ci/workflows/release_branches.py +++ b/ci/workflows/release_branches.py @@ -37,7 +37,7 @@ *[ job for job in JobConfigs.special_build_jobs - if any(t in job.name for t in ("darwin",)) + if any(t in job.name for t in ("darwin", "compat")) ], *JobConfigs.sign_macos_binary_jobs, JobConfigs.docker_server, From 30013ca2d9b5a3fc427c9a230ff5082ee151643b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 9 Nov 2026 09:58:11 +0000 Subject: [PATCH 135/219] Backport #129893 to 26.8: Fix `CANNOT_SCHEDULE_TASK` in parallel `INSERT` by sharing one Parquet encoder thread pool --- src/IO/SharedThreadPools.h | 1 + .../Formats/Impl/ParquetBlockOutputFormat.cpp | 49 ++++++++++--------- .../Formats/Impl/ParquetBlockOutputFormat.h | 12 +++-- ...rquet_encoder_shared_thread_pool.reference | 2 + ...5565_parquet_encoder_shared_thread_pool.sh | 40 +++++++++++++++ 5 files changed, 79 insertions(+), 25 deletions(-) create mode 100644 tests/queries/0_stateless/05565_parquet_encoder_shared_thread_pool.reference create mode 100755 tests/queries/0_stateless/05565_parquet_encoder_shared_thread_pool.sh diff --git a/src/IO/SharedThreadPools.h b/src/IO/SharedThreadPools.h index 5982a3dc29ad9..f502cdc3aeac0 100644 --- a/src/IO/SharedThreadPools.h +++ b/src/IO/SharedThreadPools.h @@ -72,6 +72,7 @@ class StaticThreadPool M(DropDistributedCache, "DropDistributedCacheThreadPool", DropDistributedCache) \ M(FormatParsing, "FormatParsingThreadPool", FormatParsing) \ M(IcebergManifestDecode, "IcebergManifestDecodeThreadPool", IcebergManifestDecode) \ + M(ParquetEncoder, "ParquetEncoderThreadPool", ParquetEncoder) \ M(AIRequest, "AIRequestThreadPool", AIRequest) #define DECLARE_STATIC_THREAD_POOL_GETTER(SUFFIX, NAME, METRIC) StaticThreadPool & get##SUFFIX##ThreadPool(); diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index 28c3c0816d54a..b01ca4e03015a 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -7,16 +7,12 @@ #include #include #include +#include #include #include +#include -namespace CurrentMetrics -{ - extern const Metric ParquetEncoderThreads; - extern const Metric ParquetEncoderThreadsActive; - extern const Metric ParquetEncoderThreadsScheduled; -} namespace DB { @@ -32,11 +28,10 @@ ParquetBlockOutputFormat::ParquetBlockOutputFormat(WriteBuffer & out_, SharedHea : IOutputFormat(header_, out_), format_settings{format_settings_}, format_filter_info(format_filter_info_) { if (format_settings.parquet.parallel_encoding && format_settings.max_threads > 1) - pool = std::make_unique( - CurrentMetrics::ParquetEncoderThreads, - CurrentMetrics::ParquetEncoderThreadsActive, - CurrentMetrics::ParquetEncoderThreadsScheduled, - format_settings.max_threads); + { + getParquetEncoderThreadPool().initializeWithDefaultSettingsIfNotInitialized(); + pool = &getParquetEncoderThreadPool().get(); + } using C = FormatSettings::ParquetCompression; switch (format_settings.parquet.output_compression_method) @@ -82,10 +77,14 @@ ParquetBlockOutputFormat::ParquetBlockOutputFormat(WriteBuffer & out_, SharedHea ParquetBlockOutputFormat::~ParquetBlockOutputFormat() { if (pool) - { - is_stopped = true; - pool->wait(); - } + stopThreads(); +} + +void ParquetBlockOutputFormat::stopThreads() +{ + is_stopped = true; + std::unique_lock lock(mutex); + condvar.wait(lock, [this] { return threads_running == 0; }); } void ParquetBlockOutputFormat::consume(Chunk chunk) @@ -263,8 +262,7 @@ void ParquetBlockOutputFormat::resetFormatterImpl() { if (pool) { - is_stopped = true; - pool->wait(); + stopThreads(); is_stopped = false; } @@ -413,17 +411,16 @@ void ParquetBlockOutputFormat::startMoreThreadsIfNeeded(const std::unique_lock switcher; + switcher.emplace(thread_group, ThreadName::PARQUET_ENCODER); + std::unique_lock lock(mutex); while (true) @@ -457,7 +457,12 @@ void ParquetBlockOutputFormat::threadFunction() { /// The check and the decrement need to be in the same critical section, to make sure /// we never get stuck with tasks but no threads. + /// Leave the query's thread group before stopThreads() can see the decrement. + switcher.reset(); + thread_group.reset(); --threads_running; + if (threads_running == 0) + condvar.notify_all(); return; } diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h index c38b1e5d0ce7f..cc7dd4b5128f4 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h @@ -12,6 +12,9 @@ namespace DB { +class ThreadGroup; +using ThreadGroupPtr = std::shared_ptr; + class ParquetBlockOutputFormat final : public IOutputFormat { public: @@ -104,8 +107,9 @@ class ParquetBlockOutputFormat final : public IOutputFormat void writeRowGroupInOneThread(Chunk chunk); void writeRowGroupInParallel(std::vector chunks); - void threadFunction(); + void threadFunction(ThreadGroupPtr & thread_group); void startMoreThreadsIfNeeded(const std::unique_lock & lock); + void stopThreads(); /// Called in single-threaded fashion. Writes to the file. void reapCompletedRowGroups(std::unique_lock & lock); @@ -126,13 +130,15 @@ class ParquetBlockOutputFormat final : public IOutputFormat size_t base_offset = 0; // initial out.count(), just for assert std::mutex mutex; - std::condition_variable condvar; // wakes up consume() - std::unique_ptr pool; + std::condition_variable condvar; // wakes up consume() and stopThreads() + /// Shared by all Parquet writers of the process (`getParquetEncoderThreadPool`). + ThreadPool * pool = nullptr; std::atomic_bool is_stopped{false}; std::exception_ptr background_exception = nullptr; /// Invariant: if there's at least one task then there's at least one thread. + /// Counts the jobs scheduled on `pool` until they exit, queued ones included. size_t threads_running = 0; std::atomic bytes_in_flight{0}; diff --git a/tests/queries/0_stateless/05565_parquet_encoder_shared_thread_pool.reference b/tests/queries/0_stateless/05565_parquet_encoder_shared_thread_pool.reference new file mode 100644 index 0000000000000..69bc311a9dae2 --- /dev/null +++ b/tests/queries/0_stateless/05565_parquet_encoder_shared_thread_pool.reference @@ -0,0 +1,2 @@ +8000 1535808000 4 1 +4800 552844800 24 diff --git a/tests/queries/0_stateless/05565_parquet_encoder_shared_thread_pool.sh b/tests/queries/0_stateless/05565_parquet_encoder_shared_thread_pool.sh new file mode 100755 index 0000000000000..945dea9ed33a5 --- /dev/null +++ b/tests/queries/0_stateless/05565_parquet_encoder_shared_thread_pool.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# - no-fasttest: requires `IcebergLocal` (USE_AVRO build option) + +# Every Parquet writer had its own pool of `max_threads` encoder threads, so many open writers with `max_threads = 32` +# did not fit in a 500-thread global pool: 8 parallel sinks of an Iceberg INSERT with 4 partitions each, and the +# 24 partitions of one stream. +# https://github.com/ClickHouse/ClickHouse/issues/129864 + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +TABLE_DIR="${CLICKHOUSE_USER_FILES}/${CLICKHOUSE_TEST_UNIQUE_NAME}" +rm -rf "${TABLE_DIR}" +mkdir -p "${TABLE_DIR}" +trap 'rm -rf "${TABLE_DIR}"' EXIT + +COLS=$(for i in $(seq 48); do printf ', c%d Int64' "$i"; done) +EXPRS=$(for i in $(seq 48); do printf ', number * %d' "$i"; done) +SETTINGS="SET max_threads = 32, max_threads_min_free_memory_per_thread = 0, max_insert_threads_min_free_memory_per_thread = 0, + output_format_parquet_parallel_encoding = 1, output_format_parquet_row_group_size = 200, + max_block_size = 1000, min_insert_block_size_rows = 1000, min_insert_block_size_bytes = 0" + +# An Iceberg table takes its format settings from its `SETTINGS` clause, not from the session. Small row groups make +# every writer encode while the INSERT runs, not only when its file is finalized. +${CLICKHOUSE_LOCAL} --query " + ${SETTINGS}, allow_insert_into_iceberg = 1, max_insert_threads = 8; + CREATE TABLE t (p Int32 ${COLS}) ENGINE = IcebergLocal('${TABLE_DIR}/iceberg/') PARTITION BY p + SETTINGS output_format_parquet_row_group_size = 200; + INSERT INTO t SELECT number % 4 ${EXPRS} FROM numbers(8000); + SELECT count(), sum(c48), uniqExact(p), uniqExact(_path) > 4 FROM t; +" -- --max_thread_pool_size=500 --thread_pool_queue_size=500 + +${CLICKHOUSE_LOCAL} --query " + ${SETTINGS}, max_insert_threads = 1; + INSERT INTO FUNCTION file('${TABLE_DIR}/file/{_partition_id}.parquet', 'Parquet', 'p UInt8 ${COLS}') PARTITION BY p + SELECT number % 24 ${EXPRS} FROM numbers(4800); + SELECT count(), sum(c48), uniqExact(_path) FROM file('${TABLE_DIR}/file/*.parquet'); +" -- --max_thread_pool_size=500 --thread_pool_queue_size=500 From cbaeea20f739e3323ead4078c83092c45c206e0a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 9 Nov 2026 16:07:30 +0000 Subject: [PATCH 136/219] Backport #130171 to 26.8: Cast an ORC list, map or struct requested as another type instead of reading default values --- .../Impl/NativeORCBlockInputFormat.cpp | 15 ++++- ...c_requested_type_of_another_kind.reference | 18 ++++++ ...5574_orc_requested_type_of_another_kind.sh | 56 +++++++++++++++++++ 3 files changed, 86 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/05574_orc_requested_type_of_another_kind.reference create mode 100755 tests/queries/0_stateless/05574_orc_requested_type_of_another_kind.sh diff --git a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp index 08779ffb6304e..6bc08f9f19279 100644 --- a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp @@ -1127,8 +1127,9 @@ updateIncludeTypeIds(DataTypePtr type, const ORC::Type * orc_type, bool ignore_c if (array_type) { updateIncludeTypeIds(array_type->getNestedType(), orc_type->getSubtype(0), ignore_case, include_typeids); + return; } - return; + break; } case ORC::MAP: { const auto * map_type = typeid_cast(non_nullable_type.get()); @@ -1136,8 +1137,9 @@ updateIncludeTypeIds(DataTypePtr type, const ORC::Type * orc_type, bool ignore_c { updateIncludeTypeIds(map_type->getKeyType(), orc_type->getSubtype(0), ignore_case, include_typeids); updateIncludeTypeIds(map_type->getValueType(), orc_type->getSubtype(1), ignore_case, include_typeids); + return; } - return; + break; } case ORC::STRUCT: { /// To make sure tuple field pruning work fine, we should include only the fields of ORC struct type which are also contained in CH tuple types, instead of all fields of ORC struct type. @@ -1181,8 +1183,9 @@ updateIncludeTypeIds(DataTypePtr type, const ORC::Type * orc_type, bool ignore_c for (size_t i = 0; i < tuple_type->getElements().size() && i < orc_type->getSubtypeCount(); ++i) updateIncludeTypeIds(tuple_type->getElement(i), orc_type->getSubtype(i), ignore_case, include_typeids); } + return; } - return; + break; } case ORC::UNION: { /// ORC union maps to the ClickHouse Variant type. A branch that gets a forced type hint @@ -1209,6 +1212,9 @@ updateIncludeTypeIds(DataTypePtr type, const ORC::Type * orc_type, bool ignore_c default: return; } + + /// A requested type of another kind reads the whole node, and the cast to the requested type converts it or throws. + include_typeids.insert(orc_type->getColumnId()); } NativeORCBlockInputFormat::NativeORCBlockInputFormat( @@ -2737,6 +2743,9 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( const auto * orc_value_type = orc_type->getSubtype(1); auto key_column = readColumnFromORCColumn(orc_key_column, orc_key_type, "key", false, key_type_hint); + /// A map key cannot be Nullable. + if (!key_type_hint && !DataTypeMap::isValidKeyType(key_column.type)) + key_type_hint = removeNullableOrLowCardinalityNullable(key_column.type); if (key_type_hint && !key_type_hint->equals(*key_column.type)) { /// Cast key column to target type, because it can happen diff --git a/tests/queries/0_stateless/05574_orc_requested_type_of_another_kind.reference b/tests/queries/0_stateless/05574_orc_requested_type_of_another_kind.reference new file mode 100644 index 0000000000000..338bce460b793 --- /dev/null +++ b/tests/queries/0_stateless/05574_orc_requested_type_of_another_kind.reference @@ -0,0 +1,18 @@ +Code: 53 while converting column `m` from type Map(String, String) to type Tuple(value Nullable(String)) +Code: 53 while converting column `m` from type Map(String, String) to type Tuple(value Nullable(String)) +Code: 53 while converting column `l` from type Array(String) to type Tuple(value Nullable(String)) +Code: 53 while converting column `s` from type Tuple(value String) to type Map(String, String) +Code: 53 while converting column `s` from type Tuple(value String) to type Array(String) +Code: 53 while converting column `lm` from type Array(Map(String, String)) to type Array(Tuple(value String)) +Code: 53 while converting column `sl` from type Tuple(a Array(String)) to type Tuple(a Map(String, String)) +2 {\'key\':\'profile\'} +2 [('key','profile')] +2 {'k':'(\'v\')'} +['(\'v\')'] +2 {'key':'profile'} ['profile'] ('profile') {'k':('v')} [{'k':'v'}] (['x']) +('profile',0) +2 +Code: 53 while converting column `m` from type Map(LowCardinality(String), LowCardinality(Nullable(String))) to type Tuple(value Nullable(String)) +Code: 53 while converting column `lm` from type Array(Map(LowCardinality(String), LowCardinality(Nullable(String)))) to type Array(Tuple(value String)) +{\'key\':\'profile\'} +{'key':'profile'} [{'k':'v'}] diff --git a/tests/queries/0_stateless/05574_orc_requested_type_of_another_kind.sh b/tests/queries/0_stateless/05574_orc_requested_type_of_another_kind.sh new file mode 100755 index 0000000000000..f503b6d888fa4 --- /dev/null +++ b/tests/queries/0_stateless/05574_orc_requested_type_of_another_kind.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest. + +# An ORC list, map or struct requested as a type of another kind is cast to it, not read as default values. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Prints the error code and the conversion of a query that has to fail. +function error() +{ + $CLICKHOUSE_LOCAL -q "$1" 2>&1 | tr '\n' ' ' | grep -o -E 'Code: [0-9]+|while converting column [^:]* to type [^:]*' | head -n 2 | paste -sd ' ' +} + +FILE="$(realpath "$CLICKHOUSE_TMP")/${CLICKHOUSE_TEST_UNIQUE_NAME}.orc" +$CLICKHOUSE_LOCAL -q "SELECT 2::Int64 AS id, map('key', 'profile') AS m, ['profile'] AS l, + CAST(tuple('profile'), 'Tuple(value String)') AS s, map('k', CAST(tuple('v'), 'Tuple(value String)')) AS ms, + [map('k', 'v')] AS lm, CAST(tuple(['x']), 'Tuple(a Array(String))') AS sl FORMAT ORC" > "$FILE" +F="file('$FILE', ORC" + +# Conversions the cast cannot do. +error "SELECT * FROM $F, 'id Int64, m Tuple(value Nullable(String))')" +error "SELECT * FROM $F, 'id Int64, m Tuple(value Nullable(String))') SETTINGS input_format_orc_allow_missing_columns = 0" +error "SELECT * FROM $F, 'id Int64, l Tuple(value Nullable(String))')" +error "SELECT * FROM $F, 'id Int64, s Map(String, String)')" +error "SELECT * FROM $F, 'id Int64, s Array(String)')" +error "SELECT * FROM $F, 'id Int64, lm Array(Tuple(value String))')" +error "SELECT * FROM $F, 'id Int64, sl Tuple(a Map(String, String))')" + +# Conversions the cast does, also of a map value. +$CLICKHOUSE_LOCAL -q "SELECT * FROM $F, 'id Int64, m String')" +$CLICKHOUSE_LOCAL -q "SELECT * FROM $F, 'id Int64, m Array(Tuple(String, String))')" +$CLICKHOUSE_LOCAL -q "SELECT * FROM $F, 'id Int64, ms Map(String, String)')" +$CLICKHOUSE_LOCAL -q "SELECT ms.values FROM $F, 'id Int64, ms Map(String, String)')" + +# Matching kinds, a Tuple naming only some of the fields, and the other columns of a mismatched structure. +$CLICKHOUSE_LOCAL -q "SELECT * FROM $F, 'id Int64, m Map(String, String), l Array(String), s Tuple(value String), + ms Map(String, Tuple(value String)), lm Array(Map(String, String)), sl Tuple(a Array(String))')" +$CLICKHOUSE_LOCAL -q "SELECT s FROM $F, 's Tuple(value String, other Int32)')" +$CLICKHOUSE_LOCAL -q "SELECT id FROM $F, 'id Int64, m Tuple(value Nullable(String))')" + +DICT_FILE="$(realpath "$CLICKHOUSE_TMP")/${CLICKHOUSE_TEST_UNIQUE_NAME}_dict.orc" +$CLICKHOUSE_LOCAL --output_format_orc_dictionary_key_size_threshold=1 -q "SELECT map('key', 'profile') AS m, [map('k', 'v')] AS lm FORMAT ORC" > "$DICT_FILE" +D="file('$DICT_FILE', ORC" +S="SETTINGS input_format_orc_dictionary_as_low_cardinality = 1" + +# A dictionary-encoded map key. +error "SELECT * FROM $D, 'm Tuple(value Nullable(String))') $S" +error "SELECT * FROM $D, 'lm Array(Tuple(value String))') $S" +$CLICKHOUSE_LOCAL -q "SELECT * FROM $D, 'm String') $S" +$CLICKHOUSE_LOCAL -q "SELECT * FROM $D, 'm Map(String, String), lm Array(Map(String, String))') $S" + +rm -f "$FILE" +rm -f "$DICT_FILE" From adcf7f7f2b1a7608b87df6b1068d5abe5e5c2fdd Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 10 Nov 2026 10:40:42 +0000 Subject: [PATCH 137/219] Backport #115847 to 26.8: Do not buffer the serialized step when computing the hash-table-stats cache key --- .../Optimizations/SipHashingWriteBuffer.h | 36 +++++++ .../calculateHashTableCacheKeys.cpp | 10 +- ...t_siphashing_write_buffer_key_identity.cpp | 98 +++++++++++++++++++ 3 files changed, 141 insertions(+), 3 deletions(-) create mode 100644 src/Processors/QueryPlan/Optimizations/SipHashingWriteBuffer.h create mode 100644 src/Processors/QueryPlan/tests/gtest_siphashing_write_buffer_key_identity.cpp diff --git a/src/Processors/QueryPlan/Optimizations/SipHashingWriteBuffer.h b/src/Processors/QueryPlan/Optimizations/SipHashingWriteBuffer.h new file mode 100644 index 0000000000000..66fe4c0c15fac --- /dev/null +++ b/src/Processors/QueryPlan/Optimizations/SipHashingWriteBuffer.h @@ -0,0 +1,36 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +/// Feeds everything written to it into a `SipHash` over a fixed-size window and discards the bytes, +/// so a stream that is only ever hash input never has to be held in memory to be hashed. +/// +/// `SipHash::update` carries state, including any sub-word remainder, across calls, so hashing a +/// stream in consecutive chunks gives the same 64-bit value as hashing one contiguous buffer holding +/// all of it: the window size is not observable in the result. +class SipHashingWriteBuffer final : public BufferWithOwnMemory +{ +public: + /// Any size is correct: `WriteBuffer` callers either loop over `available()` or check it before + /// writing into the buffer directly. + static constexpr size_t window_bytes = 32768; + + explicit SipHashingWriteBuffer(SipHash & hash_) : BufferWithOwnMemory(window_bytes), hash(hash_) { } + + /// This buffer may be destroyed with a stream still half-written: nothing reads the discarded + /// bytes, so abandoning one costs nothing and callers need not finalize. `~WriteBuffer` asserts + /// against being left neither finalized nor canceled, so cancel here. + ~SipHashingWriteBuffer() override { cancel(); } + +private: + void nextImpl() override { hash.update(working_buffer.begin(), offset()); } + + SipHash & hash; +}; + +} diff --git a/src/Processors/QueryPlan/Optimizations/calculateHashTableCacheKeys.cpp b/src/Processors/QueryPlan/Optimizations/calculateHashTableCacheKeys.cpp index 164c366fca56a..aaf7c22053231 100644 --- a/src/Processors/QueryPlan/Optimizations/calculateHashTableCacheKeys.cpp +++ b/src/Processors/QueryPlan/Optimizations/calculateHashTableCacheKeys.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -95,7 +96,11 @@ UInt64 calculateHashFromStep(const ITransformingStep & transform) && sameByteLayout(*transform.getOutputHeader(), *transform.getInputHeaders().front())) return 0; - WriteBufferFromOwnString wbuf; + /// This serialized form is only ever hash input - nothing reads the bytes back - so it is + /// hashed as it is produced rather than accumulated. A step can carry a whole constant-folded + /// literal here, which `serializeConstant` writes out in full. + SipHash hash; + SipHashingWriteBuffer wbuf(hash); SerializedSetsRegistry registry; registry.for_cache_key = true; /// The bytes are hashed in-process by the same binary that wrote them and never reach another @@ -110,8 +115,7 @@ UInt64 calculateHashFromStep(const ITransformingStep & transform) if (transform.isSerializable()) transform.serialize(ctx); - SipHash hash; - hash.update(wbuf.str()); + wbuf.finalize(); return hash.get64(); } diff --git a/src/Processors/QueryPlan/tests/gtest_siphashing_write_buffer_key_identity.cpp b/src/Processors/QueryPlan/tests/gtest_siphashing_write_buffer_key_identity.cpp new file mode 100644 index 0000000000000..cdb6c6d23698b --- /dev/null +++ b/src/Processors/QueryPlan/tests/gtest_siphashing_write_buffer_key_identity.cpp @@ -0,0 +1,98 @@ +#include + +#include +#include + +#include +#include +#include + +using namespace DB; + +namespace +{ + +/// The value `calculateHashFromStep` produced before it streamed: accumulate the whole stream, then +/// hash the contiguous result in one call. +UInt64 bufferedKey(const std::vector & writes) +{ + WriteBufferFromOwnString out; + for (const auto & w : writes) + out.write(w.data(), w.size()); + + SipHash hash; + hash.update(out.str()); + return hash.get64(); +} + +/// The value it produces now: hash each chunk as it is written, keeping no copy of the stream. +UInt64 streamedKey(const std::vector & writes) +{ + SipHash hash; + SipHashingWriteBuffer out(hash); + for (const auto & w : writes) + out.write(w.data(), w.size()); + out.finalize(); + return hash.get64(); +} + +/// Writes that between them cross the sink's flush boundary at every alignment: sub-word chunks +/// (which exercise `SipHash::update`'s carried remainder), chunks straddling one window, and a +/// single write many windows long. +std::vector> streamShapes() +{ + const size_t w = SipHashingWriteBuffer::window_bytes; + return { + {}, + {""}, + {"a"}, + {"a", "b", "c"}, + {std::string(7, 'x')}, + {std::string(8, 'x')}, + {std::string(9, 'x')}, + {std::string(1, 'a'), std::string(w - 1, 'b')}, + {std::string(w - 1, 'a'), std::string(1, 'b')}, + {std::string(w, 'a')}, + {std::string(w, 'a'), std::string(1, 'b')}, + {std::string(w + 1, 'a')}, + {std::string(3, 'a'), std::string(w, 'b'), std::string(5, 'c')}, + {std::string(4 * w + 3, 'a')}, + }; +} + +} + +/// The property the whole change rests on: streaming the key must not change its value. If this +/// fails, every cached hash-table-statistics entry and every parallel-replicas node match silently +/// stops agreeing with the plan that produced it. +TEST(SipHashingWriteBuffer, KeyIsIdenticalToBufferedHash) +{ + for (const auto & writes : streamShapes()) + ASSERT_EQ(streamedKey(writes), bufferedKey(writes)) << "stream shape with " << writes.size() << " write(s)"; +} + +/// Negative control for the test above. Perturbing the stream by a single byte in one shape must +/// change the key, otherwise `KeyIsIdenticalToBufferedHash` would pass for a sink that hashes +/// nothing at all. +TEST(SipHashingWriteBuffer, KeyDiffersWhenStreamIsPerturbed) +{ + const std::vector writes{std::string(SipHashingWriteBuffer::window_bytes + 1, 'a')}; + + auto perturbed = writes; + perturbed.front().back() = 'b'; + + ASSERT_NE(streamedKey(writes), streamedKey(perturbed)); + ASSERT_NE(streamedKey(writes), streamedKey({})); +} + +/// Destroying the sink with a stream still half-written must not trip `~WriteBuffer`'s "neither +/// finalized nor canceled" assertion, since callers are free to abandon a key half-computed. +TEST(SipHashingWriteBuffer, DestroyedWithoutFinalizeAfterPartialWrite) +{ + SipHash hash; + { + SipHashingWriteBuffer out(hash); + const std::string chunk(SipHashingWriteBuffer::window_bytes + 17, 'x'); + out.write(chunk.data(), chunk.size()); + } +} From fa0db9434e605848d35625b2c96b6f1d15edd79b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 10 Nov 2026 10:54:48 +0000 Subject: [PATCH 138/219] Backport #128649 to 26.8: Cache the logger of `ThreadStatus` instead of looking it up for every thread --- src/Common/ThreadStatus.cpp | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/src/Common/ThreadStatus.cpp b/src/Common/ThreadStatus.cpp index eb4970f9b0b5e..419a52981100d 100644 --- a/src/Common/ThreadStatus.cpp +++ b/src/Common/ThreadStatus.cpp @@ -145,7 +145,10 @@ ThreadStatus::ThreadStatus() /// memory_tracker is already parented to total_memory_tracker, so a thread that never attaches /// to a group still honors total_memory_tracker_sample_probability. resolveMemorySampleConfig(); - log = getLogger("ThreadStatus"); + /// A `ThreadStatus` is created for every job of a thread pool, and `getLogger` locks the global mutex of the + /// registry of loggers, so all threads of the pools would contend on it. + static const LoggerPtr thread_status_log = getLogger("ThreadStatus"); + log = thread_status_log; current_thread = this; From e9b0a6ad77a929fb1f89af51bc0b83bdbbf3ea8e Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 10 Nov 2026 11:13:12 +0000 Subject: [PATCH 139/219] Backport #129044 to 26.8: Fix identity of dynamic disks with server credential markers --- src/Disks/DiskFromAST.cpp | 335 ++++++++++++++---- src/Disks/DiskFromAST.h | 13 +- src/Disks/IDisk.h | 9 +- src/Disks/getDiskConfigurationFromAST.cpp | 40 ++- src/Disks/getDiskConfigurationFromAST.h | 9 + src/Storages/MergeTree/MergeTreeData.cpp | 7 +- src/Storages/MergeTree/MergeTreeSettings.cpp | 23 +- src/Storages/MergeTree/MergeTreeSettings.h | 12 +- .../MergeTree/registerStorageMergeTree.cpp | 6 +- src/Storages/StorageDistributed.cpp | 2 +- src/Storages/StorageFactory.cpp | 17 + src/Storages/StorageFactory.h | 9 + src/Storages/StorageJoin.cpp | 2 +- src/Storages/StorageLog.cpp | 2 +- src/Storages/StorageSet.cpp | 2 +- src/Storages/StorageStripeLog.cpp | 2 +- ...ntity_without_credentials_marker.reference | 25 ++ ...isk_identity_without_credentials_marker.sh | 85 +++++ ...apper_of_server_credentials_disk.reference | 21 ++ ...disk_wrapper_of_server_credentials_disk.sh | 74 ++++ 20 files changed, 592 insertions(+), 103 deletions(-) create mode 100644 tests/queries/0_stateless/05494_custom_disk_identity_without_credentials_marker.reference create mode 100755 tests/queries/0_stateless/05494_custom_disk_identity_without_credentials_marker.sh create mode 100644 tests/queries/0_stateless/05496_dynamic_disk_wrapper_of_server_credentials_disk.reference create mode 100755 tests/queries/0_stateless/05496_dynamic_disk_wrapper_of_server_credentials_disk.sh diff --git a/src/Disks/DiskFromAST.cpp b/src/Disks/DiskFromAST.cpp index 4053efc211b5b..4057e1092eeda 100644 --- a/src/Disks/DiskFromAST.cpp +++ b/src/Disks/DiskFromAST.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -18,48 +19,76 @@ #include #include +#include namespace DB { namespace ErrorCodes { + extern const int ACCESS_DENIED; extern const int BAD_ARGUMENTS; } -static std::string getOrCreateCustomDisk( +/// Resolve `include`, `from_env` and `from_zk` in a disk definition. +static Poco::AutoPtr resolveDiskConfiguration( + Poco::AutoPtr xml_document, ContextPtr context, bool attach) +{ + Poco::AutoPtr name_pool(new Poco::XML::NamePool()); + Poco::XML::DOMParser dom_parser(name_pool); + + std::vector> substitutions; + zkutil::ZooKeeperNodeCache zk_node_cache([&]() { return context->getZooKeeper(); }); + + ConfigProcessor::processIncludes( + xml_document, + substitutions, + context->getConfigRef().getString("include_from", ""), + /* throw_on_bad_incl= */!attach, + dom_parser, + getLogger("getOrCreateCustomDisk"), + /*contributing_zk_paths=*/ {}, + /*contributing_files=*/ {}, + &zk_node_cache); + + Poco::AutoPtr config(new Poco::Util::XMLConfiguration()); + config->load(xml_document); + return config; +} + +static std::string getCredentialsMarker(const ASTPtr & arg) +{ + const auto * eq = arg->as(); + if (!eq || eq->name != "equals" || !eq->arguments || eq->arguments->children.size() < 2) + return {}; + const auto * key = eq->arguments->children[0]->as(); + if (key && key->name() == S3_SERVER_CREDENTIALS_MARKER) + return key->name(); + return {}; +} + +static bool hasCredentialsMarker(const ASTs & args, const std::string & marker) +{ + return std::ranges::any_of(args, [&](const ASTPtr & arg) { return getCredentialsMarker(arg) == marker; }); +} + +/// `serializations` holds the definition, then other forms of it (without the credentials marker, other disks below it). +/// Returns the indexes and names of the registered disks the forms match, the one used first. The others are returned +/// only if this session may use them. If none matches, the first form is created. +static std::vector> getOrCreateCustomDisk( const ASTs & disk_args, - const std::string & serialization, + const std::vector & serializations, ContextPtr context, bool attach, bool for_system_database) { - const auto & server_config = context->getConfigRef(); - std::string include_from_path = server_config.getString("include_from", ""); - - Poco::AutoPtr config(new Poco::Util::XMLConfiguration()); + Poco::AutoPtr config; + bool uses_server_credentials = false; + bool restriction_exempt = false; { DynamicS3DiskCredentialInfo s3_disk_info; auto xml_document = getDiskConfigurationFromASTImpl(disk_args, context, attach, &s3_disk_info, for_system_database); - - Poco::AutoPtr name_pool(new Poco::XML::NamePool()); - Poco::XML::DOMParser dom_parser(name_pool); - - std::vector> substitutions; - zkutil::ZooKeeperNodeCache zk_node_cache([&]() { return context->getZooKeeper(); }); - - ConfigProcessor::processIncludes( - xml_document, - substitutions, - include_from_path, - /* throw_on_bad_incl= */!attach, - dom_parser, - getLogger("getOrCreateCustomDisk"), - /*contributing_zk_paths=*/ {}, - /*contributing_files=*/ {}, - &zk_node_cache); - - config->load(xml_document); + config = resolveDiskConfiguration(xml_document, context, attach); /// Applied after `processIncludes`. If the pre-resolution check already decided the disk must load /// anonymously, enforce it now (an `include` cannot re-introduce server credentials). Otherwise @@ -69,8 +98,13 @@ static std::string getOrCreateCustomDisk( forceAnonymousS3DiskConfig(*config); else validateResolvedS3DiskCredentials(*config, context, attach, s3_disk_info); + + uses_server_credentials = diskUsesServerCredentials(*config, s3_disk_info); + restriction_exempt = s3_disk_info.restriction_exempt; } + const bool restricted = !attach && !restriction_exempt && context->shouldRestrictUserQueryS3Credentials(); + Poco::Util::AbstractConfiguration::Keys disk_settings_keys; config->keys(disk_settings_keys); /// Check that no settings are defined when disk from the config is referred. @@ -83,28 +117,64 @@ static std::string getOrCreateCustomDisk( throw Exception( ErrorCodes::BAD_ARGUMENTS, "Disk function `{}` must have other arguments apart from `name`, which describe disk configuration. Invalid disk description.", - serialization); - - auto disk_settings_hash = sipHash128(serialization.data(), serialization.size()); + serializations[0]); - std::string disk_name; - if (config->has("name")) - { - disk_name = config->getString("name"); - } - else + auto get_disk_name = [&](UInt128 hash) -> std::string { + if (config->has("name")) + return config->getString("name"); /// We need a unique name for a created custom disk, but it needs to be the same /// after table is reattached or server is restarted, so take a hash of the disk /// configuration serialized ast as a disk name suffix. - disk_name = DiskSelector::TMP_INTERNAL_DISK_PREFIX + toString(disk_settings_hash); + return DiskSelector::TMP_INTERNAL_DISK_PREFIX + toString(hash); + }; + + std::vector hashes; + for (const auto & serialization : serializations) + hashes.push_back(sipHash128(serialization.data(), serialization.size())); + + /// A disk declared by an older version is registered under the form of the definition that version stored. Reusing it + /// is safe: this session passed the credential checks for its definition, the forms differ only in markers and in the + /// equivalent disks below, and the caller stores the form of the reused disk. + std::vector> matches; + if (!attach) + { + const auto disks = context->getDisksMap(); + for (size_t i = 0; i < serializations.size(); ++i) + { + auto name = get_disk_name(hashes[i]); + const auto it = disks.find(name); + if (it == disks.end() || !it->second->isCustomDisk() || it->second->getCustomDiskSettings() != hashes[i]) + continue; + if (!matches.empty() && restricted && it->second->customDiskUsesServerCredentials()) + continue; + matches.emplace_back(i, std::move(name)); + } } + if (matches.empty()) + matches.emplace_back(0, get_disk_name(hashes[0])); + + const auto disk_settings_hash = hashes[matches.front().first]; + const auto & disk_name = matches.front().second; auto disk = context->getOrCreateDisk(disk_name, [&](const DisksMap & disks_map) -> DiskPtr { + /// A wrapper (`encrypted`, `cache`) names a disk another session may have created with the opt-in. It is looked up + /// in the map the wrapper is created from, so that it cannot be registered in between. + const auto wrapped = disks_map.find(config->getString(".disk", "")); + const bool wraps_server_credentials = wrapped != disks_map.end() && wrapped->second->customDiskUsesServerCredentials(); + if (wraps_server_credentials && restricted) + throw Exception( + ErrorCodes::ACCESS_DENIED, + "A dynamic disk created from user SQL may not wrap the disk `{}`, which uses the server's own credentials." +#if !CLICKHOUSE_CLOUD + " Enable the setting `s3_allow_server_credentials_in_user_queries` to allow it." +#endif + , wrapped->first); + auto result = DiskFactory::instance().create( disk_name, *config, /* config_path */"", context, disks_map, /* attach */attach, /* custom_disk */true); /// Mark that disk can be used without storage policy. - result->markDiskAsCustom(disk_settings_hash); + result->markDiskAsCustom(disk_settings_hash, uses_server_credentials || wraps_server_credentials); return result; }); @@ -138,7 +208,18 @@ static std::string getOrCreateCustomDisk( disk_path_expected_prefix); } - return disk_name; + /// The disk may have been created by a session with the opt-in, from a configuration that resolved differently then + /// (`include` or `from_zk` changed since), so the check of this session's definition does not cover it. + if (restricted && disk->customDiskUsesServerCredentials()) + throw Exception( + ErrorCodes::ACCESS_DENIED, + "The disk `{}` uses the server's own credentials, which a dynamic disk created from user SQL may not use." +#if !CLICKHOUSE_CLOUD + " Enable the setting `s3_allow_server_credentials_in_user_queries` to allow it." +#endif + , disk_name); + + return matches; } template @@ -150,29 +231,129 @@ static void visitBottomUp(ASTPtr & ast, typename Matcher::Data & data) Matcher::visit(ast, data); } -class DiskConfigurationFlattener + +static ASTs & getDiskFunctionArguments(const ASTPtr & ast) { -public: - struct Data + return assert_cast(*ast->as().arguments).children; +} + +static ASTPtr makeCredentialsMarker(const std::string & marker) +{ + return makeASTFunction("equals", make_intrusive(marker), make_intrusive(static_cast(1))); +} + + +/// How a form of a disk definition stored by an older version differs from the current one. +struct CredentialsMarkersChange +{ + std::set drop; +}; + +static void applyCredentialsMarkersChange(ASTs & args, const CredentialsMarkersChange & change) +{ + args.erase(std::remove_if(args.begin(), args.end(), [&](const ASTPtr & arg) { return change.drop.contains(getCredentialsMarker(arg)); }), args.end()); +} + +struct DiskConfigurationFlattenerData +{ + ContextPtr context; + bool attach; + bool for_system_database; + bool stored_definition; +}; + +/// A registered disk a disk function may resolve to, and the changes of credentials markers in the stored definition +/// (of the function and of the disk functions below it) that give it. +struct DiskAlternative +{ + std::string name; + std::vector> edits; +}; + +/// A disk function already replaced by the name of its disk, and the disks it may resolve to, the used one first. +struct FlattenedDisk +{ + ASTPtr * literal; + std::vector alternatives; +}; + +/// Replace every disk function in `flat` (bottom-up) with the name of its disk. `original` is the AST `flat` was cloned +/// from, which is stored in metadata; the caller applies the edits of the alternative it uses to it. Returns the disk +/// functions directly in `flat`, or `flat` itself if it is one. +static std::vector flattenDiskConfiguration(ASTPtr & flat, const ASTPtr & original, const DiskConfigurationFlattenerData & data) +{ + checkStackSize(); + std::vector below; + for (size_t i = 0; i < flat->children.size(); ++i) + std::ranges::move(flattenDiskConfiguration(flat->children[i], original->children[i], data), std::back_inserter(below)); + + if (!isDiskFunction(flat)) + return below; + + const auto & args = getDiskFunctionArguments(flat); + std::vector changes{{}}; + if (!data.attach && !data.stored_definition) { - ContextPtr context; - bool attach; - bool for_system_database; + /// The form stored by older versions: versions before 26.7 had no marker. + if (hasCredentialsMarker(args, S3_SERVER_CREDENTIALS_MARKER)) + changes.push_back({.drop = {S3_SERVER_CREDENTIALS_MARKER}}); + } + + /// Every combination of the disks below (an older wrapper names the disk of the older form), the used ones first. + std::vector> combinations{{}}; + for (const auto & disk : below) + { + std::vector> extended; + for (size_t alternative = 0; alternative < disk.alternatives.size(); ++alternative) + { + for (const auto & combination : combinations) + { + extended.push_back(combination); + extended.back().push_back(alternative); + } + } + combinations = std::move(extended); + } + + auto set_disks_below = [&](const std::vector & combination) + { + for (size_t i = 0; i < below.size(); ++i) + *below[i].literal = make_intrusive(below[i].alternatives[combination[i]].name); }; - static void visit(ASTPtr & ast, Data & data) + std::vector serializations; + std::vector forms; + for (const auto & combination : combinations) { - if (isDiskFunction(ast)) + set_disks_below(combination); + for (const auto & change : changes) { - const auto * function = ast->as(); - const auto * function_args_expr = assert_cast(function->arguments.get()); - const auto & function_args = function_args_expr->children; - auto disk_setting_string = function->formatWithSecretsOneLine(); - auto disk_name = getOrCreateCustomDisk(function_args, disk_setting_string, data.context, data.attach, data.for_system_database); - ast = make_intrusive(disk_name); + auto candidate = flat->clone(); + applyCredentialsMarkersChange(getDiskFunctionArguments(candidate), change); + serializations.push_back(candidate->formatWithSecretsOneLine()); + + DiskAlternative form{.name = {}, .edits = {}}; + if (!change.drop.empty()) + form.edits.emplace_back(original, change); + for (size_t i = 0; i < below.size(); ++i) + std::ranges::copy(below[i].alternatives[combination[i]].edits, std::back_inserter(form.edits)); + forms.push_back(std::move(form)); } } -}; + /// The disk is created, if it is, from the disks used below. + set_disks_below(combinations.front()); + + const auto matches = getOrCreateCustomDisk(args, serializations, data.context, data.attach, data.for_system_database); + + std::vector alternatives; + for (const auto & [index, name] : matches) + { + alternatives.push_back(std::move(forms[index])); + alternatives.back().name = name; + } + flat = make_intrusive(alternatives.front().name); + return {FlattenedDisk{.literal = &flat, .alternatives = std::move(alternatives)}}; +} /// Persist the credential opt-in into the stored disk definition. For each leaf dynamic S3 disk that resolves /// server-managed credentials and is currently allowed (the session opted in), add a `_server_credentials_allowed` @@ -198,20 +379,11 @@ class ServerCredentialMarkerInjector return; auto & args = args_expr->children; - auto is_marker = [](const ASTPtr & arg) - { - const auto * eq = arg->as(); - if (!eq || eq->name != "equals" || !eq->arguments || eq->arguments->children.size() != 2) - return false; - const auto * key = eq->arguments->children[0]->as(); - return key && key->name() == "_server_credentials_allowed"; - }; - /// Strip any marker the user put in the definition. This runs only on a fresh create (not on metadata /// load), so a server-written marker in stored metadata is never stripped. As a result a persisted /// marker is only ever one this injector wrote below for a disk that actually needed server credentials /// while the session had the opt-in -- a user cannot pre-seed it to bypass the restriction on reload. - args.erase(std::remove_if(args.begin(), args.end(), is_marker), args.end()); + applyCredentialsMarkersChange(args, {.drop = {S3_SERVER_CREDENTIALS_MARKER}}); /// A wrapper disk (a nested `disk = disk(...)`) is not itself an S3 disk; the marker belongs on the /// inner leaf S3 disk, which the visitor reaches separately. (Markers were already stripped above.) @@ -226,15 +398,13 @@ class ServerCredentialMarkerInjector DynamicS3DiskCredentialInfo info; getDiskConfigurationFromASTImpl(args, data.context, /* is_loading_from_existing_metadata */ false, &info, data.for_system_database); if (info.persist_server_credentials_allowance) - args.push_back(makeASTFunction( - "equals", - make_intrusive("_server_credentials_allowed"), - make_intrusive(static_cast(1)))); + args.push_back(makeCredentialsMarker(S3_SERVER_CREDENTIALS_MARKER)); } }; -std::string DiskFromAST::createCustomDisk(const ASTPtr & disk_function_ast, ContextPtr context, bool attach, bool for_system_database) +std::string DiskFromAST::createCustomDisk( + const ASTPtr & disk_function_ast, ContextPtr context, bool attach, bool for_system_database, bool stored_definition) { if (!isDiskFunction(disk_function_ast)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected a disk function"); @@ -244,7 +414,7 @@ std::string DiskFromAST::createCustomDisk(const ASTPtr & disk_function_ast, Cont /// disk's name is a hash of its definition: the marker has to be part of that definition so the disk /// resolves to the same name (and storage path) when reloaded from metadata. The marker itself is an unknown /// key to the disk factory and is ignored when building the client. - if (!attach) + if (!attach && !stored_definition) { ASTPtr to_mark = disk_function_ast; ServerCredentialMarkerInjector::Data inject_data{context, for_system_database}; @@ -252,13 +422,36 @@ std::string DiskFromAST::createCustomDisk(const ASTPtr & disk_function_ast, Cont } auto ast = disk_function_ast->clone(); - - DiskConfigurationFlattener::Data data{context, attach, for_system_database}; - visitBottomUp(ast, data); + const auto disks = flattenDiskConfiguration(ast, disk_function_ast, {context, attach, for_system_database, stored_definition}); + for (const auto & [disk_function, change] : disks.front().alternatives.front().edits) + applyCredentialsMarkersChange(getDiskFunctionArguments(disk_function), change); return assert_cast(*ast).value.safeGet(); } +void DiskFromAST::ensureDiskIsAllowed(const DiskPtr & disk, ContextPtr context) +{ + /// A custom disk can be named by another session than the one that created it. + if (disk->customDiskUsesServerCredentials() && context->shouldRestrictUserQueryS3Credentials()) + throw Exception( + ErrorCodes::ACCESS_DENIED, + "The disk `{}` uses the server's own credentials, which may not be used from user SQL." +#if !CLICKHOUSE_CLOUD + " Enable the setting `s3_allow_server_credentials_in_user_queries` to allow it." +#endif + , disk->getName()); +} + +void DiskFromAST::ensureStoragePolicyIsAllowed(const std::string & storage_policy_name, ContextPtr context) +{ + /// `__` is the policy a table on a disk gets, so it names a custom disk as well. + if (const auto storage_policy = context->tryGetStoragePolicy(storage_policy_name)) + { + for (const auto & disk : storage_policy->getDisks()) + ensureDiskIsAllowed(disk, context); + } +} + void DiskFromAST::ensureDiskIsNotCustom(const std::string & disk_name, ContextPtr context) { auto disk = context->getDisk(disk_name); diff --git a/src/Disks/DiskFromAST.h b/src/Disks/DiskFromAST.h index cd0f52b731cdb..22a9e7220b35c 100644 --- a/src/Disks/DiskFromAST.h +++ b/src/Disks/DiskFromAST.h @@ -1,4 +1,5 @@ #pragma once +#include #include #include #include @@ -6,11 +7,21 @@ namespace DB { +class IDisk; +using DiskPtr = std::shared_ptr; + namespace DiskFromAST { void ensureDiskIsNotCustom(const std::string & name, ContextPtr context); - std::string createCustomDisk(const ASTPtr & disk_function, ContextPtr context, bool attach, bool for_system_database = false); + /// Refuse, for a session without `s3_allow_server_credentials_in_user_queries`, a custom disk that uses the server's own + /// credentials, or a storage policy with one. For a table definition selecting a disk or a policy by name. + void ensureDiskIsAllowed(const DiskPtr & disk, ContextPtr context); + void ensureStoragePolicyIsAllowed(const std::string & storage_policy_name, ContextPtr context); + /// `stored_definition`: the definition was read from the metadata the server stored for the table (short `ATTACH`), + /// so its credentials markers are kept as they are: they keep the identity of the disk, and are not honored unless `attach`. + std::string createCustomDisk( + const ASTPtr & disk_function, ContextPtr context, bool attach, bool for_system_database = false, bool stored_definition = false); } } diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 478795f523f1f..b7ce685fdc3dd 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -570,7 +570,13 @@ class IDisk : public Space /// Was disk created to be used without storage configuration? bool isCustomDisk() const { return custom_disk_settings_hash != 0; } UInt128 getCustomDiskSettings() const { return custom_disk_settings_hash; } - void markDiskAsCustom(UInt128 settings_hash) { custom_disk_settings_hash = settings_hash; } + void markDiskAsCustom(UInt128 settings_hash, bool uses_server_credentials) + { + custom_disk_settings_hash = settings_hash; + custom_disk_uses_server_credentials = uses_server_credentials; + } + /// A custom disk that uses the server's own credentials, directly or through the disk it wraps. + bool customDiskUsesServerCredentials() const { return custom_disk_uses_server_credentials; } virtual DiskPtr getDelegateDiskIfExists() const { return nullptr; } @@ -608,6 +614,7 @@ class IDisk : public Space std::unique_ptr copying_thread_pool; // 0 means the disk is not custom, the disk is predefined in the config UInt128 custom_disk_settings_hash = 0; + bool custom_disk_uses_server_credentials = false; /// True if underlying filesystem is case-insensitive, /// e.g. file_name and FILE_NAME are the same files. diff --git a/src/Disks/getDiskConfigurationFromAST.cpp b/src/Disks/getDiskConfigurationFromAST.cpp index 1f3bc0d903b51..e9ab532c30afc 100644 --- a/src/Disks/getDiskConfigurationFromAST.cpp +++ b/src/Disks/getDiskConfigurationFromAST.cpp @@ -175,7 +175,7 @@ Poco::AutoPtr getDiskConfigurationFromASTImpl(const ASTs & use_environment_credentials_off = !indirect && (value_str == "0" || boost::iequals(value_str, "false")); else if (key == "role_arn") has_role_arn = !value_str.empty() && !indirect; - else if (key == "_server_credentials_allowed") + else if (key == S3_SERVER_CREDENTIALS_MARKER) has_server_credentials_marker = !indirect && value_str != "0" && !boost::iequals(value_str, "false"); if (key == "include") @@ -260,6 +260,7 @@ Poco::AutoPtr getDiskConfigurationFromASTImpl(const ASTs & /// (the session opted in), so the disk is not re-restricted on reload. info->persist_server_credentials_allowance = !is_loading_from_existing_metadata && relies_on_server_credentials && !has_server_credentials_marker && !context->shouldRestrictUserQueryS3Credentials(); + info->uses_server_credentials = relies_on_server_credentials; } if (relies_on_server_credentials && context->shouldRestrictUserQueryS3Credentials() && !restriction_exempt) @@ -277,7 +278,10 @@ Poco::AutoPtr getDiskConfigurationFromASTImpl(const ASTs & /// Force the disk anonymous once `include` is resolved (see forceAnonymousS3DiskConfig); we /// cannot rewrite the DOM here because DiskFromAST processes `include` after this returns. if (info) + { info->load_anonymously = true; + info->uses_server_credentials = false; + } } else throw Exception( @@ -349,6 +353,20 @@ void forceAnonymousS3DiskConfigAtPrefix(Poco::Util::AbstractConfiguration & conf namespace { +/// The disk root and every `locations.` child of a resolved dynamic disk configuration. +std::vector getS3BackendPrefixes(const Poco::Util::AbstractConfiguration & config) +{ + std::vector prefixes{""}; + if (config.has("locations")) + { + Poco::Util::AbstractConfiguration::Keys locations; + config.keys("locations", locations); + for (const auto & location : locations) + prefixes.push_back("locations." + location + "."); + } + return prefixes; +} + /// Whether one resolved object-storage backend (the disk root, or a single `locations.` child selected by /// `prefix`) is an S3 backend that resolved to credentials the AST did not prove explicit, and so must be /// downgraded to anonymous or rejected. Returns false for a non-S3 backend or one whose resolved auth is safe. @@ -419,17 +437,8 @@ void validateResolvedS3DiskCredentials( /// Check the disk root and every `locations.` child: a multi-location `DiskObjectStorage` builds one /// object storage per child, each with its own `object_storage_type`, so an `include` can hide an S3 child /// (built with `for_disk_s3 = true`) behind a root that proves non-S3. - std::vector prefixes{""}; - if (config.has("locations")) - { - Poco::Util::AbstractConfiguration::Keys locations; - config.keys("locations", locations); - for (const auto & location : locations) - prefixes.push_back("locations." + location + "."); - } - std::vector restricted_prefixes; - for (const auto & prefix : prefixes) + for (const auto & prefix : getS3BackendPrefixes(config)) if (resolvedS3BackendIsRestricted(config, prefix, /* is_root */ prefix.empty(), info)) restricted_prefixes.push_back(prefix); @@ -462,6 +471,15 @@ void validateResolvedS3DiskCredentials( ); } +bool diskUsesServerCredentials(const Poco::Util::AbstractConfiguration & config, const DynamicS3DiskCredentialInfo & info) +{ + if (info.uses_server_credentials) + return true; + /// Every backend counts, because a multi-location disk writes to all of them. + return std::ranges::any_of(getS3BackendPrefixes(config), + [&](const String & prefix) { return resolvedS3BackendIsRestricted(config, prefix, /* is_root */ prefix.empty(), info); }); +} + DiskConfigurationPtr getDiskConfigurationFromAST(const ASTs & disk_args, ContextPtr context) { DynamicS3DiskCredentialInfo info; diff --git a/src/Disks/getDiskConfigurationFromAST.h b/src/Disks/getDiskConfigurationFromAST.h index e7c428d965ff9..0c1572a763a8a 100644 --- a/src/Disks/getDiskConfigurationFromAST.h +++ b/src/Disks/getDiskConfigurationFromAST.h @@ -40,6 +40,8 @@ struct DynamicS3DiskCredentialInfo /// should persist a `_server_credentials_allowed` marker in the stored definition. The marker is honored /// (only) on later metadata loads, so the disk keeps working across restart without re-opting in. bool persist_server_credentials_allowance = false; + /// The disk is built with server-managed S3 credentials. + bool uses_server_credentials = false; /// The safe credential forms the AST itself supplied with literal values; the resolved auth mode is /// validated against these. bool ast_has_explicit_key_pair = false; /// literal `access_key_id` + `secret_access_key` @@ -71,6 +73,13 @@ void forceAnonymousS3DiskConfigAtPrefix(Poco::Util::AbstractConfiguration & conf /// anonymous when loading from existing metadata (see `s3_load_table_anonymously_if_credentials_restricted`). void validateResolvedS3DiskCredentials(Poco::Util::AbstractConfiguration & config, ContextPtr context, bool is_loading_from_existing_metadata, const DynamicS3DiskCredentialInfo & info); +/// Persisted in the stored definition of a dynamic S3 disk created with `s3_allow_server_credentials_in_user_queries`. +inline constexpr auto S3_SERVER_CREDENTIALS_MARKER = "_server_credentials_allowed"; + +/// Whether a resolved dynamic disk may use the server's own credentials on any of its backends (the root or any +/// `locations.` child). The disk a wrapper names is not checked here. +bool diskUsesServerCredentials(const Poco::Util::AbstractConfiguration & config, const DynamicS3DiskCredentialInfo & info); + /* * A reverse function. */ diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index 86b56fc5fc33d..fdec410a6bf92 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -6163,7 +6164,11 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context if (setting_name == "storage_policy") { - checkStoragePolicy(local_context->getStoragePolicy(new_value.safeGet())); + const auto new_storage_policy = local_context->getStoragePolicy(new_value.safeGet()); + /// Only a changed policy is checked, so a table that already has it can still be altered otherwise. + if (new_storage_policy != getStoragePolicy()) + DiskFromAST::ensureStoragePolicyIsAllowed(new_storage_policy->getName(), local_context); + checkStoragePolicy(new_storage_policy); found_storage_policy_setting = true; } else if (setting_name == "disk") diff --git a/src/Storages/MergeTree/MergeTreeSettings.cpp b/src/Storages/MergeTree/MergeTreeSettings.cpp index 091afbbed4dd3..4d9115458d8de 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -2482,7 +2482,7 @@ DECLARE_SETTINGS_TRAITS(MergeTreeSettingsTraits, LIST_OF_MERGE_TREE_SETTINGS, ME struct MergeTreeSettingsImpl : public BaseSettings { /// NOTE: will rewrite the AST to add immutable settings. - void loadFromQuery(ASTStorage & storage_def, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database); + void loadFromQuery(ASTStorage & storage_def, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database, bool stored_definition); /// Check that the values are sane taking also query-level settings into account. void sanityCheck(size_t background_pool_tasks, bool background_pool_auto_lowered) const; @@ -2520,7 +2520,8 @@ static void validateTableDisk(const DiskPtr & disk) IMPLEMENT_SETTINGS_TRAITS_CUSTOM_IMPL(MergeTreeSettingsTraits, LIST_OF_MERGE_TREE_SETTINGS, MergeTreeSettings, MergeTreeSetting) -void MergeTreeSettingsImpl::loadFromQuery(ASTStorage & storage_def, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database) +void MergeTreeSettingsImpl::loadFromQuery( + ASTStorage & storage_def, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database, bool stored_definition) { if (storage_def.settings) { @@ -2532,7 +2533,7 @@ void MergeTreeSettingsImpl::loadFromQuery(ASTStorage & storage_def, ContextPtr c DiskPtr disk; auto changes = storage_def.settings->changes; - MergeTreeSettings::resolveDiskSetting(changes, context, is_loading_from_existing_metadata, for_system_database); + MergeTreeSettings::resolveDiskSetting(changes, context, is_loading_from_existing_metadata, for_system_database, stored_definition); for (const auto & [name, value, _] : changes) { @@ -2874,13 +2875,15 @@ void MergeTreeSettings::applyChange(const SettingChange & change, ContextPtr con impl->applyChange(resolved_change); } -void MergeTreeSettings::resolveDiskSetting(SettingsChanges & changes, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database) +void MergeTreeSettings::resolveDiskSetting( + SettingsChanges & changes, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database, bool stored_definition) { for (auto & change : changes) - resolveDiskSetting(change, context, is_loading_from_existing_metadata, for_system_database); + resolveDiskSetting(change, context, is_loading_from_existing_metadata, for_system_database, stored_definition); } -void MergeTreeSettings::resolveDiskSetting(SettingChange & change, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database) +void MergeTreeSettings::resolveDiskSetting( + SettingChange & change, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database, bool stored_definition) { if (change.name != "disk") return; @@ -2892,7 +2895,8 @@ void MergeTreeSettings::resolveDiskSetting(SettingChange & change, ContextPtr co if (value_as_custom_ast && isDiskFunction(value_as_custom_ast)) { - auto disk_name = DiskFromAST::createCustomDisk(value_as_custom_ast, context, is_loading_from_existing_metadata, for_system_database); + auto disk_name = DiskFromAST::createCustomDisk( + value_as_custom_ast, context, is_loading_from_existing_metadata, for_system_database, stored_definition); LOG_DEBUG(getLogger("MergeTreeSettings"), "Created custom disk {}", disk_name); change.value = disk_name; } @@ -2985,9 +2989,10 @@ SettingsTierType MergeTreeSettings::getTier(std::string_view name) const return impl->getTier(name); } -void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database) +void MergeTreeSettings::loadFromQuery( + ASTStorage & storage_def, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database, bool stored_definition) { - impl->loadFromQuery(storage_def, context, is_loading_from_existing_metadata, for_system_database); + impl->loadFromQuery(storage_def, context, is_loading_from_existing_metadata, for_system_database, stored_definition); } void MergeTreeSettings::loadFromConfig(const String & config_elem, const Poco::Util::AbstractConfiguration & config) diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 1a7910394b21f..609589144aee1 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -110,7 +110,9 @@ struct MergeTreeSettings void applyCompatibilitySetting(const String & compatibility_value); /// NOTE: will rewrite the AST to add immutable settings. - void loadFromQuery(ASTStorage & storage_def, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database = false); + void loadFromQuery( + ASTStorage & storage_def, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database = false, + bool stored_definition = false); void loadFromConfig(const String & config_elem, const Poco::Util::AbstractConfiguration & config); bool needSyncPart(size_t input_rows, size_t input_bytes) const; @@ -132,8 +134,12 @@ struct MergeTreeSettings static bool isPartFormatSetting(const String & name); static bool isDiskSettingChanged(const SettingsChanges & old_changes, const SettingsChanges & new_changes); - static void resolveDiskSetting(SettingsChanges & changes, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database = false); - static void resolveDiskSetting(SettingChange & change, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database = false); + static void resolveDiskSetting( + SettingsChanges & changes, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database = false, + bool stored_definition = false); + static void resolveDiskSetting( + SettingChange & change, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database = false, + bool stored_definition = false); /// Cloud only static bool isSMTReadonlySetting(const String & name); diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index 6569cb6f28e0e..b80c017d5e9e3 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -889,7 +889,7 @@ static StoragePtr create(const StorageFactory::Arguments & args) /// `LoadingStrictnessLevel::ATTACH` and must still be subject to these checks. storage_settings->loadFromQuery( *args.storage_def, args.getLocalContext(), isLoadingFromExistingMetadata(args.mode), - args.table_id.database_name == DatabaseCatalog::SYSTEM_DATABASE); + args.table_id.database_name == DatabaseCatalog::SYSTEM_DATABASE, /* stored_definition */ args.query.attach_short_syntax); /// What this query changes from the settings the server has in effect, which already include the /// `merge_tree` config section and `compatibility`: those are not changes made by the query. A @@ -898,6 +898,10 @@ static StoragePtr create(const StorageFactory::Arguments & args) args.getLocalContext()->checkMergeTreeSettingsConstraints( initial_storage_settings, storage_settings->changesFrom(initial_storage_settings)); + /// After the constraints, so a forbidden `storage_policy` is reported as such and not as an unknown policy. + if ((*storage_settings)[MergeTreeSetting::storage_policy].changed) + args.getStoragePolicy((*storage_settings)[MergeTreeSetting::storage_policy]); + /// `MergeTreeData` runs `sanityCheck` on the settings it is given when the mode says `CREATE`, so a /// full-definition `ATTACH` is the fresh definition left to check. Without this it can state a value /// such as `index_granularity = 0` that the same `CREATE` refuses, and the table is created broken. diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index cd54bdce2cc3d..7d6d2c37e6e11 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -2306,7 +2306,7 @@ void registerStorageDistributed(StorageFactory & factory) if (engine_args.size() >= 5) { engine_args[4] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[4], local_context); - storage_policy = checkAndGetLiteralArgument(engine_args[4], "storage_policy"); + storage_policy = args.getStoragePolicy(checkAndGetLiteralArgument(engine_args[4], "storage_policy"))->getName(); } /// TODO: move some arguments from the arguments to the SETTINGS. diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp index bef934fa0c04e..921a6495154b7 100644 --- a/src/Storages/StorageFactory.cpp +++ b/src/Storages/StorageFactory.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -56,6 +57,22 @@ ContextMutablePtr StorageFactory::Arguments::getLocalContext() const return ptr; } +DiskPtr StorageFactory::Arguments::getDisk(const String & name) const +{ + auto disk = getContext()->getDisk(name); + if (!isLoadingFromExistingMetadata(mode)) + DiskFromAST::ensureDiskIsAllowed(disk, getLocalContext()); + return disk; +} + +StoragePolicyPtr StorageFactory::Arguments::getStoragePolicy(const String & name) const +{ + auto storage_policy = getContext()->getStoragePolicy(name); + if (!isLoadingFromExistingMetadata(mode)) + DiskFromAST::ensureStoragePolicyIsAllowed(name, getLocalContext()); + return storage_policy; +} + void StorageFactory::registerStorage(const std::string & name, CreatorFn creator_fn, StorageFeatures features, Documentation documentation) { diff --git a/src/Storages/StorageFactory.h b/src/Storages/StorageFactory.h index e8859f00cec95..bf1784b0a6988 100644 --- a/src/Storages/StorageFactory.h +++ b/src/Storages/StorageFactory.h @@ -17,6 +17,10 @@ namespace DB class Context; class ASTCreateQuery; class ASTStorage; +class IDisk; +using DiskPtr = std::shared_ptr; +class IStoragePolicy; +using StoragePolicyPtr = std::shared_ptr; class AccessRightsElements; class SettingsChanges; struct StorageID; @@ -61,6 +65,11 @@ class StorageFactory : private boost::noncopyable, public IHints<> ContextMutablePtr getContext() const; ContextMutablePtr getLocalContext() const; + + /// A disk or a storage policy the table definition names. Refused, unless loading from existing metadata, when the + /// query may not use it (see `DiskFromAST::ensureDiskIsAllowed`). + DiskPtr getDisk(const String & name) const; + StoragePolicyPtr getStoragePolicy(const String & name) const; }; /// Analog of the IStorage::supports*() helpers diff --git a/src/Storages/StorageJoin.cpp b/src/Storages/StorageJoin.cpp index cdc7a5c14c2f6..501c1b60d0a68 100644 --- a/src/Storages/StorageJoin.cpp +++ b/src/Storages/StorageJoin.cpp @@ -454,7 +454,7 @@ void registerStorageJoin(StorageFactory & factory) } } - DiskPtr disk = args.getContext()->getDisk(disk_name); + DiskPtr disk = args.getDisk(disk_name); if (engine_args.size() < 3) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index 9fc213141e70e..a46099190cd6f 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -1415,7 +1415,7 @@ void registerStorageLog(StorageFactory & factory) args.engine_name, args.engine_args.size()); String disk_name = getDiskName(*args.storage_def, args.getContext()); - DiskPtr disk = args.getContext()->getDisk(disk_name); + DiskPtr disk = args.getDisk(disk_name); return std::make_shared( args.engine_name, diff --git a/src/Storages/StorageSet.cpp b/src/Storages/StorageSet.cpp index 525941a5a0710..51626384a86f0 100644 --- a/src/Storages/StorageSet.cpp +++ b/src/Storages/StorageSet.cpp @@ -360,7 +360,7 @@ void registerStorageSet(StorageFactory & factory) if (has_settings) set_settings.loadFromQuery(*args.storage_def); - DiskPtr disk = args.getContext()->getDisk(set_settings[SetSetting::disk]); + DiskPtr disk = args.getDisk(set_settings[SetSetting::disk]); return std::make_shared( disk, args.relative_data_path, args.table_id, args.columns, args.constraints, args.comment, set_settings[SetSetting::persistent]); }, StorageFactory::StorageFeatures{ .supports_settings = true, .has_builtin_setting_fn = SetSettings::hasBuiltin, }, diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp index 81211ad319eed..61c049df78f88 100644 --- a/src/Storages/StorageStripeLog.cpp +++ b/src/Storages/StorageStripeLog.cpp @@ -765,7 +765,7 @@ void registerStorageStripeLog(StorageFactory & factory) args.engine_name, args.engine_args.size()); String disk_name = getDiskName(*args.storage_def, args.getContext()); - DiskPtr disk = args.getContext()->getDisk(disk_name); + DiskPtr disk = args.getDisk(disk_name); return std::make_shared( disk, diff --git a/tests/queries/0_stateless/05494_custom_disk_identity_without_credentials_marker.reference b/tests/queries/0_stateless/05494_custom_disk_identity_without_credentials_marker.reference new file mode 100644 index 0000000000000..d6f64353077fd --- /dev/null +++ b/tests/queries/0_stateless/05494_custom_disk_identity_without_credentials_marker.reference @@ -0,0 +1,25 @@ +--- created_before_26_7 +reuse after upgrade +named_ok +unnamed_ok +named_disk +unnamed +the new tables store the definition of the existing tables +a2 same as a +b2 same as b +after restart +1 +1 +1 +1 +named_disk +unnamed +--- permanently_detached +1 +attach +2 +after restart +2 +--- mixed +wrapper_ok +w2 same as w diff --git a/tests/queries/0_stateless/05494_custom_disk_identity_without_credentials_marker.sh b/tests/queries/0_stateless/05494_custom_disk_identity_without_credentials_marker.sh new file mode 100755 index 0000000000000..c62bc4af79251 --- /dev/null +++ b/tests/queries/0_stateless/05494_custom_disk_identity_without_credentials_marker.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A disk definition that uses `from_env` is stored with the `_server_credentials_allowed` marker. Tables created by +# versions before 26.7 store it without the marker. After an upgrade, declaring the same disk again in a new table must +# reuse the existing disk, and the new table must load after a restart. +# `type = 'from_env T'` with `T=local` counts as a disk that may use server credentials, without needing S3. + +export T=local + +run_case() +{ + local name=$1 strip=$2 type="'from_env T'" + local dir="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_${name}" + rm -rf "${dir}" + local named="disk(name = 'named_disk', type = ${type}, path = '${dir}/disk_named/')" + local unnamed="disk(type = ${type}, path = '${dir}/disk_unnamed/')" + + run() { ${CLICKHOUSE_LOCAL} --path "${dir}/data" --dynamic_disk_allow_from_env=1 -mn -q "$1" -- --custom_local_disks_base_directory="${dir}/" 2>&1 | grep -oE '^[0-9]+$|^[a-z0-9_]+$|\([A-Z_]+\)'; } + disks="SELECT if(name = 'named_disk', name, 'unnamed') FROM system.disks WHERE name = 'named_disk' OR name LIKE '__tmp_internal_%' ORDER BY name DESC" + + echo "--- ${name}" + run "CREATE TABLE a (x UInt8) ENGINE = MergeTree ORDER BY x SETTINGS disk = ${named}; INSERT INTO a VALUES (1); + CREATE TABLE b (x UInt8) ENGINE = MergeTree ORDER BY x SETTINGS disk = ${unnamed}; INSERT INTO b VALUES (1);" + # Make the stored definitions look as if an older version had written them. + sed -i "${strip}" "${dir}"/data/store/*/*/{a,b}.sql + + echo "reuse after upgrade" + run "CREATE TABLE a2 (x UInt8) ENGINE = MergeTree ORDER BY x SETTINGS disk = ${named}; SELECT 'named_ok'" + run "CREATE TABLE b2 (x UInt8) ENGINE = MergeTree ORDER BY x SETTINGS disk = ${unnamed}; SELECT 'unnamed_ok'" + run "INSERT INTO a2 VALUES (2); INSERT INTO b2 VALUES (2); ${disks}" + + echo "the new tables store the definition of the existing tables" + for t in a b; do + cmp -s <(grep -o "disk = disk(.*)" "${dir}"/data/store/*/*/${t}.sql) <(grep -o "disk = disk(.*)" "${dir}"/data/store/*/*/${t}2.sql) && echo "${t}2 same as ${t}" + done + + echo "after restart" + run "SELECT count() FROM a; SELECT count() FROM a2; SELECT count() FROM b; SELECT count() FROM b2; ${disks}" + rm -rf "${dir}" +} + +run_case created_before_26_7 's/, equals(_server_credentials_allowed, 1)//' + +# A table detached permanently is not loaded at startup, so its disk is not registered when it is attached again. The +# short `ATTACH` must still find the disk by the definition stored by the older version, not create an empty one. +# `local_blob_storage` keeps its metadata in a directory named after the disk. +export T2=local_blob_storage +dir="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_permanently_detached" +rm -rf "${dir}" +run() { ${CLICKHOUSE_LOCAL} --path "${dir}/data" --dynamic_disk_allow_from_env=1 -mn -q "$1" -- --custom_local_disks_base_directory="${dir}/" 2>&1 | grep -oE '^[0-9]+$|\([A-Z_]+\)'; } +echo "--- permanently_detached" +run "CREATE DATABASE d ENGINE = Atomic; CREATE TABLE d.p (x UInt8) ENGINE = MergeTree ORDER BY x SETTINGS disk = disk(type = 'from_env T2', path = '${dir}/blobs/'); INSERT INTO d.p VALUES (7)" +old_disk=$(ls "${dir}/data/disks" | grep __tmp_internal_) +sed -i 's/, equals(_server_credentials_allowed, 1)//' "${dir}"/data/store/*/*/p.sql +# Load the definition once to learn the name an older version gave the disk, and move the metadata there. +run "SELECT 1" > /dev/null +legacy_disk=$(ls "${dir}/data/disks" | grep __tmp_internal_ | grep -v "${old_disk}") +rm -rf "${dir}/data/disks/${legacy_disk}" +mv "${dir}/data/disks/${old_disk}" "${dir}/data/disks/${legacy_disk}" +run "SELECT count() FROM d.p" +run "DETACH TABLE d.p PERMANENTLY" +echo "attach" +run "ATTACH TABLE d.p; INSERT INTO d.p VALUES (8); SELECT count() FROM d.p" +echo "after restart" +run "SELECT count() FROM d.p" +rm -rf "${dir}" + +# A wrapper declared by an older version names the disk of the older form below it, while a newer table can have +# registered the disk of the newer form too. Declaring the same wrapper again must find the wrapper. +dir="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_mixed" +rm -rf "${dir}" +run() { ${CLICKHOUSE_LOCAL} --path "${dir}/data" --dynamic_disk_allow_from_env=1 -mn -q "$1" -- --custom_local_disks_base_directory="${dir}/" --custom_cached_disks_base_directory="${dir}/caches/" 2>&1 | grep -oE '^[0-9a-z_]+$|\([A-Z_]+\)'; } +leaf="disk(type = 'from_env T2', path = '${dir}/blobs/')" +wrapper="disk(name = 'wrapper_disk', type = cache, path = '${dir}/caches/w/', max_size = '1Gi', disk = ${leaf})" +echo "--- mixed" +run "CREATE TABLE w (x UInt8) ENGINE = MergeTree ORDER BY x SETTINGS disk = ${wrapper}; CREATE TABLE l (x UInt8) ENGINE = MergeTree ORDER BY x SETTINGS disk = ${leaf}" +sed -i 's/, equals(_server_credentials_allowed, 1)//' "${dir}"/data/store/*/*/w.sql +run "CREATE TABLE w2 (x UInt8) ENGINE = MergeTree ORDER BY x SETTINGS disk = ${wrapper}; SELECT 'wrapper_ok'" +w2_definition=$(grep -o "disk = disk(.*)" "${dir}"/data/store/*/*/w2.sql) +[ -n "${w2_definition}" ] && [ "${w2_definition}" = "$(grep -o "disk = disk(.*)" "${dir}"/data/store/*/*/w.sql)" ] && echo "w2 same as w" +rm -rf "${dir}" diff --git a/tests/queries/0_stateless/05496_dynamic_disk_wrapper_of_server_credentials_disk.reference b/tests/queries/0_stateless/05496_dynamic_disk_wrapper_of_server_credentials_disk.reference new file mode 100644 index 0000000000000..c3d951aa50dcc --- /dev/null +++ b/tests/queries/0_stateless/05496_dynamic_disk_wrapper_of_server_credentials_disk.reference @@ -0,0 +1,21 @@ +with the opt-in: create a disk with server credentials: OK +with the opt-in: wrap it: OK +with the opt-in: create a disk without server credentials: OK +without the opt-in: wrap the disk with server credentials: (ACCESS_DENIED) +without the opt-in: wrap its wrapper: (ACCESS_DENIED) +without the opt-in: wrap the disk without server credentials: OK +with the opt-in: wrap the disk with server credentials: OK +with the opt-in: wrap it with from_zk: OK +without the opt-in: the same definition after the node changed: (ACCESS_DENIED) +without the opt-in: the storage policy of the disk with server credentials: (ACCESS_DENIED) +without the opt-in: the storage policy of the disk without server credentials: OK +without the opt-in: change to the storage policy of the disk with server credentials: (ACCESS_DENIED) +with the opt-in: the storage policy of the disk with server credentials: OK +without the opt-in: alter that table otherwise: OK +without the opt-in: change another setting of that table: OK +without the opt-in: Log on the disk with server credentials: (ACCESS_DENIED) +without the opt-in: StripeLog on its storage policy: (ACCESS_DENIED) +without the opt-in: Set on the disk: (ACCESS_DENIED) +without the opt-in: Join on the disk: (ACCESS_DENIED) +without the opt-in: Distributed on its storage policy: (ACCESS_DENIED) +without the opt-in: Log on the disk without server credentials: OK diff --git a/tests/queries/0_stateless/05496_dynamic_disk_wrapper_of_server_credentials_disk.sh b/tests/queries/0_stateless/05496_dynamic_disk_wrapper_of_server_credentials_disk.sh new file mode 100755 index 0000000000000..ad47a415bdc3f --- /dev/null +++ b/tests/queries/0_stateless/05496_dynamic_disk_wrapper_of_server_credentials_disk.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Tags: zookeeper, no-fasttest +# Tag zookeeper: the disk type is read with `from_zk`. +# Tag no-fasttest: needs the `encrypted` disk, which requires SSL. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A session without `s3_allow_server_credentials_in_user_queries` may not wrap, by name, a disk that a session with it +# created with the server's own credentials, nor a wrapper of such a disk. A `type` read with `from_zk` counts as a +# disk that may use server credentials; here it resolves to an `encrypted` disk, so no object storage is needed. +# The `encrypted` disks wrap local disks under `custom_local_disks_base_directory`. A disk stays registered when its +# table is dropped, so every table is dropped right away: a table naming a disk of another table could not be loaded +# after a restart. + +ZK_PATH="/clickhouse/${CLICKHOUSE_TEST_UNIQUE_NAME}" +KEY="1234567812345678" +SERVER_DISK="server_${CLICKHOUSE_DATABASE}" +WRAPPER_DISK="wrapper_${CLICKHOUSE_DATABASE}" +PLAIN_DISK="plain_${CLICKHOUSE_DATABASE}" + +run() +{ + local out + if out=$(${CLICKHOUSE_CLIENT} --dynamic_disk_allow_from_zk=1 "$@" 2>&1); then echo "OK"; else echo "$out" | grep -oE '\([A-Z_]+\)' | head -1; fi + ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS created SYNC" +} +create() { echo "CREATE TABLE $1 (x UInt8) ENGINE = MergeTree ORDER BY x SETTINGS disk = $2"; } +wrap() { echo "disk(type = encrypted, disk = '$1', path = '$2/', key = '${KEY}')"; } + +OPT_IN="--s3_allow_server_credentials_in_user_queries=1" +local_disk() { echo "disk(type = local, path = '${CLICKHOUSE_DISKS_FILES}/${CLICKHOUSE_TEST_UNIQUE_NAME}_$1/')"; } + +$CLICKHOUSE_KEEPER_CLIENT -q "create '${ZK_PATH}' 'disk'" +$CLICKHOUSE_KEEPER_CLIENT -q "create '${ZK_PATH}/type' 'encrypted'" + +echo "with the opt-in: create a disk with server credentials: $(run ${OPT_IN} -q "$(create created "disk(name = '${SERVER_DISK}', type = 'from_zk ${ZK_PATH}/type', disk = $(local_disk server), path = 'encrypted/', key = '${KEY}')")")" +echo "with the opt-in: wrap it: $(run ${OPT_IN} -q "$(create created "disk(name = '${WRAPPER_DISK}', type = encrypted, disk = '${SERVER_DISK}', path = 'wrapper/', key = '${KEY}')")")" +echo "with the opt-in: create a disk without server credentials: $(run ${OPT_IN} -q "$(create created "disk(name = '${PLAIN_DISK}', type = encrypted, disk = $(local_disk plain), path = 'encrypted/', key = '${KEY}')")")" + +echo "without the opt-in: wrap the disk with server credentials: $(run -q "$(create created "$(wrap "${SERVER_DISK}" t1)")")" +echo "without the opt-in: wrap its wrapper: $(run -q "$(create created "$(wrap "${WRAPPER_DISK}" t2)")")" +echo "without the opt-in: wrap the disk without server credentials: $(run -q "$(create created "$(wrap "${PLAIN_DISK}" t3)")")" +echo "with the opt-in: wrap the disk with server credentials: $(run ${OPT_IN} -q "$(create created "$(wrap "${SERVER_DISK}" t4)")")" + +# The wrapped disk is read with `from_zk`: after the node changes, the same definition no longer resolves to the disk with +# server credentials, but it still names the disk created from it. +$CLICKHOUSE_KEEPER_CLIENT -q "create '${ZK_PATH}/wrapped' '${SERVER_DISK}'" +INDIRECT_WRAPPER="disk(type = encrypted, disk = 'from_zk ${ZK_PATH}/wrapped', path = 'indirect/', key = '${KEY}')" +echo "with the opt-in: wrap it with from_zk: $(run ${OPT_IN} -q "$(create created "${INDIRECT_WRAPPER}")")" +$CLICKHOUSE_KEEPER_CLIENT -q "set '${ZK_PATH}/wrapped' '${PLAIN_DISK}'" +echo "without the opt-in: the same definition after the node changed: $(run -q "$(create created "${INDIRECT_WRAPPER}")")" + +# A table on a disk gets the storage policy `__`, which names the disk as well. +echo "without the opt-in: the storage policy of the disk with server credentials: $(run -q "CREATE TABLE created (x UInt8) ENGINE = MergeTree ORDER BY x SETTINGS storage_policy = '__${SERVER_DISK}'")" +echo "without the opt-in: the storage policy of the disk without server credentials: $(run -q "CREATE TABLE created (x UInt8) ENGINE = MergeTree ORDER BY x SETTINGS storage_policy = '__${PLAIN_DISK}'")" +echo "without the opt-in: change to the storage policy of the disk with server credentials: $(run -q "CREATE TABLE created (x UInt8) ENGINE = MergeTree ORDER BY x SETTINGS storage_policy = '__${PLAIN_DISK}'; ALTER TABLE created MODIFY SETTING storage_policy = '__${SERVER_DISK}'")" + +# A table that already has the policy can still be altered otherwise. +echo "with the opt-in: the storage policy of the disk with server credentials: $(run ${OPT_IN} -q "CREATE TABLE t_policy (x UInt8) ENGINE = MergeTree ORDER BY x SETTINGS storage_policy = '__${SERVER_DISK}'")" +echo "without the opt-in: alter that table otherwise: $(run -q "ALTER TABLE t_policy ADD COLUMN y UInt8")" +echo "without the opt-in: change another setting of that table: $(run -q "ALTER TABLE t_policy MODIFY SETTING parts_to_throw_insert = 500")" +${CLICKHOUSE_CLIENT} -q "DROP TABLE t_policy SYNC" + +# Other engines name a disk or a storage policy as well. +echo "without the opt-in: Log on the disk with server credentials: $(run -q "CREATE TABLE created (x UInt8) ENGINE = Log SETTINGS disk = '${SERVER_DISK}'")" +echo "without the opt-in: StripeLog on its storage policy: $(run -q "CREATE TABLE created (x UInt8) ENGINE = StripeLog SETTINGS storage_policy = '__${SERVER_DISK}'")" +echo "without the opt-in: Set on the disk: $(run -q "CREATE TABLE created (x UInt8) ENGINE = Set SETTINGS disk = '${SERVER_DISK}'")" +echo "without the opt-in: Join on the disk: $(run -q "CREATE TABLE created (x UInt8) ENGINE = Join(ANY, LEFT, x) SETTINGS disk = '${SERVER_DISK}'")" +echo "without the opt-in: Distributed on its storage policy: $(run -q "CREATE TABLE created (x UInt8) ENGINE = Distributed('test_shard_localhost', currentDatabase(), t_plain, rand(), '__${SERVER_DISK}')")" +echo "without the opt-in: Log on the disk without server credentials: $(run -q "CREATE TABLE created (x UInt8) ENGINE = Log SETTINGS disk = '${PLAIN_DISK}'")" + +$CLICKHOUSE_KEEPER_CLIENT -q "rmr '${ZK_PATH}'" From 21cd62344bde00aeee6d84dfa81c425268ed7196 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 10 Nov 2026 11:27:09 +0000 Subject: [PATCH 140/219] Backport #120476 to 26.8: Fix flaky test_kafka_many_materialized_views by re-attaching the Kafka table --- .../test_storage_kafka/test_batch_fast.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/tests/integration/test_storage_kafka/test_batch_fast.py b/tests/integration/test_storage_kafka/test_batch_fast.py index f55249b5a7d63..f4d206a9099f5 100644 --- a/tests/integration/test_storage_kafka/test_batch_fast.py +++ b/tests/integration/test_storage_kafka/test_batch_fast.py @@ -1259,6 +1259,8 @@ def test_kafka_many_materialized_views(kafka_cluster, create_query_generator): consumer_group=f"{topic_name}-group", ) + # A streaming loop keeps the materialized views it started with, so one that began before the + # second view existed commits without it; detaching and re-attaching joins it before producing. instance.query(f""" DROP TABLE IF EXISTS test.{kafka_table}_view1; DROP TABLE IF EXISTS test.{kafka_table}_view2; @@ -1275,11 +1277,10 @@ def test_kafka_many_materialized_views(kafka_cluster, create_query_generator): SELECT * FROM test.{kafka_table}; CREATE MATERIALIZED VIEW test.{kafka_table}_consumer2 TO test.{kafka_table}_view2 AS SELECT * FROM test.{kafka_table}; - """) - # we have to wait > kafka_poll_timeout_ms before producing data, - # otherwise it is expected that data might go via the first MV only - time.sleep(3) + DETACH TABLE test.{kafka_table} SYNC; + ATTACH TABLE test.{kafka_table}; + """) messages = [] for i in range(50): @@ -1310,8 +1311,8 @@ def test_kafka_many_materialized_views(kafka_cluster, create_query_generator): DROP TABLE test.{kafka_table}_view2; """) - k.kafka_check_result(result1, True) - k.kafka_check_result(result2, True) + assert k.kafka_check_result(result1), f"view1 got: {result1!r}" + assert k.kafka_check_result(result2), f"view2 got: {result2!r}" @pytest.mark.parametrize( "create_query_generator", From 70e13d2fb2f18e6c357d5fd7b18964ca1bf70425 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 10 Nov 2026 11:22:59 +0000 Subject: [PATCH 141/219] Backport #129969 to 26.8: Refuse MongoDB connection strings that authenticate with the server's own credentials --- src/Core/Settings.cpp | 4 +- src/Dictionaries/MongoDBDictionarySource.cpp | 7 +- src/Storages/StorageMongoDB.cpp | 58 +++++++++++ src/Storages/StorageMongoDB.h | 3 + ...05567_mongodb_server_credentials.reference | 27 ++++++ .../05567_mongodb_server_credentials.sh | 95 +++++++++++++++++++ ...db_dictionary_server_credentials.reference | 12 +++ ...8_mongodb_dictionary_server_credentials.sh | 85 +++++++++++++++++ 8 files changed, 289 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/05567_mongodb_server_credentials.reference create mode 100755 tests/queries/0_stateless/05567_mongodb_server_credentials.sh create mode 100644 tests/queries/0_stateless/05568_mongodb_dictionary_server_credentials.reference create mode 100755 tests/queries/0_stateless/05568_mongodb_dictionary_server_credentials.sh diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 2ec93b883a1c8..21c96f7a0d4cd 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -4716,7 +4716,7 @@ Allow using `from_zk` substitutions in the dynamic disk configuration (i.e. in t Disabled by default. )", 0) \ DECLARE(Bool, s3_allow_server_credentials_in_user_queries, false, R"( -Allow S3 access that originates from user SQL to use server-managed credentials. +Allow S3 and MongoDB access that originates from user SQL to use server-managed credentials. When disabled (the default), the `s3`/`s3Cluster` table functions, the `S3`/`S3Queue` engines, S3 named collections, dynamic `disk(type=s3, ...)` definitions, `BACKUP`/`RESTORE TO S3`, DataLake table-data reads, and `DataLakeCatalog` databases (Glue, BigLake) may not resolve credentials from the environment, instance metadata (IMDS), IRSA, ECS, instance profile, SSO, AWS config/credentials files, or the GCP OAuth metadata service. A request that asks for one of those server-managed sources (for example `use_environment_credentials = 1` or `http_client = gcp_oauth`) without supplying usable explicit credentials is rejected with `ACCESS_DENIED`. A request that asks for none of them is sent unsigned (anonymous), the same as if `NOSIGN` had been given. @@ -4736,6 +4736,8 @@ For `BACKUP`/`RESTORE ... ON CLUSTER` the initiator's value of this setting is p Durability for persistent `S3` and `S3Queue` tables: enabling this only per session or profile is not durable across a restart. When the server reloads such a table from its stored definition (startup or `RESTORE`) it rebuilds the S3 client and re-applies the restriction with the startup context, so a table that relied on server-managed credentials and was created only under a session/profile `s3_allow_server_credentials_in_user_queries = 1` is created successfully but becomes inaccessible after a restart (the table is left in place; queries against it fail until its credentials resolve to a permitted source again). The server itself still starts. Give such tables explicit credentials for durable access; alternatively, enabling the setting server-wide keeps them loading across restarts, at the cost of relaxing the restriction for all reloads. +MongoDB is covered by this setting as well. When it is disabled, a query that reads with the `mongodb` table function or from a `MongoDB` table is rejected with `ACCESS_DENIED` if the connection string makes the server authenticate with its own credentials: with the authentication mechanism `MONGODB-OIDC` (the identity the server runs under: a Kubernetes service account token, or a token from the Azure or GCP instance metadata service), `MONGODB-AWS`, `GSSAPI` or `MONGODB-X509`, or with the connection option `tlsCertificateKeyFile` (or its deprecated alias `sslClientCertificateKeyFile`), which makes the server present a client certificate from its own filesystem. Only `SCRAM-SHA-1`, `SCRAM-SHA-256` and `PLAIN` are accepted, or no `authMechanism` at all. The check uses the settings of the session that reads, also for a table created by another user, so a session where this setting is enabled should not read MongoDB tables, or views of them, defined by untrusted users. Creating a `MongoDB` table is not checked. A MongoDB dictionary created with a DDL query may not use these mechanisms or `tlsCertificateKeyFile` whatever the value of this setting (except in `clickhouse-local`), because the server loads it with its own context, after the named collection it refers to may have changed; define such a dictionary in the server configuration files. Dictionaries defined in the server configuration files are not restricted, also when they refer to a named collection created with a DDL query, so whoever may alter that collection controls their connection string. + To keep it disabled for untrusted users, pin it in their profile by both setting the value explicitly to `0` and marking it `readonly`: ```xml diff --git a/src/Dictionaries/MongoDBDictionarySource.cpp b/src/Dictionaries/MongoDBDictionarySource.cpp index 3ea596d5a5e25..6ff705c680ebf 100644 --- a/src/Dictionaries/MongoDBDictionarySource.cpp +++ b/src/Dictionaries/MongoDBDictionarySource.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -46,7 +47,7 @@ void registerDictionarySourceMongoDB(DictionarySourceFactory & factory) Block & sample_block, ContextPtr context, const std::string & /* default_database */, - bool /* created_from_ddl */) + bool created_from_ddl) { const auto config_prefix = root_config_prefix + ".mongodb"; auto configuration = std::make_shared(); @@ -103,6 +104,10 @@ void registerDictionarySourceMongoDB(DictionarySourceFactory & factory) configuration->checkHosts(context); configuration->checkCollection(); + /// A DDL dictionary is loaded by the server with the global context, so no session setting can permit server credentials here. + if (created_from_ddl && context->getApplicationType() != Context::ApplicationType::LOCAL) + configuration->checkServerCredentialsForDictionary(); + return std::make_unique(dict_struct, std::move(configuration), std::make_shared(sample_block)); }; #else diff --git a/src/Storages/StorageMongoDB.cpp b/src/Storages/StorageMongoDB.cpp index b1a68b4c56ea1..e902d3ca65d68 100644 --- a/src/Storages/StorageMongoDB.cpp +++ b/src/Storages/StorageMongoDB.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,7 @@ #include #include +#include #include using bsoncxx::builder::basic::document; @@ -57,6 +59,7 @@ MongoDBInstanceHolder & MongoDBInstanceHolder::instance() namespace ErrorCodes { + extern const int ACCESS_DENIED; extern const int BAD_ARGUMENTS; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int NOT_IMPLEMENTED; @@ -86,6 +89,58 @@ void MongoDBConfiguration::checkCollection() const throw Exception(ErrorCodes::BAD_ARGUMENTS, "MongoDB collection name must be non-empty and must not contain NUL characters"); } +namespace +{ + +void throwIfServerCredentials(const mongocxx::uri & uri, std::string_view remedy) +{ + /// Only these take the credentials from the connection string; `MONGODB-OIDC`, `MONGODB-AWS`, `GSSAPI` and + /// `MONGODB-X509` use the server's. Without a mechanism the driver uses SCRAM or no authentication. Names are case-insensitive. + static constexpr std::string_view allowed_mechanisms[] = {"SCRAM-SHA-1", "SCRAM-SHA-256", "PLAIN"}; + + const String mechanism = uri.auth_mechanism(); + const bool is_allowed_mechanism = mechanism.empty() + || std::ranges::any_of(allowed_mechanisms, [&](std::string_view allowed) { return equalsCaseInsensitive(mechanism, allowed); }); + + if (!is_allowed_mechanism) + throw Exception( + ErrorCodes::ACCESS_DENIED, + "MongoDB authentication mechanism '{}' is not allowed, because it authenticates with " + "credentials of the server process. Use SCRAM-SHA-1, SCRAM-SHA-256 or PLAIN with a username and a password " + "in the connection string{}.", + mechanism, remedy); + + /// The driver stores the deprecated `sslClientCertificateKeyFile` under the canonical name, so this covers both. + if (uri.tls_certificate_key_file()) + throw Exception( + ErrorCodes::ACCESS_DENIED, + "MongoDB connection option `tlsCertificateKeyFile` is not allowed, because it makes the server present a " + "client certificate from its own filesystem. Use SCRAM-SHA-1, SCRAM-SHA-256 or PLAIN with a username and a " + "password in the connection string{}.", + remedy); +} + +} + +void MongoDBConfiguration::checkServerCredentialsForTable(const ContextPtr & context) const +{ + if (!context->shouldRestrictUserQueryS3Credentials()) + return; + + throwIfServerCredentials(*uri, +#if CLICKHOUSE_CLOUD + "" +#else + ", or enable the setting `s3_allow_server_credentials_in_user_queries`" +#endif + ); +} + +void MongoDBConfiguration::checkServerCredentialsForDictionary() const +{ + throwIfServerCredentials(*uri, ", or define the dictionary in the server configuration files"); +} + StorageMongoDB::StorageMongoDB( const StorageID & table_id_, MongoDBConfiguration configuration_, @@ -121,6 +176,9 @@ Pipe StorageMongoDB::read( size_t max_block_size, size_t /*num_streams*/) { + /// Checked per read, with the query's settings: a table created with `mongodb(...)` is built with the global context. + configuration.checkServerCredentialsForTable(context); + storage_snapshot->check(column_names); Block sample_block; diff --git a/src/Storages/StorageMongoDB.h b/src/Storages/StorageMongoDB.h index 65c6e5a508d7d..de1aa96ab48f5 100644 --- a/src/Storages/StorageMongoDB.h +++ b/src/Storages/StorageMongoDB.h @@ -71,6 +71,9 @@ struct MongoDBConfiguration void checkHosts(const ContextPtr & context) const; void checkCollection() const; + void checkServerCredentialsForTable(const ContextPtr & context) const; + void checkServerCredentialsForDictionary() const; + bool isOidColumn(const std::string & name) const { return oid_fields.contains(name); diff --git a/tests/queries/0_stateless/05567_mongodb_server_credentials.reference b/tests/queries/0_stateless/05567_mongodb_server_credentials.reference new file mode 100644 index 0000000000000..d12c1befdb9c3 --- /dev/null +++ b/tests/queries/0_stateless/05567_mongodb_server_credentials.reference @@ -0,0 +1,27 @@ +Mechanisms that take the credentials from the server, and a client certificate file +refused +refused +refused +refused +refused +refused +refused +refused +Credentials in the connection string +connects +connects +connects +The setting permits the others +connects +connects +A MongoDB table is checked when it is read, with the setting of the session that reads +refused +connects +refused +So is a table created with CREATE TABLE ... AS mongodb(...) +x UInt8 +refused +connects +And the table function with a named collection +refused +connects diff --git a/tests/queries/0_stateless/05567_mongodb_server_credentials.sh b/tests/queries/0_stateless/05567_mongodb_server_credentials.sh new file mode 100755 index 0000000000000..ffd10d4a2a51d --- /dev/null +++ b/tests/queries/0_stateless/05567_mongodb_server_credentials.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: the MongoDB table function and table engine need USE_MONGODB + +# A query that reads with the `mongodb` table function or from a `MongoDB` table is refused with `ACCESS_DENIED` if the +# connection string makes the server authenticate with credentials of the server process, while +# `s3_allow_server_credentials_in_user_queries` is disabled in the session that reads. Nothing listens on 127.0.0.1:1, +# so a query that is not refused fails to connect. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +HOST="127.0.0.1:1" +TIMEOUTS="connectTimeoutMS=100&serverSelectionTimeoutMS=100" +OIDC_URI="mongodb://${HOST}/db?${TIMEOUTS}&authMechanism=MONGODB-OIDC" +NC="nc_05567_${CLICKHOUSE_DATABASE}" + +# Prints `refused` for `ACCESS_DENIED`, `connects` if the query got as far as connecting, or else the output. +classify() +{ + local output + output=$(cat) + if grep -q ACCESS_DENIED <<< "$output"; then + echo refused + elif grep -q 'No suitable servers found' <<< "$output"; then + echo connects + else + echo "$output" + fi +} + +# Runs the query $2 with the setting $1. +check() +{ + $CLICKHOUSE_CLIENT --s3_allow_server_credentials_in_user_queries "$1" -q "$2" 2>&1 | classify +} + +read_uri() +{ + check "$1" "SELECT count() FROM mongodb('mongodb://${HOST}/db?${TIMEOUTS}&$2', 'c', 'x UInt8')" +} + +cleanup() +{ + $CLICKHOUSE_CLIENT -m -q " + SET ast_fuzzer_any_query = 0, ignore_drop_queries_probability = 0; + DROP TABLE IF EXISTS t_engine; + DROP TABLE IF EXISTS t_as_function; + DROP NAMED COLLECTION IF EXISTS ${NC}; + " +} +trap cleanup EXIT +cleanup + +echo 'Mechanisms that take the credentials from the server, and a client certificate file' +read_uri 0 'authMechanism=MONGODB-OIDC' +read_uri 0 'authmechanism=mongodb-oidc&authMechanismProperties=ENVIRONMENT:test' +read_uri 0 'authMechanism=MONGODB-AWS' +read_uri 0 'authMechanism=MONGODB-X509' +check 0 "SELECT count() FROM mongodb('mongodb://user@${HOST}/db?${TIMEOUTS}&authMechanism=GSSAPI', 'c', 'x UInt8')" +read_uri 0 'tls=true&tlsCertificateKeyFile=/nonexistent.pem' +read_uri 0 'ssl=true&sslClientCertificateKeyFile=/nonexistent.pem' +check 0 "SELECT count() FROM mongodb('${HOST}', 'db', 'c', '', '', 'x UInt8', '${TIMEOUTS}&authMechanism=MONGODB-OIDC')" + +echo 'Credentials in the connection string' +check 0 "SELECT count() FROM mongodb('mongodb://user:pass@${HOST}/db?${TIMEOUTS}', 'c', 'x UInt8')" +check 0 "SELECT count() FROM mongodb('mongodb://user:pass@${HOST}/db?${TIMEOUTS}&authMechanism=SCRAM-SHA-256', 'c', 'x UInt8')" +check 0 "SELECT count() FROM mongodb('mongodb://user:pass@${HOST}/db?${TIMEOUTS}&authMechanism=plain', 'c', 'x UInt8')" + +echo 'The setting permits the others' +read_uri 1 'authMechanism=MONGODB-OIDC' +read_uri 1 'tls=true&tlsCertificateKeyFile=/nonexistent.pem' + +echo 'A MongoDB table is checked when it is read, with the setting of the session that reads' +$CLICKHOUSE_CLIENT --s3_allow_server_credentials_in_user_queries 0 -q "CREATE TABLE t_engine (x UInt8) ENGINE = MongoDB('${OIDC_URI}', 'c')" +check 0 "SELECT count() FROM t_engine" +check 1 "SELECT count() FROM t_engine" +$CLICKHOUSE_CLIENT -m -q "DETACH TABLE t_engine; ATTACH TABLE t_engine;" +check 0 "SELECT count() FROM t_engine" + +echo 'So is a table created with CREATE TABLE ... AS mongodb(...)' +$CLICKHOUSE_CLIENT -m -q " + CREATE TABLE t_as_function AS mongodb('${OIDC_URI}', 'c', 'x UInt8'); + DETACH TABLE t_as_function; + ATTACH TABLE t_as_function; + DESCRIBE t_as_function SETTINGS describe_compact_output = 1; +" +check 0 "SELECT count() FROM t_as_function" +check 1 "SELECT count() FROM t_as_function" + +echo 'And the table function with a named collection' +$CLICKHOUSE_CLIENT -q "CREATE NAMED COLLECTION ${NC} AS uri = '${OIDC_URI}', collection = 'c', structure = 'x UInt8'" +check 0 "SELECT count() FROM mongodb(${NC})" +check 1 "SELECT count() FROM mongodb(${NC})" diff --git a/tests/queries/0_stateless/05568_mongodb_dictionary_server_credentials.reference b/tests/queries/0_stateless/05568_mongodb_dictionary_server_credentials.reference new file mode 100644 index 0000000000000..2704615274d8e --- /dev/null +++ b/tests/queries/0_stateless/05568_mongodb_dictionary_server_credentials.reference @@ -0,0 +1,12 @@ +Refused even in a session where the setting is enabled +refused +refused +Credentials in the connection string +connects +The options that a dictionary adds to a named collection +refused +A named collection that changes after the dictionary is created +connects +refused +clickhouse-local is not restricted +connects diff --git a/tests/queries/0_stateless/05568_mongodb_dictionary_server_credentials.sh b/tests/queries/0_stateless/05568_mongodb_dictionary_server_credentials.sh new file mode 100755 index 0000000000000..cd6e077856978 --- /dev/null +++ b/tests/queries/0_stateless/05568_mongodb_dictionary_server_credentials.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: the MongoDB dictionary source needs USE_MONGODB + +# A MongoDB dictionary created with a DDL query may never make the server authenticate with credentials of the server +# process, whatever `s3_allow_server_credentials_in_user_queries` is: the source is built when the server loads the +# dictionary, also after the named collection it refers to has changed. The dictionaries are loaded by `dictGet`. +# Nothing listens on 127.0.0.1:1, so a dictionary that is not refused fails to connect. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +HOST="127.0.0.1:1" +TIMEOUTS="connectTimeoutMS=100&serverSelectionTimeoutMS=100" +OIDC_URI="mongodb://${HOST}/db?${TIMEOUTS}&authMechanism=MONGODB-OIDC" +SCRAM_URI="mongodb://user:pass@${HOST}/db?${TIMEOUTS}&authMechanism=SCRAM-SHA-256" +NC="nc_05568_${CLICKHOUSE_DATABASE}" + +# Prints `refused` for `ACCESS_DENIED`, `connects` if the query got as far as connecting, or else the output. +classify() +{ + local output + output=$(cat) + if grep -q ACCESS_DENIED <<< "$output"; then + echo refused + elif grep -q 'No suitable servers found' <<< "$output"; then + echo connects + else + echo "$output" + fi +} + +# Runs the queries $1 in a session where the setting is enabled. +check() +{ + $CLICKHOUSE_CLIENT --s3_allow_server_credentials_in_user_queries 1 -m -q "$1" 2>&1 | classify +} + +# Creates the dictionary $1 with the source $2 and prints whether reading it is refused. +check_source() +{ + $CLICKHOUSE_CLIENT -q "CREATE DICTIONARY $1 (_id String, v String) PRIMARY KEY _id SOURCE(MONGODB($2 COLLECTION 'c')) LAYOUT(COMPLEX_KEY_DIRECT())" + check "SELECT dictGet('$1', 'v', tuple('k'))" +} + +cleanup() +{ + $CLICKHOUSE_CLIENT -m -q " + SET ast_fuzzer_any_query = 0, ignore_drop_queries_probability = 0; + DROP DICTIONARY IF EXISTS d_uri; + DROP DICTIONARY IF EXISTS d_options; + DROP DICTIONARY IF EXISTS d_scram; + DROP DICTIONARY IF EXISTS d_named_options; + DROP DICTIONARY IF EXISTS d_named_altered; + DROP NAMED COLLECTION IF EXISTS ${NC}; + " +} +trap cleanup EXIT +cleanup + +echo 'Refused even in a session where the setting is enabled' +check_source d_uri "URI '${OIDC_URI}'" +check_source d_options "HOST '127.0.0.1' PORT 1 DB 'db' USER '' PASSWORD '' OPTIONS '${TIMEOUTS}&authMechanism=MONGODB-OIDC'" + +echo 'Credentials in the connection string' +check_source d_scram "URI '${SCRAM_URI}'" + +echo 'The options that a dictionary adds to a named collection' +$CLICKHOUSE_CLIENT -q "CREATE NAMED COLLECTION ${NC} AS host = '127.0.0.1', port = '1', db = 'db', user = '', password = ''" +check_source d_named_options "NAME ${NC} OPTIONS '${TIMEOUTS}&authMechanism=MONGODB-OIDC'" +$CLICKHOUSE_CLIENT -q "DROP DICTIONARY d_named_options" + +echo 'A named collection that changes after the dictionary is created' +$CLICKHOUSE_CLIENT -q "CREATE DICTIONARY d_named_altered (_id String, v String) PRIMARY KEY _id SOURCE(MONGODB(NAME ${NC} COLLECTION 'c')) LAYOUT(COMPLEX_KEY_DIRECT())" +check "SELECT dictGet('d_named_altered', 'v', tuple('k'))" +$CLICKHOUSE_CLIENT -q "ALTER NAMED COLLECTION ${NC} SET options = '${TIMEOUTS}&authMechanism=MONGODB-OIDC'" +check "SYSTEM RELOAD DICTIONARY d_named_altered; SELECT dictGet('d_named_altered', 'v', tuple('k'))" + +echo 'clickhouse-local is not restricted' +$CLICKHOUSE_LOCAL -m -q " + CREATE DICTIONARY d (_id String, v String) PRIMARY KEY _id + SOURCE(MONGODB(URI '${OIDC_URI}' COLLECTION 'c')) LAYOUT(COMPLEX_KEY_DIRECT()); + SELECT dictGet('d', 'v', tuple('k')); +" 2>&1 | classify From ad058f3a9a89b61fd57979f6b077ab9cb984795a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 10 Nov 2026 11:44:26 +0000 Subject: [PATCH 142/219] Backport #126947 to 26.8: Fix use-after-free when the source table of a `DIRECT` dictionary is truncated during a read of the dictionary --- src/Dictionaries/DirectDictionary.cpp | 12 +++-- ..._dictionary_read_truncate_source.reference | 4 ++ ..._direct_dictionary_read_truncate_source.sh | 45 +++++++++++++++++++ 3 files changed, 57 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/05427_direct_dictionary_read_truncate_source.reference create mode 100755 tests/queries/0_stateless/05427_direct_dictionary_read_truncate_source.sh diff --git a/src/Dictionaries/DirectDictionary.cpp b/src/Dictionaries/DirectDictionary.cpp index 05d98fc6c70a9..c51159bfebda8 100644 --- a/src/Dictionaries/DirectDictionary.cpp +++ b/src/Dictionaries/DirectDictionary.cpp @@ -347,7 +347,7 @@ class SourceFromQueryPipeline : public ISource public: explicit SourceFromQueryPipeline(QueryPipeline & pipeline_) : ISource(std::make_shared(pipeline_.getSharedHeader()->cloneEmpty())) - , executor(pipeline_) + , executor(std::in_place, pipeline_) { pipeline_.setConcurrencyControl(false); } @@ -355,7 +355,7 @@ class SourceFromQueryPipeline : public ISource explicit SourceFromQueryPipeline(BlockIO io) : ISource(std::make_shared(io.pipeline.getSharedHeader()->cloneEmpty())) , io_holder(std::move(io)) - , executor(io_holder->pipeline) + , executor(std::in_place, io_holder->pipeline) { io_holder->pipeline.setConcurrencyControl(false); } @@ -370,7 +370,7 @@ class SourceFromQueryPipeline : public ISource try { Chunk chunk; - while (executor.pull(chunk)) + while (executor->pull(chunk)) { if (chunk) return chunk; @@ -380,6 +380,7 @@ class SourceFromQueryPipeline : public ISource { if (io_holder) { + executor.reset(); io_holder->onException(); } throw; @@ -387,6 +388,7 @@ class SourceFromQueryPipeline : public ISource if (io_holder) { + executor.reset(); io_holder->onFinish(); } return {}; @@ -400,7 +402,9 @@ class SourceFromQueryPipeline : public ISource /// * We need to run BlockIO::onFinish or BlockIO::onException. std::optional io_holder; - TExecutor executor; + /// Reset before `io_holder` tears its pipeline down: the executor shares the pipeline's processors, + /// and they must not outlive the table locks that `onFinish` / `onException` release. + std::optional executor; }; Pipe getSourcePipe(QueryPipeline & pipeline, const bool use_async_executor) diff --git a/tests/queries/0_stateless/05427_direct_dictionary_read_truncate_source.reference b/tests/queries/0_stateless/05427_direct_dictionary_read_truncate_source.reference new file mode 100644 index 0000000000000..35ed03607e4a7 --- /dev/null +++ b/tests/queries/0_stateless/05427_direct_dictionary_read_truncate_source.reference @@ -0,0 +1,4 @@ +reader started: 1 +truncate: ok +reader alive after truncate: 1 +0 diff --git a/tests/queries/0_stateless/05427_direct_dictionary_read_truncate_source.sh b/tests/queries/0_stateless/05427_direct_dictionary_read_truncate_source.sh new file mode 100755 index 0000000000000..e5100ffe12526 --- /dev/null +++ b/tests/queries/0_stateless/05427_direct_dictionary_read_truncate_source.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, use-rocksdb + +# Reading a DIRECT dictionary as a table runs its source query inside the reading query. The source query +# must release its table locks only after its processors (and the RocksDB iterator) are destroyed, or a +# TRUNCATE of the source table closes the RocksDB database under the iterator. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q " + CREATE TABLE rdb (k UInt64, v String) ENGINE = EmbeddedRocksDB PRIMARY KEY k; + INSERT INTO rdb SELECT number, toString(number) FROM numbers(300); + CREATE DICTIONARY dd (k UInt64, v String) PRIMARY KEY k + SOURCE(CLICKHOUSE(DB '${CLICKHOUSE_DATABASE}' TABLE 'rdb')) LAYOUT(DIRECT()); +" + +query_id="${CLICKHOUSE_DATABASE}_reader" + +# ORDER BY ... LIMIT consumes the whole dictionary before producing a row, so the source query has finished +# before the sleeps start; the reader then runs until it is killed below. +$CLICKHOUSE_CLIENT --query_id "${query_id}" -q " + SELECT sum(sleepEachRow(0.2)) FROM (SELECT k FROM dd ORDER BY k LIMIT 1000) + SETTINGS max_block_size = 1, max_threads = 1, function_sleep_max_microseconds_per_block = 100000000 +" > /dev/null 2>&1 & +reader_pid=$! + +started=0 +for _ in {1..200}; do + if [ "$($CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes WHERE query_id = '${query_id}' AND read_rows > 0")" = "1" ]; then + started=1 + break + fi + sleep 0.1 +done +echo "reader started: ${started}" + +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE rdb SETTINGS lock_acquire_timeout = 10" && echo "truncate: ok" +echo "reader alive after truncate: $($CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes WHERE query_id = '${query_id}'")" + +$CLICKHOUSE_CLIENT -q "KILL QUERY WHERE query_id = '${query_id}' SYNC" > /dev/null 2>&1 +wait "${reader_pid}" 2>/dev/null + +$CLICKHOUSE_CLIENT -q "SELECT count() FROM rdb" From d99fe03507c905142a9021b9b281213d5759a899 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 10 Nov 2026 11:47:51 +0000 Subject: [PATCH 143/219] Backport #120080 to 26.8: Fix privilege check bypass in text index transform expressions --- src/Interpreters/InterpreterAlterQuery.cpp | 91 ++++++++++ src/Interpreters/InterpreterCreateQuery.cpp | 54 ++++++ src/Storages/AlterCommands.cpp | 38 ++++ src/Storages/AlterCommands.h | 3 + src/Storages/IndicesDescription.cpp | 122 ++++++++++++- src/Storages/IndicesDescription.h | 15 ++ ...dd_share_nested_offsets_disabled.reference | 4 + ...ndex_add_share_nested_offsets_disabled.sql | 16 ++ ...x_add_with_invalid_alter_command.reference | 1 + ...t_index_add_with_invalid_alter_command.sql | 24 +++ ...2346_text_index_transform_access.reference | 24 +++ .../02346_text_index_transform_access.sql | 167 ++++++++++++++++++ ...sform_access_replicated_database.reference | 4 + ...x_transform_access_replicated_database.sql | 54 ++++++ ...t_index_transform_alter_segments.reference | 4 + ...346_text_index_transform_alter_segments.sh | 37 ++++ ...text_index_transform_data_access.reference | 6 + ...02346_text_index_transform_data_access.sql | 68 +++++++ ..._text_index_transform_on_cluster.reference | 14 ++ .../02346_text_index_transform_on_cluster.sh | 77 ++++++++ 20 files changed, 817 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/02346_text_index_add_share_nested_offsets_disabled.reference create mode 100644 tests/queries/0_stateless/02346_text_index_add_share_nested_offsets_disabled.sql create mode 100644 tests/queries/0_stateless/02346_text_index_add_with_invalid_alter_command.reference create mode 100644 tests/queries/0_stateless/02346_text_index_add_with_invalid_alter_command.sql create mode 100644 tests/queries/0_stateless/02346_text_index_transform_access.reference create mode 100644 tests/queries/0_stateless/02346_text_index_transform_access.sql create mode 100644 tests/queries/0_stateless/02346_text_index_transform_access_replicated_database.reference create mode 100644 tests/queries/0_stateless/02346_text_index_transform_access_replicated_database.sql create mode 100644 tests/queries/0_stateless/02346_text_index_transform_alter_segments.reference create mode 100755 tests/queries/0_stateless/02346_text_index_transform_alter_segments.sh create mode 100644 tests/queries/0_stateless/02346_text_index_transform_data_access.reference create mode 100644 tests/queries/0_stateless/02346_text_index_transform_data_access.sql create mode 100644 tests/queries/0_stateless/02346_text_index_transform_on_cluster.reference create mode 100755 tests/queries/0_stateless/02346_text_index_transform_on_cluster.sh diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index e68b420f66e3a..74ad636323416 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -17,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -29,11 +30,14 @@ #include #include #include +#include #include #include #include #include #include +#include +#include #include #include #include @@ -72,11 +76,13 @@ namespace Setting extern const SettingsTimezone session_timezone; extern const SettingsUInt64 max_parser_depth; extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 distributed_ddl_entry_format_version; } namespace ServerSetting { extern const ServerSettingsBool disable_insertion_and_mutation; + extern const ServerSettingsBool distributed_ddl_use_initial_user_and_roles; } namespace ErrorCodes @@ -96,6 +102,72 @@ namespace ErrorCodes namespace { +/// A `Replicated`-database host applies the statement without the submitting user. `ON CLUSTER` does +/// not reach this: it is refused unless its hosts run as the initiating user. +void checkAddedIndexPermissions(const ASTAlterQuery & alter, const StoragePtr & table, const ContextPtr & context) +{ + StorageMetadataHandle metadata; + if (table) + metadata = table->getInMemoryMetadataPtr(context, /*bypass_metadata_cache=*/ false); + + bool adds_transform = false; + for (const auto & child : alter.command_list->children) + { + const auto & command = child->as(); + if (command.type != ASTAlterCommand::ADD_INDEX || !command.index_decl) + continue; + + if (const auto * declaration = command.index_decl->as()) + if (hasTextIndexPreOrPostProcessor(*declaration)) + adds_transform = true; + } + + /// Against the schema the whole statement produces: a command of the same statement can change what + /// an alias in a transform expands to. Transforms only, so other statements get no extra analysis. + if (!adds_transform || !metadata) + return; + + AlterCommands commands; + /// Runs of consecutive metadata commands; `runCommandSegments` commits each run on its own. + size_t metadata_segments = 0; + bool previous_is_metadata = false; + for (const auto & child : alter.command_list->children) + { + const auto * command_ast = child->as(); + std::optional command; + if (command_ast->type != ASTAlterCommand::EXECUTE_COMMAND) + command = AlterCommand::parse(command_ast); + + if (command) + { + if (!previous_is_metadata) + ++metadata_segments; + commands.emplace_back(std::move(*command)); + } + previous_is_metadata = command.has_value(); + } + + /// Applying every command at once would reject statements the executor, which commits segment by + /// segment, accepts. `AlterCommands::apply` authorizes each segment as the user anyway. + if (metadata_segments > 1) + return; + + /// The same sequence as `runCommandSegments`: `apply` relies on the checks before it. + commands.validate(table, context); + + bool share_nested = true; + if (auto * merge_tree = castStorage(table, DeferredTable::Load).get()) + share_nested = (*merge_tree->getSettings())[MergeTreeSetting::share_nested_offsets]; + + StorageInMemoryMetadata metadata_copy = *metadata; + commands.prepare(metadata_copy, share_nested); + table->checkAlterIsPossible(commands, context); + /// As the storages do before `apply`: an explicit `ADD STATISTICS` replaces an implicit one. + removeImplicitStatistics(metadata_copy.columns); + commands.apply(metadata_copy, context, share_nested); + +} + using CommandSegment = std::variant; using CommandSegments = std::vector; @@ -521,6 +593,23 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) checkNoRowPolicyForExport( alter, alter.getDatabase().empty() ? getContext()->getCurrentDatabase() : alter.getDatabase(), getContext()); + /// The hosts run the entry without the user, unless `distributed_ddl_use_initial_user_and_roles` is on + /// and the entry format carries the initiator's user, so a text index transform cannot be authorized there. + const bool workers_use_initiator_user + = getContext()->getServerSettings()[ServerSetting::distributed_ddl_use_initial_user_and_roles] + && getContext()->getSettingsRef()[Setting::distributed_ddl_entry_format_version] >= DDLLogEntry::INITIATOR_USER_VERSION; + if (!workers_use_initiator_user) + { + for (const auto & child : alter.command_list->children) + { + const auto & command = child->as(); + if (command.type != ASTAlterCommand::ADD_INDEX || !command.index_decl) + continue; + if (const auto * declaration = command.index_decl->as()) + rejectTextIndexTransformWithoutInitiatorUser(*declaration); + } + } + return executeDDLQueryOnCluster(query_ptr, getContext(), params); } @@ -531,6 +620,8 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database {} does not exist", backQuoteIfNeed(alter.getDatabase())); DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_id.database_name); + checkAddedIndexPermissions(alter, table, getContext()); + if (database->shouldReplicateQuery(getContext(), query_ptr)) { auto guard = DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name, database.get()); diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index afc133008a0d2..2e39a90b830b1 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -48,7 +49,9 @@ #include #include +#include #include +#include #include #include #include @@ -155,6 +158,7 @@ namespace Setting namespace ServerSetting { + extern const ServerSettingsBool distributed_ddl_use_initial_user_and_roles; extern const ServerSettingsBool ignore_empty_sql_security_in_create_view_query; } @@ -1065,6 +1069,14 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti if (!create.columns_list) create.set(create.columns_list, make_intrusive()); + /// Screened for every definition the user supplies now. A replay of stored metadata is not screened, + /// so such a table still attaches. + if (isFreshTableDefinition(mode, create.attach_short_syntax)) + { + for (const auto & index : properties.indices) + checkTextIndexPreAndPostProcessorPermissions(index, getContext()); + } + ASTPtr new_columns = formatColumns(properties.columns); ASTPtr new_indices = formatIndices(properties.indices); ASTPtr new_constraints = formatConstraints(properties.constraints); @@ -3519,6 +3531,19 @@ BlockIO InterpreterCreateQuery::executeQueryOnCluster(ASTCreateQuery & create) prepareOnClusterQuery(create, getContext(), create.cluster); DDLQueryOnClusterParams params; params.access_to_check = getRequiredAccess(); + + /// The hosts run the entry without the user, unless `distributed_ddl_use_initial_user_and_roles` is on + /// and the entry format carries the initiator's user, so a text index transform cannot be authorized there. + const bool workers_use_initiator_user + = getContext()->getServerSettings()[ServerSetting::distributed_ddl_use_initial_user_and_roles] + && getContext()->getSettingsRef()[Setting::distributed_ddl_entry_format_version] >= DDLLogEntry::INITIATOR_USER_VERSION; + if (!workers_use_initiator_user && create.columns_list && create.columns_list->indices) + { + for (const auto & index : create.columns_list->indices->children) + if (const auto * declaration = index->as()) + rejectTextIndexTransformWithoutInitiatorUser(*declaration); + } + return executeDDLQueryOnCluster(query_ptr, getContext(), params); } @@ -3585,6 +3610,35 @@ BlockIO InterpreterCreateQuery::execute() /// the worker inherits the move destination of the source too, and runs with no user checkExportAccess(inherited); + + /// A worker copies the source's indices with no user, so declare their transforms here. + if (inherited.storage && inherited.storage->engine && endsWith(inherited.storage->engine->name, "MergeTree")) + { + /// Before resolving it: an error below would otherwise disclose a source the user + /// cannot see. + getContext()->checkAccess(AccessType::SHOW_COLUMNS, create.as_database, create.as_table); + StoragePtr as_storage = resolveStorageProxyLoading( + DatabaseCatalog::instance().getTable({create.as_database, create.as_table}, getContext())); + if (const auto * alias = as_storage->as(); + alias && !alias->isTargetTableGranted(getContext(), AccessType::SHOW_COLUMNS, {})) + throw Exception(ErrorCodes::ACCESS_DENIED, "Not enough privileges to describe metadata exposed by {}", + StorageID{create.as_database, create.as_table}.getNameForLogs()); + /// The query dispatches `AS src`, so each host materializes its own source, which + /// may define the transform differently from the one authorized here. + const bool workers_use_initiator_user + = getContext()->getServerSettings()[ServerSetting::distributed_ddl_use_initial_user_and_roles] + && on_cluster_version >= DDLLogEntry::INITIATOR_USER_VERSION; + + auto as_storage_metadata = as_storage->getInMemoryMetadataPtr(getContext(), false); + for (const auto & index : as_storage_metadata->getSecondaryIndices()) + { + if (index.isImplicitlyCreated()) + continue; + if (!workers_use_initiator_user) + rejectTextIndexTransformWithoutInitiatorUser(index); + checkTextIndexPreAndPostProcessorPermissions(index, getContext()); + } + } } return executeQueryOnCluster(create); diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 8641a82d04018..6b37e1e372151 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -48,6 +48,7 @@ #include #include #include +#include #include #include #include @@ -1734,6 +1735,33 @@ bool AlterCommands::hasVectorSimilarityIndex(const StorageInMemoryMetadata & met return false; } +NameSet AlterCommands::getAddedIndexNames(const IndicesDescription & existing) const +{ + NameSet present; + for (const auto & index : existing) + present.insert(index.name); + + NameSet added; + for (const AlterCommand & command : *this) + { + if (command.ignore) + continue; + + if (command.type == AlterCommand::ADD_INDEX) + { + /// Otherwise `AlterCommand::apply` either skips it, with `IF NOT EXISTS`, or throws. + if (present.insert(command.index_name).second) + added.insert(command.index_name); + } + else if (command.type == AlterCommand::DROP_INDEX && !command.partition && !command.clear) + { + present.erase(command.index_name); + added.erase(command.index_name); + } + } + return added; +} + void AlterCommands::apply(StorageInMemoryMetadata & metadata, ContextPtr context, bool share_nested_offsets) const { if (!prepared) @@ -1793,6 +1821,16 @@ void AlterCommands::apply(StorageInMemoryMetadata & metadata, ContextPtr context } } + /// Only the declared ones, and only now: the rebuild above resolves them against the schema every + /// command produced, which is what gets stored. Existing indices stay unauthorised on purpose. + const NameSet declared_indices = getAddedIndexNames(metadata.secondary_indices); + + for (const auto & index : metadata_copy.secondary_indices) + { + if (declared_indices.contains(index.name)) + checkTextIndexPreAndPostProcessorPermissions(index, context); + } + /// Changes in columns may lead to changes in projections ProjectionsDescription new_projections; for (const auto & projection : metadata_copy.projections) diff --git a/src/Storages/AlterCommands.h b/src/Storages/AlterCommands.h index 04031c1ee8b59..546cabef6c02c 100644 --- a/src/Storages/AlterCommands.h +++ b/src/Storages/AlterCommands.h @@ -239,6 +239,9 @@ class AlterCommands : public std::vector /// stay consistent with prepare()/validate() for nested columns (see AlterCommand::apply). void apply(StorageInMemoryMetadata & metadata, ContextPtr context, bool share_nested_offsets = true) const; + /// Indices the commands actually add: `ADD INDEX IF NOT EXISTS` of an existing name adds nothing. + NameSet getAddedIndexNames(const IndicesDescription & existing) const; + /// At least one command modify settings or comments. bool hasNonReplicatedAlterCommand() const; diff --git a/src/Storages/IndicesDescription.cpp b/src/Storages/IndicesDescription.cpp index 8b4846aa8463f..e2780da4b1bba 100644 --- a/src/Storages/IndicesDescription.cpp +++ b/src/Storages/IndicesDescription.cpp @@ -1,12 +1,20 @@ +#include #include #include #include +#include #include +#include +#include +#include #include #include #include #include +#include +#include +#include #include #include #include @@ -28,27 +36,129 @@ namespace ErrorCodes namespace { -/// Only the `preprocessor` and `postprocessor` arguments contain column expressions. -void expandTextIndexTransformAliases(const ASTPtr & arguments, const ColumnsDescription & columns) +/// Calls `fn` with the value of each `preprocessor` / `postprocessor` argument of a text index. +template +void forEachTransformArgument(const ASTPtr & arguments, F && fn) { - using ReplaceAliasToExprVisitor = InDepthNodeVisitor; for (const auto & child : arguments->children) { - const auto * func = child->as(); + auto * func = child->as(); if (!func || func->name != "equals" || !func->arguments || func->arguments->children.size() != 2) continue; const auto * key = func->arguments->children[0]->as(); if (key && (key->name() == "preprocessor" || key->name() == "postprocessor")) + fn(func->arguments->children[1]); + } +} + +void expandTextIndexTransformAliases(const ASTPtr & arguments, const ColumnsDescription & columns) +{ + using ReplaceAliasToExprVisitor = InDepthNodeVisitor; + forEachTransformArgument(arguments, [&](ASTPtr & expression) + { + ReplaceAliasToExprVisitor::Data data{columns, {}, /*reject_lambda_capture=*/ true}; + ReplaceAliasToExprVisitor{data}.visit(expression); + }); +} + +/// Resolving a name runs the function's `create`, where `checkAccess` lives. Names only: building the +/// expression would report its type errors from here, not from the validator. +/// +/// A transform is analysed under the global context, so reading a table or a dictionary during analysis +/// would happen with full access. Such access depends on the arguments, not the function name, so +/// authorizing names cannot cover it, and these constructs are refused. Only here, where a user declares +/// the index: metadata stored by an older version must still load and stay alterable. +void checkFunctionsAsUser(const ASTPtr & ast, const ContextPtr & context) +{ + if (ast->as() || ast->as() || ast->as() || ast->as()) + throw Exception(ErrorCodes::INCORRECT_QUERY, "A text index transform expression must not contain subqueries or table references"); + + if (const auto * function = ast->as()) + { + if (functionIsInOrGlobalInOperator(function->name) && function->arguments && function->arguments->children.size() == 2) { - ReplaceAliasToExprVisitor::Data data{columns, {}, /*reject_lambda_capture=*/ true}; - ReplaceAliasToExprVisitor{data}.visit(func->arguments->children[1]); + const auto & set = function->arguments->children[1]; + if (set->as() || set->as()) + throw Exception(ErrorCodes::INCORRECT_QUERY, + "A text index transform expression must not use a table or a set as the right side of `{}`", function->name); } + + /// Only the name, which for `dictGet` and the like does not touch the dictionary; analysis would. + if (auto resolver = FunctionFactory::instance().tryGet(function->name, context); resolver && !resolver->isDeterministic()) + throw Exception(ErrorCodes::INCORRECT_QUERY, + "A text index transform expression must not contain non-deterministic functions. Got `{}`", function->name); } + + for (const auto & child : ast->children) + checkFunctionsAsUser(child, context); +} + +void checkTransformPermissions(const ASTPtr & arguments, const ContextPtr & context) +{ + forEachTransformArgument(arguments, [&](ASTPtr & argument) + { + auto expression = argument->clone(); + /// Inlines SQL UDF bodies, as `TreeRewriter` does, so a function used only inside one is seen. + if (!UserDefinedSQLFunctionFactory::instance().empty()) + UserDefinedSQLFunctionVisitor::visit(expression, context); + + checkFunctionsAsUser(expression, context); + }); } } +void checkTextIndexPreAndPostProcessorPermissions(const IndexDescription & index, const ContextPtr & context) +{ + if (index.type == TEXT_INDEX_NAME && index.arguments) + checkTransformPermissions(index.arguments, context); +} + +bool hasTextIndexPreOrPostProcessor(const IndexDescription & index) +{ + if (index.type != TEXT_INDEX_NAME || !index.arguments) + return false; + + bool found = false; + forEachTransformArgument(index.arguments, [&](ASTPtr &) { found = true; }); + return found; +} + +[[noreturn]] static void throwTransformNeedsInitiatorUser() +{ + throw Exception( + ErrorCodes::INCORRECT_QUERY, + "A text index preprocessor or postprocessor is authorized on the host the query is submitted to, and the " + "other hosts resolve it against their own schema without that user. Let every host run the query as the " + "initiating user instead: enable the server setting distributed_ddl_use_initial_user_and_roles on every " + "host, and set distributed_ddl_entry_format_version to at least {}", + DDLLogEntry::INITIATOR_USER_VERSION); +} + +void rejectTextIndexTransformWithoutInitiatorUser(const ASTIndexDeclaration & declaration) +{ + if (hasTextIndexPreOrPostProcessor(declaration)) + throwTransformNeedsInitiatorUser(); +} + +void rejectTextIndexTransformWithoutInitiatorUser(const IndexDescription & index) +{ + if (hasTextIndexPreOrPostProcessor(index)) + throwTransformNeedsInitiatorUser(); +} + +bool hasTextIndexPreOrPostProcessor(const ASTIndexDeclaration & declaration) +{ + const auto * type = declaration.getType() ? declaration.getType()->as() : nullptr; + if (!type || Poco::toLower(type->name) != TEXT_INDEX_NAME || !type->arguments) + return false; + + bool found = false; + forEachTransformArgument(type->arguments, [&](ASTPtr &) { found = true; }); + return found; +} + IndexDescription::IndexDescription(const IndexDescription & other) : definition_ast(other.definition_ast ? other.definition_ast->clone() : nullptr) , expression_list_ast(other.expression_list_ast ? other.expression_list_ast->clone() : nullptr) diff --git a/src/Storages/IndicesDescription.h b/src/Storages/IndicesDescription.h index 5c3cf33b3191e..f53855ae21717 100644 --- a/src/Storages/IndicesDescription.h +++ b/src/Storages/IndicesDescription.h @@ -16,6 +16,7 @@ namespace DB class ExpressionActions; using ExpressionActionsPtr = std::shared_ptr; +class ASTIndexDeclaration; /// Description of non-primary index for Storage struct IndexDescription @@ -86,6 +87,20 @@ struct IndexDescription bool isSimpleSingleColumnIndex() const; }; +/// Checks that the functions referenced by the text index preprocessor and postprocessor expressions +/// may run with the permissions of `context`, and that they read no table, set or dictionary. Call only +/// where a user declares an index, on the node they submitted it to; elsewhere these resolve under the +/// global context, and stored metadata is not re-checked. +void checkTextIndexPreAndPostProcessorPermissions(const IndexDescription & index, const ContextPtr & context); + +/// Refuses a transform when the other hosts of an `ON CLUSTER` query do not run it as the initiator. +void rejectTextIndexTransformWithoutInitiatorUser(const ASTIndexDeclaration & declaration); +void rejectTextIndexTransformWithoutInitiatorUser(const IndexDescription & index); + +/// Whether the declaration is a text index with a `preprocessor` or a `postprocessor` argument. +bool hasTextIndexPreOrPostProcessor(const ASTIndexDeclaration & declaration); +bool hasTextIndexPreOrPostProcessor(const IndexDescription & index); + /// All secondary indices in storage struct IndicesDescription : public std::vector, IHints<> { diff --git a/tests/queries/0_stateless/02346_text_index_add_share_nested_offsets_disabled.reference b/tests/queries/0_stateless/02346_text_index_add_share_nested_offsets_disabled.reference new file mode 100644 index 0000000000000..7dd2827664547 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_add_share_nested_offsets_disabled.reference @@ -0,0 +1,4 @@ +id +n +n.a +idx (n) diff --git a/tests/queries/0_stateless/02346_text_index_add_share_nested_offsets_disabled.sql b/tests/queries/0_stateless/02346_text_index_add_share_nested_offsets_disabled.sql new file mode 100644 index 0000000000000..4dff81df89552 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_add_share_nested_offsets_disabled.sql @@ -0,0 +1,16 @@ +-- The `ADD INDEX` permission preflight applies the commands as the table does, so with +-- `share_nested_offsets = 0` a column `n` is independent of `n.a` and is added before the index uses it. + +DROP TABLE IF EXISTS t; + +CREATE TABLE t (id UInt64, `n.a` Array(UInt32)) +ENGINE = MergeTree ORDER BY id SETTINGS share_nested_offsets = 0; + +ALTER TABLE t + ADD COLUMN IF NOT EXISTS n UInt32, + ADD INDEX idx(n) TYPE minmax GRANULARITY 1; + +SELECT name FROM system.columns WHERE database = currentDatabase() AND table = 't' ORDER BY name; +SELECT name, expr FROM system.data_skipping_indices WHERE database = currentDatabase() AND table = 't'; + +DROP TABLE t; diff --git a/tests/queries/0_stateless/02346_text_index_add_with_invalid_alter_command.reference b/tests/queries/0_stateless/02346_text_index_add_with_invalid_alter_command.reference new file mode 100644 index 0000000000000..0f9d2713f87d5 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_add_with_invalid_alter_command.reference @@ -0,0 +1 @@ +idx text diff --git a/tests/queries/0_stateless/02346_text_index_add_with_invalid_alter_command.sql b/tests/queries/0_stateless/02346_text_index_add_with_invalid_alter_command.sql new file mode 100644 index 0000000000000..a629b362ee8eb --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_add_with_invalid_alter_command.sql @@ -0,0 +1,24 @@ +-- An `ALTER` that adds an index is authorized against the schema the whole statement produces. +-- That must not skip the validation of its other commands and run into an exception. + +SET enable_full_text_index = 1; + +DROP TABLE IF EXISTS tab; +CREATE TABLE tab (id UInt64, val String) ENGINE = MergeTree ORDER BY id; + +ALTER TABLE tab + RENAME COLUMN missing TO renamed, + ADD INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1; -- { serverError NOT_FOUND_COLUMN_IN_BLOCK } + +ALTER TABLE tab + RENAME COLUMN val TO a, + RENAME COLUMN a TO b, + ADD INDEX idx(id) TYPE minmax GRANULARITY 1; -- { serverError NOT_IMPLEMENTED } + +ALTER TABLE tab + RENAME COLUMN IF EXISTS missing TO renamed, + ADD INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = lower(val)) GRANULARITY 1; + +SELECT name, type FROM system.data_skipping_indices WHERE database = currentDatabase() AND table = 'tab'; + +DROP TABLE tab; diff --git a/tests/queries/0_stateless/02346_text_index_transform_access.reference b/tests/queries/0_stateless/02346_text_index_transform_access.reference new file mode 100644 index 0000000000000..3da5718983273 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_transform_access.reference @@ -0,0 +1,24 @@ +1. Direct call denied. +2. Denied in a preprocessor. +3. Denied in a postprocessor. +4. Denied when nested under a String result. +5. Denied via ALTER ADD INDEX. +6. Unprivileged expression unaffected. +1 +7. Accepted with the privilege, so not a blocklist. +1 +8. Existing index stays readable without the privilege. +1 +9. Unrelated ALTER does not re-authorize. +1 +10. Neither does DETACH/ATTACH. +1 +11. An ALIAS column hiding a privileged function is denied. +12. So is an alias renamed by an earlier command of the same ALTER. +13. Redeclaring with a privileged function is denied. +14. The transform preflight does not reject what a plain ALTER accepts. +idx_pre +15. Replacing a privileged alias in the same statement is allowed. +idx_replaced +16. A statement without a transform is not analyzed at all. +idx_minmax diff --git a/tests/queries/0_stateless/02346_text_index_transform_access.sql b/tests/queries/0_stateless/02346_text_index_transform_access.sql new file mode 100644 index 0000000000000..fb873cb6ebebc --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_transform_access.sql @@ -0,0 +1,167 @@ +-- Text index `preprocessor`/`postprocessor` expressions must only use functions the user declaring +-- the index is permitted to run. +-- +-- The test needs a function the current user may not run. Introspection functions fit: they are +-- refused both without the grant and, as here, with `allow_introspection_functions = 0`. Using the +-- setting keeps the test in plain SQL, with no second user to create. + +DROP TABLE IF EXISTS tab; + +-- The test configuration sets this to 1. +SET allow_introspection_functions = 0; + +SELECT '1. Direct call denied.'; + +SELECT demangle('_ZNK2DB7Context9getAccessEv'); -- { serverError FUNCTION_NOT_ALLOWED } + +SELECT '2. Denied in a preprocessor.'; + +CREATE TABLE tab +( + id UInt64, + val String, + INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = demangle(val)) +) +ENGINE = MergeTree ORDER BY id; -- { serverError FUNCTION_NOT_ALLOWED } + +SELECT '3. Denied in a postprocessor.'; + +CREATE TABLE tab +( + id UInt64, + val String, + INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', postprocessor = demangle(val)) +) +ENGINE = MergeTree ORDER BY id; -- { serverError FUNCTION_NOT_ALLOWED } + +SELECT '4. Denied when nested under a String result.'; + +CREATE TABLE tab +( + id UInt64, + val String, + INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = concat(val, demangle(val))) +) +ENGINE = MergeTree ORDER BY id; -- { serverError FUNCTION_NOT_ALLOWED } + +SELECT '5. Denied via ALTER ADD INDEX.'; + +CREATE TABLE tab (id UInt64, val String) ENGINE = MergeTree ORDER BY id; + +ALTER TABLE tab ADD INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = demangle(val)); -- { serverError FUNCTION_NOT_ALLOWED } + +DROP TABLE tab; + +SELECT '6. Unprivileged expression unaffected.'; + +CREATE TABLE tab +( + id UInt64, + val String, + INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = lower(val)) +) +ENGINE = MergeTree ORDER BY id; + +INSERT INTO tab VALUES (1, 'Hello World'); +SELECT count() FROM tab WHERE hasAllTokens(val, ['hello']); + +DROP TABLE tab; + +SELECT '7. Accepted with the privilege, so not a blocklist.'; + +SET allow_introspection_functions = 1; + +-- `demangle` returns a non-mangled argument unchanged. +CREATE TABLE tab +( + id UInt64, + val String, + INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', postprocessor = demangle(val)) +) +ENGINE = MergeTree ORDER BY id; + +INSERT INTO tab VALUES (1, 'hello world'); +SELECT count() FROM tab WHERE hasAllTokens(val, ['hello']); + +SELECT '8. Existing index stays readable without the privilege.'; + +SET allow_introspection_functions = 0; + +SELECT count() FROM tab WHERE hasAllTokens(val, ['hello']); + +SELECT '9. Unrelated ALTER does not re-authorize.'; + +ALTER TABLE tab ADD COLUMN extra UInt8 DEFAULT 0; +ALTER TABLE tab RENAME COLUMN val TO val2; +SELECT count() FROM tab WHERE hasAllTokens(val2, ['hello']); + +SELECT '10. Neither does DETACH/ATTACH.'; + +DETACH TABLE tab; +ATTACH TABLE tab; +SELECT count() FROM tab WHERE hasAllTokens(val2, ['hello']); + +SELECT '11. An ALIAS column hiding a privileged function is denied.'; + +-- Its own table: a column may carry only one text index, and that is checked before permissions are. +SET allow_introspection_functions = 1; +CREATE TABLE tab_alias (id UInt64, val String, mangled String ALIAS demangle(val)) ENGINE = MergeTree ORDER BY id; +SET allow_introspection_functions = 0; + +ALTER TABLE tab_alias ADD INDEX idx_alias(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = mangled) GRANULARITY 1; -- { serverError FUNCTION_NOT_ALLOWED } + +SELECT '12. So is an alias renamed by an earlier command of the same ALTER.'; + +-- The rename is applied before the index is added, so `renamed` is what expands to `demangle(val)`. +ALTER TABLE tab_alias RENAME COLUMN mangled TO renamed, + ADD INDEX idx_renamed(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = renamed) GRANULARITY 1; -- { serverError FUNCTION_NOT_ALLOWED } + +DROP TABLE tab_alias; + +SELECT '13. Redeclaring with a privileged function is denied.'; + +ALTER TABLE tab DROP INDEX idx, + ADD INDEX idx(val2) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = demangle(val2)) GRANULARITY 1; -- { serverError FUNCTION_NOT_ALLOWED } + +DROP TABLE tab; + +SELECT '14. The transform preflight does not reject what a plain ALTER accepts.'; + +CREATE TABLE tab_preflight (id UInt64, val String, num UInt64) +ENGINE = MergeTree ORDER BY id SETTINGS auto_statistics_types = 'minmax'; + +-- The preflight must drop implicit statistics too, or `ADD STATISTICS` looks like a duplicate. +ALTER TABLE tab_preflight ADD INDEX idx_pre(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = lower(val)) GRANULARITY 1, + ADD STATISTICS num TYPE minmax; + +SELECT name FROM system.data_skipping_indices +WHERE database = currentDatabase() AND table = 'tab_preflight' ORDER BY name; + +DROP TABLE tab_preflight; + +SELECT '15. Replacing a privileged alias in the same statement is allowed.'; + +SET allow_introspection_functions = 1; +CREATE TABLE tab_replace (id UInt64, val String, transformed String ALIAS demangle(val)) +ENGINE = MergeTree ORDER BY id; +SET allow_introspection_functions = 0; + +-- The installed transform is `lower(val)`; the alias it replaces must not be demanded. +ALTER TABLE tab_replace MODIFY COLUMN transformed String ALIAS lower(val), + ADD INDEX idx_replaced(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = transformed) GRANULARITY 1; + +SELECT name FROM system.data_skipping_indices +WHERE database = currentDatabase() AND table = 'tab_replace' ORDER BY name; + +DROP TABLE tab_replace; + +SELECT '16. A statement without a transform is not analyzed at all.'; + +SET allow_statistics = 1; +CREATE TABLE tab_plain (x UInt64) ENGINE = MergeTree ORDER BY x SETTINGS auto_statistics_types = 'basic'; + +ALTER TABLE tab_plain ADD INDEX idx_minmax(x) TYPE minmax GRANULARITY 1, ADD STATISTICS x TYPE basic; + +SELECT name FROM system.data_skipping_indices WHERE database = currentDatabase() AND table = 'tab_plain'; + +DROP TABLE tab_plain; diff --git a/tests/queries/0_stateless/02346_text_index_transform_access_replicated_database.reference b/tests/queries/0_stateless/02346_text_index_transform_access_replicated_database.reference new file mode 100644 index 0000000000000..3a9622171b277 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_transform_access_replicated_database.reference @@ -0,0 +1,4 @@ +1. ALTER ADD INDEX is denied. +2. CREATE TABLE is denied. +3. An alias renamed by an earlier command of the same ALTER is denied. +4. Accepted with the privilege. diff --git a/tests/queries/0_stateless/02346_text_index_transform_access_replicated_database.sql b/tests/queries/0_stateless/02346_text_index_transform_access_replicated_database.sql new file mode 100644 index 0000000000000..492afc0a12edd --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_transform_access_replicated_database.sql @@ -0,0 +1,54 @@ +-- Tags: zookeeper, no-replicated-database +-- no-replicated-database: the test creates its own Replicated database + +-- A Replicated database hands the DDL to replicas that apply it without the submitting user, so the +-- permission check happens on the node it was submitted to. See 02346_text_index_transform_access +-- for the checks themselves, and 02346_text_index_transform_on_cluster for `ON CLUSTER`. + +SET allow_introspection_functions = 0; + +-- {CLICKHOUSE_DATABASE_2} is shared by every test in the client, hence the DROP. +DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE_2:Identifier}; +CREATE DATABASE {CLICKHOUSE_DATABASE_2:Identifier} +ENGINE = Replicated('/clickhouse/02346_text_index_transform_access/{database}', 'shard1', 'replica1') FORMAT Null; + +CREATE TABLE {CLICKHOUSE_DATABASE_2:Identifier}.tab (id UInt64, val String) +ENGINE = MergeTree ORDER BY id FORMAT Null; + +SELECT '1. ALTER ADD INDEX is denied.'; + +ALTER TABLE {CLICKHOUSE_DATABASE_2:Identifier}.tab + ADD INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = demangle(val)) GRANULARITY 1; -- { serverError FUNCTION_NOT_ALLOWED } + +SELECT '2. CREATE TABLE is denied.'; + +CREATE TABLE {CLICKHOUSE_DATABASE_2:Identifier}.tab_denied +( + id UInt64, + val String, + INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = demangle(val)) +) +ENGINE = MergeTree ORDER BY id; -- { serverError FUNCTION_NOT_ALLOWED } + +SELECT '3. An alias renamed by an earlier command of the same ALTER is denied.'; + +-- The initiator has to authorize against the schema the whole statement produces: the hosts run it +-- without the user, and only after the rename does `renamed` expand to `demangle(val)`. +SET allow_introspection_functions = 1; +CREATE TABLE {CLICKHOUSE_DATABASE_2:Identifier}.tab_alias (id UInt64, val String, mangled String ALIAS demangle(val)) +ENGINE = MergeTree ORDER BY id FORMAT Null; +CREATE TABLE tab_on_cluster_alias (id UInt64, val String, mangled String ALIAS demangle(val)) ENGINE = MergeTree ORDER BY id; +SET allow_introspection_functions = 0; + +ALTER TABLE {CLICKHOUSE_DATABASE_2:Identifier}.tab_alias + RENAME COLUMN mangled TO renamed, + ADD INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = renamed) GRANULARITY 1; -- { serverError FUNCTION_NOT_ALLOWED } + +SELECT '4. Accepted with the privilege.'; + +SET allow_introspection_functions = 1; + +ALTER TABLE {CLICKHOUSE_DATABASE_2:Identifier}.tab + ADD INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = demangle(val)) GRANULARITY 1 FORMAT Null; + +DROP DATABASE {CLICKHOUSE_DATABASE_2:Identifier} FORMAT Null; diff --git a/tests/queries/0_stateless/02346_text_index_transform_alter_segments.reference b/tests/queries/0_stateless/02346_text_index_transform_alter_segments.reference new file mode 100644 index 0000000000000..78c5393ccd90a --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_transform_alter_segments.reference @@ -0,0 +1,4 @@ +1. An ordinary index added after a mutation segment, with a rename in each metadata segment. +idx +2. The same with a text index transform, on the local host. +idx_text diff --git a/tests/queries/0_stateless/02346_text_index_transform_alter_segments.sh b/tests/queries/0_stateless/02346_text_index_transform_alter_segments.sh new file mode 100755 index 0000000000000..9039ed2f53914 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_transform_alter_segments.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Tags: no-replicated-database +# Tag no-replicated-database: a single `ALTER` of several types is prohibited there, which is exactly what this test needs. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The text index transform authorization of an `ALTER` must not reject a statement the executor accepts. + +echo '1. An ordinary index added after a mutation segment, with a rename in each metadata segment.' +$CLICKHOUSE_CLIENT -n -q " + CREATE TABLE seg (id UInt64, a String, m UInt64 MATERIALIZED 1) ENGINE = MergeTree ORDER BY id; + ALTER TABLE seg + RENAME COLUMN a TO b, + MATERIALIZE COLUMN m, + RENAME COLUMN b TO c, + ADD INDEX idx(c) TYPE minmax GRANULARITY 1; + SELECT name FROM system.data_skipping_indices WHERE database = currentDatabase() AND table = 'seg'; +" + +echo '2. The same with a text index transform, on the local host.' +$CLICKHOUSE_CLIENT -n -q " + SET enable_full_text_index = 1; + CREATE TABLE seg_text (id UInt64, a String, m UInt64 MATERIALIZED 1) ENGINE = MergeTree ORDER BY id; + ALTER TABLE seg_text + RENAME COLUMN a TO b, + MATERIALIZE COLUMN m, + RENAME COLUMN b TO c, + ADD INDEX idx_text(c) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = lower(c)); + SELECT name FROM system.data_skipping_indices WHERE database = currentDatabase() AND table = 'seg_text'; +" + +$CLICKHOUSE_CLIENT -n -q " + DROP TABLE seg; + DROP TABLE seg_text; +" diff --git a/tests/queries/0_stateless/02346_text_index_transform_data_access.reference b/tests/queries/0_stateless/02346_text_index_transform_data_access.reference new file mode 100644 index 0000000000000..3954d852fafaa --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_transform_data_access.reference @@ -0,0 +1,6 @@ +1. A set from a table. +2. A subquery. +3. A dictionary lookup. +4. The same via ALTER ADD INDEX. +5. A literal set is still accepted. +1 diff --git a/tests/queries/0_stateless/02346_text_index_transform_data_access.sql b/tests/queries/0_stateless/02346_text_index_transform_data_access.sql new file mode 100644 index 0000000000000..839daa8bca690 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_transform_data_access.sql @@ -0,0 +1,68 @@ +-- Text index transforms are analysed under the global context, so anything that reads a table or a +-- dictionary while being analysed would do it with full access, and constant folding could reveal +-- what it read. A transform must be a pure function of the indexed value, so these are rejected +-- before any analysis. See 02346_text_index_transform_access for the function permission checks. + +DROP TABLE IF EXISTS tab; +DROP TABLE IF EXISTS set_source; +DROP DICTIONARY IF EXISTS dict; +DROP TABLE IF EXISTS dict_source; + +CREATE TABLE set_source (x String) ENGINE = Set; +INSERT INTO set_source VALUES ('secret'); + +CREATE TABLE dict_source (id UInt64, value String) ENGINE = MergeTree ORDER BY id; +INSERT INTO dict_source VALUES (1, 'secret'); +CREATE DICTIONARY dict (id UInt64, value String) PRIMARY KEY id +SOURCE(CLICKHOUSE(TABLE 'dict_source')) LAYOUT(FLAT()) LIFETIME(0); + +SELECT '1. A set from a table.'; + +CREATE TABLE tab +( + id UInt64, + val String, + INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', postprocessor = if(val IN set_source, val, '')) +) +ENGINE = MergeTree ORDER BY id; -- { serverError INCORRECT_QUERY } + +SELECT '2. A subquery.'; + +CREATE TABLE tab +( + id UInt64, + val String, + INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = concat(val, (SELECT any(value) FROM dict_source))) +) +ENGINE = MergeTree ORDER BY id; -- { serverError INCORRECT_QUERY } + +SELECT '3. A dictionary lookup.'; + +CREATE TABLE tab +( + id UInt64, + val String, + INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', + preprocessor = concat(val, toString(toUInt64(dictGetString(currentDatabase() || '.dict', 'value', toUInt64(1)))))) +) +ENGINE = MergeTree ORDER BY id; -- { serverError INCORRECT_QUERY } + +SELECT '4. The same via ALTER ADD INDEX.'; + +CREATE TABLE tab (id UInt64, val String) ENGINE = MergeTree ORDER BY id; + +ALTER TABLE tab ADD INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', postprocessor = if(val IN set_source, val, '')); -- { serverError INCORRECT_QUERY } +ALTER TABLE tab ADD INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', postprocessor = if(val IN (SELECT x FROM set_source), val, '')); -- { serverError INCORRECT_QUERY } +ALTER TABLE tab ADD INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', + preprocessor = concat(val, toString(toUInt64(dictGetString(currentDatabase() || '.dict', 'value', toUInt64(1)))))); -- { serverError INCORRECT_QUERY } + +SELECT '5. A literal set is still accepted.'; + +ALTER TABLE tab ADD INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', postprocessor = if(val IN ('hello', 'world'), val, '')); +INSERT INTO tab VALUES (1, 'hello there'); +SELECT count() FROM tab WHERE hasAllTokens(val, ['hello']); + +DROP TABLE tab; +DROP DICTIONARY dict; +DROP TABLE dict_source; +DROP TABLE set_source; diff --git a/tests/queries/0_stateless/02346_text_index_transform_on_cluster.reference b/tests/queries/0_stateless/02346_text_index_transform_on_cluster.reference new file mode 100644 index 0000000000000..91fa6db492144 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_transform_on_cluster.reference @@ -0,0 +1,14 @@ +1. ALTER ADD INDEX with a transform is refused. +1 +2. CREATE with a transform is refused. +1 +3. Refused on the legacy entry-format path too, which normalizes on the host. +1 +4. Indices inherited by CREATE AS on the legacy path are refused. +1 +5. An index without a transform is unaffected. +idx_plain +6. CREATE AS with an explicit engine and no privilege on the source. +SHOW COLUMNS ON default.tab +7. The same for a source that does not exist. +SHOW COLUMNS ON default.missing diff --git a/tests/queries/0_stateless/02346_text_index_transform_on_cluster.sh b/tests/queries/0_stateless/02346_text_index_transform_on_cluster.sh new file mode 100755 index 0000000000000..15b49c8bf03b8 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_transform_on_cluster.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Tags: zookeeper, no-replicated-database +# no-replicated-database: ON CLUSTER is not allowed for a Replicated database + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The hosts of an `ON CLUSTER` query resolve a text index transform against their own schema and +# without the submitting user, so it is refused unless they run as that user. + +USER="user_${CLICKHOUSE_DATABASE}" + +$CLICKHOUSE_CLIENT -n -q " + DROP USER IF EXISTS $USER; + CREATE TABLE tab (id UInt64, val String, lowered String ALIAS lower(val)) ENGINE = MergeTree ORDER BY id; +" + +echo '1. ALTER ADD INDEX with a transform is refused.' +$CLICKHOUSE_CLIENT --distributed_ddl_output_mode none -q " + ALTER TABLE tab ON CLUSTER test_shard_localhost + ADD INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = lowered) +" 2>&1 | grep -c -m1 "distributed_ddl_use_initial_user_and_roles" + +echo '2. CREATE with a transform is refused.' +$CLICKHOUSE_CLIENT --distributed_ddl_output_mode none -q " + CREATE TABLE ${CLICKHOUSE_DATABASE}.dst_transform ON CLUSTER test_shard_localhost + (id UInt64, val String, INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = lower(val))) + ENGINE = MergeTree ORDER BY id +" 2>&1 | grep -c -m1 "distributed_ddl_use_initial_user_and_roles" + +echo '3. Refused on the legacy entry-format path too, which normalizes on the host.' +$CLICKHOUSE_CLIENT --distributed_ddl_entry_format_version 2 --distributed_ddl_output_mode none -q " + CREATE TABLE ${CLICKHOUSE_DATABASE}.dst_legacy ON CLUSTER test_shard_localhost + (id UInt64, val String, INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = lower(val))) + ENGINE = MergeTree ORDER BY id +" 2>&1 | grep -c -m1 "distributed_ddl_use_initial_user_and_roles" + +echo '4. Indices inherited by CREATE AS on the legacy path are refused.' +$CLICKHOUSE_CLIENT --allow_introspection_functions 1 -q " + CREATE TABLE ${CLICKHOUSE_DATABASE}.src_transform (id UInt64, val String, + INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = demangle(val))) ENGINE = MergeTree ORDER BY id" +$CLICKHOUSE_CLIENT --allow_introspection_functions 1 --distributed_ddl_entry_format_version 2 --distributed_ddl_output_mode none -q " + CREATE TABLE ${CLICKHOUSE_DATABASE}.dst_inherited ON CLUSTER test_shard_localhost + AS ${CLICKHOUSE_DATABASE}.src_transform ENGINE = MergeTree ORDER BY id +" 2>&1 | grep -c -m1 "distributed_ddl_use_initial_user_and_roles" + +echo '5. An index without a transform is unaffected.' +$CLICKHOUSE_CLIENT --distributed_ddl_output_mode none -q " + ALTER TABLE tab ON CLUSTER test_shard_localhost + ADD INDEX idx_plain(val) TYPE text(tokenizer = 'splitByNonAlpha') +" +$CLICKHOUSE_CLIENT -q "SELECT name FROM system.data_skipping_indices WHERE database = currentDatabase() AND table = 'tab' ORDER BY name" + +# With an explicit engine `setEngine` does not look at `AS src`, and the inherited indices are inspected +# on the initiator, so describing the source has to be authorized first. +$CLICKHOUSE_CLIENT -n -q " + CREATE USER $USER; + GRANT CREATE TABLE ON ${CLICKHOUSE_DATABASE}.dst TO $USER; + GRANT TABLE ENGINE ON MergeTree TO $USER; +" + +echo '6. CREATE AS with an explicit engine and no privilege on the source.' +$CLICKHOUSE_CLIENT --user "$USER" --distributed_ddl_entry_format_version 2 --distributed_ddl_output_mode none -q " + CREATE TABLE ${CLICKHOUSE_DATABASE}.dst ON CLUSTER test_shard_localhost AS ${CLICKHOUSE_DATABASE}.tab ENGINE = MergeTree ORDER BY tuple() +" 2>&1 | grep -o -m1 "SHOW COLUMNS ON ${CLICKHOUSE_DATABASE}.tab" + +echo '7. The same for a source that does not exist.' +$CLICKHOUSE_CLIENT --user "$USER" --distributed_ddl_entry_format_version 2 --distributed_ddl_output_mode none -q " + CREATE TABLE ${CLICKHOUSE_DATABASE}.dst ON CLUSTER test_shard_localhost AS ${CLICKHOUSE_DATABASE}.missing ENGINE = MergeTree ORDER BY tuple() +" 2>&1 | grep -o -m1 "SHOW COLUMNS ON ${CLICKHOUSE_DATABASE}.missing" + +$CLICKHOUSE_CLIENT -n -q " + DROP USER $USER; + DROP TABLE tab; + DROP TABLE src_transform; +" From 18eb8e822fe4b6e642db94b39399e4d76b0d927c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 10 Nov 2026 13:24:29 +0000 Subject: [PATCH 144/219] Backport #130389 to 26.8: Update grpc to 1.85.1 --- contrib/grpc | 2 +- contrib/grpc-cmake/grpc.cmake | 119 ++++++++++++++++++++-------------- 2 files changed, 71 insertions(+), 50 deletions(-) diff --git a/contrib/grpc b/contrib/grpc index 68b2fe0b2b131..f205cf1fc40ff 160000 --- a/contrib/grpc +++ b/contrib/grpc @@ -1 +1 @@ -Subproject commit 68b2fe0b2b1318f18aadee9027e86e8ceaa20af2 +Subproject commit f205cf1fc40ffe44a9517529ec0cf32112c99f0d diff --git a/contrib/grpc-cmake/grpc.cmake b/contrib/grpc-cmake/grpc.cmake index 419802aeb70b2..8393bacbb8d7a 100644 --- a/contrib/grpc-cmake/grpc.cmake +++ b/contrib/grpc-cmake/grpc.cmake @@ -180,6 +180,7 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/call/call_spine.cc ${_gRPC_SOURCE_DIR}/src/core/call/call_state.cc ${_gRPC_SOURCE_DIR}/src/core/call/client_call.cc + ${_gRPC_SOURCE_DIR}/src/core/call/evaluate_args.cc ${_gRPC_SOURCE_DIR}/src/core/call/interception_chain.cc ${_gRPC_SOURCE_DIR}/src/core/call/message.cc ${_gRPC_SOURCE_DIR}/src/core/call/metadata.cc @@ -217,6 +218,7 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/client_channel/retry_service_config.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/retry_throttle.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel.cc + ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel_metrics.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel_pool_interface.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel_stream_client.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel_stream_limiter.cc @@ -231,6 +233,7 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/credentials/call/external/file_external_account_credentials.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/call/external/url_external_account_credentials.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/call/gcp_service_account_identity/gcp_service_account_identity_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/credentials/call/gdch_service_account/gdch_service_account_credentials.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/call/iam/iam_credentials.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/call/json_util.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/call/jwt/json_token.cc @@ -301,7 +304,6 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/chttp2_plugin.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/chttp2_connector.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/server/chttp2_server.cc - ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/bin_decoder.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/bin_encoder.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/call_tracer_wrapper.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -334,6 +336,7 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_callbacks.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_promise.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_rate_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/reclaimer.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/stream_lists.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/transport_common.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/varint.cc @@ -346,10 +349,20 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/cel/expr/syntax.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/config/common/mutation_rules/v3/mutation_rules.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/config/core/v3/cel.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/extensions/filters/http/ext_proc/v3/ext_proc.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/extensions/filters/http/ext_proc/v3/processing_mode.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/extensions/grpc_service/call_credentials/access_token/v3/access_token_credentials.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/extensions/grpc_service/channel_credentials/tls/v3/tls_credentials.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/extensions/grpc_service/channel_credentials/xds/v3/xds_credentials.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/channelz/channelz.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/service/ext_proc/v3/external_processor.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/channelz/v1/channelz.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/gcp/altscontext.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/gcp/handshaker.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/gcp/transport_security_common.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/health/v1/health.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/lb/v1/load_balancer.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/lookup/v1/rls.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/lookup/v1/rls_config.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/channelz/v2/channelz.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/channelz/v2/service.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/cel/expr/checked.upbdefs.c @@ -485,20 +498,12 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/api/http.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/api/httpbody.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/any.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/duration.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/empty.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/struct.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/timestamp.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/wrappers.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/rpc/status.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/gcp/altscontext.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/gcp/handshaker.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/gcp/transport_security_common.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/health/v1/health.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/lb/v1/load_balancer.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/lookup/v1/rls.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/lookup/v1/rls_config.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/udpa/annotations/migrate.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/udpa/annotations/security.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/udpa/annotations/sensitive.upb_minitable.c @@ -598,6 +603,8 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/envoy/data/accesslog/v3/accesslog.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/envoy/extensions/clusters/aggregate/v3/cluster.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/envoy/extensions/filters/common/fault/v3/fault.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/ext_proc/v3/ext_proc.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/ext_proc/v3/processing_mode.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/fault/v3/fault.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/gcp_authn/v3/gcp_authn.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/rbac/v3/rbac.upbdefs.c @@ -654,12 +661,12 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/google/protobuf/timestamp.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/google/protobuf/wrappers.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/google/rpc/status.upbdefs.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/src/proto/grpc/channelz/channelz.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/grpc/channelz/v1/channelz.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/grpc/lookup/v1/rls_config.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/src/proto/grpc/channelz/v2/channelz.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/src/proto/grpc/channelz/v2/promise.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/src/proto/grpc/channelz/v2/property_list.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/src/proto/grpc/channelz/v2/service.upbdefs.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/src/proto/grpc/lookup/v1/rls_config.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/udpa/annotations/migrate.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/udpa/annotations/security.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/udpa/annotations/sensitive.upbdefs.c @@ -693,6 +700,8 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/filter/auth/client_auth_filter.cc ${_gRPC_SOURCE_DIR}/src/core/filter/auth/server_auth_filter.cc ${_gRPC_SOURCE_DIR}/src/core/filter/composite/composite_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/filter/ext_proc/ext_proc_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/filter/ext_proc/ext_proc_messages.cc ${_gRPC_SOURCE_DIR}/src/core/filter/fused_filters.cc ${_gRPC_SOURCE_DIR}/src/core/handshaker/http_connect/http_connect_client_handshaker.cc ${_gRPC_SOURCE_DIR}/src/core/handshaker/security/pipelined_secure_endpoint.cc @@ -708,13 +717,18 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/telemetry.cc ${_gRPC_SOURCE_DIR}/src/core/lib/resource_tracker/resource_tracker.cc ${_gRPC_SOURCE_DIR}/src/core/lib/transport/promise_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/load_balancing/autosharding/autosharding.cc ${_gRPC_SOURCE_DIR}/src/core/net/socket_mutator.cc + ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/dns_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/service_config_helper.cc ${_gRPC_SOURCE_DIR}/src/core/server/xds_server_config_fetcher_legacy.cc ${_gRPC_SOURCE_DIR}/src/core/telemetry/context_list_entry.cc ${_gRPC_SOURCE_DIR}/src/core/telemetry/instrument.cc ${_gRPC_SOURCE_DIR}/src/core/transport/message_size_service_config.cc ${_gRPC_SOURCE_DIR}/src/core/transport/session_endpoint.cc ${_gRPC_SOURCE_DIR}/src/core/tsi/ssl_telemetry_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/tls_telemetry.cc + ${_gRPC_SOURCE_DIR}/src/core/util/address_sorting_init.cc ${_gRPC_SOURCE_DIR}/src/core/util/grpc_check.cc ${_gRPC_SOURCE_DIR}/src/core/util/postmortem_emit.cc ${_gRPC_SOURCE_DIR}/src/core/util/wait_for_single_owner.cc @@ -825,9 +839,6 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set.cc ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set_windows.cc ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_windows.cc - ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address.cc - ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_posix.cc - ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_windows.cc ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/sockaddr_utils_posix.cc ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_factory_posix.cc ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_mutator.cc @@ -872,7 +883,6 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/thread_quota.cc ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/audit_logging.cc ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/authorization_policy_provider_vtable.cc - ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/evaluate_args.cc ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/grpc_authorization_engine.cc ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/grpc_server_authz_filter.cc ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/matchers.cc @@ -941,16 +951,6 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/load_balancing/xds/xds_wrr_locality.cc ${_gRPC_SOURCE_DIR}/src/core/plugin_registry/grpc_plugin_registry.cc ${_gRPC_SOURCE_DIR}/src/core/plugin_registry/grpc_plugin_registry_extra.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/c_ares/dns_resolver_ares.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/c_ares/grpc_ares_wrapper.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/dns_resolver_plugin.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/event_engine/event_engine_client_channel_resolver.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/event_engine/service_config_helper.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/native/dns_resolver.cc ${_gRPC_SOURCE_DIR}/src/core/resolver/endpoint_addresses.cc ${_gRPC_SOURCE_DIR}/src/core/resolver/fake/fake_resolver.cc ${_gRPC_SOURCE_DIR}/src/core/resolver/google_c2p/google_c2p_resolver.cc @@ -1024,7 +1024,6 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/util/http_client/parser.cc ${_gRPC_SOURCE_DIR}/src/core/util/json/json_object_loader.cc ${_gRPC_SOURCE_DIR}/src/core/util/json/json_reader.cc - ${_gRPC_SOURCE_DIR}/src/core/util/json/json_util.cc ${_gRPC_SOURCE_DIR}/src/core/util/json/json_writer.cc ${_gRPC_SOURCE_DIR}/src/core/util/latent_see.cc ${_gRPC_SOURCE_DIR}/src/core/util/load_file.cc @@ -1044,8 +1043,10 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/util/work_serializer.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/certificate_provider_store.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/file_watcher_certificate_provider_factory.cc + ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/streaming_call_promise_wrapper.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_audit_logger_registry.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_bootstrap_grpc.cc + ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_bootstrap_grpc_builder.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_certificate_provider.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_client_grpc.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_cluster.cc @@ -1055,8 +1056,10 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_common_types_parser.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_endpoint.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_endpoint_parser.cc + ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_grpc_service_parser.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_health_status.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_http_composite_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_http_ext_proc_filter.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_http_fault_filter.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_http_filter.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_http_filter_registry.cc @@ -1077,6 +1080,8 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_route_config_parser.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_routing.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_server_grpc.cc + ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_tls_context.cc + ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_tls_context_parser.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_transport_grpc.cc ${_gRPC_SOURCE_DIR}/src/core/xds/xds_client/lrs_client.cc ${_gRPC_SOURCE_DIR}/src/core/xds/xds_client/xds_api.cc @@ -1135,6 +1140,7 @@ add_library(grpc_unsecure ${_gRPC_SOURCE_DIR}/src/core/call/call_spine.cc ${_gRPC_SOURCE_DIR}/src/core/call/call_state.cc ${_gRPC_SOURCE_DIR}/src/core/call/client_call.cc + ${_gRPC_SOURCE_DIR}/src/core/call/evaluate_args.cc ${_gRPC_SOURCE_DIR}/src/core/call/interception_chain.cc ${_gRPC_SOURCE_DIR}/src/core/call/message.cc ${_gRPC_SOURCE_DIR}/src/core/call/metadata.cc @@ -1169,6 +1175,7 @@ add_library(grpc_unsecure ${_gRPC_SOURCE_DIR}/src/core/client_channel/retry_service_config.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/retry_throttle.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel.cc + ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel_metrics.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel_pool_interface.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel_stream_client.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel_stream_limiter.cc @@ -1211,7 +1218,6 @@ add_library(grpc_unsecure ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/chttp2_plugin.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/chttp2_connector.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/server/chttp2_server.cc - ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/bin_decoder.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/bin_encoder.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/call_tracer_wrapper.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -1244,6 +1250,7 @@ add_library(grpc_unsecure ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_callbacks.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_promise.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_rate_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/reclaimer.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/stream_lists.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/transport_common.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/varint.cc @@ -1255,23 +1262,22 @@ add_library(grpc_unsecure ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/api/annotations.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/api/http.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/any.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/duration.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/empty.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/struct.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/timestamp.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/wrappers.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/rpc/status.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/gcp/altscontext.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/gcp/handshaker.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/gcp/transport_security_common.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/health/v1/health.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/lb/v1/load_balancer.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/lookup/v1/rls.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/channelz/v2/channelz.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/channelz/v2/promise.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/channelz/v2/property_list.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/channelz/v2/service.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/gcp/altscontext.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/gcp/handshaker.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/gcp/transport_security_common.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/health/v1/health.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/lb/v1/load_balancer.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/lookup/v1/rls.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/validate/validate.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/xds/data/orca/v3/orca_load_report.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/xds/service/orca/v3/orca.upb_minitable.c @@ -1300,11 +1306,16 @@ add_library(grpc_unsecure ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/telemetry.cc ${_gRPC_SOURCE_DIR}/src/core/lib/resource_tracker/resource_tracker.cc ${_gRPC_SOURCE_DIR}/src/core/lib/transport/promise_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/load_balancing/autosharding/autosharding.cc ${_gRPC_SOURCE_DIR}/src/core/net/socket_mutator.cc + ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/dns_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/service_config_helper.cc + ${_gRPC_SOURCE_DIR}/src/core/server/server_config_selector_filter.cc ${_gRPC_SOURCE_DIR}/src/core/telemetry/context_list_entry.cc ${_gRPC_SOURCE_DIR}/src/core/telemetry/instrument.cc ${_gRPC_SOURCE_DIR}/src/core/transport/message_size_service_config.cc ${_gRPC_SOURCE_DIR}/src/core/transport/session_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/util/address_sorting_init.cc ${_gRPC_SOURCE_DIR}/src/core/util/grpc_check.cc ${_gRPC_SOURCE_DIR}/src/core/util/postmortem_emit.cc ${_gRPC_SOURCE_DIR}/src/core/util/wait_for_single_owner.cc @@ -1414,9 +1425,6 @@ add_library(grpc_unsecure ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set.cc ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set_windows.cc ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_windows.cc - ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address.cc - ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_posix.cc - ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_windows.cc ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/sockaddr_utils_posix.cc ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_factory_posix.cc ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_mutator.cc @@ -1460,7 +1468,6 @@ add_library(grpc_unsecure ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/resource_quota.cc ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/thread_quota.cc ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/authorization_policy_provider_vtable.cc - ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/evaluate_args.cc ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/grpc_server_authz_filter.cc ${_gRPC_SOURCE_DIR}/src/core/lib/slice/percent_encoding.cc ${_gRPC_SOURCE_DIR}/src/core/lib/slice/slice.cc @@ -1520,16 +1527,6 @@ add_library(grpc_unsecure ${_gRPC_SOURCE_DIR}/src/core/load_balancing/weighted_target/weighted_target.cc ${_gRPC_SOURCE_DIR}/src/core/plugin_registry/grpc_plugin_registry.cc ${_gRPC_SOURCE_DIR}/src/core/plugin_registry/grpc_plugin_registry_noextra.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/c_ares/dns_resolver_ares.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/c_ares/grpc_ares_wrapper.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/dns_resolver_plugin.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/event_engine/event_engine_client_channel_resolver.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/event_engine/service_config_helper.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/native/dns_resolver.cc ${_gRPC_SOURCE_DIR}/src/core/resolver/endpoint_addresses.cc ${_gRPC_SOURCE_DIR}/src/core/resolver/fake/fake_resolver.cc ${_gRPC_SOURCE_DIR}/src/core/resolver/polling_resolver.cc @@ -1704,6 +1701,29 @@ target_link_libraries(upb_mini_descriptor_lib upb_mini_table_lib ) +add_library(upb_descriptor_lib + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c +) + +target_compile_features(upb_descriptor_lib PUBLIC cxx_std_17) + +target_include_directories(upb_descriptor_lib + PUBLIC ${_gRPC_SOURCE_DIR}/include + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(upb_descriptor_lib + ${_gRPC_ALLTARGETS_LIBRARIES} +) + add_library(upb_reflection_lib ${_gRPC_SOURCE_DIR}/third_party/upb/upb/message/copy.c ${_gRPC_SOURCE_DIR}/third_party/upb/upb/message/merge.c @@ -1743,6 +1763,7 @@ target_include_directories(upb_reflection_lib ) target_link_libraries(upb_reflection_lib ${_gRPC_ALLTARGETS_LIBRARIES} + upb_descriptor_lib upb_mini_descriptor_lib upb_wire_lib ) From f1e6889510f6856440ee839097e1723b8af12839 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 10 Nov 2026 17:26:47 +0000 Subject: [PATCH 145/219] Backport #128083 to 26.8: Part check ignore query cancellation --- .../MergeTreeReaderCompactSingleBuffer.cpp | 4 +- .../MergeTree/MergeTreeReaderWide.cpp | 6 +- .../MergeTree/MergeTreeSequentialSource.cpp | 2 +- src/Storages/MergeTree/checkDataPart.cpp | 27 ++++ src/Storages/MergeTree/checkDataPart.h | 3 + ...check_ignores_query_cancellation.reference | 4 + ...7_part_check_ignores_query_cancellation.sh | 144 ++++++++++++++++++ 7 files changed, 184 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/05457_part_check_ignores_query_cancellation.reference create mode 100755 tests/queries/0_stateless/05457_part_check_ignores_query_cancellation.sh diff --git a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp index 66fae3c1dbdb9..982696184e25c 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp @@ -93,7 +93,7 @@ try } catch (...) { - if (!isRetryableException(std::current_exception())) + if (!isRetryableException(std::current_exception()) && !isQueryCancellation(std::current_exception())) data_part_info_for_read->reportBroken(); /// Better diagnostics. @@ -126,7 +126,7 @@ try } catch (...) { - if (!isRetryableException(std::current_exception())) + if (!isRetryableException(std::current_exception()) && !isQueryCancellation(std::current_exception())) data_part_info_for_read->reportBroken(); throw; } diff --git a/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 6f8b49239eba8..6db282c14290d 100644 --- a/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -80,7 +80,7 @@ MergeTreeReaderWide::MergeTreeReaderWide( } catch (...) { - if (!isRetryableException(std::current_exception())) + if (!isRetryableException(std::current_exception()) && !isQueryCancellation(std::current_exception())) data_part_info_for_read->reportBroken(); throw; } @@ -106,7 +106,7 @@ void MergeTreeReaderWide::prefetchBeginOfRange(Priority priority) } catch (...) { - if (!isRetryableException(std::current_exception())) + if (!isRetryableException(std::current_exception()) && !isQueryCancellation(std::current_exception())) data_part_info_for_read->reportBroken(); throw; } @@ -232,7 +232,7 @@ size_t MergeTreeReaderWide::readRows( } catch (...) { - if (!isRetryableException(std::current_exception())) + if (!isRetryableException(std::current_exception()) && !isQueryCancellation(std::current_exception())) data_part_info_for_read->reportBroken(); /// Better diagnostics. diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp index 91ee5dfa67d6c..4db98cb754f5c 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp +++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp @@ -294,7 +294,7 @@ try catch (...) { /// Suspicion of the broken part. A part is added to the queue for verification. - if (!isRetryableException(std::current_exception())) + if (!isRetryableException(std::current_exception()) && !isQueryCancellation(std::current_exception())) read_task_info->data_part_info->reportBroken(); throw; } diff --git a/src/Storages/MergeTree/checkDataPart.cpp b/src/Storages/MergeTree/checkDataPart.cpp index 87ec70710e29c..7ed6784f38d95 100644 --- a/src/Storages/MergeTree/checkDataPart.cpp +++ b/src/Storages/MergeTree/checkDataPart.cpp @@ -56,6 +56,9 @@ namespace ErrorCodes extern const int ABORTED; extern const int CANNOT_WRITE_TO_OSTREAM; extern const int CACHE_CANNOT_WRITE_TO_CACHE_DISK; + extern const int QUERY_WAS_CANCELLED; + extern const int QUERY_WAS_CANCELLED_BY_CLIENT; + extern const int TIMEOUT_EXCEEDED; } @@ -154,6 +157,30 @@ bool isRetryableException(std::exception_ptr exception_ptr) } } +/// A query that was killed or hit `max_execution_time` can throw from inside a part read, e.g. from +/// `FileSegment::wait`, from the S3 client or from the marks loader. Such an exception says nothing +/// about the data on disk, so a reader must not report the part as broken because of it. +/// This is deliberately not part of `isRetryableException`: inside `checkDataPart` a retryable +/// exception yields empty checksums, and a cancelled `CHECK TABLE` would then pass an unchecked part. +bool isQueryCancellation(std::exception_ptr exception_ptr) +{ + try + { + rethrow_exception(exception_ptr); + } + catch (const Exception & e) + { + return e.code() == ErrorCodes::QUERY_WAS_CANCELLED + || e.code() == ErrorCodes::QUERY_WAS_CANCELLED_BY_CLIENT + || e.code() == ErrorCodes::TIMEOUT_EXCEEDED; + } + catch (...) + { + /// Ok: only a `DB::Exception` can carry a cancellation code. + return false; + } +} + static IMergeTreeDataPart::Checksums checkDataPart( MergeTreeData::DataPartPtr data_part, const IDataPartStorage & data_part_storage, diff --git a/src/Storages/MergeTree/checkDataPart.h b/src/Storages/MergeTree/checkDataPart.h index 2f9894bc70f0f..d7049f7161a7b 100644 --- a/src/Storages/MergeTree/checkDataPart.h +++ b/src/Storages/MergeTree/checkDataPart.h @@ -16,5 +16,8 @@ IMergeTreeDataPart::Checksums checkDataPart( bool isNotEnoughMemoryErrorCode(int code); bool isRetryableException(std::exception_ptr exception_ptr); +/// True for the cancellation of the current query (`KILL QUERY`, a client disconnect or `max_execution_time`), +/// which can surface from inside a part read and says nothing about the part. +bool isQueryCancellation(std::exception_ptr exception_ptr); } diff --git a/tests/queries/0_stateless/05457_part_check_ignores_query_cancellation.reference b/tests/queries/0_stateless/05457_part_check_ignores_query_cancellation.reference new file mode 100644 index 0000000000000..a50066ed3c6a0 --- /dev/null +++ b/tests/queries/0_stateless/05457_part_check_ignores_query_cancellation.reference @@ -0,0 +1,4 @@ +1 +499999500000 +waiter timed out inside the cache read +spurious part checks: 0 diff --git a/tests/queries/0_stateless/05457_part_check_ignores_query_cancellation.sh b/tests/queries/0_stateless/05457_part_check_ignores_query_cancellation.sh new file mode 100755 index 0000000000000..c8c64a04ad583 --- /dev/null +++ b/tests/queries/0_stateless/05457_part_check_ignores_query_cancellation.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-random-settings, no-random-merge-tree-settings, no-replicated-database, no-object-storage, no-parallel-replicas, zookeeper, no-shared-merge-tree +# no-parallel: enables a global pauseable failpoint which pauses every filesystem cache write on the server. +# zookeeper: a part check is requested through the broken-part callback, which only `ReplicatedMergeTree` installs. +# no-shared-merge-tree: the table pins a private cache disk, and SharedMergeTree requires a keeper-metadata disk. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A query that waits in `FileSegment::wait` for another query's cache download, and is cancelled while it +# waits, throws `TIMEOUT_EXCEEDED` or `QUERY_WAS_CANCELLED` from inside `MergeTreeReaderWide::readRows`. +# Neither code is a retryable exception, so the reader reported the part as broken and a healthy part was +# enqueued for a full data check. On object storage such a check reads the whole part, and a replica +# restart (`partialShutdown` -> `ReplicatedMergeTreePartCheckThread::stop`) blocks until it ends. +# +# Here the cancellation comes from `max_execution_time`: the cancellation checker kills the query while +# its only thread waits for the paused download, and the wait loop observes the kill and throws. +# A killed or timed-out query says nothing about the data, so no check must be requested. + +# A private cache keeps the test independent from other cache users on the server. The whole data file +# must fit in one file segment, so that the waiter below never becomes a downloader itself (it would +# pause on its own cache write, because the failpoint is disabled only after it returns). +cache_name="cache_05457_${CLICKHOUSE_DATABASE}" +table="t_part_check_cancel" + +$CLICKHOUSE_CLIENT --query " + DROP TABLE IF EXISTS $table SYNC; + CREATE TABLE $table (a UInt64) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/$table', 'r1') + ORDER BY a + SETTINGS + min_bytes_for_wide_part = 0, + disk = disk( + type = cache, + name = '$cache_name', + path = '$cache_name/', + max_size = '1Gi', + max_file_segment_size = '256Mi', + cache_on_write_operations = 0, + load_metadata_asynchronously = 0, + disk = 'local_disk'); + INSERT INTO $table SELECT number FROM numbers(1000000); +" + +# Warm up the in-memory mark cache, so that both queries below take marks from it and their only +# filesystem cache accesses are for the data file. Otherwise the paused downloader would hold the +# mark cache load token and the waiter would block on it instead of `FileSegment::wait`. +# Strictly synchronous, no prefetches and no background download: a cache write still in flight +# after this query would pause on the failpoint while holding a segment the queries below need. +$CLICKHOUSE_CLIENT --max_threads 1 \ + --remote_filesystem_read_prefetch 0 \ + --allow_prefetched_read_pool_for_remote_filesystem 0 \ + --filesystem_cache_allow_background_download 0 \ + --query "SELECT a FROM $table LIMIT 1 FORMAT Null" + +$CLICKHOUSE_CLIENT --query " + SYSTEM DROP FILESYSTEM CACHE '$cache_name'; + SYSTEM ENABLE FAILPOINT file_segment_pause_before_write; +" + +# The failpoint is server-global, so it must be disabled even if the script aborts before reaching +# the explicit disable below. Otherwise every filesystem cache write on the server stays paused and +# the following tests hang behind it, masking the failure that happened here. +trap '$CLICKHOUSE_CLIENT --query "SYSTEM DISABLE FAILPOINT file_segment_pause_before_write" > /dev/null 2>&1 || true' EXIT + +# The downloader pauses inside its first cache write, holding the file segment in DOWNLOADING state. +$CLICKHOUSE_CLIENT --max_threads 1 \ + --enable_filesystem_cache 1 \ + --read_from_filesystem_cache_if_exists_otherwise_bypass_cache 0 \ + --remote_filesystem_read_prefetch 0 \ + --allow_prefetched_read_pool_for_remote_filesystem 0 \ + --filesystem_cache_allow_background_download 0 \ + --query "SELECT sum(a) FROM $table" & +downloader_pid=$! + +# The failpoint pauses every cache write on the server, so the first pause is not necessarily the +# downloader's: additionally wait until the downloader holds the segment of this test's cache. +# Bounded, so that a downloader that never writes to the cache is reported instead of blocking +# the test until its timeout. +if ! timeout 120 $CLICKHOUSE_CLIENT --query "SYSTEM WAIT FAILPOINT file_segment_pause_before_write PAUSE" > /dev/null 2>&1; then + echo "FAIL: no cache write paused on the failpoint" +fi +downloading=0 +for _ in {1..600}; do + downloading=$($CLICKHOUSE_CLIENT --query " + SELECT count() FROM system.filesystem_cache WHERE cache_name = '$cache_name' AND state = 'DOWNLOADING'") + [[ "$downloading" == "1" ]] && break + sleep 0.1 +done +echo "$downloading" + +# The waiter needs the paused segment. Its cache wait is long enough to outlive `max_execution_time`, +# so the cancellation checker kills it while it waits, and the wait loop throws `TIMEOUT_EXCEEDED` +# from inside the read. `Cache info:` in the error proves the throw happened inside the cache read +# buffer, i.e. under the reader's catch, and not at a cancellation point of the pipeline. +# Backgrounded with a bounded wait, so that a waiter which unexpectedly pauses on the failpoint +# itself is reported instead of blocking the test until its timeout. +waiter_out="${CLICKHOUSE_TMP:-/tmp}/05457_waiter_${CLICKHOUSE_DATABASE}.out" +rm -f "$waiter_out" +waiter_query_id="05457_waiter_$(random_str 10)" +$CLICKHOUSE_CLIENT --query_id "$waiter_query_id" --max_threads 1 \ + --enable_filesystem_cache 1 \ + --read_from_filesystem_cache_if_exists_otherwise_bypass_cache 0 \ + --remote_filesystem_read_prefetch 0 \ + --allow_prefetched_read_pool_for_remote_filesystem 0 \ + --filesystem_cache_allow_background_download 0 \ + --use_uncompressed_cache 0 \ + --filesystem_cache_wait_for_concurrent_download_timeout_milliseconds 600000 \ + --max_execution_time 3 \ + --query "SELECT sum(a) FROM $table" > "$waiter_out" 2>&1 & +waiter_pid=$! + +waiter_finished=0 +for _ in {1..600}; do + if ! kill -0 "$waiter_pid" 2>/dev/null; then + waiter_finished=1 + break + fi + sleep 0.1 +done + +$CLICKHOUSE_CLIENT --query "SYSTEM DISABLE FAILPOINT file_segment_pause_before_write" +wait "$downloader_pid" +wait "$waiter_pid" 2>/dev/null || true + +waiter_result=$(cat "$waiter_out") +rm -f "$waiter_out" +if [[ "$waiter_finished" != "1" ]]; then + echo "FAIL: the waiter did not finish while the failpoint was armed: $waiter_result" +elif [[ "$waiter_result" == *"Timeout exceeded"*"Cache info:"* ]]; then + echo "waiter timed out inside the cache read" +else + echo "FAIL: unexpected waiter result: $waiter_result" +fi + +$CLICKHOUSE_CLIENT --query "SYSTEM FLUSH LOGS text_log" +checks=$($CLICKHOUSE_CLIENT --query " + SELECT count() FROM system.text_log + WHERE event_date >= yesterday() AND query_id = '$waiter_query_id' + AND message LIKE 'Enqueueing % for check after %'") +echo "spurious part checks: $checks" + +$CLICKHOUSE_CLIENT --query "DROP TABLE $table SYNC" From c369756caad4bed9246a1b325623b455590f656e Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 10 Nov 2026 17:23:43 +0000 Subject: [PATCH 146/219] Backport #130354 to 26.8: Fix ORC `TIMESTAMP` values before the first time zone transition --- .../Formats/Impl/ORC/ColumnReader.cpp | 11 ++- src/Processors/Formats/Impl/ORC/Timezone.cpp | 90 +++++++++++++++++++ src/Processors/Formats/Impl/ORC/Timezone.h | 25 ++++++ src/Processors/Formats/Impl/ORC/Writer.cpp | 9 +- ...imestamp_before_first_transition.reference | 7 ++ ...6_orc_timestamp_before_first_transition.sh | 40 +++++++++ 6 files changed, 175 insertions(+), 7 deletions(-) create mode 100644 src/Processors/Formats/Impl/ORC/Timezone.cpp create mode 100644 src/Processors/Formats/Impl/ORC/Timezone.h create mode 100644 tests/queries/0_stateless/05616_orc_timestamp_before_first_transition.reference create mode 100755 tests/queries/0_stateless/05616_orc_timestamp_before_first_transition.sh diff --git a/src/Processors/Formats/Impl/ORC/ColumnReader.cpp b/src/Processors/Formats/Impl/ORC/ColumnReader.cpp index b04bb53500921..f792e8d980ad4 100644 --- a/src/Processors/Formats/Impl/ORC/ColumnReader.cpp +++ b/src/Processors/Formats/Impl/ORC/ColumnReader.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -366,6 +367,8 @@ class TimestampColumnReader final : public ColumnReader , writer_timezone(is_instant ? DateLUT::instance("UTC") : stripe.getWriterTimezone()) , reader_timezone(is_instant ? DateLUT::instance("UTC") : stripe.getReaderTimezone()) , same_timezone(&writer_timezone == &reader_timezone || writer_timezone.getTimeZone() == reader_timezone.getTimeZone()) + , writer_offset_of(writer_timezone) + , reader_offset_of(reader_timezone) { /// The base of the seconds is 2015-01-01 00:00:00 in the time zone of the writer. static constexpr Int64 UTC_2015_01_01 = 1420070400; @@ -421,15 +424,15 @@ class TimestampColumnReader final : public ColumnReader if (!same_timezone) { /// Keep the same wall clock time when the writer and the reader time zones differ. - Int64 writer_offset = writer_timezone.timezoneOffset(writer_time); - Int64 reader_offset = reader_timezone.timezoneOffset(writer_time); + Int64 writer_offset = writer_offset_of(writer_time); + Int64 reader_offset = reader_offset_of(writer_time); if (writer_offset != reader_offset) { /// If the adjustment crosses a DST boundary, the offset has to be reevaluated. Int64 adjusted_time = 0; if (common::addOverflow(writer_time, writer_offset - reader_offset, adjusted_time)) throwInvalidTimestamp(seconds[i], encoded_nanos); - Int64 adjusted_reader_offset = reader_timezone.timezoneOffset(adjusted_time); + Int64 adjusted_reader_offset = reader_offset_of(adjusted_time); if (common::addOverflow(writer_time, writer_offset - adjusted_reader_offset, writer_time)) throwInvalidTimestamp(seconds[i], encoded_nanos); } @@ -457,6 +460,8 @@ class TimestampColumnReader final : public ColumnReader const DateLUTImpl & writer_timezone; const DateLUTImpl & reader_timezone; const bool same_timezone; + const TimezoneOffset writer_offset_of; + const TimezoneOffset reader_offset_of; Int64 epoch_offset = 0; std::unique_ptr seconds_rle; std::unique_ptr nanos_rle; diff --git a/src/Processors/Formats/Impl/ORC/Timezone.cpp b/src/Processors/Formats/Impl/ORC/Timezone.cpp new file mode 100644 index 0000000000000..4d009451dbadd --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Timezone.cpp @@ -0,0 +1,90 @@ +#include + +#include +#include + +#include + +/// The embedded time zone files, see `DateLUTImpl.cpp`. +std::string_view getTimeZone(std::string_view name); /// NOLINT(misc-use-internal-linkage) + +namespace DB +{ +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} +} + +namespace DB::ORC +{ + +TimezoneOffset::TimezoneOffset(const DateLUTImpl & lut_) + : lut(lut_) +{ + const std::string_view name = lut.getTimeZone(); + const std::string_view file = getTimeZone(name); + /// `Fixed/UTC±HH:MM:SS` zones are synthesized without a file and have no transitions. + if (file.empty()) + { + if (!name.starts_with("Fixed/")) + throw Exception(ErrorCodes::LOGICAL_ERROR, "No time zone file for {}", name); + return; + } + + /// Raw records, as `cctz::time_zone::next_transition` skips no-op ones (Europe/Lisbon has one in 1884). + static constexpr size_t header_size = 44; + size_t pos = 0; + auto check = [&](size_t size) + { + if (size > file.size() - pos) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Malformed time zone file for {}", name); + }; + auto read_count = [&](size_t index) { return unalignedLoadBigEndian(file.data() + pos + 20 + index * 4); }; + + check(header_size); + if (!file.starts_with("TZif")) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Malformed time zone file for {}", name); + size_t time_size = 4; + /// Version 2 and later repeat the data with 64-bit times after the 32-bit section. + if (file[4] != '\0') + { + const size_t v1_size = read_count(3) * 5 + read_count(4) * 6 + read_count(5) + read_count(2) * 8 + read_count(1) + read_count(0); + check(header_size + v1_size); + pos += header_size + v1_size; + check(header_size); + time_size = 8; + } + const size_t time_count = read_count(3); + const size_t type_count = read_count(4); + pos += header_size; + check(time_count * (time_size + 1) + type_count * 6); + if (time_count == 0) + return; + + const char * times = file.data() + pos; + const auto * type_indices = reinterpret_cast(times + time_count * time_size); + const char * types = times + time_count * (time_size + 1); + auto type_record = [&](size_t type) + { + if (type >= type_count) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Malformed time zone file for {}", name); + return types + type * 6; + }; + + first_transition = time_size == 8 ? unalignedLoadBigEndian(times) : unalignedLoadBigEndian(times); + /// The oldest standard time, or the first type if all of them are daylight saving time. + const char * ancient = type_record(0); + for (size_t i = 0; i < time_count; ++i) + { + const char * record = type_record(type_indices[i]); + if (record[4] == 0) + { + ancient = record; + break; + } + } + ancient_offset = unalignedLoadBigEndian(ancient); +} + +} diff --git a/src/Processors/Formats/Impl/ORC/Timezone.h b/src/Processors/Formats/Impl/ORC/Timezone.h new file mode 100644 index 0000000000000..8a7dbd7361fd6 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Timezone.h @@ -0,0 +1,25 @@ +#pragma once + +#include + +#include + +namespace DB::ORC +{ + +/// The UTC offset as in the reference implementation: before the first transition record it is the oldest +/// standard time, not the local mean time (Asia/Shanghai +08:00, not +08:05:43). Shared by reader and writer. +class TimezoneOffset +{ +public: + explicit TimezoneOffset(const DateLUTImpl & lut_); + + Int64 operator()(Int64 t) const { return t < first_transition ? ancient_offset : lut.timezoneOffset(t); } + +private: + const DateLUTImpl & lut; + Int64 first_transition = std::numeric_limits::min(); + Int64 ancient_offset = 0; +}; + +} diff --git a/src/Processors/Formats/Impl/ORC/Writer.cpp b/src/Processors/Formats/Impl/ORC/Writer.cpp index 78bd4299eb5a1..cfab65634955e 100644 --- a/src/Processors/Formats/Impl/ORC/Writer.cpp +++ b/src/Processors/Formats/Impl/ORC/Writer.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include @@ -51,7 +52,7 @@ struct WriterContext size_t block_size = 0; bool enable_index = false; double dictionary_key_size_threshold = 0; - const DateLUTImpl * timezone = nullptr; + std::optional timezone_offset; bool timezone_is_utc = true; /// 2015-01-01 00:00:00 in the time zone of the writer, as seconds since the epoch. Int64 epoch_offset = 0; @@ -1104,7 +1105,7 @@ class TimestampColumnWriter final : public PrimitiveColumnWriter Int64 utc_seconds = secs; Int64 millis = 0; Int64 encoded_seconds = 0; - if ((!is_utc && common::addOverflow(secs, static_cast(context.timezone->timezoneOffset(secs)), utc_seconds)) + if ((!is_utc && common::addOverflow(secs, (*context.timezone_offset)(secs), utc_seconds)) || common::mulOverflow(utc_seconds, Int64{1000}, millis) || common::addOverflow(millis, nano_millis, millis) || common::subOverflow(secs + (secs < 0 && nano > 999999), epoch_offset, encoded_seconds)) @@ -1337,10 +1338,10 @@ Writer::Writer(const Type & schema_, const DataTypes & column_types, OutputStrea context->block_size = options.compression_block_size; context->enable_index = options.row_index_stride > 0; context->dictionary_key_size_threshold = options.dictionary_key_size_threshold; - context->timezone = &DateLUT::instance(options.timezone_name); + context->timezone_offset.emplace(DateLUT::instance(options.timezone_name)); context->timezone_is_utc = options.timezone_name == "GMT" || options.timezone_name == "UTC"; static constexpr Int64 UTC_2015_01_01 = 1420070400; - context->epoch_offset = UTC_2015_01_01 - context->timezone->timezoneOffset(UTC_2015_01_01); + context->epoch_offset = UTC_2015_01_01 - (*context->timezone_offset)(UTC_2015_01_01); if (schema->getKind() != STRUCT) throw Exception(ErrorCodes::LOGICAL_ERROR, "The root type of ORC file must be a struct"); diff --git a/tests/queries/0_stateless/05616_orc_timestamp_before_first_transition.reference b/tests/queries/0_stateless/05616_orc_timestamp_before_first_transition.reference new file mode 100644 index 0000000000000..8c5b5ebbbffc3 --- /dev/null +++ b/tests/queries/0_stateless/05616_orc_timestamp_before_first_transition.reference @@ -0,0 +1,7 @@ +1849-12-31 16:00:00 +1881-07-06 03:13:13 +1949-12-31 16:00:00 +1849-12-31 16:00:00 +1850-01-01 00:36:45 +0 1849-12-31 16:00:00.000000000 +1 1849-12-31 16:00:00.000000000 diff --git a/tests/queries/0_stateless/05616_orc_timestamp_before_first_transition.sh b/tests/queries/0_stateless/05616_orc_timestamp_before_first_transition.sh new file mode 100755 index 0000000000000..8a44944717c9c --- /dev/null +++ b/tests/queries/0_stateless/05616_orc_timestamp_before_first_transition.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: the fast test build has no ORC format + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Before the first transition of a time zone, the wall clock is kept with the oldest standard offset +# (Asia/Shanghai +08:00, America/Los_Angeles -08:00), not with the local mean time, as in other ORC readers. + +FILE="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}.orc" + +$CLICKHOUSE_LOCAL -q " + SELECT arrayJoin([toDateTime64('1850-01-01 00:00:00', 0, 'UTC'), toDateTime64('1881-07-06 11:13:13', 0, 'UTC'), toDateTime64('1950-01-01 00:00:00', 0, 'UTC')]) AS ts + INTO OUTFILE '$FILE' TRUNCATE FORMAT ORC" +$CLICKHOUSE_LOCAL -q "SELECT ts FROM file('$FILE', ORC, 'ts DateTime64(0, \'UTC\')') SETTINGS input_format_orc_reader_time_zone_name = 'Asia/Shanghai'" + +$CLICKHOUSE_LOCAL -q " + SELECT toDateTime64('1850-01-01 00:00:00', 0, 'UTC') AS ts + INTO OUTFILE '$FILE' TRUNCATE FORMAT ORC SETTINGS output_format_orc_writer_time_zone_name = 'America/Los_Angeles'" +$CLICKHOUSE_LOCAL -q "SELECT ts FROM file('$FILE', ORC, 'ts DateTime64(0, \'UTC\')')" + +# Europe/Lisbon starts with a no-op transition record in 1884 that keeps the local mean time (-00:36:45). +$CLICKHOUSE_LOCAL -q " + SELECT toDateTime64('1850-01-01 00:00:00', 0, 'UTC') AS ts + INTO OUTFILE '$FILE' TRUNCATE FORMAT ORC" +$CLICKHOUSE_LOCAL -q "SELECT ts FROM file('$FILE', ORC, 'ts DateTime64(0, \'UTC\')') SETTINGS input_format_orc_reader_time_zone_name = 'Europe/Lisbon'" + +# The timestamp statistics must agree with the values that are read, or the filter pushdown drops the row. +$CLICKHOUSE_LOCAL -q " + SELECT toDateTime64('1850-01-01 00:00:00', 9, 'UTC') AS ts + INTO OUTFILE '$FILE' TRUNCATE FORMAT ORC SETTINGS output_format_orc_writer_time_zone_name = 'America/Los_Angeles'" +for push_down in 0 1; do + $CLICKHOUSE_LOCAL -q " + SELECT $push_down, ts FROM file('$FILE', ORC, 'ts DateTime64(9, \'UTC\')') + WHERE ts = toDateTime64('1849-12-31 16:00:00', 9, 'UTC') SETTINGS input_format_orc_filter_push_down = $push_down" +done + +rm -f "$FILE" From 7b8858e89ae84d598cf367c018b8677233857875 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ra=C3=BAl=20Mar=C3=ADn?= Date: Tue, 10 Nov 2026 18:01:58 +0000 Subject: [PATCH 147/219] Fix backport: use the `const char *` `getTimeZone` of this branch The `std::string_view` overload of the embedded time zone lookup is not in this release, so `Timezone.cpp` failed to link. Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 69b1082e31593aafd44412fcef900ad56395105d) --- src/Processors/Formats/Impl/ORC/Timezone.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Processors/Formats/Impl/ORC/Timezone.cpp b/src/Processors/Formats/Impl/ORC/Timezone.cpp index 4d009451dbadd..50c4b80e67c01 100644 --- a/src/Processors/Formats/Impl/ORC/Timezone.cpp +++ b/src/Processors/Formats/Impl/ORC/Timezone.cpp @@ -6,7 +6,7 @@ #include /// The embedded time zone files, see `DateLUTImpl.cpp`. -std::string_view getTimeZone(std::string_view name); /// NOLINT(misc-use-internal-linkage) +std::string_view getTimeZone(const char * name); /// NOLINT(misc-use-internal-linkage) namespace DB { @@ -22,8 +22,8 @@ namespace DB::ORC TimezoneOffset::TimezoneOffset(const DateLUTImpl & lut_) : lut(lut_) { - const std::string_view name = lut.getTimeZone(); - const std::string_view file = getTimeZone(name); + const std::string & name = lut.getTimeZone(); + const std::string_view file = getTimeZone(name.c_str()); /// `Fixed/UTC±HH:MM:SS` zones are synthesized without a file and have no transitions. if (file.empty()) { From c7a925647e88568a6b4d8c2b190c51df0a9f015f Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 10 Nov 2026 19:18:57 +0000 Subject: [PATCH 148/219] Backport #130517 to 26.8: Fix crash in log file rotation when the process has no free file descriptors --- .../Foundation/include/Poco/FileChannel.h | 1 + base/poco/Foundation/src/FileChannel.cpp | 40 +++--- .../gtest_file_channel_rotation_failure.cpp | 133 ++++++++++++++++++ 3 files changed, 154 insertions(+), 20 deletions(-) create mode 100644 src/Common/tests/gtest_file_channel_rotation_failure.cpp diff --git a/base/poco/Foundation/include/Poco/FileChannel.h b/base/poco/Foundation/include/Poco/FileChannel.h index 7db27772c0a26..028f1766aaa95 100644 --- a/base/poco/Foundation/include/Poco/FileChannel.h +++ b/base/poco/Foundation/include/Poco/FileChannel.h @@ -246,6 +246,7 @@ class Foundation_API FileChannel : public Channel void unsafeOpen(); private: + void unsafeRotate(); LogFile * newLogFile(); void archiveCorrupted(const std::string & path); bool setNoPurge(const std::string & value); diff --git a/base/poco/Foundation/src/FileChannel.cpp b/base/poco/Foundation/src/FileChannel.cpp index 7b3423a646afe..c473de5200fa9 100644 --- a/base/poco/Foundation/src/FileChannel.cpp +++ b/base/poco/Foundation/src/FileChannel.cpp @@ -25,6 +25,7 @@ #include "Poco/Exception.h" #include "Poco/Ascii.h" #include "Poco/CombinedRotateStrategy.h" +#include namespace Poco { @@ -111,15 +112,7 @@ void FileChannel::log(const Message& msg) if (_pRotateStrategy && _pArchiveStrategy && _pRotateStrategy->mustRotate(_pFile)) { - try - { - _pFile = _pArchiveStrategy->archive(_pFile, _streamCompress); - purge(); - } - catch (...) - { - _pFile = newLogFile(); - } + unsafeRotate(); // we must call mustRotate() again to give the // RotateByIntervalStrategy a chance to write its timestamp // to the new file. @@ -340,17 +333,24 @@ void FileChannel::unsafeOpen() { _pFile = newLogFile(); if (_rotateOnOpen && _pFile->size() > 0) - { - try - { - _pFile = _pArchiveStrategy->archive(_pFile, _streamCompress); - purge(); - } - catch (...) - { - _pFile = newLogFile(); - } - } + unsafeRotate(); + } +} + + +void FileChannel::unsafeRotate() +{ + /// `archive` deletes the current file, so `_pFile` must not dangle if opening the new one fails. + /// The raw pointer is intentional: `archive` owns `pFile` and deletes it, so it must not be deleted here. + LogFile * pFile = std::exchange(_pFile, nullptr); + try + { + _pFile = _pArchiveStrategy->archive(pFile, _streamCompress); + purge(); + } + catch (...) + { + _pFile = newLogFile(); } } diff --git a/src/Common/tests/gtest_file_channel_rotation_failure.cpp b/src/Common/tests/gtest_file_channel_rotation_failure.cpp new file mode 100644 index 0000000000000..d4ca3bee239bc --- /dev/null +++ b/src/Common/tests/gtest_file_channel_rotation_failure.cpp @@ -0,0 +1,133 @@ +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/// Log rotation must not leave `FileChannel` with a deleted `LogFile` when the process has no free file descriptors. + +namespace +{ + +/// glibc defines `RLIMIT_NOFILE` as a self-referential macro, which trips -Wdisabled-macro-expansion inside gtest macros. +const auto nofile_resource = RLIMIT_NOFILE; + +struct DescriptorTableGuard +{ + struct rlimit original_limit{}; + std::vector descriptors; + + void release() + { + for (int fd : descriptors) + ::close(fd); + descriptors.clear(); + EXPECT_EQ(0, setrlimit(nofile_resource, &original_limit)); + } + + ~DescriptorTableGuard() { release(); } +}; + +int setSoftLimit(rlim_t value, const struct rlimit & original_limit) +{ + struct rlimit limit = original_limit; + limit.rlim_cur = value; + return setrlimit(nofile_resource, &limit); +} + +int fillDescriptorTable(DescriptorTableGuard & guard) +{ + while (true) + { + int fd = ::open("/dev/null", O_RDONLY | O_CLOEXEC); + if (fd < 0) + return errno; + guard.descriptors.push_back(fd); + } +} + +Poco::Message makeMessage(const std::string & text) +{ + return Poco::Message("gtest_file_channel_rotation_failure", text, Poco::Message::PRIO_INFORMATION); +} + +void expectNoFreeDescriptors(Poco::FileChannel & channel, const std::string & text) +{ + try + { + channel.log(makeMessage(text)); + ADD_FAILURE() << "Expected EMFILE while logging: " << text; + } + catch (const Poco::FileException & e) + { + EXPECT_EQ(EMFILE, e.code()) << e.displayText(); + } +} + +std::string readFile(const std::string & path) +{ + std::ifstream in(path); + return std::string((std::istreambuf_iterator(in)), std::istreambuf_iterator()); +} + +} + +TEST(FileChannelRotation, NoFreeDescriptorsDuringRotation) +{ + const auto dir = std::filesystem::temp_directory_path() / ("gtest_file_channel_rotation_" + std::to_string(::getpid())); + std::filesystem::remove_all(dir); + std::filesystem::create_directories(dir); + struct DirectoryGuard + { + std::filesystem::path path; + ~DirectoryGuard() + { + std::error_code ec; + std::filesystem::remove_all(path, ec); + } + } directory_guard{dir}; + const std::string path = (dir / "server.log").string(); + + Poco::AutoPtr channel = new Poco::FileChannel(path); + channel->setProperty(Poco::FileChannel::PROP_ROTATION, "1K"); + channel->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number"); + channel->setProperty(Poco::FileChannel::PROP_COMPRESS, "false"); + + DescriptorTableGuard guard; + ASSERT_EQ(0, getrlimit(nofile_resource, &guard.original_limit)); + const rlim_t limit = std::min({guard.original_limit.rlim_cur, guard.original_limit.rlim_max - 1, 1024}); + + /// The log file gets descriptor `limit`, so closing it in `archive` frees no descriptor below the limit. + ASSERT_EQ(0, setSoftLimit(limit, guard.original_limit)); + ASSERT_EQ(EMFILE, fillDescriptorTable(guard)); + ASSERT_EQ(0, setSoftLimit(limit + 1, guard.original_limit)); + ASSERT_NO_THROW(channel->open()); + ASSERT_EQ(0, setSoftLimit(limit, guard.original_limit)); + + ASSERT_NO_THROW(channel->log(makeMessage(std::string(2048, 'x')))); + + expectNoFreeDescriptors(*channel, "rotation without free descriptors"); + EXPECT_TRUE(std::filesystem::exists(path + ".0")); + EXPECT_EQ(0u, channel->size()); + expectNoFreeDescriptors(*channel, "still no free descriptors"); + + guard.release(); + + const std::string marker = "logged after descriptors were released"; + ASSERT_NO_THROW(channel->log(makeMessage(marker))); + channel->close(); + EXPECT_NE(std::string::npos, readFile(path).find(marker)); +} From 709232e160016f4b1f2e671439c98fcf30ac9903 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 10 Nov 2026 19:24:09 +0000 Subject: [PATCH 149/219] Backport #124553 to 26.8: Group PREWHERE read steps by exact columns and rewrite `m['key']` to a key subcolumn only for bucketed `Map` --- .../Passes/FunctionToSubcolumnsPass.cpp | 4 + src/DataTypes/DataTypeMap.cpp | 2 + .../QueryPlan/ReadFromMergeTree.cpp | 9 +-- src/Storages/IStorage.h | 3 + .../MergeTree/MergeTreeBlockReadUtils.cpp | 3 +- src/Storages/MergeTree/MergeTreeData.cpp | 9 +++ src/Storages/MergeTree/MergeTreeData.h | 2 + .../MergeTree/MergeTreeSelectProcessor.cpp | 11 +-- .../MergeTree/MergeTreeSelectProcessor.h | 6 +- .../MergeTreeSplitPrewhereIntoReadSteps.cpp | 55 ++----------- .../MergeTreeSplitPrewhereIntoReadSteps.h | 5 +- .../MergeTree/MergeTreeWhereOptimizer.cpp | 30 ++----- .../StorageMaterializedPostgreSQL.cpp | 8 ++ .../StorageMaterializedPostgreSQL.h | 1 + src/Storages/StorageAlias.h | 5 ++ src/Storages/StorageBuffer.cpp | 7 ++ src/Storages/StorageBuffer.h | 1 + src/Storages/StorageMaterializedView.h | 1 + src/Storages/StorageMerge.cpp | 6 ++ src/Storages/StorageMerge.h | 1 + src/Storages/StorageProxy.h | 1 + .../map_bucketed_serialization.xml | 8 +- tests/performance/map_subcolumns_prewhere.xml | 10 +-- .../prewhere_subcolumn_read_steps.xml | 19 +++++ .../subcolumn_deserialization_tuple_map.xml | 8 +- ...subcolumn_function_to_subcolumns.reference | 2 +- ...9_map_subcolumn_function_to_subcolumns.sql | 11 ++- .../04000_map_subcolumns_prewhere.reference | 2 +- .../04000_map_subcolumns_prewhere.sql | 7 +- .../04039_text_index_map_values_subcolumn.sql | 5 +- ...04207_alias_map_arrayelement_subcolumn.sql | 25 ++++-- ...04513_prewhere_map_subcolumn_read_cost.sql | 9 ++- ...map_element_duplicate_keys_first_match.sql | 19 ++--- ...ndex_map_lowcardinality_subcolumn_cast.sql | 7 +- ...39_prewhere_subcolumn_read_steps.reference | 7 ++ .../05339_prewhere_subcolumn_read_steps.sql | 79 +++++++++++++++++++ ...ement_to_subcolumn_bucketed_only.reference | 30 +++++++ ...map_element_to_subcolumn_bucketed_only.sql | 76 ++++++++++++++++++ ...ubcolumn_size_without_reading_prefixes.sql | 2 +- 39 files changed, 353 insertions(+), 143 deletions(-) create mode 100644 tests/performance/prewhere_subcolumn_read_steps.xml create mode 100644 tests/queries/0_stateless/05339_prewhere_subcolumn_read_steps.reference create mode 100644 tests/queries/0_stateless/05339_prewhere_subcolumn_read_steps.sql create mode 100644 tests/queries/0_stateless/05340_map_element_to_subcolumn_bucketed_only.reference create mode 100644 tests/queries/0_stateless/05340_map_element_to_subcolumn_bucketed_only.sql diff --git a/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp b/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp index 4ba9480c79bee..4f0c70b6cfdd1 100644 --- a/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp +++ b/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp @@ -839,6 +839,10 @@ ColumnNode * resolveTrivialAliasChain(ColumnNode * column_node) /// getTypedNodesForOptimization, so their decisions cannot diverge. bool storageAllowsTransformer(const IStorage & storage, TypeIndex type_id, const String & function_name) { + /// Without buckets the key subcolumn `m.key_` reads the whole Map, so `m['key']` is not rewritten to it. + if (function_name == "arrayElement" && type_id == TypeIndex::Map && !storage.hasBucketedMapSerialization()) + return false; + if (storage.supportsOptimizationToSubcolumns()) return true; return storage.supportsOptimizationToTupleElementSubcolumns() && type_id == TypeIndex::Tuple && function_name == "tupleElement"; diff --git a/src/DataTypes/DataTypeMap.cpp b/src/DataTypes/DataTypeMap.cpp index be88ea3a19b6f..3e8346cae6449 100644 --- a/src/DataTypes/DataTypeMap.cpp +++ b/src/DataTypes/DataTypeMap.cpp @@ -388,6 +388,8 @@ When a data part is written with `with_buckets` serialization: When a query reads a specific key (`m['key']`), the optimizer rewrites the expression to a key subcolumn (`m.key_`). The serialization layer computes which bucket the requested key belongs to and reads only that single bucket from disk. +This rewrite happens only for tables that use `with_buckets` serialization: with `basic` serialization reading a key subcolumn +reads the whole `Map` anyway. When the full map is read (e.g., `SELECT m`), all buckets are read and reassembled into the original map. This is slower than `basic` serialization due to the overhead of reading and merging multiple substreams. diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 8e9295562d9e4..7f8d7c27010d9 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -652,8 +652,7 @@ Pipe ReadFromMergeTree::readFromPoolParallelReplicas( actions_settings, reader_settings, index_build_context, - lazy_materializing_rows, - &storage_snapshot->metadata->getColumns()); + lazy_materializing_rows); auto source = std::make_shared(std::move(processor), data.getLogName()); pipes.emplace_back(std::move(source)); @@ -770,8 +769,7 @@ Pipe ReadFromMergeTree::readFromPool( actions_settings, reader_settings, index_build_context, - lazy_materializing_rows, - &storage_snapshot->metadata->getColumns()); + lazy_materializing_rows); auto source = std::make_shared(std::move(processor), data.getLogName()); @@ -1047,8 +1045,7 @@ Pipe ReadFromMergeTree::readInOrder( actions_settings, reader_settings, index_build_context, - lazy_materializing_rows, - &storage_snapshot->metadata->getColumns()); + lazy_materializing_rows); processor->addPartLevelToChunk(isQueryWithFinal()); diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 4307ed1df1c03..ee27ad3647b4e 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -198,6 +198,9 @@ class IStorage : public std::enable_shared_from_this, public TypePromo /// A storage that cannot serve synthesised subcolumns such as `.null`/`.size0` as standalone /// inputs may enable this while keeping supportsOptimizationToSubcolumns() false. virtual bool supportsOptimizationToTupleElementSubcolumns() const { return supportsOptimizationToSubcolumns(); } + /// Returns true if the storage writes `Map` columns with the bucketed serialization, where a single key + /// can be read without reading the whole column. Gates the `m['key']` -> `m.key_` rewrite. + virtual bool hasBucketedMapSerialization() const { return false; } /// Returns true if the storage supports transactions for SELECT, INSERT and ALTER queries. /// Storage may throw an exception later if some query kind is not fully supported. diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index 14679f3bca78e..88f6194694df7 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -558,8 +558,7 @@ MergeTreeReadTaskColumns getReadTaskColumns( index_read_tasks, actions_settings, reader_settings.enable_multiple_prewhere_read_steps, - reader_settings.force_short_circuit_execution, - &storage_snapshot->metadata->getColumns()); + reader_settings.force_short_circuit_execution); for (const auto & step : prewhere_actions.steps) add_step(*step); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index fdec410a6bf92..a1fef4ac70323 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -363,6 +363,7 @@ namespace MergeTreeSetting extern const MergeTreeSettingsMergeTreeStringSerializationVersion string_serialization_version; extern const MergeTreeSettingsMergeTreeNullableSerializationVersion nullable_serialization_version; extern const MergeTreeSettingsMergeTreeMapSerializationVersion map_serialization_version; + extern const MergeTreeSettingsMergeTreeMapSerializationVersion map_serialization_version_for_zero_level_parts; extern const MergeTreeSettingsUInt32 min_level_for_wide_part; extern const MergeTreeSettingsBool propagate_types_serialization_versions_to_nested_types; } @@ -982,6 +983,14 @@ bool MergeTreeData::supportsFinal() const || merging_params.mode == MergingParams::VersionedCollapsing; } +bool MergeTreeData::hasBucketedMapSerialization() const +{ + /// Inserted parts use the zero-level setting, merged parts the other one. + auto settings = getSettings(); + return (*settings)[MergeTreeSetting::map_serialization_version] == MergeTreeMapSerializationVersion::WITH_BUCKETS + || (*settings)[MergeTreeSetting::map_serialization_version_for_zero_level_parts] == MergeTreeMapSerializationVersion::WITH_BUCKETS; +} + static void checkKeyExpression(const ExpressionActions & expr, const Block & sample_block, const String & key_name, bool allow_nullable_key) { if (expr.hasArrayJoin()) diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 147e3a115495a..aa97f33a44ebd 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -575,6 +575,8 @@ class MergeTreeData : public WithMutableContext, public IStorage, public IBackgr bool supportsSubcolumns() const override { return true; } + bool hasBucketedMapSerialization() const override; + bool supportsTTL() const override { return true; } bool supportsStatistics() const override { return true; } diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index e360ea2ff1a78..feb6446456375 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -163,8 +163,7 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor( const ExpressionActionsSettings & actions_settings_, const MergeTreeReaderSettings & reader_settings_, MergeTreeIndexBuildContextPtr merge_tree_index_build_context_, - LazyMaterializingRowsPtr lazy_materializing_rows_, - const ColumnsDescription * columns_) + LazyMaterializingRowsPtr lazy_materializing_rows_) : pool(std::move(pool_)) , algorithm(std::move(algorithm_)) , row_level_filter(row_level_filter_) @@ -176,8 +175,7 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor( index_read_tasks_, actions_settings, reader_settings_.enable_multiple_prewhere_read_steps, - reader_settings_.force_short_circuit_execution, - columns_)) + reader_settings_.force_short_circuit_execution)) , reader_settings(reader_settings_) , result_header(transformHeader(pool->getHeader(), row_level_filter, prewhere_info)) , merge_tree_index_build_context(std::move(merge_tree_index_build_context_)) @@ -205,8 +203,7 @@ PrewhereExprInfo MergeTreeSelectProcessor::getPrewhereActions( const IndexReadTasks & index_read_tasks, const ExpressionActionsSettings & actions_settings, bool enable_multiple_prewhere_read_steps, - bool force_short_circuit_execution, - const ColumnsDescription * columns) + bool force_short_circuit_execution) { PrewhereExprInfo prewhere_actions; @@ -239,7 +236,7 @@ PrewhereExprInfo MergeTreeSelectProcessor::getPrewhereActions( } if (prewhere_info && - (!enable_multiple_prewhere_read_steps || !tryBuildPrewhereSteps(prewhere_info, actions_settings, prewhere_actions, force_short_circuit_execution, columns))) + (!enable_multiple_prewhere_read_steps || !tryBuildPrewhereSteps(prewhere_info, actions_settings, prewhere_actions, force_short_circuit_execution))) { PrewhereExprStep prewhere_step { diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.h b/src/Storages/MergeTree/MergeTreeSelectProcessor.h index 9c04ebde42663..3e48c004ce1b3 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.h @@ -119,8 +119,7 @@ class MergeTreeSelectProcessor : private boost::noncopyable const ExpressionActionsSettings & actions_settings_, const MergeTreeReaderSettings & reader_settings_, MergeTreeIndexBuildContextPtr merge_tree_index_build_context_ = {}, - LazyMaterializingRowsPtr lazy_materializing_rows_ = {}, - const ColumnsDescription * columns_ = nullptr); + LazyMaterializingRowsPtr lazy_materializing_rows_ = {}); String getName() const; @@ -149,8 +148,7 @@ class MergeTreeSelectProcessor : private boost::noncopyable const IndexReadTasks & index_read_tasks, const ExpressionActionsSettings & actions_settings, bool enable_multiple_prewhere_read_steps, - bool force_short_circuit_execution, - const ColumnsDescription * columns = nullptr); + bool force_short_circuit_execution); void addPartLevelToChunk(bool add_part_level_) { add_part_level = add_part_level_; } diff --git a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp index cbeefa44a7ba9..0611d0d28676f 100644 --- a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp +++ b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp @@ -1,7 +1,6 @@ #include #include -#include namespace DB @@ -22,29 +21,10 @@ namespace struct NodeInfo { NameSet required_columns; - /// Column names resolved to their physical storage names (subcolumn suffix stripped). - /// Used for grouping: conditions on subcolumns of the same storage column are placed into one step. - NameSet required_storage_columns; }; -/// Resolves a column name to its storage (physical) name. -/// For subcolumns like `map.key_k0`, returns `map`. -/// For regular columns, returns the name unchanged. -String resolveStorageColumnName(const String & column_name, const ColumnsDescription * columns) -{ - if (columns) - { - if (auto col = columns->tryGetColumnOrSubcolumn(GetColumnsOptions::AllPhysical, column_name)) - return col->getNameInStorage(); - } - return column_name; -} - /// Fills the list of required columns for a node in the DAG. -void fillRequiredColumns( - const ActionsDAG::Node * node, - std::unordered_map & nodes_info, - const ColumnsDescription * columns) +void fillRequiredColumns(const ActionsDAG::Node * node, std::unordered_map & nodes_info) { if (nodes_info.contains(node)) return; @@ -54,16 +34,14 @@ void fillRequiredColumns( if (node->type == ActionsDAG::ActionType::INPUT) { node_info.required_columns.insert(node->result_name); - node_info.required_storage_columns.insert(resolveStorageColumnName(node->result_name, columns)); return; } for (const auto & child : node->children) { - fillRequiredColumns(child, nodes_info, columns); + fillRequiredColumns(child, nodes_info); const auto & child_info = nodes_info[child]; node_info.required_columns.insert(child_info.required_columns.begin(), child_info.required_columns.end()); - node_info.required_storage_columns.insert(child_info.required_storage_columns.begin(), child_info.required_storage_columns.end()); } } @@ -225,8 +203,7 @@ bool tryBuildPrewhereSteps( PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, PrewhereExprInfo & prewhere, - bool force_short_circuit_execution, - const ColumnsDescription * columns) + bool force_short_circuit_execution) { if (!prewhere_info) return true; @@ -242,37 +219,21 @@ bool tryBuildPrewhereSteps( std::unordered_map nodes_info; for (const auto & node : condition_nodes) { - fillRequiredColumns(node, nodes_info, columns); + fillRequiredColumns(node, nodes_info); } /// 3. Sort condition nodes by the number of columns used in them and the overall size of those columns /// TODO: not sorting for now because the conditions are already sorted by Where Optimizer - /// 4. Group adjacent conditions that read from the same set of physical storage columns into a single step. - /// Conditions on subcolumns of the same column (e.g. `map.key_k0` and `map.key_k1`) are placed into one group - /// when they appear next to each other in the condition list. - /// - /// Only adjacent conditions are merged to preserve the user's explicit PREWHERE evaluation order. - /// Non-adjacent conditions on the same storage column are kept in separate steps even though this - /// may cause redundant reads, because the user may have intentionally interleaved a guard predicate - /// (e.g. `PREWHERE tags['safe'] != '' AND value > 0 AND toUInt64(tags['unsafe']) > 0` — the - /// `value > 0` step must filter rows before evaluating the potentially-throwing conversion). - /// - /// For the WHERE-to-PREWHERE path this is not a problem: `MergeTreeWhereOptimizer` already groups - /// conditions by their physical storage columns, so subcolumns of the same column arrive here adjacent. + /// 4. Group conditions with the same set of columns into a single read/compute step std::vector> condition_groups; for (const auto & node : condition_nodes) { const auto & node_info = nodes_info[node]; - if (!condition_groups.empty() - && nodes_info[condition_groups.back().front()].required_storage_columns == node_info.required_storage_columns) - { - condition_groups.back().push_back(node); - } + if (!condition_groups.empty() && nodes_info[condition_groups.back().back()].required_columns == node_info.required_columns) + condition_groups.back().push_back(node); /// Add to the last group else - { - condition_groups.push_back({node}); - } + condition_groups.push_back({node}); /// Start new group } /// 5. Build DAGs for each step diff --git a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.h b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.h index 93f40b1af8c33..40d5427b2f46e 100644 --- a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.h +++ b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.h @@ -11,13 +11,10 @@ namespace DB { -class ColumnsDescription; - bool tryBuildPrewhereSteps( PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, PrewhereExprInfo & prewhere, - bool force_short_circuit_execution, - const ColumnsDescription * columns = nullptr); + bool force_short_circuit_execution); } diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index 09baddde3b971..174bf4e179e83 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -386,10 +386,6 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTree { RPNBuilderTreeNode node; NameSet columns; - /// Column names resolved to their physical storage names (subcolumn suffix stripped). - /// Used for grouping: conditions on subcolumns of the same storage column - /// (e.g. `map.key_k0` and `map.key_k1`) are grouped and moved to PREWHERE together. - NameSet storage_columns; bool has_invalid_column = false; bool may_use_primary_index = true; bool viable = false; @@ -401,20 +397,9 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTree for (const auto & conjunct : conjuncts) { - ConjunctInfo info{conjunct, {}, {}, false, true, false, false}; + ConjunctInfo info{conjunct, {}, false, true, false, false}; collectColumns(conjunct, nullptr, table_columns, info.columns, info.has_invalid_column, info.may_use_primary_index); - /// Resolve each column to its physical storage name so that subcolumns - /// of the same column (e.g. `map.key_k0`, `map.key_k1`) share one group. - const auto & storage_columns_description = storage_metadata->getColumns(); - for (const auto & col : info.columns) - { - if (auto resolved = storage_columns_description.tryGetColumnOrSubcolumn(GetColumnsOptions::AllPhysical, col)) - info.storage_columns.insert(resolved->getNameInStorage()); - else - info.storage_columns.insert(col); - } - /// Is it possible to use primary index for this condition? For conditions like `lower(country) = 'xx'`, may_use_primary_index will be false. info.viable = !info.has_invalid_column @@ -434,18 +419,15 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTree infos.push_back(std::move(info)); } - /// Group viable conjuncts by their required storage column set so same-column conditions + /// Group viable conjuncts by their required column set so same-column conditions /// are treated as a single unit and their combined selectivity can be estimated. - /// Grouping by storage columns (rather than the exact column set) keeps subcolumns of the - /// same column (e.g. `map.key_k0` and `map.key_k1`) in one group, so they are moved to - /// PREWHERE together and arrive adjacent for the prewhere-splitting step. /// Non-viable conjuncts stay as individual Conditions. /// 'expensive' conjuncts each get their own group. /// /// We use a simple linear search to find groups (WHERE clauses are short). struct Group { - NameSet storage_columns; + NameSet columns; std::vector indices; }; std::vector groups; @@ -453,7 +435,7 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTree auto find_group_idx = [&](const NameSet & cols) -> std::optional { for (size_t g = 0; g < groups.size(); ++g) - if (groups[g].storage_columns == cols && !infos[groups[g].indices.front()].expensive) + if (groups[g].columns == cols && !infos[groups[g].indices.front()].expensive) return g; return std::nullopt; }; @@ -465,11 +447,11 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTree { if (!infos[i].viable) continue; - auto g = infos[i].expensive ? std::nullopt : find_group_idx(infos[i].storage_columns); + auto g = infos[i].expensive ? std::nullopt : find_group_idx(infos[i].columns); if (!g.has_value()) { group_of[i] = groups.size(); - groups.push_back({infos[i].storage_columns, {i}}); + groups.push_back({infos[i].columns, {i}}); } else { diff --git a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp index f49ecdbd0f847..7de0d39d40044 100644 --- a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp +++ b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp @@ -346,6 +346,14 @@ bool StorageMaterializedPostgreSQL::supportsOptimizationToSubcolumns() const } +bool StorageMaterializedPostgreSQL::hasBucketedMapSerialization() const +{ + if (auto nested = tryGetNested()) + return nested->hasBucketedMapSerialization(); + return false; +} + + IStorage::ColumnSizeByName StorageMaterializedPostgreSQL::getColumnSizes() const { if (auto nested = tryGetNested()) diff --git a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.h b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.h index 0a41ea7e74ed1..8b8e5f6a8a028 100644 --- a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.h +++ b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.h @@ -105,6 +105,7 @@ class StorageMaterializedPostgreSQL final : public IStorage, WithContext bool supportedPrewhereColumnsIncludeSubcolumns() const override; bool supportsSubcolumns() const override; bool supportsOptimizationToSubcolumns() const override; + bool hasBucketedMapSerialization() const override; ColumnSizeByName getColumnSizes() const override; ColumnSizeByName getColumnSizes(const Names & columns, bool calculate_subcolumn_sizes) const override; diff --git a/src/Storages/StorageAlias.h b/src/Storages/StorageAlias.h index 219707a565637..7c60ab2006db1 100644 --- a/src/Storages/StorageAlias.h +++ b/src/Storages/StorageAlias.h @@ -162,6 +162,11 @@ class StorageAlias final : public IStorage, WithContext auto target = tryGetTargetTable(); return target && target->supportsOptimizationToTupleElementSubcolumns(); } + bool hasBucketedMapSerialization() const override + { + auto target = tryGetTargetTable(); + return target && target->hasBucketedMapSerialization(); + } bool supportsParallelInsert() const override { auto target = tryGetTargetTable(); diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index a949586df39b8..7e546320fd0d4 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -1052,6 +1052,13 @@ bool StorageBuffer::supportsOptimizationToTupleElementSubcolumns() const return false; } +bool StorageBuffer::hasBucketedMapSerialization() const +{ + if (auto destination = getDestinationTable()) + return destination->hasBucketedMapSerialization(); + return false; +} + bool StorageBuffer::checkThresholds(const Buffer & buffer, bool direct, time_t current_time, size_t additional_rows, size_t additional_bytes) const { time_t time_passed = 0; diff --git a/src/Storages/StorageBuffer.h b/src/Storages/StorageBuffer.h index 1b69b005805e7..c060bcab7133d 100644 --- a/src/Storages/StorageBuffer.h +++ b/src/Storages/StorageBuffer.h @@ -133,6 +133,7 @@ friend class BufferSink; /// Distributed). Fails closed like supportsPrewhere(): no destination means no rewrite. bool supportsOptimizationToSubcolumns() const override; bool supportsOptimizationToTupleElementSubcolumns() const override; + bool hasBucketedMapSerialization() const override; bool supportsFinal() const override { return true; } void checkAlterIsPossible(const AlterCommands & commands, ContextPtr context) const override; diff --git a/src/Storages/StorageMaterializedView.h b/src/Storages/StorageMaterializedView.h index d6c72eae7c332..c4fda9236afec 100644 --- a/src/Storages/StorageMaterializedView.h +++ b/src/Storages/StorageMaterializedView.h @@ -46,6 +46,7 @@ class StorageMaterializedView final : public StorageWithCommonVirtualColumns, Wi /// initiator must not rewrite functions to subcolumns when the target opts out (e.g. Distributed). bool supportsOptimizationToSubcolumns() const override { return getTargetTable()->supportsOptimizationToSubcolumns(); } bool supportsOptimizationToTupleElementSubcolumns() const override { return getTargetTable()->supportsOptimizationToTupleElementSubcolumns(); } + bool hasBucketedMapSerialization() const override { return getTargetTable()->hasBucketedMapSerialization(); } bool supportsColumnsWithDynamicStructure() const override; bool supportsTransactions() const override { return getTargetTable()->supportsTransactions(); } diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index cf7aed6565382..69c5fce2f15ba 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -382,6 +382,12 @@ bool StorageMerge::supportsOptimizationToTupleElementSubcolumns() const return traverseTablesUntil([](const auto & table) { return !table->supportsOptimizationToTupleElementSubcolumns(); }) == nullptr; } +bool StorageMerge::hasBucketedMapSerialization() const +{ + /// The rewritten query is sent to every underlying table, so all of them must benefit from it. + return traverseTablesUntil([](const auto & table) { return !table->hasBucketedMapSerialization(); }) == nullptr; +} + bool StorageMerge::canMoveConditionsToPrewhere() const { /// NOTE: This check and the above check are used during query analysis as condition for applying diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 1ad5d636bd395..e5fdaecaf70e9 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -56,6 +56,7 @@ class StorageMerge final : public IStorage, WithContext /// initiator rewrite functions to subcolumns, or a skip index on the shard would be missed. bool supportsOptimizationToSubcolumns() const override; bool supportsOptimizationToTupleElementSubcolumns() const override; + bool hasBucketedMapSerialization() const override; bool supportsColumnsWithDynamicStructure() const override { return true; } bool supportsPrewhere() const override; std::optional supportedPrewhereColumns() const override; diff --git a/src/Storages/StorageProxy.h b/src/Storages/StorageProxy.h index 2dc249080a2cf..b91f6d387a3d8 100644 --- a/src/Storages/StorageProxy.h +++ b/src/Storages/StorageProxy.h @@ -45,6 +45,7 @@ class StorageProxy : public IStorage /// storage that opts out of the rewrite (e.g. Distributed) does not re-advertise true. bool supportsOptimizationToSubcolumns() const override { return getNested()->supportsOptimizationToSubcolumns(); } bool supportsOptimizationToTupleElementSubcolumns() const override { return getNested()->supportsOptimizationToTupleElementSubcolumns(); } + bool hasBucketedMapSerialization() const override { return getNested()->hasBucketedMapSerialization(); } bool supportsColumnsWithDynamicStructure() const override { return getNested()->supportsColumnsWithDynamicStructure(); } /// `ReadFromMerge::getSelectedTables` prunes children by name based on this flag; a lazy /// `StorageTableProxy` around a delegating storage (`Distributed`, `Merge`, `Buffer`, `Alias`) diff --git a/tests/performance/map_bucketed_serialization.xml b/tests/performance/map_bucketed_serialization.xml index d86c681a4f3b6..e5016ad60d469 100644 --- a/tests/performance/map_bucketed_serialization.xml +++ b/tests/performance/map_bucketed_serialization.xml @@ -1,6 +1,6 @@ @@ -107,7 +107,7 @@ CREATE TABLE map_subcol_pw_mixed (id UInt64, tags Map(String, String), value UInt32) @@ -126,7 +126,7 @@ OPTIMIZE TABLE map_subcol_pw_mixed FINAL SETTINGS mutations_sync=1 - diff --git a/tests/performance/prewhere_subcolumn_read_steps.xml b/tests/performance/prewhere_subcolumn_read_steps.xml new file mode 100644 index 0000000000000..04cb68d9dd077 --- /dev/null +++ b/tests/performance/prewhere_subcolumn_read_steps.xml @@ -0,0 +1,19 @@ + + + 0 + + + CREATE TABLE t_prewhere_json (id UInt64, j JSON(max_dynamic_paths = 0)) ENGINE = MergeTree ORDER BY id + CREATE TABLE t_prewhere_map (id UInt64, m Map(String, String)) ENGINE = MergeTree ORDER BY id SETTINGS map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets', map_buckets_strategy = 'constant', max_buckets_in_map = 16, map_buckets_min_avg_size = 0 + + INSERT INTO t_prewhere_json SELECT number, concat('{{"kind":"a","n":', toString(number), ',"payload":"', repeat('x', 2000), '"}}') FROM numbers_mt(3000000) + INSERT INTO t_prewhere_map SELECT number, mapFromArrays(arrayMap(i -> 'k' || toString(i), range(16)), arrayMap(i -> repeat('v', 60), range(16))) FROM numbers_mt(10000000) + + + SELECT count() FROM t_prewhere_json WHERE j.kind = 'b' AND length(toString(j)) > 10 + SELECT count() FROM t_prewhere_json WHERE j.kind = 'b' AND length(toString(j.payload)) > 10 + SELECT count() FROM t_prewhere_map WHERE m['k1'] = 'x' AND length(toString(m)) > 10 + + DROP TABLE IF EXISTS t_prewhere_json + DROP TABLE IF EXISTS t_prewhere_map + diff --git a/tests/performance/subcolumn_deserialization_tuple_map.xml b/tests/performance/subcolumn_deserialization_tuple_map.xml index 743f640a916a0..ca5c564b23408 100644 --- a/tests/performance/subcolumn_deserialization_tuple_map.xml +++ b/tests/performance/subcolumn_deserialization_tuple_map.xml @@ -84,10 +84,10 @@ OPTIMIZE TABLE map_basic_compact FINAL - SELECT m['k0'], m['k1'], m['k2'], m['k3'], m['k4'] FROM map_basic_wide WHERE skip = 0 FORMAT Null SETTINGS max_threads = 1, preferred_block_size_bytes = 1000000000000 - SELECT m, m.keys, m.values, m['k0'], m['k1'] FROM map_basic_wide WHERE skip = 0 FORMAT Null SETTINGS max_threads = 1, preferred_block_size_bytes = 1000000000000 - SELECT m['k0'], m['k1'], m['k2'], m['k3'], m['k4'] FROM map_basic_compact FORMAT Null SETTINGS max_threads = 1, preferred_block_size_bytes = 1000000000000 - SELECT m, m.keys, m.values, m['k0'], m['k1'] FROM map_basic_compact FORMAT Null SETTINGS max_threads = 1, preferred_block_size_bytes = 1000000000000 + SELECT m.key_k0, m.key_k1, m.key_k2, m.key_k3, m.key_k4 FROM map_basic_wide WHERE skip = 0 FORMAT Null SETTINGS max_threads = 1, preferred_block_size_bytes = 1000000000000 + SELECT m, m.keys, m.values, m.key_k0, m.key_k1 FROM map_basic_wide WHERE skip = 0 FORMAT Null SETTINGS max_threads = 1, preferred_block_size_bytes = 1000000000000 + SELECT m.key_k0, m.key_k1, m.key_k2, m.key_k3, m.key_k4 FROM map_basic_compact FORMAT Null SETTINGS max_threads = 1, preferred_block_size_bytes = 1000000000000 + SELECT m, m.keys, m.values, m.key_k0, m.key_k1 FROM map_basic_compact FORMAT Null SETTINGS max_threads = 1, preferred_block_size_bytes = 1000000000000 diff --git a/tests/queries/0_stateless/03989_map_subcolumn_function_to_subcolumns.reference b/tests/queries/0_stateless/03989_map_subcolumn_function_to_subcolumns.reference index 491b7765b057a..7d4cda3c23953 100644 --- a/tests/queries/0_stateless/03989_map_subcolumn_function_to_subcolumns.reference +++ b/tests/queries/0_stateless/03989_map_subcolumn_function_to_subcolumns.reference @@ -16,7 +16,7 @@ 1 1 1 --- Optimization also works with basic (non-bucketed) serialization +-- No optimization with basic (non-bucketed) serialization 1 -- Correctness: results with optimization 7 7 70 diff --git a/tests/queries/0_stateless/03989_map_subcolumn_function_to_subcolumns.sql b/tests/queries/0_stateless/03989_map_subcolumn_function_to_subcolumns.sql index b161c8d4067d8..71b870061030e 100644 --- a/tests/queries/0_stateless/03989_map_subcolumn_function_to_subcolumns.sql +++ b/tests/queries/0_stateless/03989_map_subcolumn_function_to_subcolumns.sql @@ -151,20 +151,19 @@ SELECT count() = 0 FROM (EXPLAIN actions = 1 SELECT id FROM t_map_subcolumns WHE -- ========================================== -- Section 8: basic serialization (not with_buckets) --- The optimization still applies because DataTypeMap exposes dynamic subcolumns --- regardless of serialization version. With basic serialization, the whole map is --- read and the key is extracted; with_buckets reads only the relevant bucket. +-- The rewrite is not applied: with basic serialization the key subcolumn reads the whole map anyway. -- ========================================== DROP TABLE t_map_subcolumns; CREATE TABLE t_map_subcolumns (id UInt64, m Map(String, UInt64)) -ENGINE = MergeTree ORDER BY id; +ENGINE = MergeTree ORDER BY id +SETTINGS map_serialization_version = 'basic', map_serialization_version_for_zero_level_parts = 'basic'; INSERT INTO t_map_subcolumns SELECT number, map('key1', number) FROM numbers(10); -SELECT '-- Optimization also works with basic (non-bucketed) serialization'; -SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT id FROM t_map_subcolumns WHERE m['key1'] > 5) WHERE explain LIKE '%m.key_key1%'; +SELECT '-- No optimization with basic (non-bucketed) serialization'; +SELECT count() = 0 FROM (EXPLAIN actions = 1 SELECT id FROM t_map_subcolumns WHERE m['key1'] > 5) WHERE explain LIKE '%m.key_key1%'; -- ========================================== -- Section 9: Correctness - verify actual query results after optimization diff --git a/tests/queries/0_stateless/04000_map_subcolumns_prewhere.reference b/tests/queries/0_stateless/04000_map_subcolumns_prewhere.reference index 89024e17dd7f3..a108bf0eea76d 100644 --- a/tests/queries/0_stateless/04000_map_subcolumns_prewhere.reference +++ b/tests/queries/0_stateless/04000_map_subcolumns_prewhere.reference @@ -54,7 +54,7 @@ 98 98 2 99 99 1 -- Section 4: PREWHERE with basic serialization --- Optimization still rewrites to subcolumn +-- No rewrite to subcolumn 1 -- Correctness: PREWHERE with basic serialization 96 96 960 diff --git a/tests/queries/0_stateless/04000_map_subcolumns_prewhere.sql b/tests/queries/0_stateless/04000_map_subcolumns_prewhere.sql index 3c2e43838788d..704ed0d09ef1e 100644 --- a/tests/queries/0_stateless/04000_map_subcolumns_prewhere.sql +++ b/tests/queries/0_stateless/04000_map_subcolumns_prewhere.sql @@ -134,7 +134,7 @@ DROP TABLE t_map_prewhere; -- ========================================== -- Section 4: PREWHERE with basic serialization (non-bucketed) --- Optimization still applies, but reads the whole map internally. +-- No rewrite to the key subcolumn, it would read the whole map anyway. -- ========================================== DROP TABLE IF EXISTS t_map_prewhere; @@ -142,14 +142,15 @@ CREATE TABLE t_map_prewhere (id UInt64, m Map(String, UInt64)) ENGINE = MergeTree ORDER BY id SETTINGS map_serialization_version = 'basic', + map_serialization_version_for_zero_level_parts = 'basic', min_bytes_for_wide_part = 1, min_rows_for_wide_part = 1; INSERT INTO t_map_prewhere SELECT number, map('key1', number, 'key2', number * 10) FROM numbers(100); SELECT '-- Section 4: PREWHERE with basic serialization'; -SELECT '-- Optimization still rewrites to subcolumn'; -SELECT count() > 0 FROM ( +SELECT '-- No rewrite to subcolumn'; +SELECT count() = 0 FROM ( EXPLAIN actions = 1 SELECT id, m FROM t_map_prewhere PREWHERE m['key1'] > 95 ) WHERE explain LIKE '%m.key_key1%'; diff --git a/tests/queries/0_stateless/04039_text_index_map_values_subcolumn.sql b/tests/queries/0_stateless/04039_text_index_map_values_subcolumn.sql index e12e21f125aad..170b94727c0e1 100644 --- a/tests/queries/0_stateless/04039_text_index_map_values_subcolumn.sql +++ b/tests/queries/0_stateless/04039_text_index_map_values_subcolumn.sql @@ -1,7 +1,7 @@ -- Tags: no-random-settings -- Tests that text indexes built on mapValues(m) work correctly when the analyzer --- rewrites arrayElement(m, 'key') into the map.key_* subcolumn form. +-- rewrites arrayElement(m, 'key') into the map.key_* subcolumn form, which it does only for bucketed Maps. SET explain_query_plan_default = 'legacy'; SET enable_analyzer = 1; @@ -17,7 +17,8 @@ CREATE TABLE tab ) ENGINE = MergeTree ORDER BY id -SETTINGS index_granularity = 1, index_granularity_bytes = '10M', min_bytes_for_wide_part = 0; +SETTINGS index_granularity = 1, index_granularity_bytes = '10M', min_bytes_for_wide_part = 0, + map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO tab VALUES (0, {'service':'web-api'}), (1, {'service':'backend'}), (2, {'service':'frontend'}); diff --git a/tests/queries/0_stateless/04207_alias_map_arrayelement_subcolumn.sql b/tests/queries/0_stateless/04207_alias_map_arrayelement_subcolumn.sql index 142d8cdc7548b..bd324ef1c67f8 100644 --- a/tests/queries/0_stateless/04207_alias_map_arrayelement_subcolumn.sql +++ b/tests/queries/0_stateless/04207_alias_map_arrayelement_subcolumn.sql @@ -6,6 +6,7 @@ -- -- Without the look-through, m_alias['k'] reads the entire Map while m['k'] reads -- only the matching key subcolumn, so the alias path was orders of magnitude slower. +-- The key subcolumn rewrite applies only to bucketed Maps, so the tables with Maps use `with_buckets`. SET explain_query_plan_default = 'legacy'; SET enable_analyzer = 1; @@ -32,7 +33,8 @@ CREATE TABLE t_alias_map a_bad ALIAS mapApply((k, v) -> (k, v + 1), m), a_after_bad ALIAS a_bad ) -ENGINE = MergeTree ORDER BY id; +ENGINE = MergeTree ORDER BY id +SETTINGS map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_alias_map (id, m) SELECT number, map('key1', number, 'key2', number + 1) FROM numbers(10); @@ -153,7 +155,8 @@ CREATE TABLE t_demo some_alias ALIAS attributes, chained_alias ALIAS some_alias ) -ENGINE = MergeTree ORDER BY tuple(); +ENGINE = MergeTree ORDER BY tuple() +SETTINGS map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_demo (attributes) VALUES (map('key1810', 'value-a', 'other', 'value-b')), @@ -182,7 +185,8 @@ CREATE TABLE t_alias_map_pk m Map(String, UInt64), a ALIAS m ) -ENGINE = MergeTree ORDER BY (id, m); +ENGINE = MergeTree ORDER BY (id, m) +SETTINGS map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_alias_map_pk (id, m) SELECT number, map('key1', number) FROM numbers(10); @@ -282,7 +286,8 @@ CREATE TABLE t_alias_correctness a3 ALIAS a2, a_bad ALIAS mapApply((k, v) -> (k, v + 1), m) ) -ENGINE = MergeTree ORDER BY id; +ENGINE = MergeTree ORDER BY id +SETTINGS map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_alias_correctness (id, m) SELECT number, map('key1', number, 'key2', number * 10) FROM numbers(5); @@ -324,7 +329,8 @@ CREATE TABLE t_array_join id UInt64, arr Array(Map(String, UInt64)) ) -ENGINE = MergeTree ORDER BY id; +ENGINE = MergeTree ORDER BY id +SETTINGS map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_array_join VALUES (1, [map('key1', 10, 'key2', 20), map('key1', 11)]), @@ -345,8 +351,10 @@ DROP TABLE t_array_join; -- both sides. The walk must stop at the ListNode and not try to rewrite the join'd Map. DROP TABLE IF EXISTS t_using_left; DROP TABLE IF EXISTS t_using_right; -CREATE TABLE t_using_left (id UInt64, m Map(String, UInt64)) ENGINE = MergeTree ORDER BY id; -CREATE TABLE t_using_right (id UInt64, m Map(String, UInt64)) ENGINE = MergeTree ORDER BY id; +CREATE TABLE t_using_left (id UInt64, m Map(String, UInt64)) ENGINE = MergeTree ORDER BY id +SETTINGS map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; +CREATE TABLE t_using_right (id UInt64, m Map(String, UInt64)) ENGINE = MergeTree ORDER BY id +SETTINGS map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_using_left VALUES (1, map('key1', 10)), (2, map('key1', 20)); INSERT INTO t_using_right VALUES (1, map('key1', 10)), (2, map('key1', 20)); @@ -371,7 +379,8 @@ CREATE TABLE t_subquery_inner id UInt64, m Map(String, UInt64) ) -ENGINE = MergeTree ORDER BY id; +ENGINE = MergeTree ORDER BY id +SETTINGS map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_subquery_inner VALUES (1, map('key1', 100, 'key2', 200)), diff --git a/tests/queries/0_stateless/04513_prewhere_map_subcolumn_read_cost.sql b/tests/queries/0_stateless/04513_prewhere_map_subcolumn_read_cost.sql index 58a42ead810e4..8d0f1e6387a0a 100644 --- a/tests/queries/0_stateless/04513_prewhere_map_subcolumn_read_cost.sql +++ b/tests/queries/0_stateless/04513_prewhere_map_subcolumn_read_cost.sql @@ -1,7 +1,8 @@ -- Regression test: the PREWHERE optimizer must account for Map-key subcolumn read cost. -- When `optimize_functions_to_subcolumns` rewrites `h['k']` to the `h.key_k` subcolumn, --- the optimizer must see its actual on-disk size (the whole Map) via getSubcolumnSize --- and place the cheap `modality` filter before the expensive Map-key predicate. +-- the optimizer must see its actual on-disk size via getSubcolumnSize and place the cheap +-- `modality` filter before the expensive Map-key predicate. The rewrite applies only to bucketed +-- Maps, and with two keys per row the single bucket holds the whole Map. -- -- We disable statistics so that only columns_size drives the ordering, -- and use equality conditions on both sides so that both are "good" conditions @@ -17,7 +18,9 @@ SET explain_query_plan_default = 'legacy'; DROP TABLE IF EXISTS t_prewhere_map_cost; CREATE TABLE t_prewhere_map_cost (id UInt64, modality LowCardinality(String), h Map(String, String)) -ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +ENGINE = MergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 0, + map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_prewhere_map_cost SELECT number, if(number < 1000, 'active', ''), map('k', repeat('v', 300), 'k2', repeat('w', 300)) diff --git a/tests/queries/0_stateless/04546_map_element_duplicate_keys_first_match.sql b/tests/queries/0_stateless/04546_map_element_duplicate_keys_first_match.sql index c0ccfda377af6..e05d09fbc8aa9 100644 --- a/tests/queries/0_stateless/04546_map_element_duplicate_keys_first_match.sql +++ b/tests/queries/0_stateless/04546_map_element_duplicate_keys_first_match.sql @@ -2,6 +2,7 @@ -- map[key] must return the value of the FIRST occurrence of the key in the row, and the -- result must not depend on optimize_functions_to_subcolumns, on preceding rows in the -- block, or on whether a filter isolates the row (issue #111203). +-- The Maps are bucketed: only then is `m[key]` rewritten to the key subcolumn with optimize_functions_to_subcolumns. DROP TABLE IF EXISTS t_map_dup; @@ -9,7 +10,7 @@ DROP TABLE IF EXISTS t_map_dup; -- its match for '' sits at relative position 1. Row 2 has '' at positions 0 and 1, so a -- prediction of position 1 would wrongly return the second occurrence. CREATE TABLE t_map_dup (id UInt64, m Map(String, String)) -ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0, map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_map_dup VALUES (1, map('x', 'k', '', 'prime')), (2, map('', 'FIRST', '', 'SECOND')); SELECT 'wide, filter id=2, subcolumns=0', m[''] FROM t_map_dup WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 0; @@ -23,7 +24,7 @@ SELECT 'wide, count first value, subcolumns=1', count() FROM t_map_dup WHERE id DROP TABLE t_map_dup; -- Compact part (default min_bytes_for_wide_part) exercises the other reader path. -CREATE TABLE t_map_dup (id UInt64, m Map(String, String)) ENGINE = MergeTree ORDER BY id; +CREATE TABLE t_map_dup (id UInt64, m Map(String, String)) ENGINE = MergeTree ORDER BY id SETTINGS map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_map_dup VALUES (1, map('x', 'k', '', 'prime')), (2, map('', 'FIRST', '', 'SECOND')); SELECT 'compact, filter id=2, subcolumns=0', m[''] FROM t_map_dup WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 0; @@ -39,7 +40,7 @@ DROP TABLE IF EXISTS t_map_types; -- Nullable(String) value CREATE TABLE t_map_types (id UInt64, m Map(String, Nullable(String))) -ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0, map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_map_types VALUES (1, map('x', 'k', 'd', 'prime')), (2, map('d', 'FIRST', 'd', 'SECOND')); SELECT 'Nullable(String) subcolumns=0', m['d'] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 0; SELECT 'Nullable(String) subcolumns=1', m['d'] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 1; @@ -47,7 +48,7 @@ DROP TABLE t_map_types; -- LowCardinality(String) value CREATE TABLE t_map_types (id UInt64, m Map(String, LowCardinality(String))) -ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0, map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_map_types VALUES (1, map('x', 'k', 'd', 'prime')), (2, map('d', 'FIRST', 'd', 'SECOND')); SELECT 'LowCardinality(String) subcolumns=0', m['d'] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 0; SELECT 'LowCardinality(String) subcolumns=1', m['d'] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 1; @@ -55,7 +56,7 @@ DROP TABLE t_map_types; -- LowCardinality(Nullable(String)) value CREATE TABLE t_map_types (id UInt64, m Map(String, LowCardinality(Nullable(String)))) -ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0, map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_map_types VALUES (1, map('x', 'k', 'd', 'prime')), (2, map('d', 'FIRST', 'd', 'SECOND')); SELECT 'LowCardinality(Nullable(String)) subcolumns=0', m['d'] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 0; SELECT 'LowCardinality(Nullable(String)) subcolumns=1', m['d'] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 1; @@ -63,7 +64,7 @@ DROP TABLE t_map_types; -- UInt64 value CREATE TABLE t_map_types (id UInt64, m Map(String, UInt64)) -ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0, map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_map_types VALUES (1, map('x', 10, 'd', 20)), (2, map('d', 111, 'd', 222)); SELECT 'UInt64 subcolumns=0', m['d'] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 0; SELECT 'UInt64 subcolumns=1', m['d'] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 1; @@ -71,7 +72,7 @@ DROP TABLE t_map_types; -- Array(String) value CREATE TABLE t_map_types (id UInt64, m Map(String, Array(String))) -ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0, map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_map_types VALUES (1, map('x', ['k'], 'd', ['prime'])), (2, map('d', ['FIRST'], 'd', ['SECOND'])); SELECT 'Array(String) subcolumns=0', m['d'] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 0; SELECT 'Array(String) subcolumns=1', m['d'] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 1; @@ -81,7 +82,7 @@ DROP TABLE t_map_types; -- LowCardinality(String) key CREATE TABLE t_map_types (id UInt64, m Map(LowCardinality(String), String)) -ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0, map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_map_types VALUES (1, map('x', 'k', 'd', 'prime')), (2, map('d', 'FIRST', 'd', 'SECOND')); SELECT 'LowCardinality(String) key subcolumns=0', m['d'] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 0; SELECT 'LowCardinality(String) key subcolumns=1', m['d'] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 1; @@ -89,7 +90,7 @@ DROP TABLE t_map_types; -- UInt64 key CREATE TABLE t_map_types (id UInt64, m Map(UInt64, String)) -ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0, map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_map_types VALUES (1, map(9, 'k', 7, 'prime')), (2, map(7, 'FIRST', 7, 'SECOND')); SELECT 'UInt64 key subcolumns=0', m[7] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 0; SELECT 'UInt64 key subcolumns=1', m[7] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 1; diff --git a/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.sql b/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.sql index 9a5833c380f14..b162f0674020b 100644 --- a/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.sql +++ b/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.sql @@ -6,6 +6,7 @@ -- and the text index has to look through the cast: for an index on `mapValues(m)`. -- The same applies to every conversion that does not change a non-NULL value: adding or dropping `LowCardinality` or -- `Nullable` (adding also as `toNullable`, which `join_use_nulls` emits in pushed-down filters), at any depth of `Array`. +-- The rewrite applies only to bucketed Maps, so the tables with Maps use `with_buckets`. SET explain_query_plan_default = 'legacy'; SET enable_analyzer = 1; @@ -23,7 +24,8 @@ CREATE TABLE tab_values ) ENGINE = MergeTree ORDER BY id -SETTINGS index_granularity = 2, min_bytes_for_wide_part = 0; +SETTINGS index_granularity = 2, min_bytes_for_wide_part = 0, + map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO tab_values VALUES (1, {'level':'error','msg':'disk is full'}), (2, {'level':'warn','msg':'disk is slow'}), (3, {'level':'error','msg':'network is down'}), (4, {}); @@ -57,7 +59,8 @@ CREATE TABLE tab_values_n ) ENGINE = MergeTree ORDER BY id -SETTINGS index_granularity = 2, min_bytes_for_wide_part = 0; +SETTINGS index_granularity = 2, min_bytes_for_wide_part = 0, + map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO tab_values_n VALUES (1, {'level':'error'}), (2, {'level':'error'}), (3, {'level':'warn'}), (4, {'level':NULL}); diff --git a/tests/queries/0_stateless/05339_prewhere_subcolumn_read_steps.reference b/tests/queries/0_stateless/05339_prewhere_subcolumn_read_steps.reference new file mode 100644 index 0000000000000..7a770acba3a2d --- /dev/null +++ b/tests/queries/0_stateless/05339_prewhere_subcolumn_read_steps.reference @@ -0,0 +1,7 @@ +json_whole 1 +json_whole_prewhere 1 +json_other_path 1 +tuple_whole 1 +map_size_and_key 1 +maps_other_map 1 +map_key_and_whole 1 diff --git a/tests/queries/0_stateless/05339_prewhere_subcolumn_read_steps.sql b/tests/queries/0_stateless/05339_prewhere_subcolumn_read_steps.sql new file mode 100644 index 0000000000000..a87623566a816 --- /dev/null +++ b/tests/queries/0_stateless/05339_prewhere_subcolumn_read_steps.sql @@ -0,0 +1,79 @@ +-- Tags: no-parallel-replicas +-- no-parallel-replicas: read_bytes in the initiator's query_log does not include the replicas' reads. + +-- A condition on a cheap subcolumn gets its own PREWHERE read step, so the column it is combined with is read +-- only for the rows it keeps. No row passes the first condition, so a case reads about the bytes of that one alone. + +SET enable_analyzer = 1; +SET optimize_functions_to_subcolumns = 1; +SET optimize_move_to_prewhere = 1; +SET query_plan_optimize_prewhere = 1; +SET move_all_conditions_to_prewhere = 1; +SET enable_multiple_prewhere_read_steps = 1; +SET use_query_condition_cache = 0; + +DROP TABLE IF EXISTS t_json; +DROP TABLE IF EXISTS t_tuple; +DROP TABLE IF EXISTS t_map; +DROP TABLE IF EXISTS t_maps; + +CREATE TABLE t_json (id UInt64, j JSON) ENGINE = MergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; +CREATE TABLE t_tuple (id UInt64, t Tuple(kind String, payload String)) ENGINE = MergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; +-- Bucketed, with as many buckets as keys, so that `m['k1']` reads one bucket and not the whole Map. +CREATE TABLE t_map (id UInt64, m Map(String, String)) ENGINE = MergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0, + map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets', + map_buckets_strategy = 'constant', max_buckets_in_map = 16, map_buckets_min_avg_size = 0; +CREATE TABLE t_maps (id UInt64, c Tuple(left Map(String, String), right Map(String, String))) ENGINE = MergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; + +-- Random values, so that the size of a column on disk, by which the conditions are ordered, reflects the cost of reading it. +INSERT INTO t_json SELECT number, concat('{"kind":"a","payload":"', hex(randomString(500)), '"}') FROM numbers(20000); +INSERT INTO t_tuple SELECT number, tuple('a', hex(randomString(500))) FROM numbers(20000); +INSERT INTO t_map SELECT number, mapFromArrays(arrayMap(i -> 'k' || toString(i), range(16)), arrayMap(i -> hex(randomString(30)), range(16))) FROM numbers(20000); +INSERT INTO t_maps SELECT number, tuple(map('k', 'a'), map('k', hex(randomString(500)))) FROM numbers(20000); + +SELECT count() FROM t_json WHERE j.kind = 'b' FORMAT Null SETTINGS log_comment = 'json_cheap'; +SELECT count() FROM t_tuple WHERE t.kind = 'b' FORMAT Null SETTINGS log_comment = 'tuple_cheap'; +SELECT count() FROM t_map WHERE length(m) = 3 FORMAT Null SETTINGS log_comment = 'map_cheap'; +SELECT count() FROM t_map WHERE m['k1'] = 'x' FORMAT Null SETTINGS log_comment = 'map_key_cheap'; +SELECT count() FROM t_maps WHERE c.left['k'] = 'b' FORMAT Null SETTINGS log_comment = 'maps_cheap'; + +SELECT count() FROM t_json WHERE j.kind = 'b' AND length(toString(j)) > 10 FORMAT Null SETTINGS log_comment = 'json_whole'; +SELECT count() FROM t_json PREWHERE j.kind = 'b' AND length(toString(j)) > 10 FORMAT Null SETTINGS log_comment = 'json_whole_prewhere'; +SELECT count() FROM t_json WHERE j.kind = 'b' AND length(toString(j.payload)) > 10 FORMAT Null SETTINGS log_comment = 'json_other_path'; +SELECT count() FROM t_tuple WHERE t.kind = 'b' AND length(toString(t)) > 10 FORMAT Null SETTINGS log_comment = 'tuple_whole'; +-- PREWHERE keeps the written order, which the WHERE optimizer may change based on the column sizes. +SELECT count() FROM t_map PREWHERE length(m) = 3 AND m['k1'] != '' FORMAT Null SETTINGS log_comment = 'map_size_and_key'; +SELECT count() FROM t_maps WHERE c.left['k'] = 'b' AND c.right['k'] != '' FORMAT Null SETTINGS log_comment = 'maps_other_map'; +SELECT count() FROM t_map WHERE m['k1'] = 'x' AND length(toString(m)) > 10 FORMAT Null SETTINGS log_comment = 'map_key_and_whole'; + +SYSTEM FLUSH LOGS query_log; + +SELECT c.case_name, q.bytes < 2 * base.bytes +FROM VALUES('idx UInt8, case_name String, base_name String', + (1, 'json_whole', 'json_cheap'), + (2, 'json_whole_prewhere', 'json_cheap'), + (3, 'json_other_path', 'json_cheap'), + (4, 'tuple_whole', 'tuple_cheap'), + (5, 'map_size_and_key', 'map_cheap'), + (6, 'maps_other_map', 'maps_cheap'), + (7, 'map_key_and_whole', 'map_key_cheap')) AS c +INNER JOIN +( + SELECT log_comment, max(read_bytes) AS bytes FROM system.query_log + WHERE current_database = currentDatabase() AND type = 'QueryFinish' GROUP BY log_comment +) AS q ON q.log_comment = c.case_name +INNER JOIN +( + SELECT log_comment, max(read_bytes) AS bytes FROM system.query_log + WHERE current_database = currentDatabase() AND type = 'QueryFinish' GROUP BY log_comment +) AS base ON base.log_comment = c.base_name +ORDER BY c.idx; + +DROP TABLE t_json; +DROP TABLE t_tuple; +DROP TABLE t_map; +DROP TABLE t_maps; diff --git a/tests/queries/0_stateless/05340_map_element_to_subcolumn_bucketed_only.reference b/tests/queries/0_stateless/05340_map_element_to_subcolumn_bucketed_only.reference new file mode 100644 index 0000000000000..fcc2c16d9ebd0 --- /dev/null +++ b/tests/queries/0_stateless/05340_map_element_to_subcolumn_bucketed_only.reference @@ -0,0 +1,30 @@ +bucketed MergeTree +1 +MergeTree with bucketed zero-level parts +1 +basic MergeTree +1 +Memory +1 +explicit key subcolumn of a basic Map +8 8 +9 9 +results +8 8 +9 9 +8 8 +9 9 +8 8 +9 9 +Merge over a bucketed table +1 +MaterializedView with a bucketed target +1 +Buffer with a bucketed destination +1 +Merge over a bucketed and a basic table +1 +8 8 +8 8 +9 9 +9 9 diff --git a/tests/queries/0_stateless/05340_map_element_to_subcolumn_bucketed_only.sql b/tests/queries/0_stateless/05340_map_element_to_subcolumn_bucketed_only.sql new file mode 100644 index 0000000000000..f1fdfafeb55a6 --- /dev/null +++ b/tests/queries/0_stateless/05340_map_element_to_subcolumn_bucketed_only.sql @@ -0,0 +1,76 @@ +-- `m['key']` is rewritten to the Map key subcolumn `m.key_` only when the table serializes Maps +-- with buckets, because only then can a single key be read without the whole Map. +-- Both serialization settings are named in every table, clickhouse-test randomizes them. + +SET enable_analyzer = 1; +SET optimize_functions_to_subcolumns = 1; +SET explain_query_plan_default = 'legacy'; + +DROP TABLE IF EXISTS t_map_buffer; +DROP TABLE IF EXISTS t_map_mv; +DROP TABLE IF EXISTS t_map_merge; +DROP TABLE IF EXISTS t_map_memory; +DROP TABLE IF EXISTS t_map_basic; +DROP TABLE IF EXISTS t_map_zero_level_bucketed; +DROP TABLE IF EXISTS t_map_bucketed; + +CREATE TABLE t_map_bucketed (id UInt64, m Map(String, UInt64)) ENGINE = MergeTree ORDER BY id +SETTINGS map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; +CREATE TABLE t_map_zero_level_bucketed (id UInt64, m Map(String, UInt64)) ENGINE = MergeTree ORDER BY id +SETTINGS map_serialization_version = 'basic', map_serialization_version_for_zero_level_parts = 'with_buckets'; +CREATE TABLE t_map_basic (id UInt64, m Map(String, UInt64)) ENGINE = MergeTree ORDER BY id +SETTINGS map_serialization_version = 'basic', map_serialization_version_for_zero_level_parts = 'basic'; +CREATE TABLE t_map_memory (id UInt64, m Map(String, UInt64)) ENGINE = Memory; + +INSERT INTO t_map_bucketed SELECT number, map('k', number) FROM numbers(10); +INSERT INTO t_map_zero_level_bucketed SELECT number, map('k', number) FROM numbers(10); +INSERT INTO t_map_basic SELECT number, map('k', number) FROM numbers(10); +INSERT INTO t_map_memory SELECT number, map('k', number) FROM numbers(10); + +SELECT 'bucketed MergeTree'; +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT id FROM t_map_bucketed WHERE m['k'] > 7) WHERE explain LIKE '%m.key_k%'; +SELECT 'MergeTree with bucketed zero-level parts'; +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT id FROM t_map_zero_level_bucketed WHERE m['k'] > 7) WHERE explain LIKE '%m.key_k%'; +SELECT 'basic MergeTree'; +SELECT count() = 0 FROM (EXPLAIN actions = 1 SELECT id FROM t_map_basic WHERE m['k'] > 7) WHERE explain LIKE '%m.key_k%'; +SELECT 'Memory'; +SELECT count() = 0 FROM (EXPLAIN actions = 1 SELECT id FROM t_map_memory WHERE m['k'] > 7) WHERE explain LIKE '%m.key_k%'; + +SELECT 'explicit key subcolumn of a basic Map'; +SELECT id, m.key_k FROM t_map_basic WHERE m.key_k > 7 ORDER BY id; + +SELECT 'results'; +SELECT id, m['k'] FROM t_map_bucketed WHERE m['k'] > 7 ORDER BY id; +SELECT id, m['k'] FROM t_map_basic WHERE m['k'] > 7 ORDER BY id; +SELECT id, m['k'] FROM t_map_memory WHERE m['k'] > 7 ORDER BY id; + +-- Engines that pass the analyzed query to other tables answer for those tables. +CREATE TABLE t_map_merge (id UInt64, m Map(String, UInt64)) ENGINE = Merge(currentDatabase(), '^t_map_bucketed$'); +CREATE MATERIALIZED VIEW t_map_mv (id UInt64, m Map(String, UInt64)) ENGINE = MergeTree ORDER BY id +SETTINGS map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets' +AS SELECT id, m FROM t_map_bucketed; +CREATE TABLE t_map_buffer (id UInt64, m Map(String, UInt64)) +ENGINE = Buffer(currentDatabase(), t_map_bucketed, 1, 10, 100, 10000, 1000000, 10000000, 100000000); + +SELECT 'Merge over a bucketed table'; +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT id FROM t_map_merge WHERE m['k'] > 7) WHERE explain LIKE '%m.key_k%'; +SELECT 'MaterializedView with a bucketed target'; +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT id FROM t_map_mv WHERE m['k'] > 7) WHERE explain LIKE '%m.key_k%'; +SELECT 'Buffer with a bucketed destination'; +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT id FROM t_map_buffer WHERE m['k'] > 7) WHERE explain LIKE '%m.key_k%'; + +-- The query is sent to every table of a Merge, so all of them must be bucketed. +DROP TABLE t_map_merge; +CREATE TABLE t_map_merge (id UInt64, m Map(String, UInt64)) ENGINE = Merge(currentDatabase(), '^t_map_(bucketed|basic)$'); + +SELECT 'Merge over a bucketed and a basic table'; +SELECT count() = 0 FROM (EXPLAIN actions = 1 SELECT id FROM t_map_merge WHERE m['k'] > 7) WHERE explain LIKE '%m.key_k%'; +SELECT id, m['k'] FROM t_map_merge WHERE m['k'] > 7 ORDER BY id; + +DROP TABLE t_map_buffer; +DROP TABLE t_map_mv; +DROP TABLE t_map_merge; +DROP TABLE t_map_memory; +DROP TABLE t_map_basic; +DROP TABLE t_map_zero_level_bucketed; +DROP TABLE t_map_bucketed; diff --git a/tests/queries/0_stateless/05401_subcolumn_size_without_reading_prefixes.sql b/tests/queries/0_stateless/05401_subcolumn_size_without_reading_prefixes.sql index 573667b7263e1..81c53c24c3247 100644 --- a/tests/queries/0_stateless/05401_subcolumn_size_without_reading_prefixes.sql +++ b/tests/queries/0_stateless/05401_subcolumn_size_without_reading_prefixes.sql @@ -33,7 +33,7 @@ INSERT INTO t_subcolumn_size_lc_map SELECT toString(number % 7), map('k' || toSt SELECT count() FROM system.parts WHERE database = currentDatabase() AND table = 't_subcolumn_size_lc_map' AND active AND part_type = 'Wide'; SELECT countIf(explain ILIKE '%Prewhere filter%') > 0, countIf(explain ILIKE '%m.key_k1%') > 0 -FROM (EXPLAIN actions = 1 SELECT count() FROM t_subcolumn_size_lc_map WHERE m['k1'] = '1' AND s = '3') +FROM (EXPLAIN actions = 1 SELECT count() FROM t_subcolumn_size_lc_map WHERE m.key_k1 = '1' AND s = '3') SETTINGS log_comment = '05401_subcolumn_size_without_reading_prefixes_explain'; SYSTEM FLUSH LOGS query_log; From 53ca072809086791c5410e0d9fdcf1f25fae8dd7 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 10 Nov 2026 21:08:39 +0000 Subject: [PATCH 150/219] Backport #130323 to 26.8: Bump `mongo-c-driver` from 2.5.7 to 2.5.8 --- contrib/mongo-c-driver | 2 +- contrib/mongo-c-driver-cmake/CMakeLists.txt | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/contrib/mongo-c-driver b/contrib/mongo-c-driver index 70de9e880072b..58b4e1f39c6f8 160000 --- a/contrib/mongo-c-driver +++ b/contrib/mongo-c-driver @@ -1 +1 @@ -Subproject commit 70de9e880072b120ccbb48c77de334775747d7a9 +Subproject commit 58b4e1f39c6f8bcf4da4204deb84f9b3527388d4 diff --git a/contrib/mongo-c-driver-cmake/CMakeLists.txt b/contrib/mongo-c-driver-cmake/CMakeLists.txt index 43ac3f5cd0283..6d013bc450fa9 100644 --- a/contrib/mongo-c-driver-cmake/CMakeLists.txt +++ b/contrib/mongo-c-driver-cmake/CMakeLists.txt @@ -6,12 +6,12 @@ endif() set(libbson_VERSION_MAJOR 2) set(libbson_VERSION_MINOR 5) -set(libbson_VERSION_PATCH 5) -set(libbson_VERSION 2.5.5) +set(libbson_VERSION_PATCH 8) +set(libbson_VERSION 2.5.8) set(libmongoc_VERSION_MAJOR 2) set(libmongoc_VERSION_MINOR 5) -set(libmongoc_VERSION_PATCH 5) -set(libmongoc_VERSION 2.5.5) +set(libmongoc_VERSION_PATCH 8) +set(libmongoc_VERSION 2.5.8) set(LIBBSON_SOURCES_ROOT "${ClickHouse_SOURCE_DIR}/contrib/mongo-c-driver/src") set(LIBBSON_SOURCE_DIR "${LIBBSON_SOURCES_ROOT}/libbson/src") From 61bc1fb94c046625977084f6d5e0f70f537b15bd Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 11 Nov 2026 04:45:25 +0000 Subject: [PATCH 151/219] Backport #128899 to 26.8: Raise the per-type task limit of a background schedule pool when its size is increased at runtime --- src/Core/BackgroundSchedulePool.cpp | 38 ++- src/Core/BackgroundSchedulePool.h | 8 +- .../tests/gtest_BackgroundSchedulePool.cpp | 222 ++++++++++++++++++ src/Interpreters/Context.cpp | 15 +- .../configs/small_message_broker_pool.xml | 3 + .../test_message_broker_pool_resize.py | 117 +++++++++ 6 files changed, 387 insertions(+), 16 deletions(-) create mode 100644 tests/integration/test_storage_kafka/configs/small_message_broker_pool.xml create mode 100644 tests/integration/test_storage_kafka/test_message_broker_pool_resize.py diff --git a/src/Core/BackgroundSchedulePool.cpp b/src/Core/BackgroundSchedulePool.cpp index 7fb3095594de8..fbcfcc137c5df 100644 --- a/src/Core/BackgroundSchedulePool.cpp +++ b/src/Core/BackgroundSchedulePool.cpp @@ -49,6 +49,12 @@ Poco::Timestamp::TimeDiff delayToMicroseconds(size_t milliseconds) return static_cast(milliseconds) * 1000; } +size_t maxParallelTasksPerType(size_t pool_size, double ratio) +{ + size_t limit = static_cast(static_cast(pool_size) * ratio); + return limit ? limit : pool_size; +} + } /// @@ -297,12 +303,12 @@ Coordination::WatchCallbackPtr BackgroundSchedulePoolTaskInfo::getWatchCallback( /// BackgroundSchedulePool /// -BackgroundSchedulePoolPtr BackgroundSchedulePool::create(size_t size, size_t initial_size, size_t max_parallel_tasks_per_type, CurrentMetrics::Metric tasks_metric, CurrentMetrics::Metric size_metric, ThreadName thread_name) +BackgroundSchedulePoolPtr BackgroundSchedulePool::create(size_t size, size_t initial_size, double max_parallel_tasks_per_type_ratio, CurrentMetrics::Metric tasks_metric, CurrentMetrics::Metric size_metric, ThreadName thread_name) { - return std::shared_ptr(new BackgroundSchedulePool(size, initial_size, max_parallel_tasks_per_type, tasks_metric, size_metric, thread_name)); + return std::shared_ptr(new BackgroundSchedulePool(size, initial_size, max_parallel_tasks_per_type_ratio, tasks_metric, size_metric, thread_name)); } -BackgroundSchedulePool::BackgroundSchedulePool(size_t size_, size_t initial_size_, size_t max_parallel_tasks_per_type_, CurrentMetrics::Metric tasks_metric_, CurrentMetrics::Metric size_metric_, ThreadName thread_name_) +BackgroundSchedulePool::BackgroundSchedulePool(size_t size_, size_t initial_size_, double max_parallel_tasks_per_type_ratio_, CurrentMetrics::Metric tasks_metric_, CurrentMetrics::Metric size_metric_, ThreadName thread_name_) : logger(getLogger(fmt::format("BackgroundSchedulePool/{}", toString(thread_name_)))) , tasks_metric(tasks_metric_) /// The metric reports the configured cap (so `system.server_settings` displays @@ -310,7 +316,7 @@ BackgroundSchedulePool::BackgroundSchedulePool(size_t size_, size_t initial_size /// not the number of currently spawned workers — that is tracked separately by `threads.size()`. , size_metric(size_metric_, size_) , thread_name(thread_name_) - , max_parallel_tasks_per_type(max_parallel_tasks_per_type_ ? max_parallel_tasks_per_type_ : size_) + , max_parallel_tasks_per_type_ratio(max_parallel_tasks_per_type_ratio_) { const size_t initial_size = std::min(size_, initial_size_); LOG_INFO(logger, "Create BackgroundSchedulePool with {} initial threads (grows lazily up to {})", initial_size, size_); @@ -320,6 +326,7 @@ BackgroundSchedulePool::BackgroundSchedulePool(size_t size_, size_t initial_size { std::lock_guard tasks_lock(tasks_mutex); max_size = size_; + max_parallel_tasks_per_type = maxParallelTasksPerType(size_, max_parallel_tasks_per_type_ratio); threads.reserve(size_); for (size_t i = 0; i < initial_size; ++i) spawnThreadLocked(); @@ -393,9 +400,32 @@ void BackgroundSchedulePool::increaseThreadsCount(size_t new_threads_count) return; } + /// Never lowered: a group may already run more tasks than a lower limit allows. + const size_t new_max_parallel_tasks_per_type = maxParallelTasksPerType(new_threads_count, max_parallel_tasks_per_type_ratio); + const bool raise_limit = new_max_parallel_tasks_per_type > max_parallel_tasks_per_type; + if (raise_limit) + runnable_task_types.reserve(task_groups.size()); + max_size = new_threads_count; size_metric.changeTo(new_threads_count); + if (raise_limit) + { + max_parallel_tasks_per_type = new_max_parallel_tasks_per_type; + + /// Groups paused at the old limit are runnable under the new one. + for (auto & [task_type, group] : task_groups) + { + if (!group.runnable_list_pos && !group.tasks.empty()) + { + group.runnable_list_pos = runnable_task_types.size(); + runnable_task_types.push_back(task_type); + } + } + /// Wake parked workers: the spawn below counts them as available. + tasks_cond_var.notify_all(); + } + /// Consume already-queued demand with the freshly-raised cap. A saturated pool has no parked /// workers to wake and may receive no further scheduleTask call for a while, so without spawning /// here it would keep executing at the old concurrency until some unrelated future task happens diff --git a/src/Core/BackgroundSchedulePool.h b/src/Core/BackgroundSchedulePool.h index bd418101030df..a618c8ae98eab 100644 --- a/src/Core/BackgroundSchedulePool.h +++ b/src/Core/BackgroundSchedulePool.h @@ -57,9 +57,10 @@ class BackgroundSchedulePool : public std::enable_shared_from_this; /// STYLE_CHECK_ALLOW_STD_CONTAINERS /// @param thread_name_ cannot be longer then 13 bytes (2 bytes is reserved for "/D" suffix for delayExecutionThreadFunction()) - BackgroundSchedulePool(size_t size_, size_t initial_size_, size_t max_parallel_tasks_per_type_, CurrentMetrics::Metric tasks_metric_, CurrentMetrics::Metric size_metric_, ThreadName thread_name_); + BackgroundSchedulePool(size_t size_, size_t initial_size_, double max_parallel_tasks_per_type_ratio_, CurrentMetrics::Metric tasks_metric_, CurrentMetrics::Metric size_metric_, ThreadName thread_name_); void threadFunction(); void delayExecutionThreadFunction(); @@ -154,7 +155,8 @@ class BackgroundSchedulePool : public std::enable_shared_from_thisdeactivate(), true); pool->join(); } + +/// Raising the pool size at runtime (config reload) must also raise the number of tasks of one +/// type that may run at once, including for a group already paused at the old limit. +TEST(BackgroundSchedulePool, IncreaseThreadsCountRaisesSameTypeLimit) +{ + constexpr size_t initial_threads = 2; + constexpr size_t increased_threads = 4; + constexpr size_t num_tasks = increased_threads; + + auto pool = BackgroundSchedulePool::create(initial_threads, initial_threads, 0, CurrentMetrics::end(), CurrentMetrics::end(), ThreadName::TEST_SCHEDULER); + + std::mutex mutex; + std::condition_variable running_changed; + std::condition_variable release_tasks; + size_t running = 0; + bool release = false; + + /// A single lambda expression ⇒ one closure type ⇒ all tasks share one task group. + auto task_body = [&] + { + std::unique_lock lock(mutex); + ++running; + running_changed.notify_all(); + release_tasks.wait(lock, [&] { return release; }); + }; + + std::vector tasks; + tasks.reserve(num_tasks); + for (size_t i = 0; i < num_tasks; ++i) + tasks.emplace_back(pool->createTask(StorageID::createEmpty(), "increase_same_type", task_body)); + + for (auto & task : tasks) + EXPECT_TRUE(task->activateAndSchedule()); + + bool initial_ran = false; + bool all_ran = false; + size_t running_before_increase = 0; + size_t running_after_increase = 0; + { + std::unique_lock lock(mutex); + initial_ran = running_changed.wait_for(lock, std::chrono::seconds(60), [&] { return running >= initial_threads; }); + running_before_increase = running; + if (initial_ran) + { + lock.unlock(); + pool->increaseThreadsCount(increased_threads); + lock.lock(); + all_ran = running_changed.wait_for(lock, std::chrono::seconds(60), [&] { return running >= num_tasks; }); + } + running_after_increase = running; + release = true; + release_tasks.notify_all(); + } + + /// The pool's destructor requires join(), so a fatal assertion must not precede it. + pool->join(); + + EXPECT_TRUE(initial_ran); + EXPECT_EQ(running_before_increase, initial_threads); + EXPECT_TRUE(all_ran) << "only " << running_after_increase << " of " << num_tasks << " tasks of one type run after raising the pool size"; + EXPECT_EQ(running_after_increase, num_tasks); +} + +/// With a ratio, the raised limit is the new size times the ratio, exactly as at startup. +TEST(BackgroundSchedulePool, IncreaseThreadsCountKeepsPerTypeRatio) +{ + constexpr size_t initial_threads = 4; + constexpr size_t increased_threads = 8; + constexpr double ratio = 0.5; + constexpr size_t initial_limit = 2; + constexpr size_t increased_limit = 4; + constexpr size_t num_tasks = 6; + + auto pool = BackgroundSchedulePool::create(initial_threads, initial_threads, ratio, CurrentMetrics::end(), CurrentMetrics::end(), ThreadName::TEST_SCHEDULER); + + std::mutex mutex; + std::condition_variable running_changed; + std::condition_variable release_tasks; + size_t running = 0; + bool release = false; + + /// A single lambda expression ⇒ one closure type ⇒ all tasks share one task group. + auto task_body = [&] + { + std::unique_lock lock(mutex); + ++running; + running_changed.notify_all(); + release_tasks.wait(lock, [&] { return release; }); + }; + + std::vector tasks; + tasks.reserve(num_tasks); + for (size_t i = 0; i < num_tasks; ++i) + tasks.emplace_back(pool->createTask(StorageID::createEmpty(), "increase_keeps_ratio", task_body)); + + for (auto & task : tasks) + EXPECT_TRUE(task->activateAndSchedule()); + + bool initial_ran = false; + bool increased_ran = false; + size_t running_before_increase = 0; + size_t running_after_increase = 0; + size_t running_later = 0; + { + std::unique_lock lock(mutex); + initial_ran = running_changed.wait_for(lock, std::chrono::seconds(60), [&] { return running >= initial_limit; }); + running_before_increase = running; + if (initial_ran) + { + lock.unlock(); + pool->increaseThreadsCount(increased_threads); + lock.lock(); + /// The pool still has idle workers here, so they must be woken to take the extra tasks. + increased_ran = running_changed.wait_for(lock, std::chrono::seconds(60), [&] { return running >= increased_limit; }); + running_after_increase = running; + if (increased_ran) + { + lock.unlock(); + std::this_thread::sleep_for(std::chrono::seconds(1)); + lock.lock(); + running_later = running; + } + } + release = true; + release_tasks.notify_all(); + } + + /// The pool's destructor requires join(), so a fatal assertion must not precede it. + pool->join(); + + EXPECT_TRUE(initial_ran); + EXPECT_EQ(running_before_increase, initial_limit); + EXPECT_TRUE(increased_ran) << "only " << running_after_increase << " of " << increased_limit << " tasks of one type run after raising the pool size"; + EXPECT_EQ(running_after_increase, increased_limit); + EXPECT_EQ(running_later, increased_limit) << "the per-type ratio is not kept after raising the pool size"; +} + +/// The recomputed limit can be lower than the current one (0.4 * 2 truncates to 0, which means the +/// pool size 2, while 0.4 * 3 gives 1). It must be ignored: a second wave of tasks of the type, +/// scheduled after the resize, must still run two at once. +TEST(BackgroundSchedulePool, IncreaseThreadsCountNeverLowersPerTypeLimit) +{ + constexpr size_t initial_threads = 2; + constexpr size_t increased_threads = 3; + constexpr double ratio = 0.4; + constexpr size_t limit = 2; + constexpr size_t num_tasks = 2 * limit; + + auto pool = BackgroundSchedulePool::create(initial_threads, initial_threads, ratio, CurrentMetrics::end(), CurrentMetrics::end(), ThreadName::TEST_SCHEDULER); + + std::mutex mutex; + std::condition_variable state_changed; + size_t started = 0; + size_t running = 0; + size_t released = 0; + size_t finished = 0; + + /// A single lambda expression ⇒ one closure type ⇒ all tasks share one task group. + auto task_body = [&] + { + std::unique_lock lock(mutex); + const size_t ordinal = started++; + ++running; + state_changed.notify_all(); + state_changed.wait(lock, [&] { return ordinal < released; }); + --running; + ++finished; + state_changed.notify_all(); + }; + + std::vector tasks; + tasks.reserve(num_tasks); + for (size_t i = 0; i < num_tasks; ++i) + tasks.emplace_back(pool->createTask(StorageID::createEmpty(), "increase_never_lowers", task_body)); + + for (size_t i = 0; i < limit; ++i) + EXPECT_TRUE(tasks[i]->activateAndSchedule()); + + bool first_wave_ran = false; + bool first_wave_finished = false; + bool second_wave_ran = false; + size_t running_before_increase = 0; + size_t running_in_second_wave = 0; + { + std::unique_lock lock(mutex); + first_wave_ran = state_changed.wait_for(lock, std::chrono::seconds(60), [&] { return running >= limit; }); + running_before_increase = running; + if (first_wave_ran) + { + lock.unlock(); + pool->increaseThreadsCount(increased_threads); + lock.lock(); + released = limit; + state_changed.notify_all(); + first_wave_finished = state_changed.wait_for(lock, std::chrono::seconds(60), [&] { return finished >= limit; }); + } + if (first_wave_finished) + { + lock.unlock(); + /// The pool decrements its running count after the task function returns. + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(60); + while (!pool->getTasks().empty() && std::chrono::steady_clock::now() < deadline) + std::this_thread::yield(); + for (size_t i = limit; i < num_tasks; ++i) + EXPECT_TRUE(tasks[i]->activateAndSchedule()); + lock.lock(); + second_wave_ran = state_changed.wait_for(lock, std::chrono::seconds(60), [&] { return running >= limit; }); + running_in_second_wave = running; + } + released = num_tasks; + state_changed.notify_all(); + } + + /// The pool's destructor requires join(), so a fatal assertion must not precede it. + pool->join(); + + EXPECT_TRUE(first_wave_ran); + EXPECT_EQ(running_before_increase, limit); + EXPECT_TRUE(first_wave_finished); + EXPECT_TRUE(second_wave_ran) << "only " << running_in_second_wave << " of " << limit << " tasks of one type run after raising the pool size"; + EXPECT_EQ(running_in_second_wave, limit); +} diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index e1b3e7fbceefb..184dbf95d835e 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -5624,7 +5624,7 @@ BackgroundSchedulePoolPtr Context::getBufferFlushSchedulePool() const auto created_pool = BackgroundSchedulePool::create( shared->server_settings[ServerSetting::background_buffer_flush_schedule_pool_size], shared->server_settings[ServerSetting::background_schedule_pool_initial_size], - /*max_parallel_tasks_per_type*/ 0, + /*max_parallel_tasks_per_type_ratio*/ 0, CurrentMetrics::BackgroundBufferFlushSchedulePoolTask, CurrentMetrics::BackgroundBufferFlushSchedulePoolSize, ThreadName::BACKGROUND_BUFFER_FLUSH_SCHEDULE_POOL); @@ -5685,9 +5685,6 @@ BackgroundTaskSchedulingSettings Context::getBackgroundStreamingTaskSchedulingSe BackgroundSchedulePoolPtr Context::getSchedulePool() const { - size_t max_parallel_tasks_per_type = static_cast( - static_cast(shared->server_settings[ServerSetting::background_schedule_pool_size]) - * static_cast(shared->server_settings[ServerSetting::background_schedule_pool_max_parallel_tasks_per_type_ratio])); callOnce( shared->schedule_pool_initialized, [&] @@ -5695,7 +5692,7 @@ BackgroundSchedulePoolPtr Context::getSchedulePool() const auto created_pool = BackgroundSchedulePool::create( shared->server_settings[ServerSetting::background_schedule_pool_size], shared->server_settings[ServerSetting::background_schedule_pool_initial_size], - max_parallel_tasks_per_type, + static_cast(shared->server_settings[ServerSetting::background_schedule_pool_max_parallel_tasks_per_type_ratio]), CurrentMetrics::BackgroundSchedulePoolTask, CurrentMetrics::BackgroundSchedulePoolSize, DB::ThreadName::BACKGROUND_SCHEDULE_POOL); @@ -5713,7 +5710,7 @@ BackgroundSchedulePoolPtr Context::getDistributedSchedulePool() const auto created_pool = BackgroundSchedulePool::create( shared->server_settings[ServerSetting::background_distributed_schedule_pool_size], shared->server_settings[ServerSetting::background_schedule_pool_initial_size], - /*max_parallel_tasks_per_type*/ 0, + /*max_parallel_tasks_per_type_ratio*/ 0, CurrentMetrics::BackgroundDistributedSchedulePoolTask, CurrentMetrics::BackgroundDistributedSchedulePoolSize, DB::ThreadName::DISTRIBUTED_SCHEDULE_POOL); @@ -5731,7 +5728,7 @@ BackgroundSchedulePoolPtr Context::getMessageBrokerSchedulePool() const auto created_pool = BackgroundSchedulePool::create( shared->server_settings[ServerSetting::background_message_broker_schedule_pool_size], shared->server_settings[ServerSetting::background_schedule_pool_initial_size], - /*max_parallel_tasks_per_type*/ 0, + /*max_parallel_tasks_per_type_ratio*/ 0, CurrentMetrics::BackgroundMessageBrokerSchedulePoolTask, CurrentMetrics::BackgroundMessageBrokerSchedulePoolSize, DB::ThreadName::MSG_BROKER_SCHEDULE_POOL); @@ -5749,7 +5746,7 @@ BackgroundSchedulePoolPtr Context::getIcebergSchedulePool() const auto created_pool = BackgroundSchedulePool::create( shared->server_settings[ServerSetting::iceberg_background_schedule_pool_size], shared->server_settings[ServerSetting::background_schedule_pool_initial_size], - /*max_parallel_tasks_per_type*/ 0, + /*max_parallel_tasks_per_type_ratio*/ 0, CurrentMetrics::IcebergSchedulePoolTask, CurrentMetrics::IcebergSchedulePoolSize, DB::ThreadName::ICEBERG_SCHEDULE_POOL); @@ -5767,7 +5764,7 @@ BackgroundSchedulePoolPtr Context::getStreamingSchedulePool() const auto created_pool = BackgroundSchedulePool::create( shared->server_settings[ServerSetting::background_streaming_schedule_pool_size], shared->server_settings[ServerSetting::background_schedule_pool_initial_size], - /*max_parallel_tasks_per_type*/ 0, + /*max_parallel_tasks_per_type_ratio*/ 0, CurrentMetrics::BackgroundStreamingSchedulePoolTask, CurrentMetrics::BackgroundStreamingSchedulePoolSize, DB::ThreadName::BACKGROUND_STREAMING_SCHEDULE_POOL); diff --git a/tests/integration/test_storage_kafka/configs/small_message_broker_pool.xml b/tests/integration/test_storage_kafka/configs/small_message_broker_pool.xml new file mode 100644 index 0000000000000..f94998de21f2a --- /dev/null +++ b/tests/integration/test_storage_kafka/configs/small_message_broker_pool.xml @@ -0,0 +1,3 @@ + + 2 + diff --git a/tests/integration/test_storage_kafka/test_message_broker_pool_resize.py b/tests/integration/test_storage_kafka/test_message_broker_pool_resize.py new file mode 100644 index 0000000000000..c62b7e42cc906 --- /dev/null +++ b/tests/integration/test_storage_kafka/test_message_broker_pool_resize.py @@ -0,0 +1,117 @@ +import time + +import pytest + +from helpers.cluster import ClickHouseCluster +import helpers.kafka.common as k + +cluster = ClickHouseCluster(__file__) +instance = cluster.add_instance( + "instance", + main_configs=["configs/small_message_broker_pool.xml"], + user_configs=["configs/users.xml"], + with_kafka=True, + with_zookeeper=True, # For Kafka2 + macros={ + "kafka_broker": "kafka1", + "kafka_format_json_each_row": "JSONEachRow", + }, + stay_alive=True, +) + +STARTUP_POOL_SIZE = 2 +RELOADED_POOL_SIZE = 8 +NUM_TABLES = 4 +OVERRIDE_PATH = "/etc/clickhouse-server/config.d/zz_message_broker_pool_size.xml" + + +@pytest.fixture(scope="module") +def kafka_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +@pytest.fixture(autouse=True) +def kafka_setup_teardown(): + k.clean_test_database_and_topics(instance, cluster) + yield + # The pool size can only grow at runtime, so a restart is needed to get back to the startup size. + instance.query("DROP DATABASE IF EXISTS test SYNC") + instance.exec_in_container(["bash", "-c", f"rm -f {OVERRIDE_PATH}"]) + instance.restart_clickhouse() + + +def metric(name): + return int( + instance.query( + f"SELECT value FROM system.metrics WHERE metric = '{name}'" + ).strip() + ) + + +def max_executing_tasks(duration_sec, stop_at=None): + # Only the Kafka tables of this test run tasks in the message broker pool. + deadline = time.monotonic() + duration_sec + result = 0 + while time.monotonic() < deadline: + result = max(result, metric("BackgroundMessageBrokerSchedulePoolTask")) + if stop_at is not None and result >= stop_at: + break + time.sleep(0.1) + return result + + +def pool_size_metric(): + return metric("BackgroundMessageBrokerSchedulePoolSize") + + +@pytest.mark.parametrize( + "create_query_generator", + [k.generate_old_create_table_query, k.generate_new_create_table_query], +) +def test_kafka_tasks_use_increased_message_broker_pool( + kafka_cluster, create_query_generator +): + admin_client = k.get_admin_client(kafka_cluster) + postfix = k.get_topic_postfix(create_query_generator) + topics = [f"pool_resize_{postfix}_{i}" for i in range(NUM_TABLES)] + for topic in topics: + k.kafka_create_topic(admin_client, topic) + + # The topics stay empty, so every streaming task spends most of its time + # blocked in poll, holding a pool thread. + for i, topic in enumerate(topics): + create_query = create_query_generator( + f"kafka_{i}", + "key UInt64, value UInt64", + topic_list=topic, + consumer_group=topic, + settings={ + "kafka_thread_per_consumer": k.must_use_thread_per_consumer( + create_query_generator + ), + "kafka_consumer_reschedule_ms": 100, + }, + ) + instance.query(f""" + {create_query}; + CREATE TABLE test.view_{i} (key UInt64, value UInt64) ENGINE = MergeTree ORDER BY key; + CREATE MATERIALIZED VIEW test.consumer_{i} TO test.view_{i} AS SELECT * FROM test.kafka_{i}; + """) + + assert pool_size_metric() == STARTUP_POOL_SIZE + assert max_executing_tasks(30, stop_at=STARTUP_POOL_SIZE) == STARTUP_POOL_SIZE + assert max_executing_tasks(5) == STARTUP_POOL_SIZE + + instance.replace_config( + OVERRIDE_PATH, + f"{RELOADED_POOL_SIZE}", + ) + instance.query("SYSTEM RELOAD CONFIG") + assert pool_size_metric() == RELOADED_POOL_SIZE + + # Before the fix the tasks of one type stayed capped at the startup pool size. + assert max_executing_tasks(60, stop_at=NUM_TABLES) == NUM_TABLES From f319e7932090267bbd9ec3fb5b805a69345fdeb3 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 11 Nov 2026 07:14:21 +0000 Subject: [PATCH 152/219] Backport #124557 to 26.8: Fix nested S3 request retries and read the error an S3 server sends before the whole request body --- .../Net/include/Poco/Net/HTTPClientSession.h | 4 ++ base/poco/Net/src/HTTPClientSession.cpp | 30 ++++++++++++ src/IO/S3/Client.cpp | 4 ++ src/IO/S3/PocoHTTPClient.cpp | 23 +++++++-- .../integration/helpers/s3_mocks/broken_s3.py | 34 +++++++++++++ .../test_checking_s3_blobs_paranoid/test.py | 48 +++++++++++++++++++ 6 files changed, 140 insertions(+), 3 deletions(-) diff --git a/base/poco/Net/include/Poco/Net/HTTPClientSession.h b/base/poco/Net/include/Poco/Net/HTTPClientSession.h index 8ab1c0f19bf24..c8dd216b070f2 100644 --- a/base/poco/Net/include/Poco/Net/HTTPClientSession.h +++ b/base/poco/Net/include/Poco/Net/HTTPClientSession.h @@ -290,6 +290,10 @@ namespace Net /// This method should only be called if the request contains /// a "Expect: 100-continue" header. + bool receiveEarlyResponse(HTTPResponse & response); + /// After sending the request failed, drops the rest of it and reads the response header the + /// server has already sent. If true, receiveResponse() must be called with the same response. + virtual void flushRequest(); /// Flushes the request stream. /// diff --git a/base/poco/Net/src/HTTPClientSession.cpp b/base/poco/Net/src/HTTPClientSession.cpp index 1754f59d8e5c2..6a183c563b647 100644 --- a/base/poco/Net/src/HTTPClientSession.cpp +++ b/base/poco/Net/src/HTTPClientSession.cpp @@ -427,6 +427,36 @@ bool HTTPClientSession::peekResponse(HTTPResponse& response) } +bool HTTPClientSession::receiveEarlyResponse(HTTPResponse& response) +{ + poco_assert (!_responseReceived); + + /// Drops the unsent rest of the request, which rethrows the send error. + try + { + flushRequest(); + } + catch (...) + { + } + clearException(); + + response.clear(); + HTTPHeaderInputStream his(*this); + try + { + response.read(his); + } + catch (Exception&) + { + close(); + return false; + } + _responseReceived = response.getStatus() != HTTPResponse::HTTP_CONTINUE; + return _responseReceived; +} + + void HTTPClientSession::reset() { close(); diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index b16f2316a3477..d552ea34d69d1 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -882,6 +882,10 @@ Client::doRequestWithRetryNetworkErrors(RequestType & request, RequestFn request // do not increment S3ReadRequestsErrors/S3WriteRequestsErrors here, it has been accounted in IO/S3/PocoHTTPClient.cpp + /// Unless retries are coordinated across threads, the SDK has already retried this error. + if (!client_configuration.s3_slow_all_threads_after_retryable_error) + break; + /// Retry attempts are managed by the outer loop, so the attemptedRetries argument can be ignored. if (!client_configuration.retryStrategy->ShouldRetry(outcome.GetError(), /*attemptedRetries*/ -1)) break; diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index 303afead2c090..d772eeb241471 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -637,9 +638,25 @@ void PocoHTTPClient::makeRequestInternalImpl( request.GetContentBody()->seekg(0); setTimeouts(*session, getTimeouts(method, first_attempt, /*first_byte*/ false)); - auto size = Poco::StreamCopier::copyStream(*request.GetContentBody(), request_body_stream); - if (enable_s3_requests_logging) - LOG_TEST(log, "Written {} bytes to request body", size); + try + { + auto size = Poco::StreamCopier::copyStream(*request.GetContentBody(), request_body_stream); + request_body_stream.flush(); + if (enable_s3_requests_logging) + LOG_TEST(log, "Written {} bytes to request body", size); + } + catch (const Poco::IOException &) + { + /// A server may answer with an error and close the connection before reading the whole body. + /// TLS allows no I/O after a failed write. + if (session->secure() + || DB::getSocketState(session->socket()) != DB::SocketState::DataPending + || !session->receiveEarlyResponse(poco_response) + || poco_response.getStatus() < Poco::Net::HTTPResponse::HTTP_MULTIPLE_CHOICES) + throw; + LOG_DEBUG(log, "Failed to send the request body to {}, using the response the server has already sent: {}", + uri, getCurrentExceptionMessage(/* with_stacktrace */ false)); + } } setTimeouts(*session, getTimeouts(method, first_attempt, /*first_byte*/ false)); diff --git a/tests/integration/helpers/s3_mocks/broken_s3.py b/tests/integration/helpers/s3_mocks/broken_s3.py index d0adeeb5df79d..a3fd6ab5852dc 100644 --- a/tests/integration/helpers/s3_mocks/broken_s3.py +++ b/tests/integration/helpers/s3_mocks/broken_s3.py @@ -229,6 +229,36 @@ def inject_error(self, request_handler): ) request_handler.write_error(500, data) + class NoSuchUploadAction: + # Answers directly instead of redirecting, so the upload is never completed upstream and + # whatever object the key already holds stays in place. + def __init__(self, early=None): + self.early = early == "1" + + def inject_error(self, request_handler): + data = ( + '' + "" + "NoSuchUpload" + "The specified upload does not exist. The upload ID may be invalid, " + "or the upload may have been aborted or completed." + "txfbd566d03042474888193-00608d7538" + "" + ) + if self.early: + request_handler.send_response(404) + request_handler.send_header("Content-Type", "text/xml") + request_handler.send_header("Content-Length", str(len(data))) + request_handler.send_header("Connection", "close") + request_handler.end_headers() + request_handler.wfile.write(bytes(data, "UTF-8")) + # like MinIO: stop sending, wait, then close with the body unread, which resets the connection + request_handler.connection.shutdown(socket.SHUT_WR) + time.sleep(0.5) + request_handler.connection.close() + else: + request_handler.write_error(404, data) + class SlowDownAction: def inject_error(self, request_handler): data = ( @@ -396,6 +426,10 @@ def __init__( ) elif self.action == "timeout": self.error_handler = _ServerRuntime.TimeoutAction() + elif self.action == "no_such_upload": + self.error_handler = _ServerRuntime.NoSuchUploadAction( + *self.action_args + ) else: self.error_handler = _ServerRuntime.Expected500ErrorAction() diff --git a/tests/integration/test_checking_s3_blobs_paranoid/test.py b/tests/integration/test_checking_s3_blobs_paranoid/test.py index 805e2b8e827e4..4e302288cf945 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/test.py +++ b/tests/integration/test_checking_s3_blobs_paranoid/test.py @@ -308,6 +308,7 @@ def test_when_error_is_retried(cluster, broken_s3, action_and_message): SETTINGS s3_max_single_part_upload_size=100, s3_min_upload_part_size=100, + s3_max_inflight_parts_for_one_file=1, s3_check_objects_after_upload=0 """, query_id=insert_query_id, @@ -316,6 +317,10 @@ def test_when_error_is_retried(cluster, broken_s3, action_and_message): assert "Code: 499" in error, error assert message in error, error + _, _, s3_errors = get_multipart_counters(node, insert_query_id) + # s3_retry_attempts + 1 attempts of the failing part, plus the failed abort of the fake upload + assert s3_errors == 5 + 1 + 1, s3_errors + def test_when_s3_broken_pipe_at_upload_is_retried(cluster, broken_s3): node = cluster.instances["node"] @@ -389,6 +394,49 @@ def test_when_s3_broken_pipe_at_upload_is_retried(cluster, broken_s3): ), error +def test_early_error_response_at_upload_is_not_retried(cluster, broken_s3): + node = cluster.instances["node"] + + broken_s3.setup_fake_multpartuploads() + broken_s3.setup_at_part_upload( + count=1000, + after=0, + action="no_such_upload", + action_args=["1"], + ) + + insert_query_id = randomize_query_id("TEST_EARLY_ERROR_RESPONSE_AT_UPLOAD") + error = node.query_and_get_error( + f""" + INSERT INTO + TABLE FUNCTION s3( + 'http://resolver:8083/root/data/test_early_error_response_at_upload_is_not_retried', + 'minio', '{minio_secret_key}', + 'CSV', auto, 'none' + ) + SELECT number, randomString(1000) FROM numbers(40000) + SETTINGS + s3_max_single_part_upload_size=100, + s3_min_upload_part_size=33554432, + s3_max_inflight_parts_for_one_file=1, + max_remote_write_network_bandwidth=1000000, + s3_check_objects_after_upload=0 + """, + query_id=insert_query_id, + ) + + assert "Code: 499" in error, error + assert "The specified upload does not exist" in error, error + + create_multipart, upload_parts, s3_errors = get_multipart_counters( + node, insert_query_id + ) + assert create_multipart == 1 + assert upload_parts == 1 + # the early 404 and the failed abort of the fake upload + assert s3_errors == 2, s3_errors + + @pytest.mark.parametrize("send_something", [True, False]) def test_when_s3_connection_reset_by_peer_at_upload_is_retried( cluster, broken_s3, send_something From 4f5816086d61b915de0c9345c4db30be5fbbb92a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 10 Nov 2026 16:23:21 +0000 Subject: [PATCH 153/219] Backport #130444 to 26.8: Bump `mongo-cxx-driver` from r4.6.0 to r4.6.2 --- contrib/mongo-cxx-driver | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/mongo-cxx-driver b/contrib/mongo-cxx-driver index fa9ee6ed008e8..35424a5cc5973 160000 --- a/contrib/mongo-cxx-driver +++ b/contrib/mongo-cxx-driver @@ -1 +1 @@ -Subproject commit fa9ee6ed008e84b9562a7a55d70681de8a6bc1c6 +Subproject commit 35424a5cc597391d3737ae5e00f029cd2afc03c4 From dced50c1182a0219f66439be0b4d6464f64652da Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 11 Nov 2026 10:38:47 +0000 Subject: [PATCH 154/219] Backport #126704 to 26.8: Fix `parseDateTime` next to a daylight saving time change --- src/Functions/parseDateTime.cpp | 41 ++++++++++-- ...e_date_time_daylight_saving_time.reference | 65 +++++++++++++++++++ ...0_parse_date_time_daylight_saving_time.sql | 51 +++++++++++++++ 3 files changed, 151 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/05410_parse_date_time_daylight_saving_time.reference create mode 100644 tests/queries/0_stateless/05410_parse_date_time_daylight_saving_time.sql diff --git a/src/Functions/parseDateTime.cpp b/src/Functions/parseDateTime.cpp index cdf9157e2fca9..08d1fa749849b 100644 --- a/src/Functions/parseDateTime.cpp +++ b/src/Functions/parseDateTime.cpp @@ -562,8 +562,10 @@ namespace [[nodiscard]] Int64OrError buildDateTime(const DateLUTImpl & time_zone) { - if (is_hour_of_half_day && !is_am) - hour += 12; + /// The hour of a half-day clock is stored in [0, 11], and PM adds 12 hours to it. They are added to a copy: for a + /// time zone name in the string (`z` in Joda syntax), this function also runs before the final conversion, so + /// adding them to `hour` itself added them twice. + const Int32 hour_of_day = is_hour_of_half_day && !is_am ? hour + 12 : hour; // Convert the parsed date/time into a timestamp. Int32 days_since_epoch = 0; @@ -574,11 +576,33 @@ namespace else ASSIGN_RESULT_OR_RETURN_ERROR(days_since_epoch, daysSinceEpochFromDate(year, month, day)) - Int64 seconds_since_epoch = days_since_epoch * 86400UZ + hour * 3600UZ + minute * 60UZ + second; + Int64 seconds_since_epoch = days_since_epoch * 86400UZ + hour_of_day * 3600UZ + minute * 60UZ + second; /// Time zone is not specified, use local time zone if (!has_time_zone_offset) - time_zone_offset = time_zone.timezoneOffset(seconds_since_epoch); + { + /// Convert the local time as `toDateTime` does. The offset at `seconds_since_epoch`, which is the local + /// time read as UTC, can be the one from the other side of a daylight saving time change. + auto local_year = static_cast(year); + auto local_month = static_cast(month); + auto local_day = static_cast(day); + /// For a week date or a day of the year, take the calendar date from the day number. + if (week_date_format || day_of_year_format) + { + const auto values = time_zone.getValues(ExtendedDayNum(days_since_epoch)); + local_year = values.year; + local_month = values.month; + local_day = values.day_of_month; + } + time_zone_offset = seconds_since_epoch + - time_zone.makeDateTime( + local_year, + local_month, + local_day, + static_cast(hour_of_day), + static_cast(minute), + static_cast(second)); + } /// Time zone is specified in format string. seconds_since_epoch -= time_zone_offset; @@ -1786,12 +1810,17 @@ namespace { std::string_view read_time_zone{cur, end}; const DateLUTImpl & date_time_zone = DateLUT::instance(read_time_zone); + const bool has_offset_before_time_zone = parsed_value.has_time_zone_offset; const auto result = parsed_value.buildDateTime(date_time_zone); if (result.has_value()) { - const DateLUTImpl::Time timezone_offset = date_time_zone.timezoneOffset(*result); + /// An offset parsed before the time zone name selects the instant, for example one of the two occurrences + /// of a repeated local time, and the time zone gives its offset at that instant. Without such an offset, + /// keep the offset that `buildDateTime` resolved for the local time: the offset at the resulting instant + /// differs from it for a local time that does not exist. + if (has_offset_before_time_zone) + parsed_value.time_zone_offset = date_time_zone.timezoneOffset(*result); parsed_value.has_time_zone_offset = true; - parsed_value.time_zone_offset = timezone_offset; return end; } else diff --git a/tests/queries/0_stateless/05410_parse_date_time_daylight_saving_time.reference b/tests/queries/0_stateless/05410_parse_date_time_daylight_saving_time.reference new file mode 100644 index 0000000000000..23ae65ac69b67 --- /dev/null +++ b/tests/queries/0_stateless/05410_parse_date_time_daylight_saving_time.reference @@ -0,0 +1,65 @@ +2026-10-25 01:00:00 +Europe/Berlin, clocks go back +2026-10-25 00:00:00 1792879200 1 +2026-10-25 00:30:00 1792881000 1 +2026-10-25 01:00:00 1792882800 1 +2026-10-25 01:30:00 1792884600 1 +2026-10-25 02:00:00 1792886400 1 +2026-10-25 02:30:00 1792888200 1 +2026-10-25 03:00:00 1792893600 1 +2026-10-25 03:30:00 1792895400 1 +2026-10-25 04:00:00 1792897200 1 +2026-10-25 04:30:00 1792899000 1 +Europe/Berlin, clocks go forward +2026-03-29 00:00:00 1774738800 1 +2026-03-29 00:30:00 1774740600 1 +2026-03-29 01:00:00 1774742400 1 +2026-03-29 01:30:00 1774744200 1 +2026-03-29 02:00:00 1774742400 1 +2026-03-29 02:30:00 1774744200 1 +2026-03-29 03:00:00 1774746000 1 +2026-03-29 03:30:00 1774747800 1 +2026-03-29 04:00:00 1774749600 1 +2026-03-29 04:30:00 1774751400 1 +America/New_York, clocks go back +2026-11-01 00:00:00 1793505600 1 +2026-11-01 00:30:00 1793507400 1 +2026-11-01 01:00:00 1793509200 1 +2026-11-01 01:30:00 1793511000 1 +2026-11-01 02:00:00 1793516400 1 +2026-11-01 02:30:00 1793518200 1 +2026-11-01 03:00:00 1793520000 1 +2026-11-01 03:30:00 1793521800 1 +2026-11-01 04:00:00 1793523600 1 +2026-11-01 04:30:00 1793525400 1 +other functions and formats +2026-10-25 01:00:00.500000 +2026-10-25 01:00:00 +2026-10-25 01:00:00 +2026-10-25 01:00:00 +2026-10-25 01:00:00 +2026-10-25 01:00:00 +2026-10-25 02:00:00 +America/New_York, clocks go forward, time zone name in the string +2026-03-08 00:00:00 1772946000 1 +2026-03-08 00:30:00 1772947800 1 +2026-03-08 01:00:00 1772949600 1 +2026-03-08 01:30:00 1772951400 1 +2026-03-08 02:00:00 1772949600 1 +2026-03-08 02:30:00 1772951400 1 +2026-03-08 03:00:00 1772953200 1 +2026-03-08 03:30:00 1772955000 1 +2026-03-08 04:00:00 1772956800 1 +2026-03-08 04:30:00 1772958600 1 +2026-03-08 06:30:00 +2026-10-25 00:30:00 +2026-10-25 00:30:00 +2026-10-25 01:30:00 +2026-10-25 01:30:00.5 +PM with a time zone name in the string +2026-10-25 12:30:00 +2026-10-25 11:30:00 +2026-10-24 22:30:00 +2026-10-25 12:30:00.5 +2026-03-10 13:30:00 +2026-10-25 12:30:00 diff --git a/tests/queries/0_stateless/05410_parse_date_time_daylight_saving_time.sql b/tests/queries/0_stateless/05410_parse_date_time_daylight_saving_time.sql new file mode 100644 index 0000000000000..465889c1a0cd2 --- /dev/null +++ b/tests/queries/0_stateless/05410_parse_date_time_daylight_saving_time.sql @@ -0,0 +1,51 @@ +-- `parseDateTime` converts a local time as `toDateTime` does, also next to a daylight saving time change. +-- In `Europe/Berlin` 02:00-02:59 happens twice on 2026-10-25 and does not exist on 2026-03-29. +-- https://github.com/ClickHouse/ClickHouse/issues/126667 + +SELECT parseDateTime('2026-10-25 01:00:00', '%Y-%m-%d %H:%i:%s', 'Europe/Berlin'); + +SELECT 'Europe/Berlin, clocks go back'; +SELECT s, toUnixTimestamp(parseDateTime(s, '%Y-%m-%d %H:%i:%s', 'Europe/Berlin')) AS t, t = toUnixTimestamp(toDateTime(s, 'Europe/Berlin')) +FROM (SELECT formatDateTime(toDateTime('2026-10-25 00:00:00', 'UTC') + number * 1800, '%Y-%m-%d %H:%i:%s', 'UTC') AS s FROM numbers(10)); + +SELECT 'Europe/Berlin, clocks go forward'; +SELECT s, toUnixTimestamp(parseDateTime(s, '%Y-%m-%d %H:%i:%s', 'Europe/Berlin')) AS t, t = toUnixTimestamp(toDateTime(s, 'Europe/Berlin')) +FROM (SELECT formatDateTime(toDateTime('2026-03-29 00:00:00', 'UTC') + number * 1800, '%Y-%m-%d %H:%i:%s', 'UTC') AS s FROM numbers(10)); + +-- West of UTC, the hours after the change were wrong. +SELECT 'America/New_York, clocks go back'; +SELECT s, toUnixTimestamp(parseDateTime(s, '%Y-%m-%d %H:%i:%s', 'America/New_York')) AS t, t = toUnixTimestamp(toDateTime(s, 'America/New_York')) +FROM (SELECT formatDateTime(toDateTime('2026-11-01 00:00:00', 'UTC') + number * 1800, '%Y-%m-%d %H:%i:%s', 'UTC') AS s FROM numbers(10)); + +SELECT 'other functions and formats'; +SELECT parseDateTime64('2026-10-25 01:00:00.5', '%Y-%m-%d %H:%i:%s.%f', 'Europe/Berlin'); +SELECT parseDateTimeInJodaSyntax('2026-10-25 01:00:00', 'yyyy-MM-dd HH:mm:ss', 'Europe/Berlin'); +SELECT parseDateTimeOrNull('2026-10-25 01:00:00', '%Y-%m-%d %H:%i:%s', 'Europe/Berlin'); +SELECT parseDateTime('2026-10-25 01:00:00 AM', '%Y-%m-%d %h:%i:%s %p', 'Europe/Berlin'); +SELECT parseDateTime('2026-298 01:00:00', '%Y-%j %H:%i:%s', 'Europe/Berlin'); +SELECT parseDateTimeInJodaSyntax('2026-43-7 01:00:00', 'xxxx-ww-e HH:mm:ss', 'Europe/Berlin'); + +-- An offset in the string is used as it is. +SELECT parseDateTime('2026-10-25 01:00:00 +0100', '%Y-%m-%d %H:%i:%s %z', 'Europe/Berlin'); + +-- A time zone name in the string gives the same results, also for a local time that does not exist. +SELECT 'America/New_York, clocks go forward, time zone name in the string'; +SELECT s, toUnixTimestamp(parseDateTimeInJodaSyntax(concat(s, ' America/New_York'), 'yyyy-MM-dd HH:mm:ss z', 'UTC')) AS t, + t = toUnixTimestamp(toDateTime(s, 'America/New_York')) +FROM (SELECT formatDateTime(toDateTime('2026-03-08 00:00:00', 'UTC') + number * 1800, '%Y-%m-%d %H:%i:%s', 'UTC') AS s FROM numbers(10)); +SELECT parseDateTime64InJodaSyntaxOrNull('2026-03-08 02:30:00 America/New_York', 'yyyy-MM-dd HH:mm:ss z', 'UTC'); +SELECT parseDateTimeInJodaSyntax('2026-10-25 02:30:00 Europe/Berlin', 'yyyy-MM-dd HH:mm:ss z', 'UTC'); + +-- An offset before the time zone name selects one of the two 02:30. +SELECT parseDateTimeInJodaSyntax('2026-10-25 02:30:00 +0200 Europe/Berlin', 'yyyy-MM-dd HH:mm:ss Z z', 'UTC'); +SELECT parseDateTimeInJodaSyntax('2026-10-25 02:30:00 +0100 Europe/Berlin', 'yyyy-MM-dd HH:mm:ss Z z', 'UTC'); +SELECT parseDateTime64InJodaSyntax('2026-10-25 02:30:00.5 +0100 Europe/Berlin', 'yyyy-MM-dd HH:mm:ss.S Z z', 'UTC'); + +-- A time zone name in the string builds the time once more, which added the 12 hours of PM twice. +SELECT 'PM with a time zone name in the string'; +SELECT parseDateTimeInJodaSyntax('2026-10-25 01:30:00 PM Europe/Berlin', 'yyyy-MM-dd hh:mm:ss a z', 'UTC'); +SELECT parseDateTimeInJodaSyntax('2026-10-25 12:30:00 PM Europe/Berlin', 'yyyy-MM-dd hh:mm:ss a z', 'UTC'); +SELECT parseDateTimeInJodaSyntax('2026-10-25 12:30:00 AM Europe/Berlin', 'yyyy-MM-dd hh:mm:ss a z', 'UTC'); +SELECT parseDateTime64InJodaSyntax('2026-10-25 01:30:00.5 PM Europe/Berlin', 'yyyy-MM-dd hh:mm:ss.S a z', 'UTC'); +SELECT parseDateTimeInJodaSyntax('2026-03-10 01:30:00 PM UTC', 'yyyy-MM-dd hh:mm:ss a z', 'UTC'); +SELECT parseDateTimeInJodaSyntax('2026-10-25 01:30:00 PM +0100 Europe/Berlin', 'yyyy-MM-dd hh:mm:ss a Z z', 'UTC'); From 1ec74e523e2267dd06ef17db3539f4da60f63b40 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 11 Nov 2026 11:17:25 +0000 Subject: [PATCH 155/219] Backport #130311 to 26.8: Bump vendored `expat` from 2.9.0 to 2.10.0 --- base/poco/XML/CMakeLists.txt | 1 + base/poco/XML/include/Poco/XML/expat.h | 5 +- .../XML/include/Poco/XML/expat_external.h | 2 +- base/poco/XML/src/ascii.h | 3 +- base/poco/XML/src/entities.h | 99 ++ base/poco/XML/src/internal.h | 8 +- base/poco/XML/src/xcs.h | 2 + base/poco/XML/src/xmlparse.cpp | 871 +++++++++++++++--- base/poco/XML/src/xmltok.h | 4 +- base/poco/XML/src/xmltok_impl.c | 6 +- base/poco/XML/src/xmlxcs.c | 21 + 11 files changed, 871 insertions(+), 151 deletions(-) create mode 100644 base/poco/XML/src/entities.h diff --git a/base/poco/XML/CMakeLists.txt b/base/poco/XML/CMakeLists.txt index 1d3c8d332f1b1..d0d9f26fc2684 100644 --- a/base/poco/XML/CMakeLists.txt +++ b/base/poco/XML/CMakeLists.txt @@ -9,6 +9,7 @@ add_library (Poco::XML::Expat ALIAS _poco_xml_expat) target_compile_options (_poco_xml_expat PRIVATE -Wno-cast-qual + -Wno-declaration-after-statement -Wno-disabled-macro-expansion -Wno-empty-translation-unit -Wno-extra-semi-stmt diff --git a/base/poco/XML/include/Poco/XML/expat.h b/base/poco/XML/include/Poco/XML/expat.h index 02a3fe98b2ac5..9e57096d428f7 100644 --- a/base/poco/XML/include/Poco/XML/expat.h +++ b/base/poco/XML/include/Poco/XML/expat.h @@ -21,6 +21,7 @@ Copyright (c) 2024 Taichi Haradaguchi <20001722@ymail.ne.jp> Copyright (c) 2025 Matthew Fernandez Copyright (c) 2026 Braian Plaku + Copyright (c) 2026 Xisco Fauli Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -1094,7 +1095,7 @@ enum XML_Parser_Property { # if defined(XML_TESTING) XML_PROP_INVALID = 0, # endif -# if XML_GE == 1 +# if defined(XML_GE) && XML_GE == 1 XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD = 1, // of type `uint64_t` XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION = 2, // of type `double` XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD = 3, // of type `uint64_t` @@ -1156,7 +1157,7 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); See https://semver.org */ # define XML_MAJOR_VERSION 2 -# define XML_MINOR_VERSION 9 +# define XML_MINOR_VERSION 10 # define XML_MICRO_VERSION 0 # ifdef __cplusplus diff --git a/base/poco/XML/include/Poco/XML/expat_external.h b/base/poco/XML/include/Poco/XML/expat_external.h index 9fb80149acd3a..f34750afbddca 100644 --- a/base/poco/XML/include/Poco/XML/expat_external.h +++ b/base/poco/XML/include/Poco/XML/expat_external.h @@ -12,7 +12,7 @@ Copyright (c) 2001-2002 Greg Stein Copyright (c) 2002-2006 Karl Waclawek Copyright (c) 2016 Cristian Rodríguez - Copyright (c) 2016-2025 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2017 Rhodri James Copyright (c) 2018 Yury Gribov Copyright (c) 2026 Matthew Fernandez diff --git a/base/poco/XML/src/ascii.h b/base/poco/XML/src/ascii.h index 56c8807aa5746..656c95e28ddb3 100644 --- a/base/poco/XML/src/ascii.h +++ b/base/poco/XML/src/ascii.h @@ -10,7 +10,7 @@ Copyright (c) 2000 Clark Cooper Copyright (c) 2002 Fred L. Drake, Jr. Copyright (c) 2007 Karl Waclawek - Copyright (c) 2017 Sebastian Pipping + Copyright (c) 2017-2026 Sebastian Pipping Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -113,6 +113,7 @@ #define ASCII_LT 0x3C #define ASCII_EQUALS 0x3D #define ASCII_GT 0x3E +#define ASCII_QUEST 0x3F #define ASCII_LSQB 0x5B #define ASCII_RSQB 0x5D #define ASCII_UNDERSCORE 0x5F diff --git a/base/poco/XML/src/entities.h b/base/poco/XML/src/entities.h new file mode 100644 index 0000000000000..d0cd309c1111f --- /dev/null +++ b/base/poco/XML/src/entities.h @@ -0,0 +1,99 @@ +/* Internal helpers related to entities + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2026 Sebastian Pipping + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT +*/ + +#if ! defined(EXPAT_ENTITIES_H) +# define EXPAT_ENTITIES_H 1 + +# include "expat_config.h" // for XML_GE +# include "Poco/XML/expat_external.h" // for XML_Char +# include "internal.h" // for XML_NONTESTING_STATIC +# include // for size_t +# include // for size_t + +typedef struct { + const XML_Char *name; + const XML_Char *textPtr; + size_t textLen; /* length in XML_Chars */ + size_t processed; /* # of processed bytes - when suspended */ + const XML_Char *systemId; + const XML_Char *base; + const XML_Char *publicId; + const XML_Char *notation; + bool open; + bool hasMore; /* true if entity has not been completely processed */ + /* An entity can be open while being already completely processed (!hasMore). + The reason is the delayed closing of entities until their inner + entities are processed and closed */ + bool is_param; + bool is_internal; /* true if declared in internal subset outside PE */ +# if XML_GE == 1 + /* Indirect accounting bytes charged when this entity is expanded: 1 per text + character, plus each reference's "&name;"/"%name;" token length and its own + expansion. Matches XML_ACCOUNT_ENTITY_EXPANSION, so it is not the + replacement-text size. 0 means "not computed yet" (an empty entity is 0 + too, told apart by textLen == 0). */ + XmlBigCount expandedSize; + bool beingAnalyzed; /* on the resolveEntityClosureExpandedSize depth-first + search stack; used for cycle detection */ + const XML_Char + *scanPtr; /* cursor position into the entity's replacement text; NULL when + `textPtr` is NULL, else a pointer into the entity value, + ranging from `.textPtr` to `.textPtr + .textLen`. */ +# endif /* XML_GE == 1 */ +} ENTITY; + +# if XML_GE == 1 +/* How a scanEntityText call ended. */ +enum EntityScanResult { + ENTITY_SCAN_COMPLETE, /* reached end of text; *expandedSize is final */ + ENTITY_SCAN_BLOCKED, /* stopped at a defined entity whose expanded + size has not been computed yet; *blockerOut + names the entity whose size to compute + first */ + ENTITY_SCAN_OVERFLOW /* the expanded size would exceed XmlBigCount; + callers must abort parsing */ +}; + +XML_NONTESTING_STATIC int isCDataCommentOrPIStart(const XML_Char *pos, + size_t maxLen, + size_t *openTextLen); + +XML_NONTESTING_STATIC int isPredefinedEntityName(const XML_Char *name, + size_t nameLen); + +XML_NONTESTING_STATIC enum EntityScanResult +scanEntityText(XML_Parser parser, ENTITY *entity, const XML_Char **posPtr, + XmlBigCount *expandedSize, ENTITY **blockerOut); +# endif /* XML_GE == 1 */ + +#endif // ! defined(EXPAT_ENTITIES_H) diff --git a/base/poco/XML/src/internal.h b/base/poco/XML/src/internal.h index d0465449a3991..58ea8e2b4dc21 100644 --- a/base/poco/XML/src/internal.h +++ b/base/poco/XML/src/internal.h @@ -29,6 +29,7 @@ Copyright (c) 2026 Matthew Wozniczka Copyright (c) 2026 Braian Plaku Copyright (c) 2026 Florian Schmaus + Copyright (c) 2026 Brooks Davis Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -90,11 +91,10 @@ # define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u" # endif # else -# include // PRIdPTR, PRIuPTR # define EXPAT_FMT_LLX(midpart) "%" midpart "llx" # define EXPAT_FMT_ULL(midpart) "%" midpart "llu" -# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart PRIdPTR -# define EXPAT_FMT_SIZE_T(midpart) "%" midpart PRIuPTR +# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "td" +# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "zu" # endif # ifndef UNUSED_P @@ -131,6 +131,8 @@ union expat_align { extern "C" { #endif +typedef unsigned long long XmlBigCount; + void _INTERNAL_trim_to_complete_utf8_characters(const char *from, const char **fromLimRef); diff --git a/base/poco/XML/src/xcs.h b/base/poco/XML/src/xcs.h index 1e63a9b1de2f6..2b4268716cced 100644 --- a/base/poco/XML/src/xcs.h +++ b/base/poco/XML/src/xcs.h @@ -44,6 +44,8 @@ extern size_t xcslen(const XML_Char *s); extern int xcscmp(const XML_Char *a, const XML_Char *b); extern int xcsncmp(const XML_Char *a, const XML_Char *b, size_t len); +extern const XML_Char *xcsnstr(const XML_Char *haystack, const XML_Char *needle, + size_t maxLen); #ifdef __cplusplus } diff --git a/base/poco/XML/src/xmlparse.cpp b/base/poco/XML/src/xmlparse.cpp index ac428105490cc..40f3093fd68ec 100644 --- a/base/poco/XML/src/xmlparse.cpp +++ b/base/poco/XML/src/xmlparse.cpp @@ -1,4 +1,4 @@ -/* e3ca845466942140c9af531ac57808c4650116df5d6dbeda3b6546d2548a586e (2.9.0+) +/* d0712c5efef760b5355ed61473a927256c6474aecb853016d2e335e571c64b0c (2.10.0+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -58,6 +58,8 @@ Copyright (c) 2026 Braian Plaku Copyright (c) 2026 Filippo Tedeschi Copyright (c) 2026 Junki Lee + Copyright (c) 2026 Shani Yosef + Copyright (c) 2026 Pu Cheng Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -233,11 +235,13 @@ typedef char ICHAR; /* Do safe (NULL-aware) pointer arithmetic */ #define EXPAT_SAFE_PTR_DIFF(p, q) (((p) && (q)) ? ((p) - (q)) : 0) +#define EXPAT_SAFE_PTR_ADD(p, a) ((p) ? ((p) + (a)) : NULL) #define EXPAT_MIN(a, b) (((a) < (b)) ? (a) : (b)) #include "internal.h" #include "hash_table.h" +#include "entities.h" #include "xmltok.h" #include "xmlrole.h" @@ -315,24 +319,6 @@ typedef struct tag { BINDING *bindings; } TAG; -typedef struct { - const XML_Char *name; - const XML_Char *textPtr; - int textLen; /* length in XML_Chars */ - int processed; /* # of processed bytes - when suspended */ - const XML_Char *systemId; - const XML_Char *base; - const XML_Char *publicId; - const XML_Char *notation; - bool open; - bool hasMore; /* true if entity has not been completely processed */ - /* An entity can be open while being already completely processed (!hasMore). - The reason is the delayed closing of entities until their inner - entities are processed and closed */ - bool is_param; - bool is_internal; /* true if declared in internal subset outside PE */ -} ENTITY; - typedef struct { enum XML_Content_Type type; enum XML_Content_Quant quant; @@ -371,7 +357,7 @@ typedef struct attribute_id { typedef struct { const ATTRIBUTE_ID *id; - XML_Bool isCdata; + bool isCdata; const XML_Char *value; } DEFAULT_ATTRIBUTE; @@ -461,7 +447,6 @@ enum XML_Account { }; #if XML_GE == 1 -typedef unsigned long long XmlBigCount; typedef struct accounting { XmlBigCount countBytesDirect; XmlBigCount countBytesIndirect; @@ -473,6 +458,7 @@ typedef struct accounting { typedef struct MALLOC_TRACKER { size_t bytesAllocated; size_t peakBytesAllocated; // updated live only for debug level >=2 + size_t bytesAllocatedWhenLastReset; unsigned long debugLevel; float maximumAmplificationFactor; // >=1.0 XmlBigCount activationThresholdBytes; @@ -484,6 +470,13 @@ typedef struct entity_stats { unsigned int maximumDepthSeen; unsigned long debugLevel; } ENTITY_STATS; + +/* One pending entity-analysis task on the explicit stack used by + resolveEntityClosureExpandedSize's depth-first search over the entity + reference graph. */ +typedef struct { + ENTITY *entity; /* entity currently being scanned */ +} ENTITY_ANALYSIS_TASK; #endif /* XML_GE == 1 */ typedef enum XML_Error Processor(XML_Parser parser, const char *start, @@ -543,15 +536,15 @@ static enum XML_Error addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, const XML_Char *uri, BINDING **bindingsPtr); static int defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, - XML_Bool isCdata, XML_Bool isId, - const XML_Char *value, XML_Parser parser); + bool isCdata, bool isId, const XML_Char *value, + XML_Parser parser); static enum XML_Error storeAttributeValue(XML_Parser parser, - const ENCODING *enc, XML_Bool isCdata, + const ENCODING *enc, bool isCdata, const char *ptr, const char *end, STRING_POOL *pool, enum XML_Account account); static enum XML_Error -appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, +appendAttributeValue(XML_Parser parser, const ENCODING *enc, bool isCdata, const char *ptr, const char *end, STRING_POOL *pool, enum XML_Account account, const char **nextPtr); static ATTRIBUTE_ID *getAttributeId(XML_Parser parser, const ENCODING *enc, @@ -566,6 +559,9 @@ static enum XML_Error callStoreEntityValue(XML_Parser parser, const ENCODING *enc, const char *start, const char *end, enum XML_Account account); +static enum XML_Error resolveEntityClosureExpandedSize(XML_Parser parser, + HASH_TABLE *table, + ENTITY *root); #else static enum XML_Error storeSelfEntityValue(XML_Parser parser, ENTITY *entity); #endif @@ -603,7 +599,7 @@ static const XML_Char *poolCopyString(STRING_POOL *pool, const XML_Char *s); static const XML_Char *poolCopyStringNoFinish(STRING_POOL *pool, const XML_Char *s); static const XML_Char *poolCopyStringN(STRING_POOL *pool, const XML_Char *s, - int n); + size_t n); static const XML_Char *poolAppendString(STRING_POOL *pool, const XML_Char *s); static int nextScaffoldPart(XML_Parser parser); @@ -621,12 +617,17 @@ static XML_Parser parserCreate(const XML_Char *encodingName, const XML_Char *nameSep, DTD *dtd, XML_Parser parentParser); -static void parserInit(XML_Parser parser, const XML_Char *encodingName); +static void parserInit(XML_Parser parser, const XML_Char *encodingName, + bool isReset); #if XML_GE == 1 -static float accountingGetCurrentAmplification(XML_Parser rootParser); -static void accountingReportStats(XML_Parser originParser, const char *epilog); -static void accountingOnAbort(XML_Parser originParser); +static float accountingGetCurrentAmplification(XML_Parser rootParser, + XmlBigCount extraIndirectBytes); +static void accountingReportStats(XML_Parser originParser, const char *epilog, + XmlBigCount extraIndirectBytes, + int sourceLine); +static void accountingOnAbort(XML_Parser originParser, + XmlBigCount extraIndirectBytes, int sourceLine); static void accountingReportDiff(XML_Parser rootParser, unsigned int levelsAwayFromRootParser, const char *before, const char *after, @@ -636,6 +637,9 @@ static XML_Bool accountingDiffTolerated(XML_Parser originParser, int tok, const char *before, const char *after, int source_line, enum XML_Account account); +static XML_Bool +accountingCurrentAmplificationTolerated(XML_Parser rootParser, + XmlBigCount extraIndirectBytes); static void entityTrackingReportStats(XML_Parser parser, ENTITY *entity, const char *action, int sourceLine); @@ -643,6 +647,10 @@ static void entityTrackingOnOpen(XML_Parser parser, ENTITY *entity, int sourceLine); static void entityTrackingOnClose(XML_Parser parser, ENTITY *entity, int sourceLine); +static void entityTrackingOnScanStart(XML_Parser parser, ENTITY *entity, + int sourceLine); +static void entityTrackingOnScanStop(XML_Parser parser, ENTITY *entity, + int sourceLine); #endif /* XML_GE == 1 */ static XML_Parser getRootParserOf(XML_Parser parser, @@ -798,8 +806,8 @@ struct XML_ParserStruct { const XML_Char *m_declNotationPublicId; ELEMENT_TYPE *m_declElementType; ATTRIBUTE_ID *m_declAttributeId; - XML_Bool m_declAttributeIsCdata; - XML_Bool m_declAttributeIsId; + bool m_declAttributeIsCdata; + bool m_declAttributeIsId; DTD *m_dtd; const XML_Char *m_curBase; TAG *m_tagStack; @@ -871,11 +879,22 @@ expat_heap_increase_tolerable(XML_Parser rootParser, size_t increase, size_t newTotal = 0; bool tolerable = true; + // The idea is to subtract the amount of memory that was already + // allocated the last time that `XML_ParserReset` was called for + // this parser, the memory use of one document could skew the + // memory statso of the next document. + const size_t bytesAllocated + = (rootParser->m_alloc_tracker.bytesAllocatedWhenLastReset + > rootParser->m_alloc_tracker.bytesAllocated) + ? 0 + : (rootParser->m_alloc_tracker.bytesAllocated + - rootParser->m_alloc_tracker.bytesAllocatedWhenLastReset); + // Detect integer overflow - if (SIZE_MAX - rootParser->m_alloc_tracker.bytesAllocated < increase) { + if (SIZE_MAX - bytesAllocated < increase) { tolerable = false; } else { - newTotal = rootParser->m_alloc_tracker.bytesAllocated + increase; + newTotal = bytesAllocated + increase; if (newTotal >= rootParser->m_alloc_tracker.activationThresholdBytes) { assert(newTotal > 0); @@ -1472,7 +1491,7 @@ parserCreate(const XML_Char *encodingName, poolInit(&parser->m_tempPool, parser); poolInit(&parser->m_temp2Pool, parser); - parserInit(parser, encodingName); + parserInit(parser, encodingName, /*isReset=*/false); if (encodingName && ! parser->m_protocolEncodingName) { if (dtd) { @@ -1499,7 +1518,7 @@ parserCreate(const XML_Char *encodingName, } static void -parserInit(XML_Parser parser, const XML_Char *encodingName) { +parserInit(XML_Parser parser, const XML_Char *encodingName, bool isReset) { parser->m_processor = prologInitProcessor; XmlPrologStateInit(&parser->m_prologState); if (encodingName != NULL) { @@ -1547,8 +1566,8 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_declAttributeType = NULL; parser->m_declNotationName = NULL; parser->m_declNotationPublicId = NULL; - parser->m_declAttributeIsCdata = XML_FALSE; - parser->m_declAttributeIsId = XML_FALSE; + parser->m_declAttributeIsCdata = false; + parser->m_declAttributeIsId = false; memset(&parser->m_position, 0, sizeof(POSITION)); parser->m_errorCode = XML_ERROR_NONE; parser->m_eventPtr = NULL; @@ -1567,6 +1586,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_unknownEncodingRelease = NULL; parser->m_unknownEncodingData = NULL; parser->m_parsingStatus.parsing = XML_INITIALIZED; + parser->m_parsingStatus.finalBuffer = XML_FALSE; // Reentry can only be triggered inside m_processor calls parser->m_reenter = XML_FALSE; parser->m_handlerCallDepth = 0; @@ -1580,15 +1600,24 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_hash_secret_salt_set = XML_FALSE; #if XML_GE == 1 - memset(&parser->m_accounting, 0, sizeof(ACCOUNTING)); - parser->m_accounting.debugLevel = getDebugLevel("EXPAT_ACCOUNTING_DEBUG", 0u); - parser->m_accounting.maximumAmplificationFactor - = EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT; - parser->m_accounting.activationThresholdBytes - = EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT; - - memset(&parser->m_entity_stats, 0, sizeof(ENTITY_STATS)); - parser->m_entity_stats.debugLevel = getDebugLevel("EXPAT_ENTITY_DEBUG", 0u); + parser->m_accounting.countBytesDirect = 0; + parser->m_accounting.countBytesIndirect = 0; + if (! isReset) { + parser->m_accounting.debugLevel + = getDebugLevel("EXPAT_ACCOUNTING_DEBUG", 0u); + parser->m_accounting.maximumAmplificationFactor + = EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT; + parser->m_accounting.activationThresholdBytes + = EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT; + } + + parser->m_entity_stats.countEverOpened = 0; + parser->m_entity_stats.currentDepth = 0; + parser->m_entity_stats.maximumDepthSeen = 0; + if (! isReset) + parser->m_entity_stats.debugLevel = getDebugLevel("EXPAT_ENTITY_DEBUG", 0u); +#else + UNUSED_P(isReset); #endif } @@ -1669,8 +1698,16 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { poolClear(&parser->m_temp2Pool); FREE(parser, (void *)parser->m_protocolEncodingName); parser->m_protocolEncodingName = NULL; - parserInit(parser, encodingName); + parserInit(parser, encodingName, /*isReset=*/true); dtdReset(parser->m_dtd, parser); + +#if XML_GE == 1 + parser->m_alloc_tracker.bytesAllocatedWhenLastReset + = parser->m_alloc_tracker.bytesAllocated; +#endif + + // NOTE: This effectively checks for `copyString` having failed during the + // call to `parserInit` above if (encodingName && ! parser->m_protocolEncodingName) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return XML_FALSE; @@ -2340,13 +2377,12 @@ XML_SetHashSalt16Bytes(XML_Parser parser, const uint8_t entropy[16]) { enum XML_Status XMLCALL XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { - if ((parser == NULL) || (len < 0) || ((s == NULL) && (len != 0))) { - if (parser != NULL) - parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT; + if (parser == NULL || isCalledFromInsideHandler(parser)) return XML_STATUS_ERROR; - } - if (isCalledFromInsideHandler(parser)) + if ((len < 0) || ((s == NULL) && (len != 0))) { + parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT; return XML_STATUS_ERROR; + } switch (parser->m_parsingStatus.parsing) { case XML_SUSPENDED: parser->m_errorCode = XML_ERROR_SUSPENDED; @@ -2366,8 +2402,15 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { #if XML_CONTEXT_BYTES == 0 if (parser->m_bufferPtr == parser->m_bufferEnd) { - const char *end; - int nLeftOver; + // This null-object trick allows us to stop worrying about case `s == NULL` + // within this branch, e.g. with `s + len - end`. + const char *const empty = ""; + if (s == NULL) { + assert(len == 0); + s = empty; + } + const char *end = s + len; + size_t nLeftOver; enum XML_Status result; /* Detect overflow (a+b > MAX <==> b > MAX-a) */ if ((uint64_t)len > UINT64_MAX - parser->m_parseEndByteIndex) { @@ -2384,7 +2427,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal; parser->m_errorCode - = callProcessor(parser, s, parser->m_parseEndPtr = s + len, &end); + = callProcessor(parser, s, parser->m_parseEndPtr = end, &end); if (parser->m_errorCode != XML_ERROR_NONE) { parser->m_eventEndPtr = parser->m_eventPtr; @@ -2414,8 +2457,14 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { // Back up and restore the parsing status to avoid XML_ERROR_SUSPENDED // (and XML_ERROR_FINISHED) from XML_GetBuffer. const enum XML_Parsing originalStatus = parser->m_parsingStatus.parsing; + if (nLeftOver > INT_MAX) { + parser->m_errorCode = XML_ERROR_NO_MEMORY; + parser->m_eventPtr = parser->m_eventEndPtr = NULL; + parser->m_processor = errorProcessor; + return XML_STATUS_ERROR; + } parser->m_parsingStatus.parsing = XML_PARSING; - void *const temp = XML_GetBuffer(parser, nLeftOver); + void *const temp = XML_GetBuffer(parser, (int)nLeftOver); parser->m_parsingStatus.parsing = originalStatus; // GetBuffer may have overwritten this, but we want to remember what the // app requested, not how many bytes were left over after parsing. @@ -2433,7 +2482,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { memcpy(parser->m_buffer, end, nLeftOver); } parser->m_bufferPtr = parser->m_buffer; - parser->m_bufferEnd = parser->m_buffer + nLeftOver; + parser->m_bufferEnd = EXPAT_SAFE_PTR_ADD(parser->m_buffer, nLeftOver); parser->m_positionPtr = parser->m_bufferPtr; parser->m_parseEndPtr = parser->m_bufferEnd; parser->m_eventPtr = parser->m_bufferPtr; @@ -3515,7 +3564,7 @@ externalEntityInitProcessor2(XML_Parser parser, const char *start, #if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, start, next, __LINE__, XML_ACCOUNT_DIRECT)) { - accountingOnAbort(parser); + accountingOnAbort(parser, 0, __LINE__); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; } #endif /* XML_GE == 1 */ @@ -3545,6 +3594,11 @@ externalEntityInitProcessor2(XML_Parser parser, const char *start, } parser->m_eventPtr = start; return XML_ERROR_PARTIAL_CHAR; + case XML_TOK_NONE: + if (! parser->m_parsingStatus.finalBuffer) { + *endPtr = start; + } + return XML_ERROR_NONE; } parser->m_processor = externalEntityInitProcessor3; return externalEntityInitProcessor3(parser, start, end, endPtr); @@ -3644,7 +3698,7 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, : next; if (! accountingDiffTolerated(parser, tok, s, accountAfter, __LINE__, account)) { - accountingOnAbort(parser); + accountingOnAbort(parser, 0, __LINE__); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; } #endif @@ -4076,10 +4130,12 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, const enum XML_Convert_Result convert_res = XmlConvert( enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd); *eventEndPP = s; - beforeHandler(parser); - charDataHandler(parser->m_handlerArg, parser->m_dataBuf, - (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); - afterHandler(parser); + if (parser->m_characterDataHandler) { + beforeHandler(parser); + charDataHandler(parser->m_handlerArg, parser->m_dataBuf, + (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); + afterHandler(parser); + } if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) break; @@ -4176,9 +4232,9 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, TAG_NAME *tagNamePtr, BINDING **bindingsPtr, enum XML_Account account) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ - int attIndex = 0; + size_t attIndex = 0; XML_Char *uri; - int nPrefixes = 0; + size_t nPrefixes = 0; BINDING *binding; const XML_Char *localPart; @@ -4205,8 +4261,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, return XML_ERROR_NO_MEMORY; /* get the attributes from the tokenizer */ - size_t n = (size_t)XmlGetAttributes(enc, attStr, (int)parser->m_attsSize, - parser->m_atts); + size_t n = XmlGetAttributes(enc, attStr, parser->m_attsSize, parser->m_atts); /* Detect and prevent integer overflow */ if (n > SIZE_MAX - nDefaultAtts) { @@ -4241,11 +4296,22 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* Detect and prevent integer overflow. */ if (n > (size_t)INT_MAX) return XML_ERROR_NO_MEMORY; - XmlGetAttributes(enc, attStr, (int)n, parser->m_atts); + XmlGetAttributes(enc, attStr, n, parser->m_atts); } } /* the attribute list for the application */ + // NOTE: `parser->m_atts` is used to read `ATTRIBUTE` elements + // from start to end while at the same time writing `XML_Char *` + // elements from start to end *to the same block of memory*. + // That black magic only works because the first member `.name` in + // `ATTRIBUTE` is no longer being read after `appAtts[attIndex]` + // has been written, which for the first element is the very same + // address in memory. + // And because `attIndex` is increased twice for default attributes + // further down, struct `ATTRIBUTE` must be at least twice the size + // of a pointer, or the allocation size math for `m_attsSize` starts + // to underallocate. const XML_Char **const appAtts = (const XML_Char **)parser->m_atts; for (size_t i = 0; i < n; i++) { ATTRIBUTE *currAtt = &parser->m_atts[i]; @@ -4265,9 +4331,11 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, return XML_ERROR_DUPLICATE_ATTRIBUTE; } (attId->name)[-1] = 1; + // NOTE: The next call overwrites `parser->m_atts`, so we cannot use + // e.g. `parser->m_atts[i].name;` after this line! appAtts[attIndex++] = attId->name; if (! parser->m_atts[i].normalized) { - XML_Bool isCdata = XML_TRUE; + bool isCdata = true; /* figure out whether declared as other than CDATA */ if (attId->maybeTokenized) { @@ -4319,11 +4387,16 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, } /* set-up for XML_GetSpecifiedAttributeCount and XML_GetIdAttributeIndex */ - parser->m_nSpecifiedAtts = attIndex; + // Detect and reject integer overflow + if (attIndex > INT_MAX) + return XML_ERROR_NO_MEMORY; + parser->m_nSpecifiedAtts = (int)attIndex; if (elementType->idAtt && (elementType->idAtt->name)[-1]) { - for (int i = 0; i < attIndex; i += 2) + for (size_t i = 0; i < attIndex; i += 2) if (appAtts[i] == elementType->idAtt->name) { - parser->m_idAttIndex = i; + if (i > INT_MAX) + return XML_ERROR_NO_MEMORY; + parser->m_idAttIndex = (int)i; break; } } else @@ -4356,7 +4429,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* expand prefixed attribute names, check for duplicates, and clear flags that say whether attributes were specified */ - int i = 0; + size_t i = 0; if (nPrefixes) { unsigned int j; /* hash table index */ unsigned long version = parser->m_nsAttsVersion; @@ -4912,7 +4985,7 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, int tok = XmlCdataSectionTok(enc, s, end, &next); #if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, account)) { - accountingOnAbort(parser); + accountingOnAbort(parser, 0, __LINE__); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; } #else @@ -5084,7 +5157,7 @@ doIgnoreSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, # if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, XML_ACCOUNT_DIRECT)) { - accountingOnAbort(parser); + accountingOnAbort(parser, 0, __LINE__); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; } # endif @@ -5176,7 +5249,7 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity, const char *s, #if XML_GE == 1 if (! accountingDiffTolerated(parser, XML_TOK_XML_DECL, s, next, __LINE__, XML_ACCOUNT_DIRECT)) { - accountingOnAbort(parser); + accountingOnAbort(parser, 0, __LINE__); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; } #endif @@ -5397,7 +5470,7 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, # if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, XML_ACCOUNT_DIRECT)) { - accountingOnAbort(parser); + accountingOnAbort(parser, 0, __LINE__); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; } # endif @@ -5450,7 +5523,7 @@ externalParEntProcessor(XML_Parser parser, const char *s, const char *end, else if (tok == XML_TOK_BOM) { if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, XML_ACCOUNT_DIRECT)) { - accountingOnAbort(parser); + accountingOnAbort(parser, 0, __LINE__); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; } @@ -5629,7 +5702,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, break; default: if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, account)) { - accountingOnAbort(parser); + accountingOnAbort(parser, 0, __LINE__); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; } } @@ -5726,7 +5799,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, return XML_ERROR_INVALID_TOKEN; } - if (parser->m_doctypeName) { + if (parser->m_doctypeName && parser->m_startDoctypeDeclHandler) { beforeHandler(parser); parser->m_startDoctypeDeclHandler( parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid, @@ -5792,7 +5865,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, handleDefault = XML_FALSE; } break; - case XML_ROLE_INSTANCE_START: + case XML_ROLE_INSTANCE_START: { #ifdef XML_DTD /* if there is no DOCTYPE declaration then now is the last chance to read the foreign DTD @@ -5836,6 +5909,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, #endif /* XML_DTD */ parser->m_processor = contentProcessor; return contentProcessor(parser, s, end, nextPtr); + } case XML_ROLE_ATTLIST_ELEMENT_NAME: parser->m_declElementType = getElementType(parser, enc, s, next); if (! parser->m_declElementType) @@ -5845,16 +5919,16 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, parser->m_declAttributeId = getAttributeId(parser, enc, s, next); if (! parser->m_declAttributeId) return XML_ERROR_NO_MEMORY; - parser->m_declAttributeIsCdata = XML_FALSE; + parser->m_declAttributeIsCdata = false; parser->m_declAttributeType = NULL; - parser->m_declAttributeIsId = XML_FALSE; + parser->m_declAttributeIsId = false; goto checkAttListDeclHandler; case XML_ROLE_ATTRIBUTE_TYPE_CDATA: - parser->m_declAttributeIsCdata = XML_TRUE; + parser->m_declAttributeIsCdata = true; parser->m_declAttributeType = atypeCDATA; goto checkAttListDeclHandler; case XML_ROLE_ATTRIBUTE_TYPE_ID: - parser->m_declAttributeIsId = XML_TRUE; + parser->m_declAttributeIsId = true; parser->m_declAttributeType = atypeID; goto checkAttListDeclHandler; case XML_ROLE_ATTRIBUTE_TYPE_IDREF: @@ -5942,7 +6016,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, /* ID attributes aren't allowed to have a default */ if (! defineAttribute( parser->m_declElementType, parser->m_declAttributeId, - parser->m_declAttributeIsCdata, XML_FALSE, attVal, parser)) + parser->m_declAttributeIsCdata, false, attVal, parser)) return XML_ERROR_NO_MEMORY; if (parser->m_attlistDeclHandler && parser->m_declAttributeType) { if (*parser->m_declAttributeType == XML_T(ASCII_LPAREN) @@ -5972,17 +6046,23 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, #if XML_GE == 1 // This will store the given replacement text in // parser->m_declEntity->textPtr. - enum XML_Error result = callStoreEntityValue( + const enum XML_Error result = callStoreEntityValue( parser, enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar, XML_ACCOUNT_NONE); + if (result != XML_ERROR_NONE) { + poolDiscard(&dtd->entityValuePool); + return result; + } if (parser->m_declEntity) { /* Detect and prevent signed integer overflow */ if ((size_t)poolLength(&dtd->entityValuePool) > (size_t)INT_MAX) { + poolDiscard(&dtd->entityValuePool); return XML_ERROR_NO_MEMORY; } - parser->m_declEntity->textPtr = poolStart(&dtd->entityValuePool); - parser->m_declEntity->textLen - = (int)(poolLength(&dtd->entityValuePool)); + const XML_Char *const valueStart = poolStart(&dtd->entityValuePool); + parser->m_declEntity->textPtr = valueStart; + parser->m_declEntity->textLen = poolLength(&dtd->entityValuePool); + parser->m_declEntity->scanPtr = valueStart; poolFinish(&dtd->entityValuePool); if (parser->m_entityDeclHandler) { *eventEndPP = s; @@ -5990,14 +6070,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->is_param, parser->m_declEntity->textPtr, - parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0); + (int)parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0); afterHandler(parser); handleDefault = XML_FALSE; } } else poolDiscard(&dtd->entityValuePool); - if (result != XML_ERROR_NONE) - return result; #else // This will store "&entity123;" in parser->m_declEntity->textPtr // to end up as "&entity123;" in the handler. @@ -6013,7 +6091,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->is_param, parser->m_declEntity->textPtr, - parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0); + (int)parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0); afterHandler(parser); handleDefault = XML_FALSE; } @@ -6132,6 +6210,17 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } break; case XML_ROLE_GENERAL_ENTITY_NAME: { + // NOTE: XML 1.0 fourth edition demands in section + // "4.6 Predefined Entities" that bad redefinitions of predefined + // general entities must be detected and rejected. Expat has been + // ignoring both conforming and non-conforming redefinitions and + // thus has been failing to reject bad redefinitions, for years. + // As a result, users learned that they can get away with + // `` in practice. We cannot start conforming to + // that part of the spec now, or we risk rejecting many documents + // that parsed fine for years and that may not be fixable now or + // in time. So Expat continues to discard redefinitions of + // predefined entities altogether. if (XmlPredefinedEntityName(enc, s, next)) { parser->m_declEntity = NULL; break; @@ -6658,7 +6747,7 @@ epilogProcessor(XML_Parser parser, const char *s, const char *end, #if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, XML_ACCOUNT_DIRECT)) { - accountingOnAbort(parser); + accountingOnAbort(parser, 0, __LINE__); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; } #endif @@ -6725,6 +6814,415 @@ epilogProcessor(XML_Parser parser, const char *s, const char *end, } } +#if XML_GE == 1 +// Checks `name` for being a predefined entity (i.e. one of "amp", "apos", "gt", +// "lt", "quot"). Similar in implementation to function +// `PREFIX(predefinedEntityName)` but operating on an `XML_Char` string with an +// explicit length, instead. Returns +// ASCII_AMP/ASCII_APOS/ASCII_GT/ASCII_LT/ASCII_QUOT for a match or 0 for no +// match. +XML_NONTESTING_STATIC int +isPredefinedEntityName(const XML_Char *name, size_t nameLen) { + switch (nameLen) { + case 2: + // Wanted: "lt" or "gt" + if (name[1] != XML_T(ASCII_t)) + break; + switch (name[0]) { + case XML_T(ASCII_l): + return ASCII_LT; + case XML_T(ASCII_g): + return ASCII_GT; + } + break; + case 3: + // Wanted: "amp" + if (name[0] == XML_T(ASCII_a) && name[1] == XML_T(ASCII_m) + && name[2] == XML_T(ASCII_p)) + return ASCII_AMP; + break; + case 4: + // Wanted: "apos" or "quot" + switch (name[0]) { + case XML_T(ASCII_a): + if (name[1] == XML_T(ASCII_p) && name[2] == XML_T(ASCII_o) + && name[3] == XML_T(ASCII_s)) + return ASCII_APOS; + break; + case XML_T(ASCII_q): + if (name[1] == XML_T(ASCII_u) && name[2] == XML_T(ASCII_o) + && name[3] == XML_T(ASCII_t)) + return ASCII_QUOT; + break; + } + } + return 0; +} + +static const XML_Char cdataOpen[] + = {ASCII_LT, ASCII_EXCL, ASCII_LSQB, ASCII_C, ASCII_D, + ASCII_A, ASCII_T, ASCII_A, ASCII_LSQB, '\0'}; // i.e. "" +static const XML_Char commentOpen[] + = {ASCII_LT, ASCII_EXCL, ASCII_MINUS, ASCII_MINUS, '\0'}; // i.e. "" +static const XML_Char piClose[] = {ASCII_QUEST, ASCII_GT, '\0'}; // i.e. "?>" + +static const size_t cdataOpenLen = sizeof(cdataOpen) / sizeof(cdataOpen[0]) - 1; +static const size_t cdataCloseLen + = sizeof(cdataClose) / sizeof(cdataClose[0]) - 1; +static const size_t commentOpenLen + = sizeof(commentOpen) / sizeof(commentOpen[0]) - 1; +static const size_t commentCloseLen + = sizeof(commentClose) / sizeof(commentClose[0]) - 1; +static const size_t piCloseLen = sizeof(piClose) / sizeof(piClose[0]) - 1; + +// Checks the string starting at `pos` for starting a CDATA section, a comment +// or processing instructions. Returns +// XML_TOK_CDATA_SECT_OPEN/XML_TOK_COMMENT/XML_TOK_PI for a match or +// XML_TOK_INVALID (=0) for no match. Only in the success case, the length +// of the token is written to `*openTextLen`. +XML_NONTESTING_STATIC int +isCDataCommentOrPIStart(const XML_Char *pos, size_t maxLen, + size_t *openTextLen) { + assert(pos != NULL); + assert(openTextLen != NULL); + + if (maxLen < 2) + return XML_TOK_INVALID; + + if (pos[0] != XML_T(ASCII_LT)) + return XML_TOK_INVALID; + + if (pos[1] == XML_T(ASCII_QUEST)) { + *openTextLen = 2; + return XML_TOK_PI; + } + + if (pos[1] != XML_T(ASCII_EXCL)) + return XML_TOK_INVALID; + + if (maxLen >= cdataOpenLen && xcsncmp(pos, cdataOpen, cdataOpenLen) == 0) { + *openTextLen = cdataOpenLen; + return XML_TOK_CDATA_SECT_OPEN; + } + + if (maxLen >= commentOpenLen + && xcsncmp(pos, commentOpen, commentOpenLen) == 0) { + *openTextLen = commentOpenLen; + return XML_TOK_COMMENT; + } + + return XML_TOK_INVALID; +} + +/* Scan `entity`'s body, starting at `*posPtr`, accumulating the expanded + size into `*expandedSize`. + + Return value: how the scan ended, see enum EntityScanResult. + + *blockerOut (out, set only for ENTITY_SCAN_BLOCKED): + The non-empty entity the scan stopped at because its expanded size + has not been computed yet. The caller computes that size first and + calls again with *posPtr unchanged to resume. + The pointer is borrowed from the entity hash table (via lookup), not + a new allocation, so it must not be freed. + + *posPtr (in/out, scan cursor): + Advanced to end-of-text on a complete scan, or parked at the + offending reference otherwise, so a follow-up call resumes from + the same position. Not meaningful after ENTITY_SCAN_OVERFLOW. + + *expandedSize (in/out, expanded-byte accumulator): + The expanded size of this one `entity` (its text plus the expandedSize + of entities it references), not a total across all entities. Not + meaningful after ENTITY_SCAN_OVERFLOW; the overflowing value is never + stored anywhere. */ +XML_NONTESTING_STATIC enum EntityScanResult +scanEntityText(XML_Parser parser, ENTITY *entity, const XML_Char **posPtr, + XmlBigCount *expandedSize, ENTITY **blockerOut) { + if (blockerOut != NULL) + *blockerOut = NULL; + const XML_Char refChar = entity->is_param ? XML_T('%') : XML_T('&'); + HASH_TABLE *const refTable +# ifdef XML_DTD + = entity->is_param ? &parser->m_dtd->paramEntities + : &parser->m_dtd->generalEntities; +# else + = &parser->m_dtd->generalEntities; +# endif + + const XML_Char *pos = *posPtr; + const XML_Char *const end = entity->textPtr + entity->textLen; + + while (pos < end) { + // Detect and skip CDATA/comment/PI sections + // because entities have no effect in these places + if (*pos == XML_T('<')) { + // Detect CDATA/comment/PI section start + size_t openTextLen = 0; + const int tok = isCDataCommentOrPIStart(pos, end - pos, &openTextLen); + if (tok != XML_TOK_INVALID) { + assert(openTextLen != 0); + + // Detect and avoid integer overflow + if (openTextLen > (XmlBigCount)(-1) - *expandedSize) + return ENTITY_SCAN_OVERFLOW; + + *expandedSize += openTextLen; + pos += openTextLen; + + // Detect CDATA/comment/PI section end + const XML_Char *closeText = NULL; + size_t closeLen = 0; + + switch (tok) { + case XML_TOK_CDATA_SECT_OPEN: + closeText = cdataClose; + closeLen = cdataCloseLen; + break; + case XML_TOK_COMMENT: + closeText = commentClose; + closeLen = commentCloseLen; + break; + case XML_TOK_PI: + closeText = piClose; + closeLen = piCloseLen; + break; + default: + assert(0 && "never gets here"); + break; + } + + const size_t remainderLen = end - pos; + const XML_Char *const closeStart + = xcsnstr(pos, closeText, remainderLen); + if (closeStart == NULL) { + // The CDATA/comment/PI section is NOT terminated + if (remainderLen > (XmlBigCount)(-1) - *expandedSize) + return ENTITY_SCAN_OVERFLOW; + *expandedSize += remainderLen; + *posPtr = end; + return ENTITY_SCAN_COMPLETE; + } + + // The CDATA/comment/PI section is NOT terminated + const size_t skippedLen = closeStart + closeLen - pos; + + // Detect and avoid integer overflow + if (skippedLen > (XmlBigCount)(-1) - *expandedSize) + return ENTITY_SCAN_OVERFLOW; + + *expandedSize += skippedLen; + pos += skippedLen; + + continue; + } + } + + // Detect an entity reference + if (*pos != refChar) { + // Detect and avoid integer overflow + if (*expandedSize == (XmlBigCount)(-1)) + return ENTITY_SCAN_OVERFLOW; + (*expandedSize)++; + pos++; + continue; + } + + const XML_Char *nameStart = pos + 1; + const XML_Char *nameEnd = nameStart; + while (nameEnd < end && *nameEnd != XML_T(';')) + nameEnd++; + + if (nameEnd >= end) { + /* Unterminated reference at end of text: treat the trailing + bytes as literal characters. */ + XmlBigCount addend = (XmlBigCount)(end - pos); + // Detect and avoid integer overflow + if (*expandedSize > (XmlBigCount)(-1) - addend) + return ENTITY_SCAN_OVERFLOW; + *expandedSize += addend; + *posPtr = end; + return ENTITY_SCAN_COMPLETE; + } + + const size_t nameLen = (size_t)(nameEnd - nameStart); + + /* At runtime the whole "&name;"/"%name;" token is re-scanned and charged to + the indirect accounting, then its expansion is charged separately; so + every reference below counts this token length plus its expansion. */ + const XmlBigCount refTokenLen + = (XmlBigCount)(/*sigil*/ 1 + nameLen + /*semicolon*/ 1); + + ENTITY *refEntity + = (ENTITY *)lookupWithLength(parser, refTable, nameStart, nameLen, 0); + + XmlBigCount addend = 0; + if (! refEntity) { + /* Predefined entities (amp/lt/gt/quot/apos) are general-only, hence the + `! entity->is_param` guard, and expand to a single character. */ + if (! entity->is_param && isPredefinedEntityName(nameStart, nameLen)) { + addend = refTokenLen + 1; + } else { + /* Undefined entity: expands to nothing, so only the token. */ + addend = refTokenLen; + } + } else if (refEntity->beingAnalyzed) { + /* Cycle: refEntity is already on the depth-first search stack, so its + expandedSize is not final yet; charge just the token. Do not fail the + parse here: per XML 1.0r4, + "A parsed entity MUST NOT contain a recursive reference to + itself, either directly or indirectly", but also "Only parsed + entities that are referenced directly or indirectly within the + document are required to be well-formed" - so a declared-but- + never-expanded cycle must be accepted; if the cycle is actually + expanded, the runtime recursive-ref check reports + XML_ERROR_RECURSIVE_ENTITY_REF. */ + addend = refTokenLen; + } else if (refEntity->textLen > 0 && refEntity->expandedSize == 0) { + /* Non-empty entity whose expanded size has not been computed yet; + report refEntity as a blocker so the caller can compute its size + first. */ + *posPtr = pos; + if (blockerOut != NULL) + *blockerOut = refEntity; + return ENTITY_SCAN_BLOCKED; + } else if (refEntity->expandedSize > 0) { + /* refEntity is fully resolved: expandedSize holds its final expanded + size (a non-zero value is never a partial here - the only entity that + briefly holds a partial is one currently on the resolve stack, which + is caught by the beingAnalyzed branch above). Add the token plus that + expansion. */ + if (refEntity->expandedSize > (XmlBigCount)(-1) - refTokenLen) + return ENTITY_SCAN_OVERFLOW; + addend = refTokenLen + refEntity->expandedSize; + } else { + /* Empty entity (textLen == 0): expands to nothing, so only the token. */ + addend = refTokenLen; + } + + // Detect and avoid integer overflow + if (*expandedSize > (XmlBigCount)(-1) - addend) + return ENTITY_SCAN_OVERFLOW; + *expandedSize += addend; + pos = nameEnd + 1; + } + + /* Loop only exits here with pos == end (a complete scan). */ + assert(pos == end); + *posPtr = end; + return ENTITY_SCAN_COMPLETE; +} + +/* Resolve expandedSize using an iterative depth-first search over the + reference graph. Each entity is scanned at most twice (once on push, + once after its blocker resolves), so the total work is O(N + T) where + N is the number of entities and T is the sum of all entities' textLen. + `entity->beingAnalyzed` marks the entities currently on the stack for + cycle detection; it is cleared again on pop. + Returns XML_ERROR_NO_MEMORY if the stack cannot be allocated, and + XML_ERROR_AMPLIFICATION_LIMIT_BREACH if an entity's expanded size + overflows XmlBigCount. */ +static enum XML_Error +resolveEntityClosureExpandedSize(XML_Parser parser, HASH_TABLE *table, + ENTITY *root) { + if (root == NULL || root->expandedSize != 0 || root->textPtr == NULL + || root->textLen == 0) + return XML_ERROR_NONE; + + if (table->used == 0) + return XML_ERROR_NONE; + + // Detect and avoid integer overflow + if (table->used > SIZE_MAX / sizeof(ENTITY_ANALYSIS_TASK)) + return XML_ERROR_NO_MEMORY; + + ENTITY_ANALYSIS_TASK *stack + = MALLOC(parser, sizeof(ENTITY_ANALYSIS_TASK) * table->used); + if (stack == NULL) + return XML_ERROR_NO_MEMORY; + + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(! rootParser->m_parentParser); + + size_t top = 0; + stack[top].entity = root; + root->beingAnalyzed = XML_TRUE; + top++; + rootParser->m_entity_stats.currentDepth++; + if (rootParser->m_entity_stats.currentDepth + > rootParser->m_entity_stats.maximumDepthSeen) { + rootParser->m_entity_stats.maximumDepthSeen++; + } + + while (top > 0) { + ENTITY_ANALYSIS_TASK *task = &stack[top - 1]; + ENTITY *e = task->entity; + XmlBigCount sizeSoFar = e->expandedSize; + ENTITY *blocker = NULL; + + entityTrackingOnScanStart(parser, e, __LINE__); + + const enum EntityScanResult scanResult + = scanEntityText(parser, e, &e->scanPtr, &sizeSoFar, &blocker); + switch (scanResult) { + case ENTITY_SCAN_COMPLETE: + case ENTITY_SCAN_BLOCKED: + /* Save the size accumulated so far; together with task->pos it is + the state that lets the scan resume once the blocker's size has + been computed. This is the one place where expandedSize briefly + holds neither 0 nor the final size - only while the entity is + on the stack, where the beingAnalyzed check in scanEntityText + keeps every other reader away from the intermediate value. */ + e->expandedSize = sizeSoFar; + break; + default: + break; + } + + entityTrackingOnScanStop(parser, e, __LINE__); + + switch (scanResult) { + case ENTITY_SCAN_OVERFLOW: + /* Abort parsing right away; the beingAnalyzed marks still set on + the stacked entities do not matter once the parse is dead. */ + FREE(parser, stack); + /* expandedSize overflowed; pass SIZE_MAX (unrepresentable size). */ + accountingOnAbort(parser, SIZE_MAX, __LINE__); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + + case ENTITY_SCAN_COMPLETE: + e->beingAnalyzed = XML_FALSE; + top--; + rootParser->m_entity_stats.currentDepth--; + break; + + case ENTITY_SCAN_BLOCKED: + /* Push the blocker. A blocker's expanded size is always still + uncomputed and it is never already on the stack, so each entity + is pushed at most once and top <= table->used. */ + assert(! blocker->beingAnalyzed); + assert(top < table->used); + stack[top].entity = blocker; + blocker->beingAnalyzed = XML_TRUE; + top++; + rootParser->m_entity_stats.currentDepth++; + if (rootParser->m_entity_stats.currentDepth + > rootParser->m_entity_stats.maximumDepthSeen) { + rootParser->m_entity_stats.maximumDepthSeen++; + } + break; + } + } + + FREE(parser, stack); + return XML_ERROR_NONE; +} +#endif /* XML_GE == 1 */ + static enum XML_Error processEntity(XML_Parser parser, ENTITY *entity, bool betweenDecl, enum EntityType type) { @@ -6753,6 +7251,37 @@ processEntity(XML_Parser parser, ENTITY *entity, bool betweenDecl, /* LCOV_EXCL_STOP */ } +#if XML_GE == 1 + // Compute entity size assuming immediate access + if (entity->textLen > 0 && entity->expandedSize == 0) { + HASH_TABLE *const table +# ifdef XML_DTD + = entity->is_param ? &parser->m_dtd->paramEntities + : &parser->m_dtd->generalEntities; +# else + = &parser->m_dtd->generalEntities; +# endif + const enum XML_Error result + = resolveEntityClosureExpandedSize(parser, table, entity); + if (result != XML_ERROR_NONE) + return result; + } + + /* Reject an amplification attack before acquiring openEntity, so a rejected + entity leaks nothing and leaves entity state untouched. */ + if (entity->expandedSize > 0) { + const XML_Parser rootParser = getRootParserOf(parser, NULL); + /* Overflow of expandedSize aborts parsing during resolution + (on first expansion), so the stored value here is always exact. */ + const XmlBigCount expandedBytes = entity->expandedSize; + if (accountingCurrentAmplificationTolerated(rootParser, expandedBytes) + != XML_TRUE) { + accountingOnAbort(rootParser, expandedBytes, __LINE__); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } + } +#endif /* XML_GE == 1 */ + if (*freeEntityList) { openEntity = *freeEntityList; *freeEntityList = openEntity->next; @@ -6765,7 +7294,7 @@ processEntity(XML_Parser parser, ENTITY *entity, bool betweenDecl, entity->hasMore = true; #if XML_GE == 1 entityTrackingOnOpen(parser, entity, __LINE__); -#endif +#endif /* XML_GE == 1 */ entity->processed = 0; openEntity->next = *openEntityList; *openEntityList = openEntity; @@ -6827,7 +7356,7 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, && (parser->m_parsingStatus.parsing == XML_SUSPENDED || (parser->m_parsingStatus.parsing == XML_PARSING && parser->m_reenter))) { - entity->processed = (int)(next - (const char *)entity->textPtr); + entity->processed = next - (const char *)entity->textPtr; return result; } @@ -6876,7 +7405,7 @@ errorProcessor(XML_Parser parser, const char *s, const char *end, } static enum XML_Error -storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, +storeAttributeValue(XML_Parser parser, const ENCODING *enc, bool isCdata, const char *ptr, const char *end, STRING_POOL *pool, enum XML_Account account) { const char *next = ptr; @@ -6908,8 +7437,7 @@ storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, // processed. A XML_SUSPENDED check here is not required as // appendAttributeValue will never suspend the parser. if (nextInEntity < textEnd) { - entity->processed - = (int)(nextInEntity - (const char *)entity->textPtr); + entity->processed = nextInEntity - (const char *)entity->textPtr; continue; } assert(nextInEntity == textEnd); @@ -6954,7 +7482,7 @@ storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, } static enum XML_Error -appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, +appendAttributeValue(XML_Parser parser, const ENCODING *enc, bool isCdata, const char *ptr, const char *end, STRING_POOL *pool, enum XML_Account account, const char **nextPtr) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ @@ -6968,7 +7496,7 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, int tok = XmlAttributeValueTok(enc, ptr, end, &next); #if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, ptr, next, __LINE__, account)) { - accountingOnAbort(parser); + accountingOnAbort(parser, 0, __LINE__); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; } #endif @@ -7184,7 +7712,7 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, if (! accountingDiffTolerated(parser, tok, entityTextPtr, next, __LINE__, account)) { - accountingOnAbort(parser); + accountingOnAbort(parser, 0, __LINE__); result = XML_ERROR_AMPLIFICATION_LIMIT_BREACH; goto endEntityValue; } @@ -7369,8 +7897,7 @@ callStoreEntityValue(XML_Parser parser, const ENCODING *enc, // processed. A XML_SUSPENDED check here is not required as // appendAttributeValue will never suspend the parser. if (textEnd != nextInEntity) { - entity->processed - = (int)(nextInEntity - (const char *)entity->textPtr); + entity->processed = nextInEntity - (const char *)entity->textPtr; continue; } @@ -7430,8 +7957,9 @@ storeSelfEntityValue(XML_Parser parser, ENTITY *entity) { poolDiscard(pool); return XML_ERROR_NO_MEMORY; } - entity->textPtr = poolStart(pool); - entity->textLen = (int)(poolLength(pool)); + const XML_Char *const valueStart = poolStart(pool); + entity->textPtr = valueStart; + entity->textLen = poolLength(pool); poolFinish(pool); return XML_ERROR_NONE; @@ -7564,8 +8092,8 @@ reportDefault(XML_Parser parser, const ENCODING *enc, const char *s, } static int -defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, - XML_Bool isId, const XML_Char *value, XML_Parser parser) { +defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, bool isCdata, + bool isId, const XML_Char *value, XML_Parser parser) { DEFAULT_ATTRIBUTE *att; if (value || isId) { /* The handling of default attributes gets messed up if we have @@ -7726,7 +8254,7 @@ static const XML_Char * getContext(XML_Parser parser) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ HASH_TABLE_ITER iter; - XML_Bool needSep = XML_FALSE; + bool needSep = false; if (dtd->defaultPrefix.binding) { if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS))) @@ -7757,7 +8285,7 @@ getContext(XML_Parser parser) { */ return NULL; /* LCOV_EXCL_LINE */ } - needSep = XML_TRUE; + needSep = true; } hashTableIterInit(&iter, &(dtd->prefixes)); @@ -7786,7 +8314,7 @@ getContext(XML_Parser parser) { len--; if (! poolAppendChars(&parser->m_tempPool, prefix->binding->uri, len)) return NULL; - needSep = XML_TRUE; + needSep = true; } hashTableIterInit(&iter, &(dtd->generalEntities)); @@ -7800,7 +8328,7 @@ getContext(XML_Parser parser) { return NULL; if (! poolAppendChars(&parser->m_tempPool, e->name, xcslen(e->name))) return NULL; - needSep = XML_TRUE; + needSep = true; } if (! poolAppendChar(&parser->m_tempPool, XML_T('\0'))) @@ -8208,6 +8736,9 @@ copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, return 0; newE->textPtr = tem; newE->textLen = oldE->textLen; +#if XML_GE == 1 + newE->scanPtr = tem; +#endif /* XML_GE == 1 */ } if (oldE->notation) { const XML_Char *tem = poolCopyString(newPool, oldE->notation); @@ -8530,7 +9061,7 @@ poolCopyStringNoFinish(STRING_POOL *pool, const XML_Char *s) { } static const XML_Char * -poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) { +poolCopyStringN(STRING_POOL *pool, const XML_Char *s, size_t n) { if (! pool->ptr && ! poolGrow(pool)) { /* The following line is unreachable given the current usage of * poolCopyStringN(). Currently it is called from exactly one @@ -8975,25 +9506,27 @@ copyString(const XML_Char *s, XML_Parser parser) { #if XML_GE == 1 static float -accountingGetCurrentAmplification(XML_Parser rootParser) { +accountingGetCurrentAmplification(XML_Parser rootParser, + XmlBigCount extraIndirectBytes) { // 1.........1.........12 => 22 const size_t lenOfShortestInclude = sizeof("") - 1; - const XmlBigCount countBytesOutput - = rootParser->m_accounting.countBytesDirect - + rootParser->m_accounting.countBytesIndirect; + /* Add as floats: u64 counts (max ~1.8e19) are far below FLT_MAX, so the sum + is always finite and cannot wrap the way an unsigned integer add could. */ + const float indirect = (float)rootParser->m_accounting.countBytesIndirect + + (float)extraIndirectBytes; const float amplificationFactor = rootParser->m_accounting.countBytesDirect - ? ((float)countBytesOutput + ? (((float)rootParser->m_accounting.countBytesDirect + indirect) / (float)(rootParser->m_accounting.countBytesDirect)) - : ((float)(lenOfShortestInclude - + rootParser->m_accounting.countBytesIndirect) + : (((float)lenOfShortestInclude + indirect) / (float)lenOfShortestInclude); assert(! rootParser->m_parentParser); return amplificationFactor; } static void -accountingReportStats(XML_Parser originParser, const char *epilog) { +accountingReportStats(XML_Parser originParser, const char *epilog, + XmlBigCount extraIndirectBytes, int sourceLine) { const XML_Parser rootParser = getRootParserOf(originParser, NULL); assert(! rootParser->m_parentParser); @@ -9001,19 +9534,33 @@ accountingReportStats(XML_Parser originParser, const char *epilog) { return; } + /* On an early abort the amplifying bytes have not been charged to the + indirect counter yet; fold extraIndirectBytes in (with overflow detection) + so the reported byte count and amplification reflect what was detected. */ + const XmlBigCount indirect = rootParser->m_accounting.countBytesIndirect; + const XmlBigCount indirectWithExtra + = (extraIndirectBytes > (XmlBigCount)(-1) - indirect) + ? (XmlBigCount)(-1) + : indirect + extraIndirectBytes; const float amplificationFactor - = accountingGetCurrentAmplification(rootParser); + = accountingGetCurrentAmplification(rootParser, extraIndirectBytes); fprintf(stderr, "expat: Accounting(%p): Direct " EXPAT_FMT_ULL( "10") ", indirect " EXPAT_FMT_ULL("10") ", amplification %8.2f%s", (void *)rootParser, rootParser->m_accounting.countBytesDirect, - rootParser->m_accounting.countBytesIndirect, - (double)amplificationFactor, epilog); + indirectWithExtra, (double)amplificationFactor, epilog); + /* An abort has no accountingReportDiff before it, so append the originating + source line here to show which rejection site these stats belong to. */ + if (sourceLine > 0) { + fprintf(stderr, " (xmlparse.c:%d)\n", sourceLine); + } } static void -accountingOnAbort(XML_Parser originParser) { - accountingReportStats(originParser, " ABORTING\n"); +accountingOnAbort(XML_Parser originParser, XmlBigCount extraIndirectBytes, + int sourceLine) { + accountingReportStats(originParser, " ABORTING", extraIndirectBytes, + sourceLine); } static void @@ -9053,6 +9600,32 @@ accountingReportDiff(XML_Parser rootParser, fprintf(stderr, "\"\n"); } +static XML_Bool +accountingCurrentAmplificationTolerated(XML_Parser rootParser, + XmlBigCount extraIndirectBytes) { + assert(! rootParser->m_parentParser); + + const XmlBigCount countBytesOutput + = rootParser->m_accounting.countBytesDirect + + rootParser->m_accounting.countBytesIndirect; + /* Overflow check: extraIndirectBytes can be SIZE_MAX, which would otherwise + wrap the unsigned total around to a small value and wrongly report the + output as tolerated. */ + const XmlBigCount countBytesOutputWithExtra + = (extraIndirectBytes > (XmlBigCount)(-1) - countBytesOutput) + ? (XmlBigCount)(-1) + : countBytesOutput + extraIndirectBytes; + const float amplificationFactor + = accountingGetCurrentAmplification(rootParser, extraIndirectBytes); + const XML_Bool tolerated + = (countBytesOutputWithExtra + < rootParser->m_accounting.activationThresholdBytes) + || (amplificationFactor + <= rootParser->m_accounting.maximumAmplificationFactor); + + return tolerated; +} + static XML_Bool accountingDiffTolerated(XML_Parser originParser, int tok, const char *before, const char *after, int source_line, @@ -9089,18 +9662,11 @@ accountingDiffTolerated(XML_Parser originParser, int tok, const char *before, return XML_FALSE; *additionTarget += bytesMore; - const XmlBigCount countBytesOutput - = rootParser->m_accounting.countBytesDirect - + rootParser->m_accounting.countBytesIndirect; - const float amplificationFactor - = accountingGetCurrentAmplification(rootParser); const XML_Bool tolerated - = (countBytesOutput < rootParser->m_accounting.activationThresholdBytes) - || (amplificationFactor - <= rootParser->m_accounting.maximumAmplificationFactor); + = accountingCurrentAmplificationTolerated(rootParser, 0); if (rootParser->m_accounting.debugLevel >= 2u) { - accountingReportStats(rootParser, ""); + accountingReportStats(rootParser, "", 0, 0); accountingReportDiff(rootParser, levelsAwayFromRootParser, before, after, bytesMore, source_line, account); } @@ -9146,14 +9712,21 @@ entityTrackingReportStats(XML_Parser rootParser, ENTITY *entity, = limitingNeeded ? (maxLimitedDepth - /* make space for ellipis */ 2) : candidateIndentDepth; + const ptrdiff_t scannedLength + = EXPAT_SAFE_PTR_DIFF(entity->scanPtr, entity->textPtr); + const char *const confidenceMarker + = ((size_t)scannedLength == entity->textLen) ? "!" : "?"; + fprintf( stderr, - "expat: Entities(%p): Count %9u, depth %2u/%2u %*s%s%s%s; %s length %d (xmlparse.c:%d)\n", + "expat: Entities(%p): Count %9u, depth %2u/%2u %*s%s%s%s; %s scanned " EXPAT_FMT_PTRDIFF_T( + "") "/" EXPAT_FMT_SIZE_T("") ", expanded " EXPAT_FMT_ULL("") "%s (xmlparse.c:%d)\n", (void *)rootParser, rootParser->m_entity_stats.countEverOpened, rootParser->m_entity_stats.currentDepth, rootParser->m_entity_stats.maximumDepthSeen, indentDepth * 2, "", ellipisOrEmpty, entity->is_param ? "%" : "&", entityName, action, - entity->textLen, sourceLine); + scannedLength, entity->textLen, entity->expandedSize, confidenceMarker, + sourceLine); } static void @@ -9168,6 +9741,7 @@ entityTrackingOnOpen(XML_Parser originParser, ENTITY *entity, int sourceLine) { rootParser->m_entity_stats.maximumDepthSeen++; } + // NOTE: The space at the end of "OPEN " is for length alignment with "CLOSE" entityTrackingReportStats(rootParser, entity, "OPEN ", sourceLine); } @@ -9180,6 +9754,25 @@ entityTrackingOnClose(XML_Parser originParser, ENTITY *entity, int sourceLine) { rootParser->m_entity_stats.currentDepth--; } +static void +entityTrackingOnScanStart(XML_Parser originParser, ENTITY *entity, + int sourceLine) { + const XML_Parser rootParser = getRootParserOf(originParser, NULL); + assert(! rootParser->m_parentParser); + + entityTrackingReportStats(rootParser, entity, "START", sourceLine); +} + +static void +entityTrackingOnScanStop(XML_Parser originParser, ENTITY *entity, + int sourceLine) { + const XML_Parser rootParser = getRootParserOf(originParser, NULL); + assert(! rootParser->m_parentParser); + + // NOTE: The space at the end of "STOP " is for length alignment with "START" + entityTrackingReportStats(rootParser, entity, "STOP ", sourceLine); +} + #endif /* XML_GE == 1 */ static XML_Parser diff --git a/base/poco/XML/src/xmltok.h b/base/poco/XML/src/xmltok.h index 4f6fb3850cafc..fb8329e06394a 100644 --- a/base/poco/XML/src/xmltok.h +++ b/base/poco/XML/src/xmltok.h @@ -180,8 +180,8 @@ struct encoding { const char *); int (*nameLength)(const ENCODING *, const char *); const char *(*skipS)(const ENCODING *, const char *); - int (*getAtts)(const ENCODING *enc, const char *ptr, int attsMax, - ATTRIBUTE *atts); + size_t (*getAtts)(const ENCODING *enc, const char *ptr, size_t attsMax, + ATTRIBUTE *atts); int (*charRefNumber)(const ENCODING *enc, const char *ptr); int (*predefinedEntityName)(const ENCODING *, const char *, const char *); void (*updatePosition)(const ENCODING *, const char *ptr, const char *end, diff --git a/base/poco/XML/src/xmltok_impl.c b/base/poco/XML/src/xmltok_impl.c index 571af3698af7a..67fececb1f61c 100644 --- a/base/poco/XML/src/xmltok_impl.c +++ b/base/poco/XML/src/xmltok_impl.c @@ -1505,11 +1505,11 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end, first attsMax attributes are stored in atts. */ -static int -PREFIX(getAtts)(const ENCODING *enc, const char *ptr, int attsMax, +static size_t +PREFIX(getAtts)(const ENCODING *enc, const char *ptr, size_t attsMax, ATTRIBUTE *atts) { enum { other, inName, inValue } state = inName; - int nAtts = 0; + size_t nAtts = 0; int open = 0; /* defined when state == inValue; initialization just to shut up compilers */ diff --git a/base/poco/XML/src/xmlxcs.c b/base/poco/XML/src/xmlxcs.c index ae5e042d778ff..a97647fa9d69e 100644 --- a/base/poco/XML/src/xmlxcs.c +++ b/base/poco/XML/src/xmlxcs.c @@ -88,3 +88,24 @@ xcsncmp(const XML_Char *a, const XML_Char *b, size_t len) { return strncmp(a, b, len); #endif } + +// A port of `strnstr(3bsd)` to `XML_Char` +const XML_Char * +xcsnstr(const XML_Char *haystack, const XML_Char *needle, size_t maxLen) { + // NOTE: We are not calling `xcsnlen(haystack, maxLen)` because + // `haystack` could be long and `xcsnlen` could be used in a loop + // and we need to avoid quadratic runtime behavior. + if (! *needle) + return haystack; // i.e. empty needle is always a match + const size_t needleLen = xcslen(needle); + if (needleLen > maxLen) + return NULL; + const size_t maxTries = (maxLen - needleLen) + 1; + for (size_t i = 0; i < maxTries && haystack[i]; i++) { + if (haystack[i] == needle[0] + && xcsncmp(haystack + i, needle, needleLen) == 0) { + return haystack + i; + } + } + return NULL; +} From c4282a4613e00fb18658007e5df7fb0e3c27d8e8 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 11 Nov 2026 13:47:47 +0000 Subject: [PATCH 156/219] Backport #117018 to 26.8: Convert native scalars directly in convertColumnToType (fix dictGet regression with direct dictionaries) --- src/Interpreters/convertColumnToType.cpp | 102 ++++++++++++------ src/Interpreters/convertColumnToType.h | 6 +- .../tests/gtest_convert_column_to_type.cpp | 18 ++++ ...ert_column_to_type_bool_identity.reference | 16 +++ ...9_convert_column_to_type_bool_identity.sql | 26 +++++ 5 files changed, 135 insertions(+), 33 deletions(-) create mode 100644 tests/queries/0_stateless/05059_convert_column_to_type_bool_identity.reference create mode 100644 tests/queries/0_stateless/05059_convert_column_to_type_bool_identity.sql diff --git a/src/Interpreters/convertColumnToType.cpp b/src/Interpreters/convertColumnToType.cpp index 56deff6b7e196..ea1632930e827 100644 --- a/src/Interpreters/convertColumnToType.cpp +++ b/src/Interpreters/convertColumnToType.cpp @@ -1,8 +1,10 @@ #include #include -#include #include +#include +#include +#include #include #include #include @@ -29,43 +31,70 @@ namespace ErrorCodes namespace { -/// Column-native conversion for the cases where CAST provably matches `convertFieldToType`: -/// plain native-numeric-to-native-numeric, in the default AND `strict` modes. `castColumnAccurateOrNull` -/// uses the same accurate numeric conversion (`accurate::convertNumeric`) as `convertFieldToType`'s -/// default path (out-of-range / inexact-narrowing -> NULL). For native numbers `strict` and default -/// agree with it too: `strict` only rejects additional cases for types that are already excluded here - -/// `Bool` (10 is not a valid `Bool`) and `Decimal` (scale reduction; `castColumnAccurateOrNull` would -/// round, so it must NOT be used for strict Decimal - and it isn't, `Decimal` is not a native number). -/// The strict native-number equivalence is pinned by `gtest_convert_column_to_type`. Returns: +/// Column-native conversion for native-numeric values: the same `accurate::convertNumeric` that +/// `convertFieldToType` applies, evaluated directly on the scalar instead of through a CAST function - +/// building and executing a CAST per single-row conversion dominated callers such as the `IN` set +/// builder, which converts every literal of the right-hand side separately. `exact` repeats the +/// predicate of `convertNumericTypeImpl`: an out-of-range value is never representable, and the +/// additional round-trip check that rejects an inexact conversion is relaxed only for a floating-point +/// target under `convert_inexact_floats` - conversions to an integer target stay exact in every mode, so +/// the flag must not disable the fast path wholesale (the `values` table function passes it for every +/// conversion). The equivalence with `convertFieldToType` across `strict` / `convert_inexact_floats` is +/// pinned by `gtest_convert_column_to_type`. Returns: /// - the converted size-1 column of `to` on success, /// - a null `ColumnPtr{}` when not representable, /// - std::nullopt when this fast path does not apply (caller falls back to the `Field` path). -/// Excluded: `convert_inexact_floats` mode (allows rounding, so `castColumnAccurateOrNull` differs), -/// `Bool` (clamp/validity semantics), and anything non-native-numeric +/// Excluded: `Bool` (clamp/validity semantics), and anything non-native-numeric /// (Decimal/Date/Enum/String/wide-int/wrappers/composite). std::optional tryConvertNumericColumnNative( const IColumn & value, const DataTypePtr & from, const DataTypePtr & to, + bool strict, bool convert_inexact_floats) { - if (convert_inexact_floats) - return std::nullopt; if (!isNativeNumber(from) || !isNativeNumber(to) || isBool(from) || isBool(to)) return std::nullopt; - /// `strict` is intentionally not a parameter: for native numbers it is equivalent to the default - /// accurate path (both reject non-representable values), so this fast path serves strict too. - - ColumnWithTypeAndName arg{value.getPtr(), from, ""}; - ColumnPtr casted = castColumnAccurateOrNull(arg, to); - /// `ExecutableFunctionCast` uses the default implementation for constants, so a `ColumnConst` - /// argument yields a `ColumnConst` result. Callers already pass a full column, but unwrap here too - /// so the `assert_cast` below is correct regardless of the argument's constness. - casted = casted->convertToFullColumnIfConst(); - const auto & nullable = assert_cast(*casted); - if (nullable.isNullAt(0)) - return ColumnPtr{}; - return nullable.getNestedColumnPtr(); + + ColumnPtr result; + auto convert = [&](const auto & types) + { + using Types = std::decay_t; + using From = typename Types::LeftType; + using To = typename Types::RightType; + + const From source = assert_cast &>(value).getData()[0]; + const bool exact = strict || !convert_inexact_floats || !is_floating_point; + + To converted{}; + const bool representable = exact + ? accurate::convertNumeric(source, converted) + : accurate::convertNumeric(source, converted); + + if (representable) + { + auto column = ColumnVector::create(); + column->insertValue(converted); + result = std::move(column); + } + return true; + }; + + if (!callOnBasicTypes(from->getTypeId(), to->getTypeId(), convert)) + return std::nullopt; + return result; +} + +/// `Bool` anywhere in the type tree (the shapes `retagBoolInField` normalizes, and any other nesting). +/// `Dynamic` and `JSON` count as containing `Bool`: their payload types are not children the type tree +/// exposes, yet a row of either may hold a `Bool`. +bool containsBool(const IDataType & type) +{ + auto is_bool_or_opaque = [](const IDataType & t) { return t.getName() == "Bool" || isDynamic(t) || isObject(t); }; + bool found = is_bool_or_opaque(type); + if (!found) + type.forEachChild([&](const IDataType & child) { found = found || is_bool_or_opaque(child); }); + return found; } /// `IColumn::get` reconstructs a `Field` using the storage column's `NearestFieldType`, which does not @@ -140,12 +169,25 @@ ColumnPtr convertColumnToTypeOrNull( chassert(value.size() == 1); /// Callers usually pass a `ColumnConst` (e.g. from `evaluateConstantExpressionAsColumn`); operate - /// on the underlying full column so the fast path's CAST returns a plain (non-const) column and the - /// `Field` fallback reads the value directly. - const ColumnPtr full = value.convertToFullColumnIfConst(); + /// on the underlying full column so both the native fast path and the `Field` fallback read the + /// value directly. + ColumnPtr full = value.convertToFullColumnIfConst(); const IColumn & unwrapped = *full; - if (auto native = tryConvertNumericColumnNative(unwrapped, from, to, convert_inexact_floats)) + /// Same as `convertFieldToType`, which returns the value untouched when `from` equals `to` - but + /// only for genuinely identical types that do not involve `Bool`. `IDataType::equals` ignores + /// custom names, so it also holds for `Bool` and its storage type `UInt8` (and for wrapper pairs + /// such as `Nullable(Bool)` and `Nullable(UInt8)`); like the identity check of `CAST`, require the + /// custom names to match too, comparing the full names so every nesting level is covered. And a + /// `Bool` column is a plain `ColumnUInt8` that may hold a raw byte such as 2 (e.g. from + /// `reinterpret`), which the `Field` path below normalizes to `true` through `retagBoolInField` + /// (and which `strict` rejects for a `Bool` target) - so a type containing `Bool` is never returned + /// untouched, not even for `Bool -> Bool`; `CAST` likewise treats `Bool -> Bool` as + /// `UInt8 -> Bool` normalization rather than identity. + if (from->equals(*to) && from->getName() == to->getName() && !containsBool(*from)) + return full; + + if (auto native = tryConvertNumericColumnNative(unwrapped, from, to, strict, convert_inexact_floats)) return std::move(*native); /// Fallback: materialize a `Field`, reuse `convertFieldToType`, rebuild a column. Column-native diff --git a/src/Interpreters/convertColumnToType.h b/src/Interpreters/convertColumnToType.h index 49f554739af54..18e00462153c0 100644 --- a/src/Interpreters/convertColumnToType.h +++ b/src/Interpreters/convertColumnToType.h @@ -18,9 +18,9 @@ namespace DB * column holding NULL — NOT as `ColumnPtr{}`. * * The purpose is to convert constants WITHOUT materializing a `Field`. Cases that can be done - * column-natively (currently: plain numeric-to-numeric in the default mode) go through - * `IColumn`/CAST; the rest still delegate to `convertFieldToType` (same behavior, just not yet - * `Field`-free). The behavior is pinned by `gtest_convert_column_to_type` against `convertFieldToType`, + * column-natively (currently: native numeric-to-numeric, in every `strict` / `convert_inexact_floats` + * mode) read the value straight from the column; the rest still delegate to `convertFieldToType` + * (same behavior, just not yet `Field`-free). The behavior is pinned by `gtest_convert_column_to_type` against `convertFieldToType`, * so more column-native fast paths can be added without changing results. * * The equivalence holds for scalar `Bool` and for `Bool` nested under the structural carriers diff --git a/src/Interpreters/tests/gtest_convert_column_to_type.cpp b/src/Interpreters/tests/gtest_convert_column_to_type.cpp index b67b2eb8f93ac..52cb1889f1f9f 100644 --- a/src/Interpreters/tests/gtest_convert_column_to_type.cpp +++ b/src/Interpreters/tests/gtest_convert_column_to_type.cpp @@ -192,6 +192,24 @@ TEST(ConvertColumnToType, MatchesConvertFieldToType) /// use `castColumnAccurateOrNull`, which would round `33.33` to `33.3` instead of rejecting it). {"Decimal64(2)", Field(DecimalField(Decimal64(3333), 2)), "Decimal64(1)", true}, // scale loss -> null {"Decimal64(1)", Field(DecimalField(Decimal64(333), 1)), "Decimal64(2)", true}, // widen -> 33.30 + + /// `convert_inexact_floats` native-number matrix. The flag relaxes the exactness check only for + /// a floating-point target, so the column-native fast path serves it for every native-number + /// pair rather than falling back to the `Field` path - which matters because the `values` table + /// function passes it for every single conversion it makes. + {"Float64", Field(Float64(0.1)), "Float32", true, true}, // strict wins over inexact -> null + {"Float64", Field(Float64(3.0)), "Int32", false, true}, // integer target, exact -> 3 + {"Float64", Field(Float64(3.5)), "Int32", false, true}, // integer target stays exact -> null + {"Int64", Field(Int64(9007199254740993ll)), "Float64", false, true}, // float target -> nearest + {"UInt64", Field(UInt64(5)), "UInt8", false, true}, // integer target stays exact -> 5 + {"UInt64", Field(UInt64(256)), "UInt8", false, true}, // out of range -> null + {"Int64", Field(Int64(-1)), "UInt8", false, true}, // negative -> null + + /// identical types: the value passes through untouched (including a genuine NULL) + {"UInt64", Field(UInt64(5)), "UInt64", true}, + {"String", Field(String("abc")), "String", true}, + {"Nullable(UInt8)", Field(), "Nullable(UInt8)", true}, + {"Decimal64(2)", Field(DecimalField(Decimal64(3333), 2)), "Decimal64(2)", true}, }; for (const auto & c : cases) diff --git a/tests/queries/0_stateless/05059_convert_column_to_type_bool_identity.reference b/tests/queries/0_stateless/05059_convert_column_to_type_bool_identity.reference new file mode 100644 index 0000000000000..6a0543ef42352 --- /dev/null +++ b/tests/queries/0_stateless/05059_convert_column_to_type_bool_identity.reference @@ -0,0 +1,16 @@ +1 +1 +[1] +(1) +{'a':1} +0 +1 +0 +1 +true 1 +true 1 +[true] 1 +(true) 1 +{'a':true} 1 +1 +1 diff --git a/tests/queries/0_stateless/05059_convert_column_to_type_bool_identity.sql b/tests/queries/0_stateless/05059_convert_column_to_type_bool_identity.sql new file mode 100644 index 0000000000000..782fc765296fe --- /dev/null +++ b/tests/queries/0_stateless/05059_convert_column_to_type_bool_identity.sql @@ -0,0 +1,26 @@ +-- `Bool` is a custom-named `UInt8`, so `IDataType::equals` treats `Bool` and `UInt8` (and wrapper pairs +-- such as `Nullable(Bool)` and `Nullable(UInt8)`) as the same type. The constant conversion must still +-- normalize a raw `Bool` byte (2 here, produced by `reinterpret`) to the semantic `true` rather than +-- alias it byte-for-byte. +SELECT * FROM values('x UInt8', reinterpret(toUInt8(2), 'Bool')); +SELECT * FROM values('x Nullable(UInt8)', toNullable(reinterpret(toUInt8(2), 'Bool'))); +SELECT * FROM values('x Array(UInt8)', [reinterpret(toUInt8(2), 'Bool')]); +SELECT * FROM values('x Tuple(UInt8)', tuple(reinterpret(toUInt8(2), 'Bool'))); +SELECT * FROM values('x Map(String, UInt8)', map('a', reinterpret(toUInt8(2), 'Bool'))); + +-- The `IN` set builder converts each literal strictly: a `Bool` literal is normalized to `true` before +-- it is compared with a `UInt8`, and 2 is not a valid `Bool`. +SELECT toUInt8(2) IN (reinterpret(toUInt8(2), 'Bool')); +SELECT toUInt8(1) IN (reinterpret(toUInt8(2), 'Bool')); +SELECT reinterpret(toUInt8(2), 'Bool') IN (toUInt8(2)); +SELECT reinterpret(toUInt8(2), 'Bool') IN (toUInt8(1)); + +-- Identical types containing `Bool` are not a byte-for-byte no-op either: the raw byte must still be +-- normalized, as `CAST` does for `Bool -> Bool`. +SELECT x, x = true FROM values('x Bool', reinterpret(toUInt8(2), 'Bool')); +SELECT x, x = true FROM values('x Nullable(Bool)', toNullable(reinterpret(toUInt8(2), 'Bool'))); +SELECT x, x = [true] FROM values('x Array(Bool)', [reinterpret(toUInt8(2), 'Bool')]); +SELECT x, x = tuple(true) FROM values('x Tuple(Bool)', tuple(reinterpret(toUInt8(2), 'Bool'))); +SELECT x, x = map('a', true) FROM values('x Map(String, Bool)', map('a', reinterpret(toUInt8(2), 'Bool'))); +SELECT true IN (reinterpret(toUInt8(2), 'Bool')); +SELECT reinterpret(toUInt8(3), 'Bool') IN (reinterpret(toUInt8(2), 'Bool')); From dfbd93cd75927988904aa9bc72c4fe1e6e995861 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 11 Nov 2026 14:55:29 +0000 Subject: [PATCH 157/219] Backport #130279 to 26.8: Fix reading ORC lists and maps whose element struct has none of the requested fields --- .../Impl/NativeORCBlockInputFormat.cpp | 227 ++++++++++++++++-- .../Formats/Impl/NativeORCBlockInputFormat.h | 5 +- ..._struct_without_requested_fields.reference | 159 ++++++++++++ ...nested_struct_without_requested_fields.sql | 101 ++++++++ 4 files changed, 473 insertions(+), 19 deletions(-) create mode 100644 tests/queries/0_stateless/05617_orc_nested_struct_without_requested_fields.reference create mode 100644 tests/queries/0_stateless/05617_orc_nested_struct_without_requested_fields.sql diff --git a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp index 6bc08f9f19279..21be2e8ba3859 100644 --- a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp @@ -1104,8 +1104,13 @@ static bool orcUnionBranchMatchesType(const ORC::Type * orc_branch_type, const D static bool orcUnionBranchPrefersType(const ORC::Type * orc_branch_type, const DataTypePtr & target_type); static DataTypes computeOrcUnionBranchHints(const ORC::Type * orc_type, const DataTypePtr & type_hint, bool case_insensitive_matching); -static void -updateIncludeTypeIds(DataTypePtr type, const ORC::Type * orc_type, bool ignore_case, std::unordered_set & include_typeids) +/// Also collects the ORC structs that are requested as a `Tuple`. +static void updateIncludeTypeIds( + DataTypePtr type, + const ORC::Type * orc_type, + bool ignore_case, + std::unordered_set & include_typeids, + std::vector & requested_structs) { /// Recurses the file-controlled ORC type tree in lockstep with the (parser-bounded) CH type. /// Keep a stack backstop here too: this runs in prepareFileReader, before the @@ -1126,7 +1131,7 @@ updateIncludeTypeIds(DataTypePtr type, const ORC::Type * orc_type, bool ignore_c const auto * array_type = typeid_cast(non_nullable_type.get()); if (array_type) { - updateIncludeTypeIds(array_type->getNestedType(), orc_type->getSubtype(0), ignore_case, include_typeids); + updateIncludeTypeIds(array_type->getNestedType(), orc_type->getSubtype(0), ignore_case, include_typeids, requested_structs); return; } break; @@ -1135,8 +1140,8 @@ updateIncludeTypeIds(DataTypePtr type, const ORC::Type * orc_type, bool ignore_c const auto * map_type = typeid_cast(non_nullable_type.get()); if (map_type) { - updateIncludeTypeIds(map_type->getKeyType(), orc_type->getSubtype(0), ignore_case, include_typeids); - updateIncludeTypeIds(map_type->getValueType(), orc_type->getSubtype(1), ignore_case, include_typeids); + updateIncludeTypeIds(map_type->getKeyType(), orc_type->getSubtype(0), ignore_case, include_typeids, requested_structs); + updateIncludeTypeIds(map_type->getValueType(), orc_type->getSubtype(1), ignore_case, include_typeids, requested_structs); return; } break; @@ -1148,6 +1153,7 @@ updateIncludeTypeIds(DataTypePtr type, const ORC::Type * orc_type, bool ignore_c const auto * tuple_type = typeid_cast(non_nullable_type.get()); if (tuple_type) { + requested_structs.push_back(orc_type); if (tuple_type->hasExplicitNames()) { std::unordered_map orc_field_name_to_index; @@ -1174,14 +1180,15 @@ updateIncludeTypeIds(DataTypePtr type, const ORC::Type * orc_type, bool ignore_c tuple_type->getElement(tuple_i), orc_type->getSubtype(orc_field_name_to_index[element_name]), ignore_case, - include_typeids); + include_typeids, + requested_structs); } } } else { for (size_t i = 0; i < tuple_type->getElements().size() && i < orc_type->getSubtypeCount(); ++i) - updateIncludeTypeIds(tuple_type->getElement(i), orc_type->getSubtype(i), ignore_case, include_typeids); + updateIncludeTypeIds(tuple_type->getElement(i), orc_type->getSubtype(i), ignore_case, include_typeids, requested_structs); } return; } @@ -1203,7 +1210,7 @@ updateIncludeTypeIds(DataTypePtr type, const ORC::Type * orc_type, bool ignore_c { const auto * branch_orc_type = orc_type->getSubtype(i); if (branch_hints[i]) - updateIncludeTypeIds(branch_hints[i], branch_orc_type, ignore_case, include_typeids); + updateIncludeTypeIds(branch_hints[i], branch_orc_type, ignore_case, include_typeids, requested_structs); else include_typeids.insert(branch_orc_type->getColumnId()); } @@ -1217,6 +1224,47 @@ updateIncludeTypeIds(DataTypePtr type, const ORC::Type * orc_type, bool ignore_c include_typeids.insert(orc_type->getColumnId()); } +/// Whether a column of the subtree of `orc_type`, or one of its ancestors (which reads the subtree whole), is included. +static bool isORCTypeRead( + const ORC::Type & root, const ORC::Type & orc_type, const std::unordered_set & include_typeids) +{ + for (UInt64 id = orc_type.getColumnId(); id <= orc_type.getMaximumColumnId(); ++id) + { + if (include_typeids.contains(id)) + return true; + } + + const ORC::Type * node = &root; + while (node && node->getColumnId() < orc_type.getColumnId()) + { + if (include_typeids.contains(node->getColumnId())) + return true; + + const ORC::Type * child = nullptr; + for (UInt64 i = 0; i < node->getSubtypeCount() && !child; ++i) + { + if (node->getSubtype(i)->getMaximumColumnId() >= orc_type.getColumnId()) + child = node->getSubtype(i); + } + node = child; + } + return false; +} + +/// The field of an ORC struct with the fewest columns, or the struct itself if it has no fields. +static const ORC::Type & getNarrowestORCStructField(const ORC::Type & orc_struct) +{ + const ORC::Type * narrowest = &orc_struct; + for (UInt64 i = 0; i < orc_struct.getSubtypeCount(); ++i) + { + const auto * field = orc_struct.getSubtype(i); + if (narrowest == &orc_struct + || field->getMaximumColumnId() - field->getColumnId() < narrowest->getMaximumColumnId() - narrowest->getColumnId()) + narrowest = field; + } + return *narrowest; +} + NativeORCBlockInputFormat::NativeORCBlockInputFormat( ReadBuffer & in_, SharedHeader header_, @@ -1246,27 +1294,44 @@ void NativeORCBlockInputFormat::prepareFileReader() format_filter_info->filter_actions_dag.get(), format_settings.orc.case_insensitive_column_matching)); - orc_column_to_ch_column = std::make_unique( - getPort().getHeader(), - format_settings.orc.allow_missing_columns, - format_settings.null_as_default, - format_settings.orc.case_insensitive_column_matching, - format_settings.orc.dictionary_as_low_cardinality, - format_settings.date_time_overflow_behavior); - const bool ignore_case = format_settings.orc.case_insensitive_column_matching; const auto & header = getPort().getHeader(); const auto & file_schema = file_reader->getType(); std::unordered_set include_typeids; + std::vector requested_structs; for (const auto & column : header) { auto adjusted_type = column.type; const auto * orc_type = traverseDownORCTypeByName(column.name, &file_schema, adjusted_type, ignore_case); if (orc_type) - updateIncludeTypeIds(adjusted_type, orc_type, ignore_case, include_typeids); + updateIncludeTypeIds(adjusted_type, orc_type, ignore_case, include_typeids, requested_structs); + } + + /// A requested struct none of whose requested fields is in the file is still read, through its narrowest field, + /// for its presence and the sizes of the lists and maps around it; `matchORCTupleElementsByName` then fills the + /// requested fields with defaults. Inner structs first, as a read inner one makes its parents read. + std::ranges::sort(requested_structs, std::greater{}, &ORC::Type::getColumnId); + std::unordered_set count_only_fields; + for (const auto * orc_struct : requested_structs) + { + if (isORCTypeRead(file_schema, *orc_struct, include_typeids)) + continue; + const auto & narrowest = getNarrowestORCStructField(*orc_struct); + include_typeids.insert(narrowest.getColumnId()); + if (&narrowest != orc_struct) + count_only_fields.insert(narrowest.getColumnId()); } include_indices.assign(include_typeids.begin(), include_typeids.end()); + orc_column_to_ch_column = std::make_unique( + getPort().getHeader(), + format_settings.orc.allow_missing_columns, + format_settings.null_as_default, + format_settings.orc.case_insensitive_column_matching, + format_settings.orc.dictionary_as_low_cardinality, + format_settings.date_time_overflow_behavior, + std::move(count_only_fields)); + if (format_settings.orc.filter_push_down && format_filter_info && format_filter_info->key_condition && !sargs) sargs = buildORCSearchArgument(*format_filter_info->key_condition, header, file_schema, format_settings); @@ -1482,13 +1547,15 @@ ORCColumnToCHColumn::ORCColumnToCHColumn( bool null_as_default_, bool case_insensitive_matching_, bool dictionary_as_low_cardinality_, - FormatSettings::DateTimeOverflowBehavior date_time_overflow_behavior_) + FormatSettings::DateTimeOverflowBehavior date_time_overflow_behavior_, + std::unordered_set count_only_fields_) : header(header_) , allow_missing_columns(allow_missing_columns_) , null_as_default(null_as_default_) , case_insensitive_matching(case_insensitive_matching_) , dictionary_as_low_cardinality(dictionary_as_low_cardinality_) , date_time_overflow_behavior(date_time_overflow_behavior_) + , count_only_fields(std::move(count_only_fields_)) { } @@ -2392,6 +2459,117 @@ static DataTypes computeOrcUnionBranchHints(const ORC::Type * orc_type, const Da return branch_hints; } +/// Matches the elements of the tuples in `column` to those of the named tuples in `requested_type` by name, through +/// Nullable, Array and Map: a requested element that was not read is filled with defaults, and an element that is +/// not requested is dropped. A CAST would match tuples with no common element names by position instead. +static ColumnWithTypeAndName +matchORCTupleElementsByName(const ColumnWithTypeAndName & column, const DataTypePtr & requested_type, bool case_insensitive) +{ + if (column.type->equals(*requested_type)) + return column; + + if (const auto * nullable_type = typeid_cast(column.type.get())) + { + const auto & nullable_column = assert_cast(*column.column); + auto nested = matchORCTupleElementsByName( + {nullable_column.getNestedColumnPtr(), nullable_type->getNestedType(), column.name}, + removeNullable(requested_type), + case_insensitive); + return { + ColumnNullable::create(nested.column, nullable_column.getNullMapColumnPtr()), + std::make_shared(nested.type), + column.name}; + } + + const auto requested_non_nullable_type = removeNullable(requested_type); + + if (const auto * array_type = typeid_cast(column.type.get())) + { + const auto * requested_array_type = typeid_cast(requested_non_nullable_type.get()); + if (!requested_array_type) + return column; + const auto & array_column = assert_cast(*column.column); + auto nested = matchORCTupleElementsByName( + {array_column.getDataPtr(), array_type->getNestedType(), column.name}, requested_array_type->getNestedType(), case_insensitive); + return { + ColumnArray::create(nested.column, array_column.getOffsetsPtr()), std::make_shared(nested.type), column.name}; + } + + if (const auto * map_type = typeid_cast(column.type.get())) + { + const auto * requested_map_type = typeid_cast(requested_non_nullable_type.get()); + if (!requested_map_type) + return column; + const auto & map_column = assert_cast(*column.column); + auto keys = matchORCTupleElementsByName( + {map_column.getNestedData().getColumnPtr(0), map_type->getKeyType(), "key"}, requested_map_type->getKeyType(), case_insensitive); + auto values = matchORCTupleElementsByName( + {map_column.getNestedData().getColumnPtr(1), map_type->getValueType(), "value"}, + requested_map_type->getValueType(), + case_insensitive); + return { + ColumnMap::create(keys.column, values.column, map_column.getNestedColumn().getOffsetsPtr()), + std::make_shared(keys.type, values.type), + column.name}; + } + + const auto * tuple_type = typeid_cast(column.type.get()); + const auto * requested_tuple_type = typeid_cast(requested_non_nullable_type.get()); + if (!tuple_type || !requested_tuple_type) + return column; + + const auto & tuple_column = assert_cast(*column.column); + const size_t num_rows = tuple_column.size(); + Columns elements; + DataTypes types; + Strings names; + + /// An unnamed tuple is matched by position and keeps its elements, so the CAST rejects a different number of them. + if (!requested_tuple_type->hasExplicitNames()) + { + for (size_t i = 0; i < tuple_type->getElements().size(); ++i) + { + ColumnWithTypeAndName element{tuple_column.getColumnPtr(i), tuple_type->getElement(i), tuple_type->getNameByPosition(i + 1)}; + if (i < requested_tuple_type->getElements().size()) + element = matchORCTupleElementsByName(element, requested_tuple_type->getElement(i), case_insensitive); + elements.push_back(std::move(element.column)); + types.push_back(std::move(element.type)); + } + auto result_column = elements.empty() ? ColumnTuple::create(num_rows) : ColumnTuple::create(std::move(elements)); + return {std::move(result_column), std::make_shared(std::move(types)), column.name}; + } + + for (size_t i = 0; i < requested_tuple_type->getElements().size(); ++i) + { + const auto & requested_element_type = requested_tuple_type->getElement(i); + const auto & name = requested_tuple_type->getNameByPosition(i + 1); + + auto pos = tuple_type->tryGetPositionByName(name, /*case_insensitive=*/false); + if (!pos && case_insensitive) + pos = tuple_type->tryGetPositionByName(name, /*case_insensitive=*/true); + + if (pos) + { + auto element = matchORCTupleElementsByName( + {tuple_column.getColumnPtr(*pos), tuple_type->getElement(*pos), name}, requested_element_type, case_insensitive); + elements.push_back(std::move(element.column)); + types.push_back(std::move(element.type)); + } + else + { + auto element = requested_element_type->createColumn(); + requested_element_type->insertManyDefaultsInto(*element, num_rows); + elements.push_back(std::move(element)); + types.push_back(requested_element_type); + } + names.push_back(name); + } + + auto result_type = std::make_shared(std::move(types), std::move(names)); + auto result_column = elements.empty() ? ColumnTuple::create(num_rows) : ColumnTuple::create(std::move(elements)); + return {std::move(result_column), std::move(result_type), column.name}; +} + ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( const ORC::ColumnVectorBatch * orc_column, const ORC::Type * orc_type, @@ -2746,6 +2924,8 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( /// A map key cannot be Nullable. if (!key_type_hint && !DataTypeMap::isValidKeyType(key_column.type)) key_type_hint = removeNullableOrLowCardinalityNullable(key_column.type); + if (key_type_hint) + key_column = matchORCTupleElementsByName(key_column, key_type_hint, case_insensitive_matching); if (key_type_hint && !key_type_hint->equals(*key_column.type)) { /// Cast key column to target type, because it can happen @@ -2778,6 +2958,12 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( const auto * orc_nested_type = orc_type->getSubtype(0); auto nested_column = readColumnFromORCColumn(orc_nested_column, orc_nested_type, column_name, false, nested_type_hint); + /// A struct read only for its size has no elements, which `createNested` below rejects. Otherwise the elements + /// are matched after a subcolumn is extracted, as a hint for one can name a path, such as `x.v`. + const auto * read_tuple_type = typeid_cast(removeNullable(nested_column.type).get()); + if (type_hint && isNested(type_hint) && read_tuple_type && read_tuple_type->getElements().empty()) + nested_column = matchORCTupleElementsByName(nested_column, nested_type_hint, case_insensitive_matching); + auto offsets_column = readOffsetsFromORCListColumn(orc_list_column); DataTypePtr array_type; ColumnPtr array_data_column = nested_column.column; @@ -2813,6 +2999,10 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( for (size_t i = 0; i < orc_type->getSubtypeCount(); ++i) { + /// Read only for the size and the null map of the struct, so its values are not converted. + if (count_only_fields.contains(orc_type->getSubtype(i)->getColumnId())) + continue; + auto field_name = orc_type->getFieldName(i); DataTypePtr nested_type_hint; @@ -2929,6 +3119,7 @@ void ORCColumnToCHColumn::orcColumnsToCHChunk( orc_column_with_type.first, orc_column_with_type.second, header_column.name, false, header_column.type); } + column = matchORCTupleElementsByName(column, header_column.type, case_insensitive_matching); if (null_as_default) insertNullAsDefaultIfNeeded(column, header_column, column_i, block_missing_values); diff --git a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h index c50e2d388176d..5596542988060 100644 --- a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h +++ b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h @@ -153,7 +153,8 @@ class ORCColumnToCHColumn bool null_as_default_, bool case_insensitive_matching_ = false, bool dictionary_as_low_cardinality_ = false, - FormatSettings::DateTimeOverflowBehavior date_time_overflow_behavior_ = FormatSettings::DateTimeOverflowBehavior::Ignore); + FormatSettings::DateTimeOverflowBehavior date_time_overflow_behavior_ = FormatSettings::DateTimeOverflowBehavior::Ignore, + std::unordered_set count_only_fields_ = {}); void orcTableToCHChunk( Chunk & res, @@ -180,6 +181,8 @@ class ORCColumnToCHColumn bool case_insensitive_matching; bool dictionary_as_low_cardinality; FormatSettings::DateTimeOverflowBehavior date_time_overflow_behavior; + /// ORC column ids of the struct fields that are read only for the size and the null map of their struct. + const std::unordered_set count_only_fields; }; } #endif diff --git a/tests/queries/0_stateless/05617_orc_nested_struct_without_requested_fields.reference b/tests/queries/0_stateless/05617_orc_nested_struct_without_requested_fields.reference new file mode 100644 index 0000000000000..73a3ed806c7c7 --- /dev/null +++ b/tests/queries/0_stateless/05617_orc_nested_struct_without_requested_fields.reference @@ -0,0 +1,159 @@ +-- { echo } + +SET engine_file_truncate_on_insert = 1; +SET enable_nullable_tuple_type = 1; +INSERT INTO FUNCTION file(currentDatabase() || '_05617.orc', 'ORC') SELECT + number AS id, + if(number = 0, [(1, 'a'), (2, 'b')], [])::Array(Tuple(a Int32, b String)) AS l, + if(number = 0, map('k1', (1, 'a'), 'k2', (2, 'b')), map())::Map(String, Tuple(a Int32, b String)) AS m, + map(CAST(tuple(toInt32(number)), 'Tuple(a Int32)'), 'v') AS mk, + CAST((toInt32(number), [(toInt32(number), 'x')]), 'Tuple(a Int32, l Array(Tuple(x Int32, y String)))') AS t, + [[(toInt32(number), 'x'), (1, 'y')]]::Array(Array(Tuple(a Int32, b String))) AS ll, + map('k', [(toInt32(number), 'x')])::Map(String, Array(Tuple(a Int32, b String))) AS ml, + if(number = 0, [(1, 'a'), NULL, (2, 'b')], [])::Array(Nullable(Tuple(a Int32, b String))) AS ln, + [(toInt32(number), 'c')]::Array(Tuple(A Int32, b String)) AS lc, + CAST((toInt32(number), map('k', (1, 'v'))), 'Tuple(a Int32, m Map(String, Tuple(a Int32, b String)))') AS tm, + [(1, tuple(2)), (3, tuple(4))]::Array(Tuple(x Int32, z Tuple(w Int32))) AS n, + [(1, [(1, 'a'), (2, 'b')])]::Array(Tuple(y Int32, x Array(Tuple(a Int32, b String)))) AS lx, + if(number = 0, [tuple(tuple(1)), tuple(NULL)], [tuple(NULL)])::Array(Tuple(b Nullable(Tuple(a Int32)))) AS nb, + map('k', (if(number = 0, tuple(1), NULL), if(number = 0, NULL, tuple('x'))))::Map(String, Tuple(b Nullable(Tuple(a Int32)), c Nullable(Tuple(x String)))) AS mb, + CAST((toInt32(number), [tuple(if(number = 0, tuple(1), NULL))]), 'Tuple(a Int32, l Array(Tuple(b Nullable(Tuple(a Int32)))))') AS tb, + if(number = 0, tuple(1, 'a'), NULL)::Nullable(Tuple(a Int32, b String)) AS ts, + [CAST(tuple(toDateTime64('3000-01-01 00:00:00', 0, 'UTC')), 'Tuple(a DateTime64(0, \'UTC\'))')] AS ld, + CAST([tuple(11, tuple(5))], 'Array(Tuple(w Int32, x Tuple(v Int32)))') AS an +FROM numbers(2); +-- A list element, a map value and a map key, also nested in a struct, a list or a map, and with NULL elements. +SELECT id, l FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Array(Tuple(q Nullable(Int32)))') ORDER BY id; +0 [(NULL),(NULL)] +1 [] +SELECT id, m FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, m Map(String, Tuple(q Nullable(String)))') ORDER BY id; +0 {'k1':(NULL),'k2':(NULL)} +1 {} +SELECT id, mk FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, mk Map(Tuple(z Nullable(Int32)), String)') ORDER BY id; +0 {(NULL):'v'} +1 {(NULL):'v'} +SELECT id, t FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, t Tuple(a Int32, l Array(Tuple(q Nullable(Int32))))') ORDER BY id; +0 (0,[(NULL)]) +1 (1,[(NULL)]) +SELECT id, ll FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ll Array(Array(Tuple(q Nullable(Int32))))') ORDER BY id; +0 [[(NULL),(NULL)]] +1 [[(NULL),(NULL)]] +SELECT id, ml FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ml Map(String, Array(Tuple(q Nullable(Int32))))') ORDER BY id; +0 {'k':[(NULL)]} +1 {'k':[(NULL)]} +SELECT id, ln FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ln Array(Nullable(Tuple(q Nullable(Int32))))') ORDER BY id; +0 [(NULL),NULL,(NULL)] +1 [] +SELECT id, l FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Array(Tuple(q Nullable(Int32)))') ORDER BY id SETTINGS input_format_orc_row_batch_size = 1; +0 [(NULL),(NULL)] +1 [] +-- The same through a subcolumn of a struct and a subcolumn of a Nested column. +SELECT id, `t.a`, `t.l` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, `t.a` Int32, `t.l` Array(Tuple(q Nullable(Int32)))') ORDER BY id; +0 0 [(NULL)] +1 1 [(NULL)] +SELECT id, `tm.m` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, `tm.m` Map(String, Tuple(q Nullable(String)))') ORDER BY id; +0 {'k':(NULL)} +1 {'k':(NULL)} +SELECT id, `n.z` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, `n.z` Array(Tuple(q Nullable(Int32)))') ORDER BY id; +0 [(NULL),(NULL)] +1 [(NULL),(NULL)] +SELECT id, `n.x`, `n.z` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, `n.x` Array(Int32), `n.z` Array(Tuple(q Nullable(Int32)))') ORDER BY id; +0 [1,3] [(NULL),(NULL)] +1 [1,3] [(NULL),(NULL)] +-- A struct that contains such a list is read with its fields. +SELECT id, lx FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, lx Array(Tuple(x Array(Tuple(q Nullable(Int32)))))') ORDER BY id; +0 [([(NULL),(NULL)])] +1 [([(NULL),(NULL)])] +-- A struct read whole by another column keeps its data. optimize_functions_to_subcolumns = 0 keeps `t.l` itself in the read. +SELECT id, t, length(`t.l`) FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, t String, `t.l` Array(Tuple(q String, r String))') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +0 (0,[(0,\'x\')]) 1 +1 (1,[(1,\'x\')]) 1 +-- A Tuple naming some of the fields, and case-insensitive matching. +SELECT id, l FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Array(Tuple(a Int32, q Nullable(Int32)))') ORDER BY id; +0 [(1,NULL),(2,NULL)] +1 [] +SELECT id, m FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, m Map(String, Tuple(a Int32, q Nullable(String)))') ORDER BY id; +0 {'k1':(1,NULL),'k2':(2,NULL)} +1 {} +SELECT id, lc FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, lc Array(Tuple(a Int32))') ORDER BY id SETTINGS input_format_orc_case_insensitive_column_matching = 1; +0 [(0)] +1 [(1)] +-- A non-nullable Date32 field gets the same default with and without a field that is in the file, an Enum gets its first value. +SELECT id, l, m FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Array(Tuple(q Date32, e Enum8(\'x\' = 1, \'y\' = 2))), m Map(String, Tuple(q Date32))') ORDER BY id; +0 [('1970-01-01','x'),('1970-01-01','x')] {'k1':('1970-01-01'),'k2':('1970-01-01')} +1 [] {} +SELECT id, l, m FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Array(Tuple(a Int32, q Date32)), m Map(String, Tuple(a Int32, q Date32))') ORDER BY id; +0 [(1,'1970-01-01'),(2,'1970-01-01')] {'k1':(1,'1970-01-01'),'k2':(2,'1970-01-01')} +1 [] {} +-- A requested struct field of such a struct keeps its presence, the same as when one of its fields is requested. +SELECT id, nb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, nb Array(Tuple(b Nullable(Tuple(q Int32))))') ORDER BY id; +0 [((0)),(NULL)] +1 [(NULL)] +SELECT id, nb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, nb Array(Tuple(b Nullable(Tuple(a Nullable(Int32), q Int32))))') ORDER BY id; +0 [((1,0)),(NULL)] +1 [(NULL)] +SELECT id, mb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, mb Map(String, Tuple(c Nullable(Tuple(q Int32)), b Nullable(Tuple(r String))))') ORDER BY id; +0 {'k':(NULL,(''))} +1 {'k':((0),NULL)} +SELECT id, mb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, mb Map(String, Tuple(c Nullable(Tuple(x Nullable(String), q Int32)), b Nullable(Tuple(a Nullable(Int32), r String))))') ORDER BY id; +0 {'k':(NULL,(1,''))} +1 {'k':(('x',0),NULL)} +-- A NULL struct requested as non-Nullable is rejected or read as default, the same as when one of its fields is requested. +SELECT id, nb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, nb Array(Tuple(b Tuple(q Int32)))') ORDER BY id SETTINGS input_format_null_as_default = 1; +0 [((0)),((0))] +1 [((0))] +SELECT id, nb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, nb Array(Tuple(b Tuple(a Nullable(Int32), q Int32)))') ORDER BY id SETTINGS input_format_null_as_default = 1; +0 [((1,0)),((NULL,0))] +1 [((NULL,0))] +SELECT id, nb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, nb Array(Tuple(b Tuple(q Int32)))') ORDER BY id SETTINGS input_format_null_as_default = 0; -- { serverError CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN } +SELECT id, nb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, nb Array(Tuple(b Tuple(a Nullable(Int32), q Int32)))') ORDER BY id SETTINGS input_format_null_as_default = 0; -- { serverError CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN } +-- Two columns reading the same struct with different types get the defaults of their own type. +SELECT id, t, `t.l` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, t Tuple(l Array(Tuple(q Nullable(Int32)))), `t.l` Array(Tuple(r Nullable(String), s Nullable(Int32)))') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +0 ([(NULL)]) [(NULL,NULL)] +1 ([(NULL)]) [(NULL,NULL)] +SELECT id, t, `t.l` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, t Tuple(l Array(Tuple(e Enum8(\'x\' = 1)))), `t.l` Array(Tuple(r Nullable(String)))') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +0 ([('x')]) [(NULL)] +1 ([('x')]) [(NULL)] +SELECT id, `t.l`, t FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, `t.l` Array(Tuple(r Nullable(String), s Nullable(Int32))), t Tuple(l Array(Tuple(q Nullable(Int32))))') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +0 [(NULL,NULL)] ([(NULL)]) +1 [(NULL,NULL)] ([(NULL)]) +-- Of two columns reading the same struct, one that requests a nested struct of it keeps that struct's presence. +SELECT id, tb, `tb.l` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, tb Tuple(l Array(Tuple(q Nullable(Int32)))), `tb.l` Array(Tuple(b Nullable(Tuple(q Int32))))') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +0 ([(NULL)]) [((0))] +1 ([(NULL)]) [(NULL)] +SELECT id, `tb.l`, tb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, `tb.l` Array(Tuple(b Nullable(Tuple(q Int32)))), tb Tuple(l Array(Tuple(q Nullable(Int32))))') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +0 [((0))] ([(NULL)]) +1 [(NULL)] ([(NULL)]) +SELECT id, tb, `tb.a` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, tb Tuple(l Array(Tuple(q Nullable(Int32)))), `tb.a` Int32') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +0 ([(NULL)]) 0 +1 ([(NULL)]) 1 +-- A top-level struct keeps its presence too, also when missing columns are not allowed. +SELECT id, ts FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ts Nullable(Tuple(q Int32))') ORDER BY id; +0 (0) +1 \N +SELECT id, ts FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ts Nullable(Tuple(a Int32, q Int32))') ORDER BY id; +0 (1,0) +1 \N +SELECT id, ts FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ts Nullable(Tuple(q Int32))') ORDER BY id SETTINGS input_format_orc_allow_missing_columns = 0; +0 (0) +1 \N +-- A field read only for the size of its struct is not converted. +SELECT id, ld FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ld Array(Tuple(q Int32))') ORDER BY id; +0 [(0)] +1 [(0)] +-- An unnamed Tuple is matched by position and must have as many elements as the struct. +SELECT id, ts FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ts Nullable(Tuple(Int32, String))') ORDER BY id; +0 (1,'a') +1 \N +SELECT id, ts FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ts Nullable(Tuple(Int32, String, Int32))') ORDER BY id; -- { serverError TYPE_MISMATCH } +SELECT id, l FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Array(Tuple(Int32, String, Int32))') ORDER BY id; -- { serverError TYPE_MISMATCH } +-- The same for a Nested column that is not flattened. +SELECT id, l FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Nested(q Int32)') ORDER BY id SETTINGS flatten_nested = 0; +0 [(0),(0)] +1 [] +SELECT id, ln FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ln Nested(q Nullable(Int32))') ORDER BY id SETTINGS flatten_nested = 0; +0 [(NULL),(NULL),(NULL)] +1 [] +SELECT id, ln FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ln Nested(a Nullable(Int32), q Nullable(Int32))') ORDER BY id SETTINGS flatten_nested = 0; +0 [(1,NULL),(NULL,NULL),(2,NULL)] +1 [] diff --git a/tests/queries/0_stateless/05617_orc_nested_struct_without_requested_fields.sql b/tests/queries/0_stateless/05617_orc_nested_struct_without_requested_fields.sql new file mode 100644 index 0000000000000..8dbf07d584af2 --- /dev/null +++ b/tests/queries/0_stateless/05617_orc_nested_struct_without_requested_fields.sql @@ -0,0 +1,101 @@ +-- Tags: no-fasttest +-- no-fasttest: ORC is not available in fasttest builds +-- An ORC list or map whose element, key or value struct has none of the fields of the requested Tuple keeps its +-- elements, and each of them is filled with default values. + +-- { echo } + +SET engine_file_truncate_on_insert = 1; +SET enable_nullable_tuple_type = 1; + +INSERT INTO FUNCTION file(currentDatabase() || '_05617.orc', 'ORC') SELECT + number AS id, + if(number = 0, [(1, 'a'), (2, 'b')], [])::Array(Tuple(a Int32, b String)) AS l, + if(number = 0, map('k1', (1, 'a'), 'k2', (2, 'b')), map())::Map(String, Tuple(a Int32, b String)) AS m, + map(CAST(tuple(toInt32(number)), 'Tuple(a Int32)'), 'v') AS mk, + CAST((toInt32(number), [(toInt32(number), 'x')]), 'Tuple(a Int32, l Array(Tuple(x Int32, y String)))') AS t, + [[(toInt32(number), 'x'), (1, 'y')]]::Array(Array(Tuple(a Int32, b String))) AS ll, + map('k', [(toInt32(number), 'x')])::Map(String, Array(Tuple(a Int32, b String))) AS ml, + if(number = 0, [(1, 'a'), NULL, (2, 'b')], [])::Array(Nullable(Tuple(a Int32, b String))) AS ln, + [(toInt32(number), 'c')]::Array(Tuple(A Int32, b String)) AS lc, + CAST((toInt32(number), map('k', (1, 'v'))), 'Tuple(a Int32, m Map(String, Tuple(a Int32, b String)))') AS tm, + [(1, tuple(2)), (3, tuple(4))]::Array(Tuple(x Int32, z Tuple(w Int32))) AS n, + [(1, [(1, 'a'), (2, 'b')])]::Array(Tuple(y Int32, x Array(Tuple(a Int32, b String)))) AS lx, + if(number = 0, [tuple(tuple(1)), tuple(NULL)], [tuple(NULL)])::Array(Tuple(b Nullable(Tuple(a Int32)))) AS nb, + map('k', (if(number = 0, tuple(1), NULL), if(number = 0, NULL, tuple('x'))))::Map(String, Tuple(b Nullable(Tuple(a Int32)), c Nullable(Tuple(x String)))) AS mb, + CAST((toInt32(number), [tuple(if(number = 0, tuple(1), NULL))]), 'Tuple(a Int32, l Array(Tuple(b Nullable(Tuple(a Int32)))))') AS tb, + if(number = 0, tuple(1, 'a'), NULL)::Nullable(Tuple(a Int32, b String)) AS ts, + [CAST(tuple(toDateTime64('3000-01-01 00:00:00', 0, 'UTC')), 'Tuple(a DateTime64(0, \'UTC\'))')] AS ld, + CAST([tuple(11, tuple(5))], 'Array(Tuple(w Int32, x Tuple(v Int32)))') AS an +FROM numbers(2); + +-- A list element, a map value and a map key, also nested in a struct, a list or a map, and with NULL elements. +SELECT id, l FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Array(Tuple(q Nullable(Int32)))') ORDER BY id; +SELECT id, m FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, m Map(String, Tuple(q Nullable(String)))') ORDER BY id; +SELECT id, mk FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, mk Map(Tuple(z Nullable(Int32)), String)') ORDER BY id; +SELECT id, t FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, t Tuple(a Int32, l Array(Tuple(q Nullable(Int32))))') ORDER BY id; +SELECT id, ll FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ll Array(Array(Tuple(q Nullable(Int32))))') ORDER BY id; +SELECT id, ml FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ml Map(String, Array(Tuple(q Nullable(Int32))))') ORDER BY id; +SELECT id, ln FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ln Array(Nullable(Tuple(q Nullable(Int32))))') ORDER BY id; +SELECT id, l FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Array(Tuple(q Nullable(Int32)))') ORDER BY id SETTINGS input_format_orc_row_batch_size = 1; + +-- The same through a subcolumn of a struct and a subcolumn of a Nested column. +SELECT id, `t.a`, `t.l` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, `t.a` Int32, `t.l` Array(Tuple(q Nullable(Int32)))') ORDER BY id; +SELECT id, `tm.m` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, `tm.m` Map(String, Tuple(q Nullable(String)))') ORDER BY id; +SELECT id, `n.z` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, `n.z` Array(Tuple(q Nullable(Int32)))') ORDER BY id; +SELECT id, `n.x`, `n.z` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, `n.x` Array(Int32), `n.z` Array(Tuple(q Nullable(Int32)))') ORDER BY id; + +-- A struct that contains such a list is read with its fields. +SELECT id, lx FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, lx Array(Tuple(x Array(Tuple(q Nullable(Int32)))))') ORDER BY id; + +-- A struct read whole by another column keeps its data. optimize_functions_to_subcolumns = 0 keeps `t.l` itself in the read. +SELECT id, t, length(`t.l`) FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, t String, `t.l` Array(Tuple(q String, r String))') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; + +-- A Tuple naming some of the fields, and case-insensitive matching. +SELECT id, l FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Array(Tuple(a Int32, q Nullable(Int32)))') ORDER BY id; +SELECT id, m FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, m Map(String, Tuple(a Int32, q Nullable(String)))') ORDER BY id; +SELECT id, lc FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, lc Array(Tuple(a Int32))') ORDER BY id SETTINGS input_format_orc_case_insensitive_column_matching = 1; + +-- A non-nullable Date32 field gets the same default with and without a field that is in the file, an Enum gets its first value. +SELECT id, l, m FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Array(Tuple(q Date32, e Enum8(\'x\' = 1, \'y\' = 2))), m Map(String, Tuple(q Date32))') ORDER BY id; +SELECT id, l, m FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Array(Tuple(a Int32, q Date32)), m Map(String, Tuple(a Int32, q Date32))') ORDER BY id; + +-- A requested struct field of such a struct keeps its presence, the same as when one of its fields is requested. +SELECT id, nb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, nb Array(Tuple(b Nullable(Tuple(q Int32))))') ORDER BY id; +SELECT id, nb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, nb Array(Tuple(b Nullable(Tuple(a Nullable(Int32), q Int32))))') ORDER BY id; +SELECT id, mb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, mb Map(String, Tuple(c Nullable(Tuple(q Int32)), b Nullable(Tuple(r String))))') ORDER BY id; +SELECT id, mb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, mb Map(String, Tuple(c Nullable(Tuple(x Nullable(String), q Int32)), b Nullable(Tuple(a Nullable(Int32), r String))))') ORDER BY id; + +-- A NULL struct requested as non-Nullable is rejected or read as default, the same as when one of its fields is requested. +SELECT id, nb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, nb Array(Tuple(b Tuple(q Int32)))') ORDER BY id SETTINGS input_format_null_as_default = 1; +SELECT id, nb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, nb Array(Tuple(b Tuple(a Nullable(Int32), q Int32)))') ORDER BY id SETTINGS input_format_null_as_default = 1; +SELECT id, nb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, nb Array(Tuple(b Tuple(q Int32)))') ORDER BY id SETTINGS input_format_null_as_default = 0; -- { serverError CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN } +SELECT id, nb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, nb Array(Tuple(b Tuple(a Nullable(Int32), q Int32)))') ORDER BY id SETTINGS input_format_null_as_default = 0; -- { serverError CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN } + +-- Two columns reading the same struct with different types get the defaults of their own type. +SELECT id, t, `t.l` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, t Tuple(l Array(Tuple(q Nullable(Int32)))), `t.l` Array(Tuple(r Nullable(String), s Nullable(Int32)))') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT id, t, `t.l` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, t Tuple(l Array(Tuple(e Enum8(\'x\' = 1)))), `t.l` Array(Tuple(r Nullable(String)))') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT id, `t.l`, t FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, `t.l` Array(Tuple(r Nullable(String), s Nullable(Int32))), t Tuple(l Array(Tuple(q Nullable(Int32))))') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; + +-- Of two columns reading the same struct, one that requests a nested struct of it keeps that struct's presence. +SELECT id, tb, `tb.l` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, tb Tuple(l Array(Tuple(q Nullable(Int32)))), `tb.l` Array(Tuple(b Nullable(Tuple(q Int32))))') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT id, `tb.l`, tb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, `tb.l` Array(Tuple(b Nullable(Tuple(q Int32)))), tb Tuple(l Array(Tuple(q Nullable(Int32))))') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT id, tb, `tb.a` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, tb Tuple(l Array(Tuple(q Nullable(Int32)))), `tb.a` Int32') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; + +-- A top-level struct keeps its presence too, also when missing columns are not allowed. +SELECT id, ts FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ts Nullable(Tuple(q Int32))') ORDER BY id; +SELECT id, ts FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ts Nullable(Tuple(a Int32, q Int32))') ORDER BY id; +SELECT id, ts FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ts Nullable(Tuple(q Int32))') ORDER BY id SETTINGS input_format_orc_allow_missing_columns = 0; + +-- A field read only for the size of its struct is not converted. +SELECT id, ld FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ld Array(Tuple(q Int32))') ORDER BY id; + +-- An unnamed Tuple is matched by position and must have as many elements as the struct. +SELECT id, ts FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ts Nullable(Tuple(Int32, String))') ORDER BY id; +SELECT id, ts FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ts Nullable(Tuple(Int32, String, Int32))') ORDER BY id; -- { serverError TYPE_MISMATCH } +SELECT id, l FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Array(Tuple(Int32, String, Int32))') ORDER BY id; -- { serverError TYPE_MISMATCH } + +-- The same for a Nested column that is not flattened. +SELECT id, l FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Nested(q Int32)') ORDER BY id SETTINGS flatten_nested = 0; +SELECT id, ln FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ln Nested(q Nullable(Int32))') ORDER BY id SETTINGS flatten_nested = 0; +SELECT id, ln FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ln Nested(a Nullable(Int32), q Nullable(Int32))') ORDER BY id SETTINGS flatten_nested = 0; From 282d3c7c70f7c303ba385a4090e6bfac83582769 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 11 Nov 2026 15:06:49 +0000 Subject: [PATCH 158/219] Backport #130755 to 26.8: Fix Keeper crash on `CheckWatch` request with an unknown watch type --- src/Coordination/KeeperStorageImpl.cpp | 13 ++++- .../test_keeper_persistent_watches/test.py | 55 +++++++++++++++++++ 2 files changed, 65 insertions(+), 3 deletions(-) diff --git a/src/Coordination/KeeperStorageImpl.cpp b/src/Coordination/KeeperStorageImpl.cpp index 92b5405c561c2..0ffb06436dab7 100644 --- a/src/Coordination/KeeperStorageImpl.cpp +++ b/src/Coordination/KeeperStorageImpl.cpp @@ -960,7 +960,9 @@ processLocal(const Coordination::ZooKeeperCheckWatchRequest & zk_request, Keeper ProfileEvents::increment(ProfileEvents::KeeperCheckWatchRequest); auto response = zk_request.makeResponse(); - if (!storage.containsWatch(zk_request.getPath(), zk_request.type)) + if (!magic_enum::enum_contains(zk_request.type)) + response->error = Coordination::Error::ZBADARGUMENTS; + else if (!storage.containsWatch(zk_request.getPath(), zk_request.type)) response->error = Coordination::Error::ZNOWATCHER; return response; } @@ -979,7 +981,9 @@ processLocal(const Coordination::ZooKeeperRemoveWatchRequest & zk_request, Keepe ProfileEvents::increment(ProfileEvents::KeeperRemoveWatchRequest); auto response = zk_request.makeResponse(); - if (!storage.removePersistentWatch(zk_request.path, zk_request.type, session_id)) + if (!magic_enum::enum_contains(zk_request.type)) + response->error = Coordination::Error::ZBADARGUMENTS; + else if (!storage.removePersistentWatch(zk_request.path, zk_request.type, session_id)) response->error = Coordination::Error::ZNOWATCHER; return response; @@ -999,7 +1003,10 @@ processLocal(const Coordination::ZooKeeperAddWatchRequest & zk_request, KeeperSt ProfileEvents::increment(ProfileEvents::KeeperAddWatchRequest); auto response = zk_request.makeResponse(); - storage.addPersistentWatch(zk_request.path, zk_request.mode, session_id); + if (!magic_enum::enum_contains(zk_request.mode)) + response->error = Coordination::Error::ZBADARGUMENTS; + else + storage.addPersistentWatch(zk_request.path, zk_request.mode, session_id); return response; } diff --git a/tests/integration/test_keeper_persistent_watches/test.py b/tests/integration/test_keeper_persistent_watches/test.py index 733e77063d9c5..c32d713242677 100644 --- a/tests/integration/test_keeper_persistent_watches/test.py +++ b/tests/integration/test_keeper_persistent_watches/test.py @@ -4,6 +4,7 @@ import os import threading import re +import struct import helpers.keeper_utils as keeper_utils from helpers.cluster import ClickHouseCluster @@ -341,3 +342,57 @@ def cb(_): destroy_zk_client(client) data = keeper_utils.send_4lw_cmd(cluster, node1, cmd="mntr") assert "zk_watch_count\t0" in data + +CHECK_WATCH, REMOVE_WATCH, ADD_WATCH = 17, 18, 106 +ZOK, ZBADARGUMENTS, ZNOWATCHER = 0, -8, -121 +# (opnum, type or mode, expected error): types are 1..5 and modes 0..1, so the last request of each op is valid. +UNKNOWN_TYPE_REQUESTS = [ + (CHECK_WATCH, 0, ZBADARGUMENTS), + (CHECK_WATCH, 6, ZBADARGUMENTS), + (CHECK_WATCH, 1, ZNOWATCHER), + (REMOVE_WATCH, 0, ZBADARGUMENTS), + (REMOVE_WATCH, 6, ZBADARGUMENTS), + (REMOVE_WATCH, 1, ZNOWATCHER), + (ADD_WATCH, -1, ZBADARGUMENTS), + (ADD_WATCH, 2, ZBADARGUMENTS), + (ADD_WATCH, 0, ZOK), +] + +def recv_exactly(sock, size): + data = b"" + while len(data) < size: + chunk = sock.recv(size - len(data)) + if not chunk: + raise EOFError("Keeper closed the connection") + data += chunk + return data + +def send_unknown_type_requests(): + with get_keeper_socket(node1.name) as sock: + handshake = struct.pack(">iqiqi16s", 0, 0, 30000, 0, 16, b"\0" * 16) + sock.sendall(struct.pack(">i", len(handshake)) + handshake) + reply = recv_exactly(sock, struct.unpack(">i", recv_exactly(sock, 4))[0]) + assert struct.unpack(">q", reply[8:16])[0] != 0 + path = b"/testUnknownWatchType" + for xid, (opnum, value, expected) in enumerate(UNKNOWN_TYPE_REQUESTS, start=1): + body = struct.pack(">iii", xid, opnum, len(path)) + path + struct.pack(">i", value) + sock.sendall(struct.pack(">i", len(body)) + body) + reply = recv_exactly(sock, struct.unpack(">i", recv_exactly(sock, 4))[0]) + reply_xid, _, error = struct.unpack(">iqi", reply[:16]) + assert (reply_xid, error) == (xid, expected), (opnum, value) + +def test_watch_requests_with_unknown_type(started_cluster): + # Through Raft (`quorum_reads`), then a restart that replays the logged requests, then through the local read path. + config = "/etc/clickhouse-server/config.d/keeper_config1.xml" + try: + node1.replace_in_config(config, "", "1") + node1.restart_clickhouse() + keeper_utils.wait_until_connected(cluster, node1) + assert "quorum_reads=true" in keeper_utils.send_4lw_cmd(cluster, node1, cmd="conf") + send_unknown_type_requests() + finally: + node1.replace_in_config(config, "1", "") + node1.restart_clickhouse(kill=True) + keeper_utils.wait_until_connected(cluster, node1) + assert "quorum_reads=false" in keeper_utils.send_4lw_cmd(cluster, node1, cmd="conf") + send_unknown_type_requests() From fb2d85a96946ac59d31ded3f6a5ede2c88bd4655 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 11 Nov 2026 15:09:15 +0000 Subject: [PATCH 159/219] Backport #130488 to 26.8: Validate values of deserialized `gini` states --- .../AggregateFunctionGini.cpp | 22 ++++++++++++++++--- ...5617_gini_deserialize_validation.reference | 3 +++ .../05617_gini_deserialize_validation.sql | 15 +++++++++++++ 3 files changed, 37 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/05617_gini_deserialize_validation.reference create mode 100644 tests/queries/0_stateless/05617_gini_deserialize_validation.sql diff --git a/src/AggregateFunctions/AggregateFunctionGini.cpp b/src/AggregateFunctions/AggregateFunctionGini.cpp index 336519528c92e..1994d725b86b3 100644 --- a/src/AggregateFunctions/AggregateFunctionGini.cpp +++ b/src/AggregateFunctions/AggregateFunctionGini.cpp @@ -25,6 +25,7 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int INCORRECT_DATA; extern const int TOO_LARGE_ARRAY_SIZE; } @@ -67,11 +68,12 @@ struct AggregateFunctionGiniData return static_cast(value); } - void add(const Value & x) + /// Returns false for NaN, which is skipped. Throws for unsupported values. + static bool checkValue(const Value & x) { - /// Skip NaNs as they are not compatible with comparison sorting. + /// NaNs are not compatible with comparison sorting. if (isNaN(x)) - return; + return false; if (!isFinite(x)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Aggregate function `gini` does not support infinite values"); @@ -79,6 +81,14 @@ struct AggregateFunctionGiniData if (x < Value{}) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Aggregate function `gini` does not support negative values"); + return true; + } + + void add(const Value & x) + { + if (!checkValue(x)) + return; + if (unlikely(array.size() >= GINI_MAX_ARRAY_SIZE)) throwTooLargeArraySize(); @@ -107,6 +117,12 @@ struct AggregateFunctionGiniData checkArraySize(size); array.resize(size); buf.readStrict(reinterpret_cast(array.data()), size * sizeof(array[0])); + + for (const auto & value : array) + { + if (!checkValue(value)) + throw Exception(ErrorCodes::INCORRECT_DATA, "NaN in serialized state of aggregate function `gini`"); + } } /** Gini coefficient of the collected values. For sorted values x, the sum of diff --git a/tests/queries/0_stateless/05617_gini_deserialize_validation.reference b/tests/queries/0_stateless/05617_gini_deserialize_validation.reference new file mode 100644 index 0000000000000..b1ea233d5eecd --- /dev/null +++ b/tests/queries/0_stateless/05617_gini_deserialize_validation.reference @@ -0,0 +1,3 @@ +0.25 +0.366667 +0.366667 diff --git a/tests/queries/0_stateless/05617_gini_deserialize_validation.sql b/tests/queries/0_stateless/05617_gini_deserialize_validation.sql new file mode 100644 index 0000000000000..bb90839d649e6 --- /dev/null +++ b/tests/queries/0_stateless/05617_gini_deserialize_validation.sql @@ -0,0 +1,15 @@ +-- Serialized `gini` states must be validated like ordinary input values. + +-- Int64 min and max. +SELECT finalizeAggregation(CAST(unhex('020000000000000080FFFFFFFFFFFFFF7F') AS AggregateFunction(gini, Int64))); -- { serverError BAD_ARGUMENTS } +-- Negative value: [-1, 2]. +SELECT finalizeAggregation(CAST(unhex('02FFFFFFFFFFFFFFFF0200000000000000') AS AggregateFunction(gini, Int64))); -- { serverError BAD_ARGUMENTS } +-- Infinity: [0, inf]. +SELECT finalizeAggregation(CAST(unhex('020000000000000000000000000000F07F') AS AggregateFunction(gini, Float64))); -- { serverError BAD_ARGUMENTS } +-- NaN is never stored by `add`: [0, nan]. +SELECT finalizeAggregation(CAST(unhex('020000000000000000000000000000F87F') AS AggregateFunction(gini, Float64))); -- { serverError INCORRECT_DATA } + +-- Valid states still round-trip. +SELECT finalizeAggregation(CAST(unhex('0201000000000000000300000000000000') AS AggregateFunction(gini, Int64))); +SELECT round(finalizeAggregation(CAST(unhex(hex(giniState(number))) AS AggregateFunction(gini, UInt64))), 6) FROM numbers(10); +SELECT round(finalizeAggregation(CAST(unhex(hex(giniState(toFloat64(number)))) AS AggregateFunction(gini, Float64))), 6) FROM numbers(10); From d053319ec4119d917c41ecbca62c951c561f987c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 11 Nov 2026 16:17:35 +0000 Subject: [PATCH 160/219] Backport #128978 to 26.8: Do not echo dictionary source file content in YAMLRegExpTree error --- src/Dictionaries/YAMLRegExpTreeDictionarySource.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Dictionaries/YAMLRegExpTreeDictionarySource.cpp b/src/Dictionaries/YAMLRegExpTreeDictionarySource.cpp index 20c2c6bc0d152..28f1da3ca9bbe 100644 --- a/src/Dictionaries/YAMLRegExpTreeDictionarySource.cpp +++ b/src/Dictionaries/YAMLRegExpTreeDictionarySource.cpp @@ -334,7 +334,7 @@ void parseMatchList(UInt64 parent_id, UInt64 & id, const YAML::Node & node, Resu { if (!node.IsSequence()) { - throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Configuration {} must be a yaml list of match rules", node.as()); + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Configuration must be a yaml list of match rules"); } for (const auto & child_node : node) From 07507bdd3ee8dce71bedf73fe864989dd28f4135 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 11 Nov 2026 23:36:45 +0000 Subject: [PATCH 161/219] Backport #125897 to 26.8: Do not prefetch in a two-level GROUP BY that holds few keys --- src/Common/ColumnsHashing.h | 2 +- src/Common/HashTable/Prefetching.h | 14 +++ src/Common/HashTable/StringHashTable.h | 5 + src/Interpreters/Aggregator.cpp | 2 +- .../group_by_two_level_few_keys.xml | 42 +++++++ ...p_by_two_level_few_keys_prefetch.reference | 115 ++++++++++++++++++ ...8_group_by_two_level_few_keys_prefetch.sql | 62 ++++++++++ 7 files changed, 240 insertions(+), 2 deletions(-) create mode 100644 tests/performance/group_by_two_level_few_keys.xml create mode 100644 tests/queries/0_stateless/05498_group_by_two_level_few_keys_prefetch.reference create mode 100644 tests/queries/0_stateless/05498_group_by_two_level_few_keys_prefetch.sql diff --git a/src/Common/ColumnsHashing.h b/src/Common/ColumnsHashing.h index a12369c22bfa8..f5f3a87317e76 100644 --- a/src/Common/ColumnsHashing.h +++ b/src/Common/ColumnsHashing.h @@ -528,7 +528,7 @@ struct HashMethodSerialized precomputed_hashes_initialized = true; calibration_row = first_row + PrefetchingHelper::iterationsToMeasure(); - if (min_bytes_for_prefetch != 0 && data.getBufferSizeInBytes() <= min_bytes_for_prefetch) + if (min_bytes_for_prefetch != 0 && bytesForPrefetch(data) <= min_bytes_for_prefetch) { can_precompute_hashes = false; return; diff --git a/src/Common/HashTable/Prefetching.h b/src/Common/HashTable/Prefetching.h index 6a609e1444464..e887d8a4d57eb 100644 --- a/src/Common/HashTable/Prefetching.h +++ b/src/Common/HashTable/Prefetching.h @@ -52,4 +52,18 @@ class PrefetchingHelper Stopwatch watch; }; +/// The size compared with `min_bytes_for_prefetch`. A two-level table allocates all its buckets up front, so until one +/// grows its buffer is a constant; count its keys instead, as the smallest table that holds them at load factor 1/2. +template +size_t bytesForPrefetch(const Table & table) +{ + const size_t buffer_bytes = table.getBufferSizeInBytes(); + if constexpr (requires { Table::NUM_BUCKETS; Table::Impl::initial_buffer_bytes; }) + { + if (buffer_bytes == Table::NUM_BUCKETS * Table::Impl::initial_buffer_bytes) + return std::min(buffer_bytes, 2 * table.size() * sizeof(typename Table::cell_type)); + } + return buffer_bytes; +} + } diff --git a/src/Common/HashTable/StringHashTable.h b/src/Common/HashTable/StringHashTable.h index ec1506898e940..05a933c2f05ce 100644 --- a/src/Common/HashTable/StringHashTable.h +++ b/src/Common/HashTable/StringHashTable.h @@ -200,6 +200,7 @@ struct StringHashTableEmpty // NOLINT(cppcoreguidelines-pro-type-member-init,hic void readText(DB::ReadBuffer & rb) { zeroValue()->readText(rb); } size_t size() const { return hasZero() ? 1 : 0; } bool empty() const { return !hasZero(); } + static constexpr size_t initial_buffer_bytes = sizeof(Cell); size_t getBufferSizeInBytes() const { return sizeof(Cell); } size_t getCollisions() const { return 0; } }; @@ -296,6 +297,10 @@ class StringHashTable : private boost::noncopyable using value_type = typename Ts::value_type; using cell_type = typename Ts::cell_type; + /// What `getBufferSizeInBytes` returns before any submap grows. + static constexpr size_t initial_buffer_bytes = T0::initial_buffer_bytes + T1::initial_buffer_bytes + + T2::initial_buffer_bytes + T3::initial_buffer_bytes + Ts::initial_buffer_bytes; + using LookupResult = StringHashTableLookupResult; using ConstLookupResult = StringHashTableLookupResult; diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index c41978e614779..26f52e0ecdf1f 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -1180,7 +1180,7 @@ void Aggregator::executeImpl( /// its key there (e.g. serializing all key columns) would pay its dominant per-row cost /// twice - far more than the cache miss the prefetch hides. See `has_cheap_key_holder`. const bool prefetch = State::has_cheap_key_holder && params.enable_prefetch - && (method.data.getBufferSizeInBytes() > min_bytes_for_prefetch); + && (bytesForPrefetch(method.data) > min_bytes_for_prefetch); #if USE_EMBEDDED_COMPILER if (compiled_aggregate_functions_holder && !hasSparseArguments(aggregate_instructions)) diff --git a/tests/performance/group_by_two_level_few_keys.xml b/tests/performance/group_by_two_level_few_keys.xml new file mode 100644 index 0000000000000..e2684ff54d78b --- /dev/null +++ b/tests/performance/group_by_two_level_few_keys.xml @@ -0,0 +1,42 @@ + + + + 8 + 1 + + + + CREATE TABLE group_by_two_level_few_keys + ( + lc LowCardinality(Nullable(String)), + s Nullable(String), + a UInt64, + b UInt64, + c UInt64 + ) + ENGINE = MergeTree ORDER BY tuple() + + + INSERT INTO group_by_two_level_few_keys + SELECT v, v, intHash32(intDiv(number, 16)) % 13, 7, 11 + FROM + ( + SELECT + number, + multiIf( + intHash32(intDiv(number, 16)) % 100 < 40, 'category_value_01_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + intHash32(intDiv(number, 16)) % 100 < 70, 'category_value_02_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + intHash32(intDiv(number, 16)) % 100 < 95, 'category_value_03_cccccccccccccccccccccccccccccccccccccccc', + intHash32(intDiv(number, 16)) % 100 < 97, NULL, + concat('category_value_', toString((intHash32(number) % 9) + 4), '_dddddddddddddddddddddddddddddddddddddddd')) AS v + FROM numbers_mt(300000000) + ) + + OPTIMIZE TABLE group_by_two_level_few_keys FINAL + + SELECT lc FROM group_by_two_level_few_keys GROUP BY lc FORMAT Null + SELECT s FROM group_by_two_level_few_keys GROUP BY s FORMAT Null + SELECT a, b, c, count() FROM group_by_two_level_few_keys GROUP BY a, b, c FORMAT Null + + DROP TABLE IF EXISTS group_by_two_level_few_keys + diff --git a/tests/queries/0_stateless/05498_group_by_two_level_few_keys_prefetch.reference b/tests/queries/0_stateless/05498_group_by_two_level_few_keys_prefetch.reference new file mode 100644 index 0000000000000..10172e5b7749e --- /dev/null +++ b/tests/queries/0_stateless/05498_group_by_two_level_few_keys_prefetch.reference @@ -0,0 +1,115 @@ +LowCardinality(Nullable(String)), prefetch on +category_value_01_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 120928 +category_value_02_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 88992 +category_value_03_cccccccccccccccccccccccccccccccccccccccc 74464 +category_value_10_dddddddddddddddddddddddddddddddddddddddd 1029 +category_value_11_dddddddddddddddddddddddddddddddddddddddd 1063 +category_value_12_dddddddddddddddddddddddddddddddddddddddd 1025 +category_value_4_dddddddddddddddddddddddddddddddddddddddd 959 +category_value_5_dddddddddddddddddddddddddddddddddddddddd 1012 +category_value_6_dddddddddddddddddddddddddddddddddddddddd 1037 +category_value_7_dddddddddddddddddddddddddddddddddddddddd 1043 +category_value_8_dddddddddddddddddddddddddddddddddddddddd 982 +category_value_9_dddddddddddddddddddddddddddddddddddddddd 1082 +\N 6384 +LowCardinality(Nullable(String)), prefetch off +category_value_01_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 120928 +category_value_02_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 88992 +category_value_03_cccccccccccccccccccccccccccccccccccccccc 74464 +category_value_10_dddddddddddddddddddddddddddddddddddddddd 1029 +category_value_11_dddddddddddddddddddddddddddddddddddddddd 1063 +category_value_12_dddddddddddddddddddddddddddddddddddddddd 1025 +category_value_4_dddddddddddddddddddddddddddddddddddddddd 959 +category_value_5_dddddddddddddddddddddddddddddddddddddddd 1012 +category_value_6_dddddddddddddddddddddddddddddddddddddddd 1037 +category_value_7_dddddddddddddddddddddddddddddddddddddddd 1043 +category_value_8_dddddddddddddddddddddddddddddddddddddddd 982 +category_value_9_dddddddddddddddddddddddddddddddddddddddd 1082 +\N 6384 +Nullable(String), prefetch on +category_value_01_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 120928 +category_value_02_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 88992 +category_value_03_cccccccccccccccccccccccccccccccccccccccc 74464 +category_value_10_dddddddddddddddddddddddddddddddddddddddd 1029 +category_value_11_dddddddddddddddddddddddddddddddddddddddd 1063 +category_value_12_dddddddddddddddddddddddddddddddddddddddd 1025 +category_value_4_dddddddddddddddddddddddddddddddddddddddd 959 +category_value_5_dddddddddddddddddddddddddddddddddddddddd 1012 +category_value_6_dddddddddddddddddddddddddddddddddddddddd 1037 +category_value_7_dddddddddddddddddddddddddddddddddddddddd 1043 +category_value_8_dddddddddddddddddddddddddddddddddddddddd 982 +category_value_9_dddddddddddddddddddddddddddddddddddddddd 1082 +\N 6384 +Nullable(String), prefetch off +category_value_01_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 120928 +category_value_02_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 88992 +category_value_03_cccccccccccccccccccccccccccccccccccccccc 74464 +category_value_10_dddddddddddddddddddddddddddddddddddddddd 1029 +category_value_11_dddddddddddddddddddddddddddddddddddddddd 1063 +category_value_12_dddddddddddddddddddddddddddddddddddddddd 1025 +category_value_4_dddddddddddddddddddddddddddddddddddddddd 959 +category_value_5_dddddddddddddddddddddddddddddddddddddddd 1012 +category_value_6_dddddddddddddddddddddddddddddddddddddddd 1037 +category_value_7_dddddddddddddddddddddddddddddddddddddddd 1043 +category_value_8_dddddddddddddddddddddddddddddddddddddddd 982 +category_value_9_dddddddddddddddddddddddddddddddddddddddd 1082 +\N 6384 +keys256, prefetch on +0 7 11 23584 +1 7 11 22480 +2 7 11 23504 +3 7 11 23424 +4 7 11 22992 +5 7 11 23248 +6 7 11 22848 +7 7 11 23168 +8 7 11 21952 +9 7 11 23200 +10 7 11 23456 +11 7 11 23392 +12 7 11 22752 +keys256, prefetch off +0 7 11 23584 +1 7 11 22480 +2 7 11 23504 +3 7 11 23424 +4 7 11 22992 +5 7 11 23248 +6 7 11 22848 +7 7 11 23168 +8 7 11 21952 +9 7 11 23200 +10 7 11 23456 +11 7 11 23392 +12 7 11 22752 +serialized, prefetch on + 7 6384 +category_value_01_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 7 120928 +category_value_02_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 7 88992 +category_value_03_cccccccccccccccccccccccccccccccccccccccc 7 74464 +category_value_10_dddddddddddddddddddddddddddddddddddddddd 7 1029 +category_value_11_dddddddddddddddddddddddddddddddddddddddd 7 1063 +category_value_12_dddddddddddddddddddddddddddddddddddddddd 7 1025 +category_value_4_dddddddddddddddddddddddddddddddddddddddd 7 959 +category_value_5_dddddddddddddddddddddddddddddddddddddddd 7 1012 +category_value_6_dddddddddddddddddddddddddddddddddddddddd 7 1037 +category_value_7_dddddddddddddddddddddddddddddddddddddddd 7 1043 +category_value_8_dddddddddddddddddddddddddddddddddddddddd 7 982 +category_value_9_dddddddddddddddddddddddddddddddddddddddd 7 1082 +serialized, prefetch off + 7 6384 +category_value_01_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 7 120928 +category_value_02_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 7 88992 +category_value_03_cccccccccccccccccccccccccccccccccccccccc 7 74464 +category_value_10_dddddddddddddddddddddddddddddddddddddddd 7 1029 +category_value_11_dddddddddddddddddddddddddddddddddddddddd 7 1063 +category_value_12_dddddddddddddddddddddddddddddddddddddddd 7 1025 +category_value_4_dddddddddddddddddddddddddddddddddddddddd 7 959 +category_value_5_dddddddddddddddddddddddddddddddddddddddd 7 1012 +category_value_6_dddddddddddddddddddddddddddddddddddddddd 7 1037 +category_value_7_dddddddddddddddddddddddddddddddddddddddd 7 1043 +category_value_8_dddddddddddddddddddddddddddddddddddddddd 7 982 +category_value_9_dddddddddddddddddddddddddddddddddddddddd 7 1082 +grown two-level, prefetch on and off +100000 300000 14999850000 +100000 300000 14999850000 diff --git a/tests/queries/0_stateless/05498_group_by_two_level_few_keys_prefetch.sql b/tests/queries/0_stateless/05498_group_by_two_level_few_keys_prefetch.sql new file mode 100644 index 0000000000000..231b76d6e9d6a --- /dev/null +++ b/tests/queries/0_stateless/05498_group_by_two_level_few_keys_prefetch.sql @@ -0,0 +1,62 @@ +-- A two-level `GROUP BY` table that holds few keys decides the software prefetch by its keys, not by the preallocated +-- buckets (see `bytesForPrefetch`). The decision never changes the result, so the test only checks that every two-level +-- table shape behind that gate returns the same rows with the prefetch on and off: the issue's +-- `LowCardinality(Nullable(String))` key, `Nullable(String)` (`TwoLevelStringHashMap`), `keys256`, serialized keys +-- (`HashMethodSerialized` has a gate of its own), and a two-level table that has outgrown its initial allocation. + +DROP TABLE IF EXISTS t_two_level_few_keys; + +CREATE TABLE t_two_level_few_keys +( + lc LowCardinality(Nullable(String)), + s Nullable(String), + str String, + a UInt64, + b UInt64, + c UInt64, + many UInt64 +) +ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO t_two_level_few_keys +SELECT v, v, ifNull(v, ''), intHash32(intDiv(number, 16)) % 13, 7, 11, number % 100000 +FROM +( + SELECT + number, + multiIf( + intHash32(intDiv(number, 16)) % 100 < 40, 'category_value_01_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + intHash32(intDiv(number, 16)) % 100 < 70, 'category_value_02_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + intHash32(intDiv(number, 16)) % 100 < 95, 'category_value_03_cccccccccccccccccccccccccccccccccccccccc', + intHash32(intDiv(number, 16)) % 100 < 97, NULL, + concat('category_value_', toString((intHash32(number) % 9) + 4), '_dddddddddddddddddddddddddddddddddddddddd')) AS v + FROM numbers_mt(300000) +); + +SET max_threads = 8, max_block_size = 8192, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes = 1; + +SELECT 'LowCardinality(Nullable(String)), prefetch on'; +SELECT lc, count() FROM t_two_level_few_keys GROUP BY lc ORDER BY lc SETTINGS enable_software_prefetch_in_aggregation = 1; +SELECT 'LowCardinality(Nullable(String)), prefetch off'; +SELECT lc, count() FROM t_two_level_few_keys GROUP BY lc ORDER BY lc SETTINGS enable_software_prefetch_in_aggregation = 0; + +SELECT 'Nullable(String), prefetch on'; +SELECT s, count() FROM t_two_level_few_keys GROUP BY s ORDER BY s SETTINGS enable_software_prefetch_in_aggregation = 1; +SELECT 'Nullable(String), prefetch off'; +SELECT s, count() FROM t_two_level_few_keys GROUP BY s ORDER BY s SETTINGS enable_software_prefetch_in_aggregation = 0; + +SELECT 'keys256, prefetch on'; +SELECT a, b, c, count() FROM t_two_level_few_keys GROUP BY a, b, c ORDER BY a, b, c SETTINGS enable_software_prefetch_in_aggregation = 1; +SELECT 'keys256, prefetch off'; +SELECT a, b, c, count() FROM t_two_level_few_keys GROUP BY a, b, c ORDER BY a, b, c SETTINGS enable_software_prefetch_in_aggregation = 0; + +SELECT 'serialized, prefetch on'; +SELECT str, b, count() FROM t_two_level_few_keys GROUP BY str, b ORDER BY str, b SETTINGS enable_software_prefetch_in_aggregation = 1; +SELECT 'serialized, prefetch off'; +SELECT str, b, count() FROM t_two_level_few_keys GROUP BY str, b ORDER BY str, b SETTINGS enable_software_prefetch_in_aggregation = 0; + +SELECT 'grown two-level, prefetch on and off'; +SELECT count(), sum(cnt), sum(many * cnt) FROM (SELECT many, count() AS cnt FROM t_two_level_few_keys GROUP BY many) SETTINGS enable_software_prefetch_in_aggregation = 1; +SELECT count(), sum(cnt), sum(many * cnt) FROM (SELECT many, count() AS cnt FROM t_two_level_few_keys GROUP BY many) SETTINGS enable_software_prefetch_in_aggregation = 0; + +DROP TABLE t_two_level_few_keys; From 381ac1f85a100f342aedd1b8b8740154ee40dfd1 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 11 Nov 2026 16:37:24 +0000 Subject: [PATCH 162/219] Backport #122465 to 26.8: Fix `ssd_cache` dictionaries with the same `PATH` returning each other's values --- src/Dictionaries/SSDCacheDictionaryStorage.h | 46 ++++++++++--- ...ssd_cache_dictionaries_same_path.reference | 7 ++ .../05258_ssd_cache_dictionaries_same_path.sh | 64 +++++++++++++++++++ 3 files changed, 109 insertions(+), 8 deletions(-) create mode 100644 tests/queries/0_stateless/05258_ssd_cache_dictionaries_same_path.reference create mode 100755 tests/queries/0_stateless/05258_ssd_cache_dictionaries_same_path.sh diff --git a/src/Dictionaries/SSDCacheDictionaryStorage.h b/src/Dictionaries/SSDCacheDictionaryStorage.h index 9d8f2c2a48a51..873f9d533e16c 100644 --- a/src/Dictionaries/SSDCacheDictionaryStorage.h +++ b/src/Dictionaries/SSDCacheDictionaryStorage.h @@ -12,6 +12,7 @@ #include #include #include +#include # include # include @@ -480,24 +481,53 @@ class SSDCacheFileBuffer : private boost::noncopyable ProfileEvents::increment(ProfileEvents::FileOpen); + /// Another storage with the same path may still be reading the old file, so it is replaced, not truncated. + /// rename(2) overwrites a symlink, not the file it points to, so the link is resolved first. + std::filesystem::path target_path = file_path; + for (size_t hops = 0; target_path.has_filename() && FS::isSymlinkNoThrow(target_path); ++hops) + { + if (hops == 40) + ErrnoException::throwFromPathWithErrno(ErrorCodes::CANNOT_OPEN_FILE, file_path, ELOOP, "Cannot open file {}", file_path); + target_path = target_path.parent_path() / FS::readSymlink(target_path); + } + const std::string target_file_path = target_path.string(); + + /// rename(2) ignores the replaced file's permissions: a file the server cannot read and write must not be replaced. + if (::access(target_file_path.c_str(), R_OK | W_OK) != 0 && errno != ENOENT) + ErrnoException::throwFromPath(ErrorCodes::CANNOT_OPEN_FILE, file_path, "Cannot open file {}", target_file_path); + + const std::string new_file_path = (target_path.parent_path() / fmt::format("ssd_cache_{}.tmp", randomSeed())).string(); + #if defined(OS_DARWIN) /// macOS has no O_DIRECT; F_NOCACHE (set below) is the closest equivalent. - file.fd = ::open(file_path.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0666); + file.fd = ::open(new_file_path.c_str(), O_RDWR | O_CREAT | O_EXCL, 0666); #else - file.fd = ::open(file_path.c_str(), O_RDWR | O_CREAT | O_TRUNC | O_DIRECT, 0666); + file.fd = ::open(new_file_path.c_str(), O_RDWR | O_CREAT | O_EXCL | O_DIRECT, 0666); #endif if (file.fd == -1) { auto error_code = (errno == ENOENT) ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE; - ErrnoException::throwFromPath(error_code, file_path, "Cannot open file {}", file_path); + ErrnoException::throwFromPath(error_code, file_path, "Cannot create file {} to replace {}", new_file_path, target_file_path); } - #if defined(OS_DARWIN) - if (::fcntl(file.fd, F_NOCACHE, 1) == -1) - ErrnoException::throwFromPath(ErrorCodes::CANNOT_OPEN_FILE, file_path, "Cannot set F_NOCACHE on file {}", file_path); - #endif + try + { + #if defined(OS_DARWIN) + if (::fcntl(file.fd, F_NOCACHE, 1) == -1) + ErrnoException::throwFromPath(ErrorCodes::CANNOT_OPEN_FILE, file_path, "Cannot set F_NOCACHE on file {}", file_path); + #endif + + allocateSizeForNextPartition(); - allocateSizeForNextPartition(); + if (::rename(new_file_path.c_str(), target_file_path.c_str()) != 0) + ErrnoException::throwFromPath( + ErrorCodes::CANNOT_OPEN_FILE, file_path, "Cannot rename {} to {}", new_file_path, target_file_path); + } + catch (...) + { + ::unlink(new_file_path.c_str()); + throw; + } } void allocateSizeForNextPartition() diff --git a/tests/queries/0_stateless/05258_ssd_cache_dictionaries_same_path.reference b/tests/queries/0_stateless/05258_ssd_cache_dictionaries_same_path.reference new file mode 100644 index 0000000000000..59252d9172177 --- /dev/null +++ b/tests/queries/0_stateless/05258_ssd_cache_dictionaries_same_path.reference @@ -0,0 +1,7 @@ +0 +0 +0 +0 +42 +symlink kept +65536 diff --git a/tests/queries/0_stateless/05258_ssd_cache_dictionaries_same_path.sh b/tests/queries/0_stateless/05258_ssd_cache_dictionaries_same_path.sh new file mode 100755 index 0000000000000..78bdeb18385ca --- /dev/null +++ b/tests/queries/0_stateless/05258_ssd_cache_dictionaries_same_path.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# Two ssd_cache dictionaries (and two complex_key_ssd_cache ones) with the same PATH each return their own values. + +path="$CLICKHOUSE_USER_FILES/${CLICKHOUSE_DATABASE}_same_path" +complex_path="$CLICKHOUSE_USER_FILES/${CLICKHOUSE_DATABASE}_same_path_complex" + +$CLICKHOUSE_CLIENT --query=" + CREATE TABLE src_a (id UInt64, value UInt64) ENGINE = TinyLog; + CREATE TABLE src_b (id UInt64, value UInt64) ENGINE = TinyLog; + INSERT INTO src_a SELECT number, 1000 + number FROM numbers(500); + INSERT INTO src_b SELECT number, 9000000 + number FROM numbers(500); + + CREATE DICTIONARY dict_a (id UInt64, value UInt64 DEFAULT 0) PRIMARY KEY id + SOURCE(CLICKHOUSE(TABLE 'src_a')) LIFETIME(MIN 1000 MAX 2000) + LAYOUT(SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 65536 WRITE_BUFFER_SIZE 4096 PATH '$path')); + CREATE DICTIONARY dict_b (id UInt64, value UInt64 DEFAULT 0) PRIMARY KEY id + SOURCE(CLICKHOUSE(TABLE 'src_b')) LIFETIME(MIN 1000 MAX 2000) + LAYOUT(SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 65536 WRITE_BUFFER_SIZE 4096 PATH '$path')); + + CREATE TABLE src_c (key String, value UInt64) ENGINE = TinyLog; + CREATE TABLE src_d (key String, value UInt64) ENGINE = TinyLog; + INSERT INTO src_c SELECT toString(number), 1000 + number FROM numbers(500); + INSERT INTO src_d SELECT toString(number), 9000000 + number FROM numbers(500); + + CREATE DICTIONARY dict_c (key String, value UInt64 DEFAULT 0) PRIMARY KEY key + SOURCE(CLICKHOUSE(TABLE 'src_c')) LIFETIME(MIN 1000 MAX 2000) + LAYOUT(COMPLEX_KEY_SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 65536 WRITE_BUFFER_SIZE 4096 PATH '$complex_path')); + CREATE DICTIONARY dict_d (key String, value UInt64 DEFAULT 0) PRIMARY KEY key + SOURCE(CLICKHOUSE(TABLE 'src_d')) LIFETIME(MIN 1000 MAX 2000) + LAYOUT(COMPLEX_KEY_SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 65536 WRITE_BUFFER_SIZE 4096 PATH '$complex_path')); + + SELECT sum(dictGet('dict_a', 'value', number)) FROM numbers(500) FORMAT Null; + SELECT sum(dictGet('dict_b', 'value', number)) FROM numbers(500) FORMAT Null; + SELECT sum(dictGet('dict_c', 'value', tuple(toString(number)))) FROM numbers(500) FORMAT Null; + SELECT sum(dictGet('dict_d', 'value', tuple(toString(number)))) FROM numbers(500) FORMAT Null; + + -- Only the caches can answer from here on. + TRUNCATE TABLE src_a; TRUNCATE TABLE src_b; TRUNCATE TABLE src_c; TRUNCATE TABLE src_d; + + SELECT countIf(dictGet('dict_a', 'value', number) != 1000 + number) FROM numbers(500); + SELECT countIf(dictGet('dict_b', 'value', number) != 9000000 + number) FROM numbers(500); + SELECT countIf(dictGet('dict_c', 'value', tuple(toString(number))) != 1000 + number) FROM numbers(500); + SELECT countIf(dictGet('dict_d', 'value', tuple(toString(number))) != 9000000 + number) FROM numbers(500); +" + +# A symlink at a dictionary's cache file is kept, and the file it points to becomes the cache. +ln -s "${CLICKHOUSE_DATABASE}_link_target.bin" "$CLICKHOUSE_USER_FILES/${CLICKHOUSE_DATABASE}_link.bin" + +$CLICKHOUSE_CLIENT --query=" + CREATE TABLE src_e (id UInt64, value UInt64) ENGINE = TinyLog; + INSERT INTO src_e VALUES (1, 42); + CREATE DICTIONARY dict_e (id UInt64, value UInt64 DEFAULT 0) PRIMARY KEY id + SOURCE(CLICKHOUSE(TABLE 'src_e')) LIFETIME(MIN 1000 MAX 2000) + LAYOUT(SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 65536 WRITE_BUFFER_SIZE 4096 PATH '$CLICKHOUSE_USER_FILES/${CLICKHOUSE_DATABASE}_link')); + SELECT dictGet('dict_e', 'value', toUInt64(1)); +" + +[ -L "$CLICKHOUSE_USER_FILES/${CLICKHOUSE_DATABASE}_link.bin" ] && echo "symlink kept" +wc -c < "$CLICKHOUSE_USER_FILES/${CLICKHOUSE_DATABASE}_link_target.bin" From 9c3c09686c4e4dae940519cda40e6e98c4322106 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 12 Nov 2026 05:18:42 +0000 Subject: [PATCH 163/219] Backport #130493 to 26.8: Fix flaky test_async_metadata_refresh --- src/Common/FailPoint.cpp | 1 + .../DataLakes/Iceberg/IcebergMetadata.cpp | 9 ++++ .../test_async_metadata_refresh.py | 46 +++++++++++-------- 3 files changed, 38 insertions(+), 18 deletions(-) diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index 0c00fdb7849fc..91b870818faac 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -245,6 +245,7 @@ static struct InitFiu REGULAR(slowdown_skip_index_read_result_build) \ ONCE(iceberg_writes_cleanup) \ REGULAR(iceberg_slow_manifest_read) \ + PAUSEABLE_ONCE(iceberg_metadata_prefetch_pause) \ REGULAR(storage_cluster_read_sleep) \ ONCE(backup_add_empty_memory_table) \ ONCE(backup_from_snapshot_fail_after_batch) \ diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index 67ecc542c68cf..1cb962cb4b2e1 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -82,6 +82,7 @@ #include #include +#include #include #include @@ -123,6 +124,11 @@ extern const int SUPPORT_IS_DISABLED; extern const int FILE_ALREADY_EXISTS; } +namespace FailPoints +{ +extern const char iceberg_metadata_prefetch_pause[]; +} + namespace Setting { extern const SettingsNonZeroUInt64 max_block_size; @@ -278,6 +284,9 @@ void IcebergMetadata::backgroundMetadataPrefetcherThread() try { + /// Lets a test hold the refresher before it reads the latest metadata + FailPointInjection::pauseFailPoint(FailPoints::iceberg_metadata_prefetch_pause); + if (!Context::getGlobalContextInstance()) { /// Should never happen, but if seen, this is clear indicator that the task should be started/stopped via startup/shutdown mechanism (check TODOs above) diff --git a/tests/integration/test_storage_iceberg_with_spark/test_async_metadata_refresh.py b/tests/integration/test_storage_iceberg_with_spark/test_async_metadata_refresh.py index 1108f44ca08a1..d0129ce43dc94 100644 --- a/tests/integration/test_storage_iceberg_with_spark/test_async_metadata_refresh.py +++ b/tests/integration/test_storage_iceberg_with_spark/test_async_metadata_refresh.py @@ -7,6 +7,7 @@ write_iceberg_from_df, default_upload_directory, ) +from helpers.test_tools import assert_eq_with_retry _ASYNC_CACHE_REFRESH_CONFIG_PATH = "/etc/clickhouse-server/config.d/iceberg_async_cache_refresh.xml" @@ -260,30 +261,39 @@ def test_async_metadata_refresh(started_cluster_iceberg_with_spark, storage_type "SELECT value FROM system.events WHERE name = 'S3ReadRequestsCount' SETTINGS system_events_show_zero_values=1" )) - write_iceberg_from_df( - spark, - generate_data(spark, 100, 200), - TABLE_NAME, - mode="append", - ) - default_upload_directory( - started_cluster_iceberg_with_spark, - storage_type, - f"/iceberg_data/default/{TABLE_NAME}/", - "", + # Hold the refresher before it reads the latest metadata, so the next SELECT must see stale data + instance.query("SYSTEM ENABLE FAILPOINT iceberg_metadata_prefetch_pause") + try: + instance.query("SYSTEM WAIT FAILPOINT iceberg_metadata_prefetch_pause PAUSE") + write_iceberg_from_df( + spark, + generate_data(spark, 100, 200), + TABLE_NAME, + mode="append", + ) + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + "", + ) + # the fresh metadata won't get pulled at SELECT, so we see stale data + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME} SETTINGS iceberg_metadata_staleness_ms=600000")) == 100 + finally: + instance.query("SYSTEM DISABLE FAILPOINT iceberg_metadata_prefetch_pause") + + # we don't pull fresh metadata at SELECT, but the data gets up to date because of the async refresh + assert_eq_with_retry( + instance, + f"SELECT count() FROM {TABLE_NAME} SETTINGS iceberg_metadata_staleness_ms=600000", + "200", + retry_count=60, ) - # the fresh metadata won't get pulled at SELECT, so we see stale data - assert int(instance.query(f"SELECT count() FROM {TABLE_NAME} SETTINGS iceberg_metadata_staleness_ms=600000")) == 100 - # Wait for the background async refresher to pick up the new metadata (2 periods of ASYNC_METADATA_REFRESH_PERIOD_MS) - time.sleep(ASYNC_METADATA_REFRESH_PERIOD_MS/1000 * 2) - # we expect some background activity against S3 s3_reads_after = int(instance.query( "SELECT value FROM system.events WHERE name = 'S3ReadRequestsCount' SETTINGS system_events_show_zero_values=1" )) assert s3_reads_after > s3_reads_before - # we don't pull fresh metadata at SELECT, but the data is up to date because of the async refresh - assert int(instance.query(f"SELECT count() FROM {TABLE_NAME} SETTINGS iceberg_metadata_staleness_ms=600000")) == 200 @pytest.mark.parametrize("storage_type", ["s3"]) From 3f6fbae387d186758670487cce6ab38a9c1e16a7 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 12 Nov 2026 08:26:02 +0000 Subject: [PATCH 164/219] Backport #121154 to 26.8: Fix wrong results when a predicate applies a NULL-handling function to a `Nullable` partition key --- src/Storages/MergeTree/KeyCondition.cpp | 62 +++------- ...l_nullable_partition_key_pruning.reference | 43 +++++++ ...ot_null_nullable_partition_key_pruning.sql | 115 ++++++++++++++++++ 3 files changed, 178 insertions(+), 42 deletions(-) create mode 100644 tests/queries/0_stateless/05233_assume_not_null_nullable_partition_key_pruning.reference create mode 100644 tests/queries/0_stateless/05233_assume_not_null_nullable_partition_key_pruning.sql diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index b45e96877aa3a..b01bb7f2f7719 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -5210,6 +5210,14 @@ std::optional KeyCondition::applyMonotonicFunctionsChainToRange( DataTypePtr current_type, bool single_point) { + if (functions.empty()) + return key_range; + + /// A bound where `isNull()` holds stands in for NULL or for an infinity, and is left untransformed below, + /// so no transformed value would remain to compare against an atom in the function's result domain. + if (single_point && (key_range.left.isNull() || key_range.right.isNull())) + return {}; + /// The chain was built against a recursively `LowCardinality`-stripped key type, so seed it with the /// stripped type here rather than in each caller: several of them pass the key column's raw type. current_type = recursiveRemoveLowCardinality(current_type); @@ -6252,52 +6260,23 @@ BoolMask KeyCondition::checkInHyperrectangle( } else { - Range key_range = sparse_hyperrectangle[sparse_pos]; + /// The column may be wrapped in a chain of possibly monotonic functions; for an empty chain + /// the helper returns the range unchanged. + std::optional new_range = applyMonotonicFunctionsChainToRange( + sparse_hyperrectangle[sparse_pos], + element.monotonic_functions_chain, + sparse_data_types[sparse_pos], + single_point); - /// The case when the column is wrapped in a chain of possibly monotonic functions. - if (!element.monotonic_functions_chain.empty()) + if (!new_range) { - std::optional new_range = applyMonotonicFunctionsChainToRange( - key_range, - element.monotonic_functions_chain, - sparse_data_types[sparse_pos], - single_point); - - if (!new_range) - { - /// Cannot determine monotonicity on this range – unknown. - rpn_stack.emplace_back(true, true); - } - else - { - key_range = *new_range; - - bool intersects = element.range.intersectsRange(key_range); - bool contains = element.range.containsRange(key_range); - - /// NaN doesn't satisfy any comparison condition in SQL (e.g., NaN > 0 is false/NULL). - /// In ClickHouse sort order, NaN has a defined position (after +inf), so Range-based - /// analysis may incorrectly include NaN values. - /// - If left bound is NaN: all values in the range are NaN (NaN sorts last), - /// so no comparison condition can be true. - /// - If only right bound is NaN: the range extends into NaN territory, - /// so it cannot be fully contained (NaN values don't satisfy the condition). - if (unlikely(key_range.left.isNaN())) - { - intersects = false; - contains = false; - } - else if (unlikely(key_range.right.isNaN())) - { - contains = false; - } - - rpn_stack.emplace_back(intersects, !contains); - /// we don't create bloom_filter_data if monotonic_functions_chain is present - } + /// Cannot determine monotonicity on this range – unknown. + rpn_stack.emplace_back(true, true); } else { + const Range & key_range = *new_range; + bool intersects = element.range.intersectsRange(key_range); bool contains = element.range.containsRange(key_range); @@ -6319,7 +6298,6 @@ BoolMask KeyCondition::checkInHyperrectangle( } rpn_stack.emplace_back(intersects, !contains); - } } diff --git a/tests/queries/0_stateless/05233_assume_not_null_nullable_partition_key_pruning.reference b/tests/queries/0_stateless/05233_assume_not_null_nullable_partition_key_pruning.reference new file mode 100644 index 0000000000000..02703d86a7bd1 --- /dev/null +++ b/tests/queries/0_stateless/05233_assume_not_null_nullable_partition_key_pruning.reference @@ -0,0 +1,43 @@ +the reported oracle mismatch: count(p) + count(not p) + count(p is null) = count(*) +1 +predicate, row-level truth, answer over the partitioned table +assumeNotNull(c0) = 0 3 3 +assumeNotNull(c0) < 3 4 4 +NOT (assumeNotNull(c0) > 0) 3 3 +assumeNotNull(c0) >= 0 5 5 +assumeNotNull(c0) IN (0) 3 3 +isNull and isNotNull compared with a constant +(c0 IS NULL) >= true 2 2 +(c0 IS NOT NULL) < true 2 2 +not(1 < (c0 IS NULL)) 5 5 +the partition step is the sole pruner here, and it now keeps the NULL partition +Min-Max +Condition: true +Parts: 4/4 +Partition +Condition: (isNull(c0) in [1, +Inf)) +Parts: 1/4 +ifNull and coalesce reach the same chain once the rewrite that usually hides them is off +ifNull(c0, 0) = 0 3 3 +coalesce(c0, 0) = 0 3 3 +NOT (ifNull(c0, 0) >= 7) 5 5 +NOT (coalesce(nullIf(c0, 5), 0) = 1) 4 4 +Min-Max +Parts: 4/4 +Partition +Parts: 3/4 +reverse key 3 3 +LowCardinality(Nullable) key 3 3 +SummingMergeTree, the query the fuzzer reported +1 +control: with no NULL row the partition step is the only pruner and still prunes +isZeroOrNull(c0) >= true 1 +Min-Max +Parts: 3/3 +Partition +Parts: 1/3 +control: the primary-key path already rejected the chain and is unchanged +assumeNotNull(c0) = 0 3 3 +control: a NULL-preserving wrapper maps the NULL row to NULL and still answers correctly +c0 + 1 = 1 1 1 +round(c0) = 0 1 1 diff --git a/tests/queries/0_stateless/05233_assume_not_null_nullable_partition_key_pruning.sql b/tests/queries/0_stateless/05233_assume_not_null_nullable_partition_key_pruning.sql new file mode 100644 index 0000000000000..917f89a126cc6 --- /dev/null +++ b/tests/queries/0_stateless/05233_assume_not_null_nullable_partition_key_pruning.sql @@ -0,0 +1,115 @@ +-- Tags: no-replicated-database, no-parallel-replicas +-- no-replicated-database: EXPLAIN output differs for replicated database. +-- no-parallel-replicas: EXPLAIN output differs for parallel replicas. +-- Test for https://github.com/ClickHouse/ClickHouse/issues/121133, https://github.com/ClickHouse/ClickHouse/issues/126369 and https://github.com/ClickHouse/ClickHouse/issues/127755 + +-- Randomized to 0 or 1, and the value changes the plan the EXPLAIN blocks below print. +SET parallel_replicas_local_plan = 1; +-- Statistics are a part-pruning source of their own; pinning both off keeps the Min-Max and +-- Partition steps the only pruners the EXPLAIN blocks below can attribute a Parts: count to. +SET use_statistics = 0; +SET use_statistics_for_part_pruning = 0; +SET explain_query_plan_default = 'legacy'; + +DROP TABLE IF EXISTS t_part; +DROP TABLE IF EXISTS t_flat; + +-- `t_flat` holds the same rows behind an unprunable key, so its answer is the row-level truth +-- that every other table below must reproduce. +CREATE TABLE t_part (c0 Nullable(Int)) ENGINE = MergeTree PARTITION BY (c0) ORDER BY tuple() SETTINGS allow_nullable_key = 1; +CREATE TABLE t_flat (c0 Nullable(Int)) ENGINE = MergeTree PARTITION BY tuple() ORDER BY tuple() SETTINGS allow_nullable_key = 1; +INSERT INTO t_part VALUES (NULL), (0), (1), (5), (NULL); +INSERT INTO t_flat VALUES (NULL), (0), (1), (5), (NULL); + +SELECT 'the reported oracle mismatch: count(p) + count(not p) + count(p is null) = count(*)'; +SELECT (SELECT count() FROM t_part WHERE assumeNotNull(c0) = 0) + + (SELECT count() FROM t_part WHERE NOT (assumeNotNull(c0) = 0)) + + (SELECT count() FROM t_part WHERE (assumeNotNull(c0) = 0) IS NULL) + = (SELECT count() FROM t_part) AS tlp_identity_holds; + +SELECT 'predicate, row-level truth, answer over the partitioned table'; +SELECT 'assumeNotNull(c0) = 0', (SELECT count() FROM t_flat WHERE assumeNotNull(c0) = 0), (SELECT count() FROM t_part WHERE assumeNotNull(c0) = 0); +SELECT 'assumeNotNull(c0) < 3', (SELECT count() FROM t_flat WHERE assumeNotNull(c0) < 3), (SELECT count() FROM t_part WHERE assumeNotNull(c0) < 3); +SELECT 'NOT (assumeNotNull(c0) > 0)', (SELECT count() FROM t_flat WHERE NOT (assumeNotNull(c0) > 0)), (SELECT count() FROM t_part WHERE NOT (assumeNotNull(c0) > 0)); +SELECT 'assumeNotNull(c0) >= 0', (SELECT count() FROM t_flat WHERE assumeNotNull(c0) >= 0), (SELECT count() FROM t_part WHERE assumeNotNull(c0) >= 0); +-- A set atom carries its own copy of the chain and is applied by MergeTreeSetIndex, not by KeyCondition. +SELECT 'assumeNotNull(c0) IN (0)', (SELECT count() FROM t_flat WHERE assumeNotNull(c0) IN (0)), (SELECT count() FROM t_part WHERE assumeNotNull(c0) IN (0)); + +SELECT 'isNull and isNotNull compared with a constant'; +SELECT '(c0 IS NULL) >= true', (SELECT count() FROM t_flat WHERE (c0 IS NULL) >= true), (SELECT count() FROM t_part WHERE (c0 IS NULL) >= true); +SELECT '(c0 IS NOT NULL) < true', (SELECT count() FROM t_flat WHERE (c0 IS NOT NULL) < true), (SELECT count() FROM t_part WHERE (c0 IS NOT NULL) < true); +SELECT 'not(1 < (c0 IS NULL))', (SELECT count() FROM t_flat WHERE not(1 < (c0 IS NULL))), (SELECT count() FROM t_part WHERE not(1 < (c0 IS NULL))); + +-- `isNull` reports no monotonicity, so only the single-point Partition step can use it: it is the sole +-- pruner and its Parts: denominator is the table's part count. +SELECT 'the partition step is the sole pruner here, and it now keeps the NULL partition'; +SELECT trimLeft(explain) FROM ( + EXPLAIN indexes = 1 SELECT c0 FROM t_part WHERE (c0 IS NULL) >= true + SETTINGS optimize_use_implicit_projections = 0, optimize_use_projections = 0 +) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Min-Max%' OR explain LIKE '%Partition%'; + +SELECT 'ifNull and coalesce reach the same chain once the rewrite that usually hides them is off'; +SELECT 'ifNull(c0, 0) = 0', (SELECT count() FROM t_flat WHERE ifNull(c0, 0) = 0 SETTINGS allow_key_condition_coalesce_rewrite = 0), (SELECT count() FROM t_part WHERE ifNull(c0, 0) = 0 SETTINGS allow_key_condition_coalesce_rewrite = 0); +SELECT 'coalesce(c0, 0) = 0', (SELECT count() FROM t_flat WHERE coalesce(c0, 0) = 0 SETTINGS allow_key_condition_coalesce_rewrite = 0), (SELECT count() FROM t_part WHERE coalesce(c0, 0) = 0 SETTINGS allow_key_condition_coalesce_rewrite = 0); +-- The rewrite applies only to a non-negated condition, so a negated one reaches the chain with the rewrite on. +SELECT 'NOT (ifNull(c0, 0) >= 7)', (SELECT count() FROM t_flat WHERE NOT (ifNull(c0, 0) >= 7) SETTINGS allow_key_condition_coalesce_rewrite = 1), (SELECT count() FROM t_part WHERE NOT (ifNull(c0, 0) >= 7) SETTINGS allow_key_condition_coalesce_rewrite = 1); + +-- An inner link can also produce a real NULL: `nullIf` maps the exact point 5 to one, and the chain +-- leaves it untransformed just like an infinity bound. Such a bound compares as both inside and +-- outside every range, so the negated atom is the form that can wrongly prune it; keep it negated. +SELECT 'NOT (coalesce(nullIf(c0, 5), 0) = 1)', (SELECT count() FROM t_flat WHERE NOT (coalesce(nullIf(c0, 5), 0) = 1) SETTINGS allow_key_condition_coalesce_rewrite = 0), (SELECT count() FROM t_part WHERE NOT (coalesce(nullIf(c0, 5), 0) = 1) SETTINGS allow_key_condition_coalesce_rewrite = 0); +-- Partition 1 is still pruned, so such a bound costs its own partition and no other. +SELECT trimLeft(explain) FROM ( + EXPLAIN indexes = 1 SELECT c0 FROM t_part WHERE NOT (coalesce(nullIf(c0, 5), 0) = 1) + SETTINGS optimize_use_implicit_projections = 0, optimize_use_projections = 0, use_skip_indexes = 0, + allow_key_condition_coalesce_rewrite = 0 +) WHERE explain LIKE '%Parts%' OR explain LIKE '%Min-Max%' OR explain LIKE '%Partition%'; + +DROP TABLE IF EXISTS t_rev; +CREATE TABLE t_rev (c0 Nullable(Int)) ENGINE = MergeTree PARTITION BY (c0) ORDER BY (c0 DESC) +SETTINGS allow_nullable_key = 1; +INSERT INTO t_rev VALUES (NULL), (0), (1), (5), (NULL); +SELECT 'reverse key', (SELECT count() FROM t_flat WHERE assumeNotNull(c0) = 0), (SELECT count() FROM t_rev WHERE assumeNotNull(c0) = 0); + +DROP TABLE IF EXISTS t_lc; +SET allow_suspicious_low_cardinality_types = 1; +CREATE TABLE t_lc (c0 LowCardinality(Nullable(Int))) ENGINE = MergeTree PARTITION BY (c0) ORDER BY tuple() SETTINGS allow_nullable_key = 1; +INSERT INTO t_lc VALUES (NULL), (0), (1), (5), (NULL); +SELECT 'LowCardinality(Nullable) key', (SELECT count() FROM t_flat WHERE assumeNotNull(c0) = 0), (SELECT count() FROM t_lc WHERE assumeNotNull(c0) = 0); + +DROP TABLE IF EXISTS t_smt; +CREATE TABLE t_smt (c0 Nullable(Int)) ENGINE = SummingMergeTree PARTITION BY (c0) ORDER BY tuple() SETTINGS allow_nullable_key = 1; +INSERT INTO t_smt VALUES (NULL), (0), (1), (NULL); +SELECT 'SummingMergeTree, the query the fuzzer reported'; +SELECT (SELECT count() FROM t_smt WHERE assumeNotNull(t_smt.c0) = false) + + (SELECT count() FROM t_smt WHERE NOT (assumeNotNull(t_smt.c0) = false)) + + (SELECT count() FROM t_smt WHERE (assumeNotNull(t_smt.c0) = false) IS NULL) + = (SELECT count() FROM t_smt) AS tlp_identity_holds; + +SELECT 'control: with no NULL row the partition step is the only pruner and still prunes'; +DROP TABLE IF EXISTS t_nonull; +CREATE TABLE t_nonull (c0 Nullable(Int)) ENGINE = MergeTree PARTITION BY (c0) ORDER BY tuple() SETTINGS allow_nullable_key = 1; +INSERT INTO t_nonull VALUES (0), (1), (5); +SELECT 'isZeroOrNull(c0) >= true', count() FROM t_nonull WHERE isZeroOrNull(c0) >= true; +SELECT trimLeft(explain) FROM ( + EXPLAIN indexes = 1 SELECT c0 FROM t_nonull WHERE isZeroOrNull(c0) >= true + SETTINGS optimize_use_implicit_projections = 0, optimize_use_projections = 0 +) WHERE explain LIKE '%Parts%' OR explain LIKE '%Min-Max%' OR explain LIKE '%Partition%'; + +SELECT 'control: the primary-key path already rejected the chain and is unchanged'; +DROP TABLE IF EXISTS t_pk; +CREATE TABLE t_pk (c0 Nullable(Int)) ENGINE = MergeTree ORDER BY c0 SETTINGS allow_nullable_key = 1, index_granularity = 1; +INSERT INTO t_pk VALUES (NULL), (0), (1), (5), (NULL); +SELECT 'assumeNotNull(c0) = 0', (SELECT count() FROM t_flat WHERE assumeNotNull(c0) = 0), (SELECT count() FROM t_pk WHERE assumeNotNull(c0) = 0); + +SELECT 'control: a NULL-preserving wrapper maps the NULL row to NULL and still answers correctly'; +SELECT 'c0 + 1 = 1', (SELECT count() FROM t_flat WHERE c0 + 1 = 1), (SELECT count() FROM t_part WHERE c0 + 1 = 1); +SELECT 'round(c0) = 0', (SELECT count() FROM t_flat WHERE round(c0) = 0), (SELECT count() FROM t_part WHERE round(c0) = 0); + +DROP TABLE t_part; +DROP TABLE t_flat; +DROP TABLE t_rev; +DROP TABLE t_lc; +DROP TABLE t_smt; +DROP TABLE t_nonull; +DROP TABLE t_pk; From 689289807d900af27d2c45827e6368886cbec8bf Mon Sep 17 00:00:00 2001 From: Pedro Ferreira Date: Thu, 12 Nov 2026 08:56:03 +0000 Subject: [PATCH 165/219] Do not prune a partition when a function of the partition key returns NULL for it Port of #124041, a prerequisite of #121154 on this branch: when a link of the monotonic chain turns a non-NULL bound into NULL, the bound has no position in the `Range` order, so the range is left unanalyzed instead of compared. Without it the `NOT (coalesce(nullIf(c0, 5), 0) = 1)` case of `05233_assume_not_null_nullable_partition_key_pruning` still prunes the partition of 5 (4 rows expected, 3 returned), as seen on the 26.9 backport. #124041 is written against the try/catch chain loop of #117355, which this branch does not have, so this applies only its check to the branch's loop. https://github.com/ClickHouse/ClickHouse/pull/130788 Co-Authored-By: Claude Opus 5.5 --- src/Storages/MergeTree/KeyCondition.cpp | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index b01bb7f2f7719..cc1cfddae41a3 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -5249,17 +5249,21 @@ std::optional KeyCondition::applyMonotonicFunctionsChainToRange( /// If we apply function to open interval, we can get empty intervals in result. /// E.g. for ('2020-01-03', '2020-01-20') after applying 'toYYYYMM' we will get ('202001', '202001'). /// To avoid this we make range left and right included. - /// Any function that treats NULL specially is not monotonic. - /// Thus we can safely use isNull() as an -Inf/+Inf indicator here. + /// A NULL bound is an -inf/+inf stand-in and is left as is. A function that returns NULL for a non-NULL + /// argument leaves a bound with no position in the `Range` order, so the range cannot be analyzed. if (!key_range.left.isNull()) { key_range.left = applyFunction(func, current_type, key_range.left); + if (key_range.left.isNull()) + return {}; key_range.left_included = true; } if (!key_range.right.isNull()) { key_range.right = applyFunction(func, current_type, key_range.right); + if (key_range.right.isNull()) + return {}; key_range.right_included = true; } } From 239a220e9e8995bc734b05bac98c15cd30089f49 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 12 Nov 2026 08:53:40 +0000 Subject: [PATCH 166/219] Backport #130462 to 26.8: Bound the declared length of PostgreSQL messages parsed before authentication --- src/Core/PostgreSQLProtocol.h | 28 ++++++-- src/Server/PostgreSQLHandler.cpp | 4 +- ...postgresql_protocol_message_size.reference | 4 ++ .../05054_postgresql_protocol_message_size.sh | 66 +++++++++++++++++++ 4 files changed, 96 insertions(+), 6 deletions(-) diff --git a/src/Core/PostgreSQLProtocol.h b/src/Core/PostgreSQLProtocol.h index 0adea1b15bb52..72cfde989a1bc 100644 --- a/src/Core/PostgreSQLProtocol.h +++ b/src/Core/PostgreSQLProtocol.h @@ -55,6 +55,14 @@ namespace PostgreSQLProtocol namespace Messaging { +/** Upper bound for the declared length of a message that is parsed before the client is authenticated. + * The payload of such a message is bounded by its declared length, but that length is chosen by an + * unauthenticated client, so the length itself must be bounded as well. Otherwise a client declares + * the Int32 maximum and makes the server buffer up to 2 GiB of payload before it gives up. + * PostgreSQL uses the same limit for its startup packet. + */ +constexpr Int32 max_unauthenticated_message_size = 10000; + enum class FrontMessageType : Int32 { // first message types @@ -307,14 +315,26 @@ class FrontMessage : public IMessage virtual void deserialize(ReadBuffer & in) = 0; protected: + /** Parses the payload of a message inside a buffer that ends at the declared length. + * `max_size` bounds the declared length itself. Pass it for every message that is parsed before the + * client is authenticated, because that client chooses the length. A frame rejected by `max_size` is + * left unread in the stream, so only a caller whose error closes the connection may pass a limit. + */ template - static void deserializePayload(ReadBuffer & in, std::string_view message_name, F && deserialize_payload) + static void deserializePayload( + ReadBuffer & in, + std::string_view message_name, + F && deserialize_payload, + Int32 max_size = std::numeric_limits::max()) { Int32 size = 0; readBinaryBigEndian(size, in); if (size < 4) throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT, "Wrong message length {} in {}, it must be at least 4", size, message_name); + if (size > max_size) + throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT, + "Wrong message length {} in {}, it must not exceed {}", size, message_name, max_size); const size_t payload_size = static_cast(size - 4); LimitReadBuffer payload_in(in, {.read_no_less = payload_size, .read_no_more = payload_size}); @@ -598,7 +618,7 @@ class SASLInitialResponse : public Messaging::FrontMessage "Wrong SASL mechanism length {} in SASLInitialResponse, it must not be less than -1", size_sasl_mechanism); if (size_sasl_mechanism > 0) readStringOfDeclaredSize(sasl_mechanism, size_sasl_mechanism, payload_in); - }); + }, max_unauthenticated_message_size); } MessageType getMessageType() const override @@ -646,7 +666,7 @@ class SASLResponse : public Messaging::FrontMessage deserializePayload(in, "SASLResponse message", [this](ReadBuffer & payload_in) { readStringUntilEOF(sasl_mechanism, payload_in); - }); + }, max_unauthenticated_message_size); } MessageType getMessageType() const override @@ -687,7 +707,7 @@ class PasswordMessage : FrontMessage deserializePayload(in, "PasswordMessage", [this](ReadBuffer & payload_in) { readNullTerminated(password, payload_in); - }); + }, max_unauthenticated_message_size); } MessageType getMessageType() const override diff --git a/src/Server/PostgreSQLHandler.cpp b/src/Server/PostgreSQLHandler.cpp index 653a311023ed4..87ada41e4a541 100644 --- a/src/Server/PostgreSQLHandler.cpp +++ b/src/Server/PostgreSQLHandler.cpp @@ -722,8 +722,8 @@ void PostgreSQLHandler::cancelRequest() inline std::unique_ptr PostgreSQLHandler::receiveStartupMessage(int payload_size) { /// The declared size is read from the wire before any authentication, and the message is read - /// into memory in full, so it has to be bounded. PostgreSQL uses the same limit. - static constexpr Int32 max_startup_message_size = 10000; + /// into memory in full, so it has to be bounded (`max_unauthenticated_message_size`). + static constexpr Int32 max_startup_message_size = PostgreSQLProtocol::Messaging::max_unauthenticated_message_size; std::unique_ptr message; try diff --git a/tests/queries/0_stateless/05054_postgresql_protocol_message_size.reference b/tests/queries/0_stateless/05054_postgresql_protocol_message_size.reference index ec35dc29b3b47..6be1fa0c53ad0 100644 --- a/tests/queries/0_stateless/05054_postgresql_protocol_message_size.reference +++ b/tests/queries/0_stateless/05054_postgresql_protocol_message_size.reference @@ -1,5 +1,9 @@ SASL mechanism longer than the message: error response password without a terminator: error response +password message at the Int32 maximum: error response +password message above the limit: error response +SASL initial response at the Int32 maximum: error response +SASL final response at the Int32 maximum: error response well-formed exchange: query result received query with a smuggled tail: error response query shorter than declared: error response diff --git a/tests/queries/0_stateless/05054_postgresql_protocol_message_size.sh b/tests/queries/0_stateless/05054_postgresql_protocol_message_size.sh index e5fd34c2db3ff..b5d3e9b7acef8 100755 --- a/tests/queries/0_stateless/05054_postgresql_protocol_message_size.sh +++ b/tests/queries/0_stateless/05054_postgresql_protocol_message_size.sh @@ -32,6 +32,32 @@ def connect(): return sock +def connect_short(): + """A connection whose client timeout is well below the server's `handshake_timeout_milliseconds` (30 + seconds by default), so a message the server waits for shows up as a client timeout. With a timeout + at or above the handshake timeout the server would answer the same error response on its own, and a + regression that only makes the server wait would not be noticed.""" + sock = socket.create_connection(("127.0.0.1", port), timeout=5) + sock.settimeout(5) + return sock + + +def read_message(sock): + """Reads one backend message: a type byte, a length that counts itself, and the body.""" + def recv_exactly(size): + data = b"" + while len(data) < size: + chunk = sock.recv(size - len(data)) + if not chunk: + raise AssertionError("the connection closed before a complete message") + data += chunk + return data + + message_type = recv_exactly(1) + (length,) = struct.unpack(">i", recv_exactly(4)) + return message_type, recv_exactly(length - 4) + + def startup(sock, user): payload = ("user\x00" + user + "\x00\x00").encode() sock.sendall(struct.pack(">ii", 8 + len(payload), 196608) + payload) @@ -93,6 +119,46 @@ except (BrokenPipeError, ConnectionResetError, socket.timeout): print("password without a terminator:", outcome(sock)) sock.close() +# The declared length of a message that is parsed before authentication is bounded by the server, because +# the client that declares it is not trusted yet. Otherwise the client declares the Int32 maximum and the +# server buffers up to 2 GiB of payload that never completes. +sock = connect_short() +startup(sock, user_plain) +read_message(sock) +sock.sendall(b"p" + struct.pack(">i", 0x7FFFFFFF)) +print("password message at the Int32 maximum:", outcome(sock)) +sock.close() + +# The same for a length just above the limit, which pins the boundary itself. +sock = connect_short() +startup(sock, user_plain) +read_message(sock) +sock.sendall(b"p" + struct.pack(">i", 10001)) +print("password message above the limit:", outcome(sock)) +sock.close() + +# The same for a `SASLInitialResponse`. +sock = connect_short() +startup(sock, user_scram) +read_message(sock) +sock.sendall(b"p" + struct.pack(">i", 0x7FFFFFFF)) +print("SASL initial response at the Int32 maximum:", outcome(sock)) +sock.close() + +# The same for a `SASLResponse`, the second SCRAM message. A well-formed client-first message is needed +# to get there, and its own declared lengths stay small. +sock = connect_short() +startup(sock, user_scram) +read_message(sock) +client_first = b"n,,n=,r=abcdefghijklmnopqrstuvwx" +body = b"SCRAM-SHA-256\x00" + struct.pack(">i", len(client_first)) + client_first +sock.sendall(b"p" + struct.pack(">i", 4 + len(body)) + body) +message_type, body = read_message(sock) +assert message_type == b"R" and struct.unpack(">i", body[:4])[0] == 11, body +sock.sendall(b"p" + struct.pack(">i", 0x7FFFFFFF)) +print("SASL final response at the Int32 maximum:", outcome(sock)) +sock.close() + # A well-formed exchange still works: authenticate and run a query. def read_until_ready(sock): data = b"" From 46b44405cd7528f3360f1dab2210de5ad457d883 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 11 Nov 2026 23:35:48 +0000 Subject: [PATCH 167/219] Backport #127416 to 26.8: Fix `Cannot fold actions for projection` when the join order optimizer reorders an outer join converted to INNER --- .../QueryPlan/Optimizations/optimizeJoin.cpp | 12 ++- ...converted_outer_join_type_change.reference | 16 ++++ ...order_converted_outer_join_type_change.sql | 88 +++++++++++++++++++ 3 files changed, 113 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/05443_join_reorder_converted_outer_join_type_change.reference create mode 100644 tests/queries/0_stateless/05443_join_reorder_converted_outer_join_type_change.sql diff --git a/src/Processors/QueryPlan/Optimizations/optimizeJoin.cpp b/src/Processors/QueryPlan/Optimizations/optimizeJoin.cpp index 4b191debc0fa2..2ca1a370b6f91 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeJoin.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeJoin.cpp @@ -1162,6 +1162,10 @@ static QueryPlan::Node chooseJoinOrder(QueryGraphBuilder query_graph_builder, Qu QueryGraph query_graph; query_graph.relation_stats = std::move(query_graph_builder.relation_stats); query_graph.edges = std::move(query_graph_builder.join_edges); + BitSet null_supplying_relations; + for (const auto & [relation, restriction] : query_graph_builder.join_kinds) + if (restriction.second != JoinKind::Inner) + null_supplying_relations.set(relation); query_graph.join_kinds = std::move(query_graph_builder.join_kinds); query_graph.outer_join_conditions = std::move(query_graph_builder.outer_join_conditions); @@ -1356,11 +1360,13 @@ static QueryPlan::Node chooseJoinOrder(QueryGraphBuilder query_graph_builder, Qu /// input pos -> new input node std::unordered_map current_step_type_changes; - for (auto rel_id : {left_rels.getSingleBit(), right_rels.getSingleBit()}) + /// NULL padding applies where the relation is joined alone; other type changes only after all predicates on original columns. + const bool is_last_step = entry_idx + 1 == sequence.size(); + for (const auto & [relation, new_inputs] : query_graph_builder.type_changes) { - if (!rel_id.has_value()) + const bool joined_alone = left_rels.getSingleBit() == relation || right_rels.getSingleBit() == relation; + if (!(null_supplying_relations.test(relation) ? joined_alone : is_last_step)) continue; - const auto & new_inputs = query_graph_builder.type_changes[rel_id.value()]; for (const auto * new_input : new_inputs) { const auto * input_node = trackInputColumn(new_input); diff --git a/tests/queries/0_stateless/05443_join_reorder_converted_outer_join_type_change.reference b/tests/queries/0_stateless/05443_join_reorder_converted_outer_join_type_change.reference new file mode 100644 index 0000000000000..a29660cf90cc5 --- /dev/null +++ b/tests/queries/0_stateless/05443_join_reorder_converted_outer_join_type_change.reference @@ -0,0 +1,16 @@ +a1 x1 x1 x Nullable(String) +a3 x1 x1 x Nullable(String) +a1 x1 x1 x +a3 x1 x1 x +a1 x1 x1 x +a3 x1 x1 x +a1 x1 x1 x +a3 x1 x1 x +a1 x1 x1 x +a3 x1 x1 x +a1 x1 x1 x +a3 x1 x1 x +a1 x1 x1 x +a3 x1 x1 x +a1 x1 x1 x +a3 x1 x1 x diff --git a/tests/queries/0_stateless/05443_join_reorder_converted_outer_join_type_change.sql b/tests/queries/0_stateless/05443_join_reorder_converted_outer_join_type_change.sql new file mode 100644 index 0000000000000..615e0ae1e5c38 --- /dev/null +++ b/tests/queries/0_stateless/05443_join_reorder_converted_outer_join_type_change.sql @@ -0,0 +1,88 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/127331: with join_use_nulls = 1, an outer join converted to INNER and +-- reordered so that its null-supplying side is joined first failed with "Cannot fold actions for projection". + +DROP TABLE IF EXISTS a; +DROP TABLE IF EXISTS b; +DROP TABLE IF EXISTS c; +DROP TABLE IF EXISTS d; + +CREATE TABLE a (k String, t String, v String) ENGINE = MergeTree ORDER BY k; +CREATE TABLE b (k String, t String, x String) ENGINE = MergeTree ORDER BY k; +CREATE TABLE c (x String, t String, v String) ENGINE = MergeTree ORDER BY x; +INSERT INTO a VALUES ('k1', 't1', 'a1'), ('k2', 't1', 'a2'), ('k3', 't2', 'a3'); +INSERT INTO b VALUES ('k1', 't1', 'x1'), ('k2', 't1', 'x2'), ('k3', 't2', 'x1'); +INSERT INTO c VALUES ('x1', 't1', 'x'), ('x2', 't1', 'y'), ('x1', 't2', 'x'); + +SELECT a.v, b.x, c.x, c.v, toTypeName(c.x) +FROM a JOIN b ON a.k = b.k AND a.t = b.t LEFT JOIN c ON b.x = c.x AND b.t = c.t +WHERE c.v = 'x' ORDER BY ALL +SETTINGS join_use_nulls = 1, query_plan_convert_outer_join_to_inner_join = 1, enable_join_transitive_predicates = 1, + query_plan_optimize_join_order_limit = 10, use_hash_table_stats_for_join_reordering = 0, + query_plan_optimize_join_order_algorithm = 'greedy', query_plan_optimize_join_order_randomize = 0; + +SELECT a.v, b.x, c.x, c.v +FROM a JOIN b ON a.k = b.k AND a.t = b.t LEFT JOIN c ON b.x = c.x AND b.t = c.t +WHERE c.v = 'x' ORDER BY ALL +SETTINGS join_use_nulls = 1, query_plan_convert_outer_join_to_inner_join = 1, enable_join_transitive_predicates = 1, + query_plan_optimize_join_order_limit = 10, use_hash_table_stats_for_join_reordering = 0, + query_plan_optimize_join_order_algorithm = 'dpsize', query_plan_optimize_join_order_randomize = 0; + +SELECT a.v, b.x, c.x, c.v +FROM a JOIN b ON a.k = b.k AND a.t = b.t LEFT JOIN c ON b.x = c.x AND b.t = c.t +WHERE c.v = 'x' ORDER BY ALL +SETTINGS join_use_nulls = 1, query_plan_convert_outer_join_to_inner_join = 1, enable_join_transitive_predicates = 1, + query_plan_optimize_join_order_limit = 10, use_hash_table_stats_for_join_reordering = 0, + query_plan_optimize_join_order_algorithm = 'dphyp', query_plan_optimize_join_order_randomize = 13; + +SELECT a.v, b.x, c.x, c.v +FROM a JOIN b ON a.k = b.k AND a.t = b.t LEFT JOIN c ON b.x = c.x AND b.t = c.t +WHERE c.v = 'x' ORDER BY ALL +SETTINGS join_use_nulls = 1, query_plan_convert_outer_join_to_inner_join = 1, enable_join_transitive_predicates = 1, + query_plan_optimize_join_order_limit = 10, use_hash_table_stats_for_join_reordering = 0, + query_plan_optimize_join_order_algorithm = 'dpsub', query_plan_optimize_join_order_randomize = 13, + query_plan_optimize_join_order_conflict_detector = ''; + +SELECT a.v, b.x, c.x, c.v +FROM a JOIN b ON a.k = b.k AND a.t = b.t LEFT JOIN c ON b.x = c.x AND b.t = c.t +WHERE c.v = 'x' ORDER BY ALL +SETTINGS join_use_nulls = 1, query_plan_convert_outer_join_to_inner_join = 1, enable_join_transitive_predicates = 1, + query_plan_optimize_join_order_limit = 10, use_hash_table_stats_for_join_reordering = 0, + query_plan_optimize_join_order_algorithm = 'dpsub', query_plan_optimize_join_order_randomize = 13, + query_plan_optimize_join_order_conflict_detector = 'a'; + +SELECT a.v, b.x, c.x, c.v +FROM a JOIN b ON a.k = b.k AND a.t = b.t LEFT JOIN c ON b.x = c.x AND b.t = c.t +WHERE c.v = 'x' ORDER BY ALL +SETTINGS join_use_nulls = 1, query_plan_convert_outer_join_to_inner_join = 1, enable_join_transitive_predicates = 1, + query_plan_optimize_join_order_limit = 10, use_hash_table_stats_for_join_reordering = 0, + query_plan_optimize_join_order_algorithm = 'dpsub', query_plan_optimize_join_order_randomize = 13, + query_plan_optimize_join_order_conflict_detector = 'c'; + +SELECT ab.v, ab.x, c.x, c.v +FROM c RIGHT JOIN (SELECT a.v AS v, a.t AS t, b.x AS x FROM a JOIN b ON a.k = b.k AND a.t = b.t) AS ab ON ab.x = c.x AND ab.t = c.t +WHERE c.v = 'x' ORDER BY ALL +SETTINGS join_use_nulls = 1, query_plan_convert_outer_join_to_inner_join = 1, enable_join_transitive_predicates = 1, + query_plan_optimize_join_order_limit = 10, use_hash_table_stats_for_join_reordering = 0, + query_plan_optimize_join_order_algorithm = 'greedy', query_plan_optimize_join_order_randomize = 0; + +DROP TABLE a; +DROP TABLE c; +CREATE TABLE a (k String, t String, v String) ENGINE = MergeTree ORDER BY k; +CREATE TABLE c (x String, t String, v String) ENGINE = MergeTree ORDER BY x; +CREATE TABLE d (w String) ENGINE = MergeTree ORDER BY w; +INSERT INTO a VALUES ('x1', 't1', 'a1'), ('x2', 't1', 'a2'), ('x1', 't2', 'a3'); +INSERT INTO c VALUES ('x1', 't1', 'x'), ('x2', 't1', 'y'); +INSERT INTO d VALUES ('x1'), ('x2'); + +-- `c` is joined with `a` before `b` through the equivalence `a.k = d.w = b.x = c.x`. +SELECT a.v, b.x, c.x, c.v +FROM a JOIN d ON a.k = d.w JOIN b ON d.w = b.x AND a.t = b.t LEFT JOIN c ON b.x = c.x +WHERE c.v = 'x' ORDER BY ALL +SETTINGS join_use_nulls = 1, query_plan_convert_outer_join_to_inner_join = 1, enable_join_transitive_predicates = 1, + query_plan_optimize_join_order_limit = 10, use_hash_table_stats_for_join_reordering = 0, + query_plan_optimize_join_order_algorithm = 'dpsize', query_plan_optimize_join_order_randomize = 82; + +DROP TABLE a; +DROP TABLE b; +DROP TABLE c; +DROP TABLE d; From 49d1c7a0105adc7573eb5891218db6f8af30c665 Mon Sep 17 00:00:00 2001 From: Nikita Taranov Date: Thu, 12 Nov 2026 12:43:25 +0000 Subject: [PATCH 168/219] Adapt test `05443_join_reorder_converted_outer_join_type_change` to 26.8 The setting `query_plan_optimize_join_order_conflict_detector` does not exist in 26.8. https://s3.amazonaws.com/clickhouse-test-reports/json.html?PR=130942&sha=fbb8838b5457161bb21224a066271cd08aefb56f&name_0=BackportPR&name_1=Stateless%20tests%20%28amd_asan_ubsan%2C%20distributed%20plan%2C%20parallel%29 https://github.com/ClickHouse/ClickHouse/pull/130942 Co-Authored-By: Claude Opus 5.5 --- ...converted_outer_join_type_change.reference | 4 ---- ...order_converted_outer_join_type_change.sql | 19 +------------------ 2 files changed, 1 insertion(+), 22 deletions(-) diff --git a/tests/queries/0_stateless/05443_join_reorder_converted_outer_join_type_change.reference b/tests/queries/0_stateless/05443_join_reorder_converted_outer_join_type_change.reference index a29660cf90cc5..9aab6c3c4ee44 100644 --- a/tests/queries/0_stateless/05443_join_reorder_converted_outer_join_type_change.reference +++ b/tests/queries/0_stateless/05443_join_reorder_converted_outer_join_type_change.reference @@ -10,7 +10,3 @@ a1 x1 x1 x a3 x1 x1 x a1 x1 x1 x a3 x1 x1 x -a1 x1 x1 x -a3 x1 x1 x -a1 x1 x1 x -a3 x1 x1 x diff --git a/tests/queries/0_stateless/05443_join_reorder_converted_outer_join_type_change.sql b/tests/queries/0_stateless/05443_join_reorder_converted_outer_join_type_change.sql index 615e0ae1e5c38..50b4202ef35a0 100644 --- a/tests/queries/0_stateless/05443_join_reorder_converted_outer_join_type_change.sql +++ b/tests/queries/0_stateless/05443_join_reorder_converted_outer_join_type_change.sql @@ -39,24 +39,7 @@ FROM a JOIN b ON a.k = b.k AND a.t = b.t LEFT JOIN c ON b.x = c.x AND b.t = c.t WHERE c.v = 'x' ORDER BY ALL SETTINGS join_use_nulls = 1, query_plan_convert_outer_join_to_inner_join = 1, enable_join_transitive_predicates = 1, query_plan_optimize_join_order_limit = 10, use_hash_table_stats_for_join_reordering = 0, - query_plan_optimize_join_order_algorithm = 'dpsub', query_plan_optimize_join_order_randomize = 13, - query_plan_optimize_join_order_conflict_detector = ''; - -SELECT a.v, b.x, c.x, c.v -FROM a JOIN b ON a.k = b.k AND a.t = b.t LEFT JOIN c ON b.x = c.x AND b.t = c.t -WHERE c.v = 'x' ORDER BY ALL -SETTINGS join_use_nulls = 1, query_plan_convert_outer_join_to_inner_join = 1, enable_join_transitive_predicates = 1, - query_plan_optimize_join_order_limit = 10, use_hash_table_stats_for_join_reordering = 0, - query_plan_optimize_join_order_algorithm = 'dpsub', query_plan_optimize_join_order_randomize = 13, - query_plan_optimize_join_order_conflict_detector = 'a'; - -SELECT a.v, b.x, c.x, c.v -FROM a JOIN b ON a.k = b.k AND a.t = b.t LEFT JOIN c ON b.x = c.x AND b.t = c.t -WHERE c.v = 'x' ORDER BY ALL -SETTINGS join_use_nulls = 1, query_plan_convert_outer_join_to_inner_join = 1, enable_join_transitive_predicates = 1, - query_plan_optimize_join_order_limit = 10, use_hash_table_stats_for_join_reordering = 0, - query_plan_optimize_join_order_algorithm = 'dpsub', query_plan_optimize_join_order_randomize = 13, - query_plan_optimize_join_order_conflict_detector = 'c'; + query_plan_optimize_join_order_algorithm = 'dpsub', query_plan_optimize_join_order_randomize = 13; SELECT ab.v, ab.x, c.x, c.v FROM c RIGHT JOIN (SELECT a.v AS v, a.t AS t, b.x AS x FROM a JOIN b ON a.k = b.k AND a.t = b.t) AS ab ON ab.x = c.x AND ab.t = c.t From 57eb7fb8e67ade45ec66eeaf68eedc8e97ff9704 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 12 Nov 2026 13:53:08 +0000 Subject: [PATCH 169/219] Backport #130952 to 26.8: Reject named collection DDL in readonly mode and with `allow_ddl = 0` --- src/Access/ContextAccess.cpp | 5 ++-- ...21_readonly_named_collection_ddl.reference | 13 ++++++++++ .../05621_readonly_named_collection_ddl.sh | 25 +++++++++++++++++++ 3 files changed, 41 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/05621_readonly_named_collection_ddl.reference create mode 100755 tests/queries/0_stateless/05621_readonly_named_collection_ddl.sh diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index 15d1e49dd815e..81ed1e44fa2ba 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -813,16 +813,17 @@ bool ContextAccess::checkAccessImplHelper(const ContextPtr & context, AccessFlag const AccessFlags function_ddl = AccessType::CREATE_FUNCTION | AccessType::DROP_FUNCTION; const AccessFlags workload_ddl = AccessType::CREATE_WORKLOAD | AccessType::DROP_WORKLOAD; const AccessFlags resource_ddl = AccessType::CREATE_RESOURCE | AccessType::DROP_RESOURCE; + const AccessFlags named_collection_ddl = AccessType::CREATE_NAMED_COLLECTION | AccessType::DROP_NAMED_COLLECTION | AccessType::ALTER_NAMED_COLLECTION; const AccessFlags handler_ddl = AccessType::CREATE_HANDLER | AccessType::ALTER_HANDLER | AccessType::DROP_HANDLER; const AccessFlags table_and_dictionary_ddl = table_ddl | dictionary_ddl; const AccessFlags table_and_dictionary_and_function_ddl = table_ddl | dictionary_ddl | function_ddl; const AccessFlags write_table_access = AccessType::INSERT | AccessType::OPTIMIZE; const AccessFlags write_dcl_access = AccessType::ACCESS_MANAGEMENT - AccessType::SHOW_ACCESS; - const AccessFlags not_readonly_flags = write_table_access | table_and_dictionary_and_function_ddl | workload_ddl | resource_ddl | handler_ddl | write_dcl_access | AccessType::SYSTEM | AccessType::KILL_QUERY; + const AccessFlags not_readonly_flags = write_table_access | table_and_dictionary_and_function_ddl | workload_ddl | resource_ddl | named_collection_ddl | handler_ddl | write_dcl_access | AccessType::SYSTEM | AccessType::KILL_QUERY; const AccessFlags not_readonly_1_flags = AccessType::CREATE_TEMPORARY_TABLE; - const AccessFlags ddl_flags = table_ddl | dictionary_ddl | function_ddl | workload_ddl | resource_ddl | handler_ddl; + const AccessFlags ddl_flags = table_ddl | dictionary_ddl | function_ddl | workload_ddl | resource_ddl | named_collection_ddl | handler_ddl; const AccessFlags introspection_flags = AccessType::INTROSPECTION; }; static const PrecalculatedFlags precalc; diff --git a/tests/queries/0_stateless/05621_readonly_named_collection_ddl.reference b/tests/queries/0_stateless/05621_readonly_named_collection_ddl.reference new file mode 100644 index 0000000000000..f5cee83fbf7a9 --- /dev/null +++ b/tests/queries/0_stateless/05621_readonly_named_collection_ddl.reference @@ -0,0 +1,13 @@ +--readonly 1 +READONLY +READONLY +READONLY +--readonly 2 +READONLY +READONLY +READONLY +--allow_ddl 0 +QUERY_IS_PROHIBITED +QUERY_IS_PROHIBITED +QUERY_IS_PROHIBITED +nc_existing diff --git a/tests/queries/0_stateless/05621_readonly_named_collection_ddl.sh b/tests/queries/0_stateless/05621_readonly_named_collection_ddl.sh new file mode 100755 index 0000000000000..10e845dafb275 --- /dev/null +++ b/tests/queries/0_stateless/05621_readonly_named_collection_ddl.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Named collection DDL is rejected in readonly mode and with allow_ddl = 0, like other DDL. + +nc="nc_${CLICKHOUSE_DATABASE}" + +$CLICKHOUSE_CLIENT -q "DROP NAMED COLLECTION IF EXISTS ${nc}_existing" +$CLICKHOUSE_CLIENT -q "DROP NAMED COLLECTION IF EXISTS ${nc}_new" +$CLICKHOUSE_CLIENT -q "CREATE NAMED COLLECTION ${nc}_existing AS key = 'value'" + +for setting in "--readonly 1" "--readonly 2" "--allow_ddl 0"; do + echo "$setting" + $CLICKHOUSE_CLIENT $setting -q "CREATE NAMED COLLECTION ${nc}_new AS key = 'value'" 2>&1 | grep -o -m1 -E 'READONLY|QUERY_IS_PROHIBITED' + $CLICKHOUSE_CLIENT $setting -q "ALTER NAMED COLLECTION ${nc}_existing SET key = 'other'" 2>&1 | grep -o -m1 -E 'READONLY|QUERY_IS_PROHIBITED' + $CLICKHOUSE_CLIENT $setting -q "DROP NAMED COLLECTION ${nc}_existing" 2>&1 | grep -o -m1 -E 'READONLY|QUERY_IS_PROHIBITED' +done + +# Nothing was created or dropped. +$CLICKHOUSE_CLIENT -q "SELECT name FROM system.named_collections WHERE name LIKE '${nc}%' ORDER BY name" | sed "s/${nc}/nc/" + +$CLICKHOUSE_CLIENT -q "DROP NAMED COLLECTION ${nc}_existing" From 5873d663abb906b3deaf8c0017ddf148cb45715e Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 12 Nov 2026 14:05:25 +0000 Subject: [PATCH 170/219] Backport #126549 to 26.8: Limit the total size of text bloom filter indexes and track their memory --- src/Interpreters/BloomFilter.cpp | 2 -- src/Interpreters/BloomFilter.h | 7 +++- src/Storages/MergeTree/MergeTreeData.cpp | 13 ++++++++ .../MergeTreeIndexBloomFilterText.cpp | 30 +++++++++++++++++ src/Storages/MergeTree/MergeTreeIndices.h | 5 +++ ...oom_filter_text_index_total_size.reference | 2 ++ ...421_bloom_filter_text_index_total_size.sql | 32 +++++++++++++++++++ ...ter_text_index_total_size_attach.reference | 4 +++ ...oom_filter_text_index_total_size_attach.sh | 20 ++++++++++++ 9 files changed, 112 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/05421_bloom_filter_text_index_total_size.reference create mode 100644 tests/queries/0_stateless/05421_bloom_filter_text_index_total_size.sql create mode 100644 tests/queries/0_stateless/05422_bloom_filter_text_index_total_size_attach.reference create mode 100755 tests/queries/0_stateless/05422_bloom_filter_text_index_total_size_attach.sh diff --git a/src/Interpreters/BloomFilter.cpp b/src/Interpreters/BloomFilter.cpp index 95a6c163f2762..240c2a2ad7e17 100644 --- a/src/Interpreters/BloomFilter.cpp +++ b/src/Interpreters/BloomFilter.cpp @@ -21,8 +21,6 @@ namespace ErrorCodes static constexpr UInt64 SEED_GEN_A = 845897321; static constexpr UInt64 SEED_GEN_B = 217728422; -static constexpr UInt64 MAX_BLOOM_FILTER_SIZE = 1 << 30; - namespace { using BloomDivider = libdivide::divider; diff --git a/src/Interpreters/BloomFilter.h b/src/Interpreters/BloomFilter.h index 5cf7d318ea5a8..3a70a4999b92b 100644 --- a/src/Interpreters/BloomFilter.h +++ b/src/Interpreters/BloomFilter.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -12,6 +13,10 @@ namespace DB { + +/// The maximum size in bytes of a single bloom filter. +static constexpr size_t MAX_BLOOM_FILTER_SIZE = 1 << 30; + struct BloomFilterHashPair { UInt64 hash1; @@ -35,7 +40,7 @@ class BloomFilter public: using UnderType = UInt64; - using Container = std::vector; + using Container = VectorWithMemoryTracking; explicit BloomFilter(const BloomFilterParameters & params); /// size -- size of filter in bytes. diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index a1fef4ac70323..d382d77c1fa4a 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1183,6 +1183,19 @@ void MergeTreeData::checkProperties( if (!attach && !allow_minmax_index_for_json) checkMinMaxIndexForJSON(index); MergeTreeIndexFactory::instance().validate(index, attach, *getSettings()); + + if (!attach) + { + /// The create path and the projection recursion pass the same metadata object as both arguments, + /// so nothing can be inherited there. + const IndexDescription * old_index = nullptr; + if (&old_metadata != &new_metadata) + for (const auto & candidate : old_metadata.secondary_indices) + if (candidate.name == index.name) + old_index = &candidate; + + bloomFilterIndexTextCheckTotalSize(index, old_index); + } } catch (Exception & e) { diff --git a/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp b/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp index 0ef8f588cc09b..8c8360b3c75e5 100644 --- a/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp @@ -1013,4 +1013,34 @@ void bloomFilterIndexTextValidator(const IndexDescription & index, bool /*attach args[first_bf_param_idx+2].safeGet()); } +static size_t bloomFilterIndexTextTotalSize(const IndexDescription & index) +{ + if (index.type != NgramsTokenizer::getName() + && index.type != SplitByNonAlphaTokenizer::getName() + && index.type != SparseGramsTokenizer::getBloomFilterIndexName()) + return 0; + + /// The size of bloom filter is the first of the last three arguments, whatever the tokenizer. + /// The arguments are already checked by `bloomFilterIndexTextValidator`, and the size is at most `MAX_BLOOM_FILTER_SIZE`, + /// so the multiplication cannot overflow. + FieldVector args = getFieldsFromIndexArgumentsAST(index.arguments); + return args.at(args.size() - 3).safeGet() * index.column_names.size(); +} + +void bloomFilterIndexTextCheckTotalSize(const IndexDescription & index, const IndexDescription * old_index) +{ + const size_t total_size = bloomFilterIndexTextTotalSize(index); + if (total_size <= MAX_BLOOM_FILTER_SIZE) + return; + + /// An index inherited from the old metadata may have been created before this check, so it is accepted as long as it does not grow. + if (old_index && bloomFilterIndexTextTotalSize(*old_index) >= total_size) + return; + + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "The total size of bloom filters in a granule of index {} cannot be more than {}: " + "the index has {} columns and the total size of their bloom filters is {}", + backQuote(index.name), MAX_BLOOM_FILTER_SIZE, index.column_names.size(), total_size); +} + } diff --git a/src/Storages/MergeTree/MergeTreeIndices.h b/src/Storages/MergeTree/MergeTreeIndices.h index f749aaf3302ec..6c85669609e34 100644 --- a/src/Storages/MergeTree/MergeTreeIndices.h +++ b/src/Storages/MergeTree/MergeTreeIndices.h @@ -408,6 +408,11 @@ void setIndexValidator(const IndexDescription & index, bool attach, const MergeT MergeTreeIndexPtr bloomFilterIndexTextCreator(StorageMetadataPtr metadata_snapshot, const IndexDescription & index, const MergeTreeSettings & settings); void bloomFilterIndexTextValidator(const IndexDescription & index, bool attach, const MergeTreeSettings & settings); +/// Every granule of a text bloom filter index eagerly allocates one filter per indexed column, so the total size of the filters +/// (not only the size of a single filter) is limited. The check is done only for indexes being declared or changed, not on `ATTACH` +/// and not for indexes inherited unchanged from `old_index`, so that tables created before the check still load and can be altered. +/// Does nothing for other index types. +void bloomFilterIndexTextCheckTotalSize(const IndexDescription & index, const IndexDescription * old_index); MergeTreeIndexPtr bloomFilterIndexCreator(StorageMetadataPtr metadata_snapshot, const IndexDescription & index, const MergeTreeSettings & settings); void bloomFilterIndexValidator(const IndexDescription & index, bool attach, const MergeTreeSettings & settings); diff --git a/tests/queries/0_stateless/05421_bloom_filter_text_index_total_size.reference b/tests/queries/0_stateless/05421_bloom_filter_text_index_total_size.reference new file mode 100644 index 0000000000000..b261da18d51a3 --- /dev/null +++ b/tests/queries/0_stateless/05421_bloom_filter_text_index_total_size.reference @@ -0,0 +1,2 @@ +1 +0 diff --git a/tests/queries/0_stateless/05421_bloom_filter_text_index_total_size.sql b/tests/queries/0_stateless/05421_bloom_filter_text_index_total_size.sql new file mode 100644 index 0000000000000..0a25cd09791d7 --- /dev/null +++ b/tests/queries/0_stateless/05421_bloom_filter_text_index_total_size.sql @@ -0,0 +1,32 @@ +-- A granule of a text bloom filter index allocates one filter per indexed column, +-- so the total size of filters (filter size multiplied by the number of columns) is limited by 1 GiB. + +DROP TABLE IF EXISTS t_bf_total_size; + +-- A single filter of the maximum size is still allowed. +CREATE TABLE t_bf_total_size (a String, INDEX idx a TYPE tokenbf_v1(1073741824, 3, 0) GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple(); +DROP TABLE t_bf_total_size; + +-- Two columns with filters of half of the maximum size are allowed. +CREATE TABLE t_bf_total_size (a String, b String, INDEX idx (a, b) TYPE tokenbf_v1(536870912, 3, 0) GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple(); +DROP TABLE t_bf_total_size; + +-- One byte more is too much. +CREATE TABLE t_bf_total_size (a String, b String, INDEX idx (a, b) TYPE tokenbf_v1(536870913, 3, 0) GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS } +CREATE TABLE t_bf_total_size (a String, b String, INDEX idx (a, b) TYPE ngrambf_v1(3, 536870913, 3, 0) GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS } +CREATE TABLE t_bf_total_size (a String, b String, c String, d String, e String, f String, g String, h String, + INDEX idx (a, b, c, d, e, f, g, h) TYPE tokenbf_v1(268435456, 3, 0) GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS } + +-- The same check is done for ALTER TABLE ... ADD INDEX. +CREATE TABLE t_bf_total_size (a String, b String) ENGINE = MergeTree ORDER BY tuple(); +ALTER TABLE t_bf_total_size ADD INDEX idx (a, b) TYPE tokenbf_v1(536870913, 3, 0) GRANULARITY 1; -- { serverError BAD_ARGUMENTS } +ALTER TABLE t_bf_total_size ADD INDEX idx (a, b) TYPE tokenbf_v1(1024, 3, 0) GRANULARITY 1; +INSERT INTO t_bf_total_size VALUES ('hello', 'world'); +SELECT count() FROM t_bf_total_size WHERE hasToken(a, 'hello'); +DROP TABLE t_bf_total_size; + +-- The memory of bloom filters is tracked, so a too large filter fails with an exception instead of exhausting the memory. +CREATE TABLE t_bf_total_size (a String, INDEX idx a TYPE tokenbf_v1(268435456, 3, 0) GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_bf_total_size SETTINGS max_memory_usage = '100Mi', async_insert = 0 VALUES ('hello world'); -- { serverError MEMORY_LIMIT_EXCEEDED } +SELECT count() FROM t_bf_total_size; +DROP TABLE t_bf_total_size; diff --git a/tests/queries/0_stateless/05422_bloom_filter_text_index_total_size_attach.reference b/tests/queries/0_stateless/05422_bloom_filter_text_index_total_size_attach.reference new file mode 100644 index 0000000000000..51aa977e86033 --- /dev/null +++ b/tests/queries/0_stateless/05422_bloom_filter_text_index_total_size_attach.reference @@ -0,0 +1,4 @@ +a +d +c +idx diff --git a/tests/queries/0_stateless/05422_bloom_filter_text_index_total_size_attach.sh b/tests/queries/0_stateless/05422_bloom_filter_text_index_total_size_attach.sh new file mode 100755 index 0000000000000..b1fd11bddf6d4 --- /dev/null +++ b/tests/queries/0_stateless/05422_bloom_filter_text_index_total_size_attach.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A text bloom filter index with the total size of filters over the limit cannot be created anymore, +# but a table created before the check still loads and can be altered, as long as its index does not grow. + +UUID=$($CLICKHOUSE_CLIENT -q "SELECT generateUUIDv4()") + +$CLICKHOUSE_CLIENT -m -q " +ATTACH TABLE t_bf_total_size_attach UUID '$UUID' (a String, b String, INDEX idx (a, b) TYPE tokenbf_v1(536870913, 3, 0) GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple(); +ALTER TABLE t_bf_total_size_attach ADD COLUMN c String; +ALTER TABLE t_bf_total_size_attach RENAME COLUMN b TO d; +ALTER TABLE t_bf_total_size_attach ADD INDEX idx2 (a, c) TYPE tokenbf_v1(536870913, 3, 0) GRANULARITY 1; -- { serverError BAD_ARGUMENTS } +SELECT name FROM system.columns WHERE database = currentDatabase() AND table = 't_bf_total_size_attach' ORDER BY position; +SELECT name FROM system.data_skipping_indices WHERE database = currentDatabase() AND table = 't_bf_total_size_attach'; +DROP TABLE t_bf_total_size_attach; +" From 40946fe75d4600518be2ec520215359465eabf4b Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 11 Nov 2026 08:33:44 +0000 Subject: [PATCH 171/219] Backport #130192 to 26.8: Keep the `NULL` partition in Delta Lake partition pruning when the filter's set holds `NULL` --- .../DataLakes/DeltaLake/PartitionPruner.cpp | 27 ++++-- .../DataLakes/DeltaLake/PartitionPruner.h | 3 + ...ke_partition_pruning_null_in_set.reference | 16 ++++ ...elta_lake_partition_pruning_null_in_set.sh | 82 +++++++++++++++++++ 4 files changed, 123 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.reference create mode 100755 tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.sh diff --git a/src/Storages/ObjectStorage/DataLakes/DeltaLake/PartitionPruner.cpp b/src/Storages/ObjectStorage/DataLakes/DeltaLake/PartitionPruner.cpp index 182015d014d7d..afd0e08b3de94 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeltaLake/PartitionPruner.cpp +++ b/src/Storages/ObjectStorage/DataLakes/DeltaLake/PartitionPruner.cpp @@ -46,7 +46,8 @@ namespace DB::ColumnsDescription getPartitionColumnsDescription( const DB::Names & partition_columns, - const DB::NamesAndTypesList & table_schema) + const DB::NamesAndTypesList & table_schema, + bool keep_nullable) { DB::NamesAndTypesList names_and_types; for (const auto & column_name : partition_columns) @@ -54,7 +55,7 @@ namespace auto column = table_schema.tryGetByName(column_name); if (!column.has_value()) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Not found partition column in schema: {}", column_name); - names_and_types.emplace_back(column_name, removeNullable(column->type)); + names_and_types.emplace_back(column_name, keep_nullable ? column->type : removeNullable(column->type)); } return DB::ColumnsDescription(names_and_types); } @@ -70,7 +71,8 @@ PartitionPruner::PartitionPruner( { if (!partition_columns_.empty()) { - const auto partition_columns_description = getPartitionColumnsDescription(partition_columns_, table_schema_); + const auto partition_columns_description + = getPartitionColumnsDescription(partition_columns_, table_schema_, /* keep_nullable */ false); const auto partition_key_ast = createPartitionKeyAST(partition_columns_); partition_key = DB::KeyDescription::getKeyFromAST( @@ -78,10 +80,17 @@ PartitionPruner::PartitionPruner( partition_columns_description, {}, context); + nullable_partition_key = DB::KeyDescription::getKeyFromAST( + partition_key_ast, + getPartitionColumnsDescription(partition_columns_, table_schema_, /* keep_nullable */ true), + {}, + context); DB::ActionsDAGWithInversionPushDown inverted_dag(filter_dag.getOutputs().front(), context, /* boolean_context */ true); key_condition.emplace( inverted_dag, context, partition_key.column_names, partition_key.expression, true /* single_point */); + nullable_key_condition.emplace( + inverted_dag, context, nullable_partition_key.column_names, nullable_partition_key.expression, true /* single_point */); } if (!physical_names_map_.empty()) { @@ -110,20 +119,28 @@ bool PartitionPruner::canBePruned(const DB::ObjectInfo & object_info) const DB::Row partition_key_values; partition_key_values.reserve(partition_values.size()); + bool holds_null = false; for (auto && value : partition_values) { if (value.isNull()) + { + holds_null = true; partition_key_values.push_back(DB::POSITIVE_INFINITY); /// NULL_LAST + } else partition_key_values.push_back(std::move(value)); } + /// The key without Nullable drops the NULL elements of a set, so it is exact only for a tuple without NULL. + const auto & key = holds_null ? nullable_partition_key : partition_key; + const auto & condition = holds_null ? *nullable_key_condition : *key_condition; + std::vector partition_key_values_ref(partition_key_values.begin(), partition_key_values.end()); - bool can_be_true = key_condition->mayBeTrueInRange( + bool can_be_true = condition.mayBeTrueInRange( partition_key_values_ref.size(), partition_key_values_ref.data(), partition_key_values_ref.data(), - partition_key.data_types); + key.data_types); return !can_be_true; } diff --git a/src/Storages/ObjectStorage/DataLakes/DeltaLake/PartitionPruner.h b/src/Storages/ObjectStorage/DataLakes/DeltaLake/PartitionPruner.h index 1ff9e2e5559bf..ef89f33852bc6 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeltaLake/PartitionPruner.h +++ b/src/Storages/ObjectStorage/DataLakes/DeltaLake/PartitionPruner.h @@ -32,6 +32,9 @@ class PartitionPruner private: std::optional key_condition; DB::KeyDescription partition_key; + /// The same key over the schema's Nullable types, for a partition tuple that holds NULL. + std::optional nullable_key_condition; + DB::KeyDescription nullable_partition_key; DB::Names physical_partition_columns; }; diff --git a/tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.reference b/tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.reference new file mode 100644 index 0000000000000..dd6c40870201f --- /dev/null +++ b/tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.reference @@ -0,0 +1,16 @@ +t: has([NULL, 1], p) [2] [2] 3 +t: has([NULL, 5], p) [1,2] [1,2] 2 +t: NOT has([NULL, 1], p) [1,3,4] [1,3,4] 0 +t: has([NULL, 1]::Array(Nullable(Int64)), p) [2] [2] 3 +t: NOT has([NULL, 1]::Array(Nullable(Int64)), p) [1,3,4] [1,3,4] 1 +t: p IN (NULL, 1), transform_null_in = 1 [2] [2] 3 +t: p IS NULL, delta_lake_enable_engine_predicate = 0 [2] [2] 3 +t: p = 7, delta_lake_enable_engine_predicate = 0 [3] [3] 3 +t: p < 6, delta_lake_enable_engine_predicate = 0 [1] [1] 3 +t: p IS NOT NULL, delta_lake_enable_engine_predicate = 0 [1,3,4] [1,3,4] 1 +t: has([5, 7], p) [1,3] [1,3] 2 +t: ifNull(p, 0) IN (0) [2] [2] 3 +n: has([NULL, 5], p) [1] [1] 2 +n: NOT has([NULL, 5], p) [3,4] [3,4] 1 +t2: has([NULL, 1], a) [1,2,3] [1,2,3] 0 +s: has([NULL, 'a'], s) [1,2] [1,2] 1 diff --git a/tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.sh b/tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.sh new file mode 100755 index 0000000000000..8626c203cd6c1 --- /dev/null +++ b/tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.sh @@ -0,0 +1,82 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-msan +# Tag no-fasttest: delta-kernel pulls in extra dependencies. +# Tag no-msan: delta-kernel-rs (Rust) is not built under MSan, so DeltaLakeLocal is absent. +# Partition pruning must keep the data file of the NULL partition when the filter's set holds NULL, and prune the other files as before. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh +set -e + +ROOT="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_delta_null_set" +trap 'rm -rf "${ROOT}" 2>/dev/null' EXIT +rm -rf "${ROOT}" + +# An empty Delta table: a v0 log with only protocol + metaData, as ClickHouse cannot create a Delta log itself. +bootstrap() { + local table=$1 schema=$2 partition_columns=$3 + mkdir -p "${ROOT}/${table}/_delta_log" + cat > "${ROOT}/${table}/_delta_log/00000000000000000000.json" < [settings]: prints the rows read with no filter at the source (LIMIT blocks the push-down), +# the rows read with partition pruning, and the number of data files pruned by the second query. +arm() { + local table=$1 condition=$2 settings=${3:+ SETTINGS $3} + LABELS+=("${table}: ${condition}${3:+, $3}") + QUERIES+=" + SELECT arraySort(groupArray(v)) FROM (SELECT * FROM deltaLakeLocal('${ROOT}/${table}') LIMIT 1000) WHERE ${condition}${settings}; + SELECT sumIf(value, event = 'DeltaLakePartitionPrunedFiles') FROM system.events; + SELECT arraySort(groupArray(v)) FROM deltaLakeLocal('${ROOT}/${table}') WHERE ${condition}${settings}; + SELECT sumIf(value, event = 'DeltaLakePartitionPrunedFiles') FROM system.events;" +} + +arm t "has([NULL, 1], p)" +arm t "has([NULL, 5], p)" +arm t "NOT has([NULL, 1], p)" +arm t "has([NULL, 1]::Array(Nullable(Int64)), p)" +arm t "NOT has([NULL, 1]::Array(Nullable(Int64)), p)" +arm t "p IN (NULL, 1)" "transform_null_in = 1" +# The engine predicate would skip these files before the partition pruner counts them. +arm t "p IS NULL" "delta_lake_enable_engine_predicate = 0" +arm t "p = 7" "delta_lake_enable_engine_predicate = 0" +arm t "p < 6" "delta_lake_enable_engine_predicate = 0" +arm t "p IS NOT NULL" "delta_lake_enable_engine_predicate = 0" +arm t "has([5, 7], p)" +arm t "ifNull(p, 0) IN (0)" +arm n "has([NULL, 5], p)" +arm n "NOT has([NULL, 5], p)" +arm t2 "has([NULL, 1], a)" +arm s "has([NULL, 'a'], s)" + +mapfile -t LINES < <(${CLICKHOUSE_LOCAL} --allow_delta_lake_writes=1 --query "${QUERIES}" < /dev/null) +if [[ "${#LINES[@]}" != "$(( 4 * ${#LABELS[@]} ))" ]]; then + printf 'Expected %s lines, got %s:\n%s\n' "$(( 4 * ${#LABELS[@]} ))" "${#LINES[@]}" "$(printf '%s\n' "${LINES[@]}")" >&2 + exit 1 +fi + +for i in "${!LABELS[@]}"; do + printf '%s\t%s\t%s\t%s\n' "${LABELS[$i]}" "${LINES[4 * i]}" "${LINES[4 * i + 2]}" "$(( LINES[4 * i + 3] - LINES[4 * i + 1] ))" +done From 05aea30104b8de8977c0fd386eb3c5a6c3173207 Mon Sep 17 00:00:00 2001 From: Groene AI <270696204+groeneai@users.noreply.github.com> Date: Wed, 11 Nov 2026 11:43:20 +0000 Subject: [PATCH 172/219] Adapt 05576 to 26.8: drop `ifNull(p, 0) IN (0)`, `has([5, 7], p)` prunes 1 file On 26.8 `ifNull(p, 0) IN (0)` returns no rows instead of [2] with and without the backported fix: partition pruning drops the NULL partition, and a MergeTree table partitioned by the same Nullable column returns the same wrong rows on 26.8. The case tests another bug, so this backport's test does not keep it. On 26.8 the fix keeps the NULL partition's file for `has([5, 7], p)`, so that case prunes 1 file instead of 2 with the same rows: the untyped [5, 7] is not used by the Nullable key condition there, while has([5, 7]::Array(Int64), p) and p IN (5, 7) still prune 2. 26.8's MergeTree partition pruning does not use the untyped has on a Nullable key either. Co-Authored-By: Claude Opus 5.5 --- .../05576_delta_lake_partition_pruning_null_in_set.reference | 3 +-- .../05576_delta_lake_partition_pruning_null_in_set.sh | 1 - 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.reference b/tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.reference index dd6c40870201f..626195e3c0a6a 100644 --- a/tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.reference +++ b/tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.reference @@ -8,8 +8,7 @@ t: p IS NULL, delta_lake_enable_engine_predicate = 0 [2] [2] 3 t: p = 7, delta_lake_enable_engine_predicate = 0 [3] [3] 3 t: p < 6, delta_lake_enable_engine_predicate = 0 [1] [1] 3 t: p IS NOT NULL, delta_lake_enable_engine_predicate = 0 [1,3,4] [1,3,4] 1 -t: has([5, 7], p) [1,3] [1,3] 2 -t: ifNull(p, 0) IN (0) [2] [2] 3 +t: has([5, 7], p) [1,3] [1,3] 1 n: has([NULL, 5], p) [1] [1] 2 n: NOT has([NULL, 5], p) [3,4] [3,4] 1 t2: has([NULL, 1], a) [1,2,3] [1,2,3] 0 diff --git a/tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.sh b/tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.sh index 8626c203cd6c1..2e388a0166240 100755 --- a/tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.sh +++ b/tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.sh @@ -65,7 +65,6 @@ arm t "p = 7" "delta_lake_enable_engine_predicate = 0" arm t "p < 6" "delta_lake_enable_engine_predicate = 0" arm t "p IS NOT NULL" "delta_lake_enable_engine_predicate = 0" arm t "has([5, 7], p)" -arm t "ifNull(p, 0) IN (0)" arm n "has([NULL, 5], p)" arm n "NOT has([NULL, 5], p)" arm t2 "has([NULL, 1], a)" From 4b466a15a3860202980dd5b255258f9729282758 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 12 Nov 2026 18:18:20 +0000 Subject: [PATCH 173/219] Backport #130541 to 26.8: Update `AMQP-CPP` submodule --- contrib/AMQP-CPP | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/contrib/AMQP-CPP b/contrib/AMQP-CPP index 4fccd7f84318f..5f4c447d305af 160000 --- a/contrib/AMQP-CPP +++ b/contrib/AMQP-CPP @@ -1 +1 @@ -Subproject commit 4fccd7f84318fe3a022c63a5b621a831c99e3a32 +Subproject commit 5f4c447d305afd3d8ec4001683ff855ccb4a8ef4 From 3669c5c92b26c7bfb71fe6db4ea40e2e0c017350 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 12 Nov 2026 18:26:57 +0000 Subject: [PATCH 174/219] Backport #130578 to 26.8: Skip the range analysis of a MergeTree read under `LIMIT 0` --- .../optimizePrimaryKeyConditionAndLimit.cpp | 6 ++ .../QueryPlan/ReadFromMergeTree.cpp | 34 +++++++++-- src/Processors/QueryPlan/ReadFromMergeTree.h | 7 ++- .../MergeTree/MergeTreeDataSelectExecutor.cpp | 55 ++++++++++-------- .../MergeTree/MergeTreeDataSelectExecutor.h | 3 + .../limit_zero_primary_key_analysis.xml | 19 +++++++ ..._limit_zero_skips_range_analysis.reference | 43 ++++++++++++++ .../05617_limit_zero_skips_range_analysis.sql | 56 +++++++++++++++++++ 8 files changed, 194 insertions(+), 29 deletions(-) create mode 100644 tests/performance/limit_zero_primary_key_analysis.xml create mode 100644 tests/queries/0_stateless/05617_limit_zero_skips_range_analysis.reference create mode 100644 tests/queries/0_stateless/05617_limit_zero_skips_range_analysis.sql diff --git a/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyConditionAndLimit.cpp b/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyConditionAndLimit.cpp index ef3608c98a99b..20bddff29a92f 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyConditionAndLimit.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizePrimaryKeyConditionAndLimit.cpp @@ -2,6 +2,7 @@ #include #include #include +#include #include #include @@ -52,6 +53,11 @@ void optimizePrimaryKeyConditionAndLimit(const Stack & stack) else if (auto * limit_step = typeid_cast(iter->node->step.get())) { source_step_with_filter->setLimit(limit_step->getLimitForSorting()); + + /// `LIMIT 0` takes no row unless it reads till end, for totals or `exact_rows_before_limit`. + if (limit_step->getLimit() == 0 && limit_step->getOffset() == 0 && !limit_step->alwaysReadTillEnd()) + if (auto * read_from_merge_tree = typeid_cast(source_step_with_filter)) + read_from_merge_tree->setNoRowsNeeded(); break; } else if (auto * expression_step = typeid_cast(iter->node->step.get())) diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 7f8d7c27010d9..7d558999fcb13 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -242,8 +242,11 @@ namespace Setting extern const SettingsBool force_creating_set_partitions_independently; extern const SettingsBool force_distinct_partitions_independently; extern const SettingsBool force_window_partitions_independently; + extern const SettingsBool force_optimize_projection; + extern const SettingsString force_optimize_projection_name; extern const SettingsBool force_primary_key; extern const SettingsString ignore_data_skipping_indices; + extern const SettingsBool make_distributed_plan; extern const SettingsUInt64 max_number_of_partitions_for_independent_aggregation; extern const SettingsUInt64 max_number_of_partitions_for_independent_distinct; extern const SettingsUInt64 max_number_of_partitions_for_independent_window; @@ -2699,6 +2702,12 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::selectRangesToRead(bool find_exact_ranges) const { + const auto & settings = context->getSettingsRef(); + /// The ranges are still needed to coordinate replicas or tasks, and to choose a forced projection. + const bool skip_ranges_analysis = no_rows_needed && !is_parallel_reading_from_replicas + && !settings[Setting::make_distributed_plan] && !settings[Setting::force_optimize_projection] + && settings[Setting::force_optimize_projection_name].value.empty(); + analyzed_result_ptr = selectRangesToRead( getParts(), mutations_snapshot, @@ -2718,7 +2727,8 @@ ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::selectRangesToRead(bool is_parallel_reading_from_replicas, allow_query_condition_cache, supportsSkipIndexesOnDataRead(), - /*check_row_limits=*/true); + /*check_row_limits=*/true, + skip_ranges_analysis); return analyzed_result_ptr; } @@ -2746,7 +2756,8 @@ ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::estimateRangesToReadWith is_parallel_reading_from_replicas, /*allow_query_condition_cache_=*/false, supportsSkipIndexesOnDataRead(), - /*check_row_limits=*/true); + /*check_row_limits=*/true, + /*skip_ranges_analysis=*/false); } ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::selectRangesToReadForEstimation() const @@ -2770,7 +2781,8 @@ ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::selectRangesToReadForEst is_parallel_reading_from_replicas, allow_query_condition_cache, supportsSkipIndexesOnDataRead(), - /*check_row_limits=*/false); + /*check_row_limits=*/false, + /*skip_ranges_analysis=*/false); } namespace @@ -3333,7 +3345,8 @@ ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::selectRangesToRead( bool is_parallel_reading_from_replicas_, bool allow_query_condition_cache_, bool supports_skip_indexes_on_data_read, - bool check_row_limits) + bool check_row_limits, + bool skip_ranges_analysis) { ProfileEvents::increment(ProfileEvents::IndexAnalysisRounds); @@ -3463,6 +3476,18 @@ ReadFromMergeTree::AnalysisResultPtr ReadFromMergeTree::selectRangesToRead( total_marks_pk += part.data_part->index_granularity->getMarksCountWithoutFinal(); parts_before_pk = res_parts.size(); + /// Before the query condition cache below, which records the ranges missing from the selection as not matching. + if (skip_ranges_analysis) + { + if (indexes->use_skip_indexes) + MergeTreeDataSelectExecutor::checkForcedSkipIndexes(indexes->skip_indexes, settings); + LOG_DEBUG(log, "Skipping the analysis of ranges to read: the query takes no row from this read"); + result.total_parts = total_parts; + result.parts_before_pk = parts_before_pk; + result.total_marks_pk = total_marks_pk; + return std::make_shared(std::move(result)); + } + /// Check if we have projections or exact-range analysis, as that can determine whether we fail /// during reading parts or analyze projection / exact-count candidates to serve the query more @@ -4560,6 +4585,7 @@ QueryPlanStepPtr ReadFromMergeTree::clone() const /// materialized only by this task map, and losing it makes the clone evaluate the rewritten filter /// without the index readers (`optimizeLazyFinal` copies the same map onto its synthetic reads). cloned_step->index_read_tasks = index_read_tasks; + cloned_step->no_rows_needed = no_rows_needed; cloned_step->setStepDescription(*this); return cloned_step; } diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index 2c6ad2410aaea..9d45e8f9cefd1 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -333,7 +333,8 @@ class ReadFromMergeTree final : public SourceStepWithFilter bool is_parallel_reading_from_replicas_, bool allow_query_condition_cache_, bool supports_skip_indexes_on_data_read, - bool check_row_limits); + bool check_row_limits, + bool skip_ranges_analysis); AnalysisResultPtr selectRangesToRead(bool find_exact_ranges = false) const; @@ -356,6 +357,9 @@ class ReadFromMergeTree final : public SourceStepWithFilter /// columns (query-wide part numbering, catalog names, disk placement). static bool filterDependsOnNonDeterministicVirtuals(const VirtualColumnsDescription & virtuals, const SelectQueryInfo & query_info_); + /// The query takes no row from this read, so `selectRangesToRead` does not analyze its ranges. + void setNoRowsNeeded() { no_rows_needed = true; } + /// Returns `false` if requested reading cannot be performed. bool requestReadingInOrder(size_t prefix_size, int direction, size_t read_limit, size_t query_limit = 0); bool setVirtualRowConversions(ActionsDAG virtual_row_conversion_); @@ -712,6 +716,7 @@ class ReadFromMergeTree final : public SourceStepWithFilter std::optional read_task_callback; bool enable_vertical_final = false; bool allow_query_condition_cache = true; + bool no_rows_needed = false; LazyMaterializingRowsPtr lazy_materializing_rows; diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index f7b12e9f78026..bae6fb67cdc0e 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -964,6 +964,33 @@ static bool partHasStaleTopKIndex( return false; } +void MergeTreeDataSelectExecutor::checkForcedSkipIndexes(const UsefulSkipIndexes & skip_indexes, const Settings & settings) +{ + if (!settings[Setting::force_data_skipping_indices].changed) + return; + + const auto & indices_str = settings[Setting::force_data_skipping_indices].toString(); + auto forced_indices = parseIdentifiersOrStringLiterals(indices_str, settings); + + if (forced_indices.empty()) + throw Exception(ErrorCodes::CANNOT_PARSE_TEXT, "No indices parsed from force_data_skipping_indices ('{}')", indices_str); + + std::unordered_set useful_indices_names; + for (const auto & useful_index : skip_indexes.useful_indices) + useful_indices_names.insert(useful_index.index->index.name); + + for (const auto & index_name : forced_indices) + { + if (!useful_indices_names.contains(index_name)) + { + throw Exception( + ErrorCodes::INDEX_NOT_USED, + "Index {} is not used and setting 'force_data_skipping_indices' contains it", + backQuote(index_name)); + } + } +} + RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipIndexes(IndexAnalysisContext & filter_context, RangesInDataParts parts_with_ranges, ReadFromMergeTree::IndexStats & index_stats) { auto & metadata_snapshot = filter_context.metadata_snapshot; @@ -991,29 +1018,8 @@ RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPrimaryKeyAndSkipInd const auto original_num_parts = parts_with_ranges.size(); const Settings & settings = context->getSettingsRef(); - if (use_skip_indexes && settings[Setting::force_data_skipping_indices].changed) - { - const auto & indices_str = settings[Setting::force_data_skipping_indices].toString(); - auto forced_indices = parseIdentifiersOrStringLiterals(indices_str, settings); - - if (forced_indices.empty()) - throw Exception(ErrorCodes::CANNOT_PARSE_TEXT, "No indices parsed from force_data_skipping_indices ('{}')", indices_str); - - std::unordered_set useful_indices_names; - for (const auto & useful_index : skip_indexes.useful_indices) - useful_indices_names.insert(useful_index.index->index.name); - - for (const auto & index_name : forced_indices) - { - if (!useful_indices_names.contains(index_name)) - { - throw Exception( - ErrorCodes::INDEX_NOT_USED, - "Index {} is not used and setting 'force_data_skipping_indices' contains it", - backQuote(index_name)); - } - } - } + if (use_skip_indexes) + checkForcedSkipIndexes(skip_indexes, settings); struct IndexStat { @@ -1904,7 +1910,8 @@ ReadFromMergeTree::AnalysisResultPtr MergeTreeDataSelectExecutor::estimateNumMar /*is_parallel_reading_from_replicas*/false, use_query_condition_cache, /*supports_skip_indexes_on_data_read*/false, - /*check_row_limits=*/true); + /*check_row_limits=*/true, + /*skip_ranges_analysis=*/false); } QueryPlanStepPtr MergeTreeDataSelectExecutor::readFromParts( diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index c925b34dbdaaf..f8fd9d4f6e2f3 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -261,6 +261,9 @@ class MergeTreeDataSelectExecutor /// For every part, select mark ranges to read. static RangesInDataParts filterPartsByPrimaryKeyAndSkipIndexes(IndexAnalysisContext & filter_context, RangesInDataParts parts_with_ranges, ReadFromMergeTree::IndexStats & index_stats); + /// Throws if an index in `force_data_skipping_indices` is not among the useful ones. + static void checkForcedSkipIndexes(const UsefulSkipIndexes & skip_indexes, const Settings & settings); + /// Filter parts using query condition cache. static void filterPartsByQueryConditionCache( RangesInDataParts & parts_with_ranges, diff --git a/tests/performance/limit_zero_primary_key_analysis.xml b/tests/performance/limit_zero_primary_key_analysis.xml new file mode 100644 index 0000000000000..1f571047337d0 --- /dev/null +++ b/tests/performance/limit_zero_primary_key_analysis.xml @@ -0,0 +1,19 @@ + + + CREATE TABLE limit_zero_pk (c1 DateTime, c2 UInt8, c3 LowCardinality(String), c4 UInt32) + ENGINE = MergeTree PRIMARY KEY (c1, c2, c3) ORDER BY (c1, c2, c3, c4) + + + + INSERT INTO limit_zero_pk + SELECT toDateTime('2026-08-01 00:00:00') + intDiv(number, 100000), (number % 100000) >= 50000, + concat('ex', toString(cityHash64(number) % 20)), toUInt32(cityHash64(number, 1) % 50) + FROM numbers_mt(600000000) + + OPTIMIZE TABLE limit_zero_pk FINAL + + SELECT c1, c4 FROM limit_zero_pk WHERE c4 = 1 AND c2 LIMIT 0 + SELECT c1, c4 FROM limit_zero_pk WHERE c4 = 1 AND c2 LIMIT 0 SETTINGS compatibility = '25.12' + + DROP TABLE IF EXISTS limit_zero_pk + diff --git a/tests/queries/0_stateless/05617_limit_zero_skips_range_analysis.reference b/tests/queries/0_stateless/05617_limit_zero_skips_range_analysis.reference new file mode 100644 index 0000000000000..aac675144b3ad --- /dev/null +++ b/tests/queries/0_stateless/05617_limit_zero_skips_range_analysis.reference @@ -0,0 +1,43 @@ +-- { echoOn } + +SELECT count() FROM (EXPLAIN indexes = 1 SELECT k FROM t WHERE flag LIMIT 0) WHERE explain ILIKE '%PrimaryKey%'; +0 +SELECT count() FROM (EXPLAIN indexes = 1 SELECT k FROM t WHERE flag LIMIT 1) WHERE explain ILIKE '%PrimaryKey%'; +1 +SELECT count() FROM (EXPLAIN indexes = 1 SELECT k FROM t WHERE flag LIMIT 0 SETTINGS exact_rows_before_limit = 1) WHERE explain ILIKE '%PrimaryKey%'; +1 +SELECT k FROM t WHERE flag LIMIT 0 SETTINGS exact_rows_before_limit = 1, output_format_write_statistics = 0 FORMAT JSONCompact; +{ + "meta": + [ + { + "name": "k", + "type": "UInt64" + } + ], + + "data": + [ + + ], + + "rows": 0, + + "rows_before_limit_at_least": 50000 +} +SELECT k FROM t WHERE v = 1 LIMIT 0 SETTINGS force_primary_key = 1; -- { serverError INDEX_NOT_USED } +SELECT k FROM t WHERE flag LIMIT 0 SETTINGS force_index_by_date = 1; -- { serverError INDEX_NOT_USED } +SELECT k FROM t WHERE flag LIMIT 0 SETTINGS force_data_skipping_indices = 'iv'; -- { serverError INDEX_NOT_USED } +SET use_query_condition_cache = 1; +SELECT k FROM t WHERE k < 10 AND flag AND v < 50 LIMIT 0; +SELECT count() FROM t WHERE k < 10 AND flag AND v < 50; +2500 +SELECT k FROM tp WHERE v = 7 LIMIT 0 SETTINGS optimize_use_projections = 1, force_optimize_projection_name = 'p'; +SELECT k FROM tp WHERE v = 7 LIMIT 0 SETTINGS optimize_use_projections = 1, force_optimize_projection = 1; +SELECT k, v FROM tr FINAL WHERE k > 10 LIMIT 0 FORMAT TSVWithNamesAndTypes; +k v +UInt64 UInt64 +SELECT k FROM t WHERE flag LIMIT 0 SETTINGS max_rows_to_read = 10; +SELECT k FROM t WHERE flag LIMIT 1 SETTINGS max_rows_to_read = 10 FORMAT Null; -- { serverError TOO_MANY_ROWS } +SELECT count() FROM t WHERE k IN (SELECT k FROM vs) SETTINGS max_rows_to_read = 10; +0 diff --git a/tests/queries/0_stateless/05617_limit_zero_skips_range_analysis.sql b/tests/queries/0_stateless/05617_limit_zero_skips_range_analysis.sql new file mode 100644 index 0000000000000..3b73dd82a1b8c --- /dev/null +++ b/tests/queries/0_stateless/05617_limit_zero_skips_range_analysis.sql @@ -0,0 +1,56 @@ +-- Tags: no-parallel-replicas + +-- A `LIMIT 0` over a MergeTree read skips the analysis of its ranges, and keeps the checks that do not depend on them. + +DROP VIEW IF EXISTS vs; +DROP TABLE IF EXISTS t; +DROP TABLE IF EXISTS tp; +DROP TABLE IF EXISTS tr; + +CREATE TABLE t (k UInt64, flag UInt8, v UInt64, INDEX iv v TYPE minmax GRANULARITY 1) +ENGINE = MergeTree PARTITION BY k % 2 ORDER BY (k, flag) SETTINGS index_granularity = 64; +INSERT INTO t SELECT intDiv(number, 1000), number % 2, number % 100 FROM numbers(100000); + +CREATE TABLE tp (k UInt64, v UInt64, PROJECTION p (SELECT * ORDER BY v)) +ENGINE = MergeTree ORDER BY k SETTINGS index_granularity = 64; +INSERT INTO tp SELECT number, number % 1000 FROM numbers(100000); + +CREATE TABLE tr (k UInt64, v UInt64) ENGINE = ReplacingMergeTree ORDER BY k; +INSERT INTO tr SELECT number, 1 FROM numbers(100); +INSERT INTO tr SELECT number, 2 FROM numbers(50, 100); + +-- A sealed view: its plan is optimized on its own, and cloned to build the set below. +CREATE VIEW vs DEFINER = CURRENT_USER SQL SECURITY DEFINER AS SELECT k FROM t WHERE flag LIMIT 0; + +-- { echoOn } + +SELECT count() FROM (EXPLAIN indexes = 1 SELECT k FROM t WHERE flag LIMIT 0) WHERE explain ILIKE '%PrimaryKey%'; +SELECT count() FROM (EXPLAIN indexes = 1 SELECT k FROM t WHERE flag LIMIT 1) WHERE explain ILIKE '%PrimaryKey%'; +SELECT count() FROM (EXPLAIN indexes = 1 SELECT k FROM t WHERE flag LIMIT 0 SETTINGS exact_rows_before_limit = 1) WHERE explain ILIKE '%PrimaryKey%'; + +SELECT k FROM t WHERE flag LIMIT 0 SETTINGS exact_rows_before_limit = 1, output_format_write_statistics = 0 FORMAT JSONCompact; + +SELECT k FROM t WHERE v = 1 LIMIT 0 SETTINGS force_primary_key = 1; -- { serverError INDEX_NOT_USED } +SELECT k FROM t WHERE flag LIMIT 0 SETTINGS force_index_by_date = 1; -- { serverError INDEX_NOT_USED } +SELECT k FROM t WHERE flag LIMIT 0 SETTINGS force_data_skipping_indices = 'iv'; -- { serverError INDEX_NOT_USED } + +SET use_query_condition_cache = 1; +SELECT k FROM t WHERE k < 10 AND flag AND v < 50 LIMIT 0; +SELECT count() FROM t WHERE k < 10 AND flag AND v < 50; + +SELECT k FROM tp WHERE v = 7 LIMIT 0 SETTINGS optimize_use_projections = 1, force_optimize_projection_name = 'p'; +SELECT k FROM tp WHERE v = 7 LIMIT 0 SETTINGS optimize_use_projections = 1, force_optimize_projection = 1; + +SELECT k, v FROM tr FINAL WHERE k > 10 LIMIT 0 FORMAT TSVWithNamesAndTypes; + +SELECT k FROM t WHERE flag LIMIT 0 SETTINGS max_rows_to_read = 10; +SELECT k FROM t WHERE flag LIMIT 1 SETTINGS max_rows_to_read = 10 FORMAT Null; -- { serverError TOO_MANY_ROWS } + +SELECT count() FROM t WHERE k IN (SELECT k FROM vs) SETTINGS max_rows_to_read = 10; + +-- { echoOff } + +DROP VIEW vs; +DROP TABLE t; +DROP TABLE tp; +DROP TABLE tr; From 622c1227e78d805e2217d7c5f78ac0caaf56b844 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 12 Nov 2026 19:44:54 +0000 Subject: [PATCH 175/219] Backport #131054 to 26.8: Keep a PREWHERE guard on a subcolumn ahead of a throwing condition on its sibling --- .../MergeTree/MergeTreeWhereOptimizer.cpp | 68 ++++++++++++++++++- ...prewhere_guard_sibling_subcolumn.reference | 3 + ...05627_prewhere_guard_sibling_subcolumn.sql | 35 ++++++++++ 3 files changed, 103 insertions(+), 3 deletions(-) create mode 100644 tests/queries/0_stateless/05627_prewhere_guard_sibling_subcolumn.reference create mode 100644 tests/queries/0_stateless/05627_prewhere_guard_sibling_subcolumn.sql diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index 174bf4e179e83..c7f4576b487a9 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -365,6 +365,24 @@ static void collectConjuncts(const RPNBuilderTreeNode & node, std::vectortype == ActionsDAG::ActionType::FUNCTION && node->function_base) + { + DataTypesWithConstInfo argument_types; + argument_types.reserve(node->children.size()); + for (const auto * child : node->children) + argument_types.push_back({child->result_type, child->column != nullptr}); + if (node->function_base->isSuitableForShortCircuitArgumentsExecution(argument_types)) + return true; + } + + return std::ranges::any_of(node->children, [](const auto * child) { return conditionMayThrow(child); }); +} + /// Analyze the WHERE expression and populate `res` with one `Condition` per conjunct group. /// Algorithm: /// 1. Flatten the top-level AND chain into individual conjuncts with `collectConjuncts`. @@ -373,6 +391,8 @@ static void collectConjuncts(const RPNBuilderTreeNode & node, std::vector infos; infos.reserve(conjuncts.size()); + const auto & storage_columns_description = storage_metadata->getColumns(); for (const auto & conjunct : conjuncts) { - ConjunctInfo info{conjunct, {}, false, true, false, false}; + ConjunctInfo info{conjunct, {}, {}, false, true, false, false}; collectColumns(conjunct, nullptr, table_columns, info.columns, info.has_invalid_column, info.may_use_primary_index); + for (const auto & column : info.columns) + { + if (auto resolved = storage_columns_description.tryGetColumnOrSubcolumn(GetColumnsOptions::AllPhysical, column)) + info.storage_columns.insert(resolved->getNameInStorage()); + else + info.storage_columns.insert(column); + } + /// Is it possible to use primary index for this condition? For conditions like `lower(country) = 'xx'`, may_use_primary_index will be false. info.viable = !info.has_invalid_column @@ -424,7 +455,37 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTree /// Non-viable conjuncts stay as individual Conditions. /// 'expensive' conjuncts each get their own group. /// + /// A condition that may throw is guarded by the conditions written before it (`and` evaluates it only on + /// the rows they pass), but separate groups are reordered by cost. So when such a condition reads the same + /// physical columns as an earlier condition, e.g. `t.safe != '' AND toUInt64(t.val) > 50` on subcolumns + /// of one `Tuple`, all conditions over these physical columns are grouped by them instead of by the + /// exact columns. The group keeps the written order, and `tryBuildPrewhereSteps` splits it into steps + /// so that the throwing condition runs only on the rows the guard passes. + /// /// We use a simple linear search to find groups (WHERE clauses are short). + std::vector storage_columns_guarding_throwing_condition; + for (size_t i = 0; i < infos.size(); ++i) + { + if (!infos[i].viable || infos[i].expensive || !conditionMayThrow(infos[i].node.getDAGNode())) + continue; + for (size_t j = 0; j < i; ++j) + { + if (infos[j].viable && !infos[j].expensive && infos[j].storage_columns == infos[i].storage_columns) + { + storage_columns_guarding_throwing_condition.push_back(infos[i].storage_columns); + break; + } + } + } + + auto group_key = [&](const ConjunctInfo & info) -> const NameSet & + { + if (std::ranges::find(storage_columns_guarding_throwing_condition, info.storage_columns) + != storage_columns_guarding_throwing_condition.end()) + return info.storage_columns; + return info.columns; + }; + struct Group { NameSet columns; @@ -447,11 +508,12 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTree { if (!infos[i].viable) continue; - auto g = infos[i].expensive ? std::nullopt : find_group_idx(infos[i].columns); + const auto & key = group_key(infos[i]); + auto g = infos[i].expensive ? std::nullopt : find_group_idx(key); if (!g.has_value()) { group_of[i] = groups.size(); - groups.push_back({infos[i].columns, {i}}); + groups.push_back({key, {i}}); } else { diff --git a/tests/queries/0_stateless/05627_prewhere_guard_sibling_subcolumn.reference b/tests/queries/0_stateless/05627_prewhere_guard_sibling_subcolumn.reference new file mode 100644 index 0000000000000..307310c0c0f39 --- /dev/null +++ b/tests/queries/0_stateless/05627_prewhere_guard_sibling_subcolumn.reference @@ -0,0 +1,3 @@ +3266 +3266 +3266 diff --git a/tests/queries/0_stateless/05627_prewhere_guard_sibling_subcolumn.sql b/tests/queries/0_stateless/05627_prewhere_guard_sibling_subcolumn.sql new file mode 100644 index 0000000000000..6319990ed7914 --- /dev/null +++ b/tests/queries/0_stateless/05627_prewhere_guard_sibling_subcolumn.sql @@ -0,0 +1,35 @@ +-- A condition on one subcolumn keeps guarding a throwing condition on a sibling subcolumn of the same column +-- when WHERE is moved to PREWHERE, although the throwing condition reads less and would be ordered first. +-- https://github.com/ClickHouse/ClickHouse/issues/130086 + +SET optimize_move_to_prewhere = 1, query_plan_optimize_prewhere = 1, allow_reorder_prewhere_conditions = 1; +SET optimize_functions_to_subcolumns = 1, use_statistics = 0, move_all_conditions_to_prewhere = 1; + +DROP TABLE IF EXISTS t_guard_tuple; +DROP TABLE IF EXISTS t_guard_json; +DROP TABLE IF EXISTS t_guard_map; + +CREATE TABLE t_guard_tuple (id UInt64, t Tuple(safe String, val String)) +ENGINE = MergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; +INSERT INTO t_guard_tuple SELECT number, if(number % 3 = 0, ('', 'n/a'), (hex(MD5(toString(number))), toString(number % 100))) FROM numbers(10000); + +CREATE TABLE t_guard_json (id UInt64, j JSON(safe String, val String)) +ENGINE = MergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; +INSERT INTO t_guard_json SELECT number, if(number % 3 = 0, '{"safe": "", "val": "n/a"}', toJSONString(map('safe', hex(MD5(toString(number))), 'val', toString(number % 100)))) FROM numbers(10000); + +CREATE TABLE t_guard_map (id UInt64, tags Map(String, String)) +ENGINE = MergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0, + map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets', + map_buckets_strategy = 'constant', max_buckets_in_map = 2, map_buckets_min_avg_size = 0; +INSERT INTO t_guard_map SELECT number, if(number % 3 = 0, map('safe', '', 'val', 'n/a'), map('safe', hex(MD5(toString(number))), 'val', toString(number % 100))) FROM numbers(10000); + +SELECT count() FROM t_guard_tuple WHERE t.safe != '' AND toUInt64(t.val) > 50; +SELECT count() FROM t_guard_json WHERE j.safe != '' AND toUInt64(j.val) > 50; +SELECT count() FROM t_guard_map WHERE tags['safe'] != '' AND toUInt64(tags['val']) > 50; + +DROP TABLE t_guard_tuple; +DROP TABLE t_guard_json; +DROP TABLE t_guard_map; From a1ac4f908553472722f2d801e7ed98d18a61806f Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 12 Nov 2026 19:58:46 +0000 Subject: [PATCH 176/219] Backport #129965 to 26.8: Hide secrets of the inner query of `COPY (query) TO STDOUT` in logs --- src/Common/QueryFuzzer.cpp | 5 ++++ src/Parsers/ASTCopyQuery.cpp | 24 +++++++++++++-- src/Parsers/ASTCopyQuery.h | 13 ++++++++ src/Parsers/ParserCopyQuery.cpp | 7 +++++ src/Parsers/tests/gtest_Parser.cpp | 28 +++++++++++++++++ ...5567_copy_subquery_hides_secrets.reference | 11 +++++++ .../05567_copy_subquery_hides_secrets.sh | 30 +++++++++++++++++++ 7 files changed, 116 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/05567_copy_subquery_hides_secrets.reference create mode 100755 tests/queries/0_stateless/05567_copy_subquery_hides_secrets.sh diff --git a/src/Common/QueryFuzzer.cpp b/src/Common/QueryFuzzer.cpp index fa45aaeb1a3ab..a22c41f724860 100644 --- a/src/Common/QueryFuzzer.cpp +++ b/src/Common/QueryFuzzer.cpp @@ -41,6 +41,7 @@ #include #include #include +#include #include #include #include @@ -8525,6 +8526,10 @@ void QueryFuzzer::fuzz(ASTPtr & ast) } fuzzChildrenWithAlias(*describe, describe->table_expression); } + else if (auto * copy_query = typeid_cast(ast.get())) + { + fuzzChildrenWithAlias(*copy_query, copy_query->subquery_ast); + } else if (auto * drop_function = typeid_cast(ast.get())) { if (fuzz_rand() % 10 == 0) diff --git a/src/Parsers/ASTCopyQuery.cpp b/src/Parsers/ASTCopyQuery.cpp index 4e15e01210ac4..a1c6e71860d05 100644 --- a/src/Parsers/ASTCopyQuery.cpp +++ b/src/Parsers/ASTCopyQuery.cpp @@ -5,18 +5,38 @@ namespace DB { -void ASTCopyQuery::formatImpl(WriteBuffer & ostr, const FormatSettings &, FormatState &, FormatStateStacked) const +void ASTCopyQuery::formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState &, FormatStateStacked) const { - ostr << table_name; + if (subquery_ast) + { + /// The inner query is a standalone query in its own parentheses. Format it on one line, with + /// the default settings, except that its secrets are hidden unless `show_secrets` is set. + /// Only `COPY (query) TO` is parsed, so the direction is always `TO STDOUT`. + FormatSettings subquery_settings(/*one_line_=*/true); + subquery_settings.show_secrets = settings.show_secrets; + ostr << "COPY ("; + subquery_ast->format(ostr, subquery_settings); + ostr << ") TO STDOUT"; + } + else + ostr << table_name; } ASTPtr ASTCopyQuery::clone() const { auto res = make_intrusive(*this); res->children.clear(); + res->subquery_ast = nullptr; + if (subquery_ast) + res->setSubquery(subquery_ast->clone()); return res; } +void ASTCopyQuery::setSubquery(ASTPtr subquery_ast_) +{ + set(subquery_ast, std::move(subquery_ast_)); +} + String toString(ASTCopyQuery::Formats format) { switch (format) diff --git a/src/Parsers/ASTCopyQuery.h b/src/Parsers/ASTCopyQuery.h index f6c95190f663d..28a8ad0b4c7b5 100644 --- a/src/Parsers/ASTCopyQuery.h +++ b/src/Parsers/ASTCopyQuery.h @@ -24,6 +24,9 @@ class ASTCopyQuery : public IAST } type{}; String table_name; + /// The inner query of `COPY (query) TO STDOUT`, null for `COPY table`. It is the only child of this + /// node, so the secrets in it are found and hidden as in any other query. + ASTPtr subquery_ast; Strings column_names; enum class Formats : uint8_t { @@ -38,8 +41,18 @@ class ASTCopyQuery : public IAST QueryKind getQueryKind() const override { return QueryKind::Copy; } + /// Set `subquery_ast` and add it to `children`. + void setSubquery(ASTPtr subquery_ast_); + protected: void formatImpl(WriteBuffer & ostr, const FormatSettings & settings, FormatState & state, FormatStateStacked frame) const override; + + /// `subquery_ast` is a separate pointer to the node that `children` holds too, so a visitor that + /// replaces the child has to be able to repair it. + void forEachPointerToChild(std::function *)> f) override + { + f(nullptr, &subquery_ast); + } }; String toString(ASTCopyQuery::Formats format); diff --git a/src/Parsers/ParserCopyQuery.cpp b/src/Parsers/ParserCopyQuery.cpp index 77e2a5dddf0ba..36e68eff62518 100644 --- a/src/Parsers/ParserCopyQuery.cpp +++ b/src/Parsers/ParserCopyQuery.cpp @@ -116,6 +116,13 @@ bool ParserCopyQuery::parseImpl(Pos & pos, ASTPtr & node, Expected & expected) return false; } + /// `COPY (query) TO STDOUT` - keep the inner query as the child, so that the `COPY` is formatted and + /// logged from it and the secrets in it are hidden. + ASTPtr inner_query = name_or_expr; + if (const auto * subquery_node = name_or_expr->as()) + inner_query = subquery_node->children.at(0); + copy_element->setSubquery(inner_query); + saved_pos = pos; if (s_to.ignore(pos, expected)) { diff --git a/src/Parsers/tests/gtest_Parser.cpp b/src/Parsers/tests/gtest_Parser.cpp index 2039740746589..de122e63ae8e3 100644 --- a/src/Parsers/tests/gtest_Parser.cpp +++ b/src/Parsers/tests/gtest_Parser.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -55,6 +56,33 @@ TEST(Lexer, NullInputWithMaxQuerySize) EXPECT_EQ(TokenType::EndOfStream, token.type); } +/// The inner query of `COPY (query) TO STDOUT` is the only child of the node, also after `clone`. +TEST(ParserCopyQuery, SubqueryIsTheOnlyChild) +{ + const String query = "COPY (SELECT a, b FROM db.t WHERE x = 1) TO STDOUT WITH (FORMAT csv)"; + const String formatted = "COPY (SELECT a, b FROM db.t WHERE x = 1) TO STDOUT"; + + ParserQuery parser(query.data() + query.size()); + ASTPtr ast = parseQuery(parser, query, "", 0, 0, 0); + ASSERT_NE(nullptr, ast); + + const auto * copy_query = ast->as(); + ASSERT_NE(nullptr, copy_query); + ASSERT_NE(nullptr, copy_query->subquery_ast.get()); + ASSERT_EQ(1u, copy_query->children.size()); + EXPECT_EQ(copy_query->subquery_ast.get(), copy_query->children[0].get()); + EXPECT_EQ(formatted, ast->formatWithSecretsOneLine()); + + ASTPtr cloned = ast->clone(); + const auto * cloned_copy_query = cloned->as(); + ASSERT_NE(nullptr, cloned_copy_query); + ASSERT_EQ(1u, cloned_copy_query->children.size()); + EXPECT_EQ(cloned_copy_query->subquery_ast.get(), cloned_copy_query->children[0].get()); + EXPECT_NE(copy_query->subquery_ast.get(), cloned_copy_query->subquery_ast.get()); + EXPECT_EQ(formatted, cloned->formatWithSecretsOneLine()); + EXPECT_EQ(ast->getTreeHash(/*ignore_aliases=*/ false), cloned->getTreeHash(/*ignore_aliases=*/ false)); +} + /// The output-option children (INTO OUTFILE, COMPRESSION, FORMAT, SETTINGS) must end up /// in the same canonical order whether the AST is freshly parsed, cloned, or obtained by /// a format+reparse roundtrip. Otherwise the tree hash differs across these paths, which diff --git a/tests/queries/0_stateless/05567_copy_subquery_hides_secrets.reference b/tests/queries/0_stateless/05567_copy_subquery_hides_secrets.reference new file mode 100644 index 0000000000000..cf9fad47c615c --- /dev/null +++ b/tests/queries/0_stateless/05567_copy_subquery_hides_secrets.reference @@ -0,0 +1,11 @@ +-- formatted +COPY (SELECT * FROM remote('127.0.0.1', 'db', 't', 'u', '[HIDDEN]')) TO STDOUT +COPY (SELECT * FROM remote('127.0.0.1', 'db', 't', 'u', '[HIDDEN]')) TO STDOUT +COPY (SELECT * FROM remote('127.0.0.1', 'db', 't', 'u', 'CopySubquerySecret05567')) TO STDOUT +-- logged +UNKNOWN_TYPE_OF_QUERY +ExceptionBeforeStart COPY (SELECT * FROM remote(\'127.0.0.1\', \'db\', \'t\', \'u\', \'[HIDDEN]\')) TO STDOUT 1 1 +0 1 +-- explain +QueryStart 1 1 +QueryFinish 1 1 diff --git a/tests/queries/0_stateless/05567_copy_subquery_hides_secrets.sh b/tests/queries/0_stateless/05567_copy_subquery_hides_secrets.sh new file mode 100755 index 0000000000000..58a431c164b5d --- /dev/null +++ b/tests/queries/0_stateless/05567_copy_subquery_hides_secrets.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The secrets in the inner query of `COPY (query) TO STDOUT` are hidden when the query is formatted +# and logged, as they are in the same query outside of `COPY`. +secret='CopySubquerySecret05567' +copy_query="COPY (SELECT * FROM remote('127.0.0.1', 'db', 't', 'u', '$secret')) TO STDOUT" + +echo "-- formatted" +$CLICKHOUSE_FORMAT --oneline --query "$copy_query" +$CLICKHOUSE_FORMAT --oneline --query "$copy_query WITH (FORMAT csv, HEADER)" +$CLICKHOUSE_FORMAT --oneline --show_secrets --query "$copy_query" + +# Only the PostgreSQL protocol serves `COPY`, so the native protocol rejects it after the query is logged. +echo "-- logged" +query_id="${CLICKHOUSE_DATABASE}_copy_subquery_hides_secrets" +$CLICKHOUSE_CLIENT --log_queries=1 --log_formatted_queries=1 --query_id "$query_id" --query "$copy_query" 2>&1 | grep -o -m1 UNKNOWN_TYPE_OF_QUERY +$CLICKHOUSE_CLIENT --query "SYSTEM FLUSH LOGS query_log, text_log" +$CLICKHOUSE_CLIENT --query "SELECT type, query, position(formatted_query, '[HIDDEN]') > 0, position(concat(query, formatted_query), '$secret') = 0 FROM system.query_log WHERE current_database = currentDatabase() AND query_id = '$query_id' AND event_date >= yesterday() SETTINGS enable_parallel_replicas = 0" +$CLICKHOUSE_CLIENT --query "SELECT countIf(message LIKE '%$secret%'), countIf(logger_name = 'executeQuery' AND message LIKE '%[HIDDEN]%') > 0 FROM system.text_log WHERE query_id = '$query_id' AND event_date >= yesterday() SETTINGS enable_parallel_replicas = 0" + +# `EXPLAIN AST` accepts `COPY`, so this one runs on the native protocol. +echo "-- explain" +explain_query_id="${CLICKHOUSE_DATABASE}_copy_subquery_explain" +$CLICKHOUSE_CLIENT --log_queries=1 --log_formatted_queries=1 --query_id "$explain_query_id" --query "EXPLAIN AST $copy_query" > /dev/null +$CLICKHOUSE_CLIENT --query "SYSTEM FLUSH LOGS query_log" +$CLICKHOUSE_CLIENT --query "SELECT type, position(query, '[HIDDEN]') > 0, position(concat(query, formatted_query), '$secret') = 0 FROM system.query_log WHERE current_database = currentDatabase() AND query_id = '$explain_query_id' AND event_date >= yesterday() ORDER BY event_time_microseconds SETTINGS enable_parallel_replicas = 0" From 01d1aa0989becbd52ca22a33b5cb62445a2a12dc Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Thu, 12 Nov 2026 20:05:58 +0000 Subject: [PATCH 177/219] Backport #130997 to 26.8: Keep a delayed task valid when overwriting its deadline fails --- src/Core/BackgroundSchedulePool.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Core/BackgroundSchedulePool.cpp b/src/Core/BackgroundSchedulePool.cpp index fbcfcc137c5df..029c398c8329a 100644 --- a/src/Core/BackgroundSchedulePool.cpp +++ b/src/Core/BackgroundSchedulePool.cpp @@ -593,10 +593,12 @@ void BackgroundSchedulePool::scheduleDelayedTask(TaskInfo & task, size_t ms, std { std::lock_guard lock(delayed_tasks_mutex); + /// Insert before erasing the old entry: if the insertion throws, the task keeps its old entry and `task.iterator` stays valid. + auto it = delayed_tasks.emplace(current_time + delayToMicroseconds(ms), task.shared_from_this()); if (task.delayed) delayed_tasks.erase(task.iterator); - task.iterator = delayed_tasks.emplace(current_time + delayToMicroseconds(ms), task.shared_from_this()); + task.iterator = it; task.delayed = true; } From 333e7ab6d09b673305bce71ac65968321dd93bd9 Mon Sep 17 00:00:00 2001 From: Alexei Fedotov Date: Fri, 13 Nov 2026 04:55:41 +0100 Subject: [PATCH 178/219] Regenerate release_branches.yml Co-Authored-By: Claude Sonnet 5.5 --- .github/workflows/release_branches.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index e09c72f617696..db5db0386c040 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -2185,7 +2185,6 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Stress test (arm_msan)' --workflow "ReleaseBranchCI" --ci --timestamp - finish_workflow: runs-on: [self-hosted, style-checker-aarch64] needs: [build_amd_asan_ubsan, build_amd_compat, build_amd_darwin, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_darwin, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_arm_v80compat, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_1_8, integration_tests_amd_asan_ubsan_db_disk_2_8, integration_tests_amd_asan_ubsan_db_disk_3_8, integration_tests_amd_asan_ubsan_db_disk_4_8, integration_tests_amd_asan_ubsan_db_disk_5_8, integration_tests_amd_asan_ubsan_db_disk_6_8, integration_tests_amd_asan_ubsan_db_disk_7_8, integration_tests_amd_asan_ubsan_db_disk_8_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, sign_macos_binary_amd_darwin, sign_macos_binary_arm_darwin, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_3_3, stateless_tests_amd_asan_ubsan_distributed_plan_parallel, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, unit_tests_asan_ubsan, unit_tests_msan, unit_tests_tsan] From 0d9b90c64059f5cf957c1d421e047fd587a35933 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 13 Nov 2026 14:04:10 +0000 Subject: [PATCH 179/219] Backport #118697 to 26.8: Fix optimize_rewrite_aggregate_function_with_if defeating short-circuit evaluation --- .../RewriteAggregateFunctionWithIfPass.cpp | 71 +++++++++++++++++++ ...e_function_with_if_short_circuit.reference | 22 ++++++ ...gregate_function_with_if_short_circuit.sql | 60 ++++++++++++++++ 3 files changed, 153 insertions(+) create mode 100644 tests/queries/0_stateless/05137_rewrite_aggregate_function_with_if_short_circuit.reference create mode 100644 tests/queries/0_stateless/05137_rewrite_aggregate_function_with_if_short_circuit.sql diff --git a/src/Analyzer/Passes/RewriteAggregateFunctionWithIfPass.cpp b/src/Analyzer/Passes/RewriteAggregateFunctionWithIfPass.cpp index c335644ebf235..6984c8f4e670e 100644 --- a/src/Analyzer/Passes/RewriteAggregateFunctionWithIfPass.cpp +++ b/src/Analyzer/Passes/RewriteAggregateFunctionWithIfPass.cpp @@ -22,6 +22,7 @@ namespace DB namespace Setting { extern const SettingsBool optimize_rewrite_aggregate_function_with_if; + extern const SettingsShortCircuitFunctionEvaluation short_circuit_function_evaluation; } namespace @@ -67,6 +68,64 @@ bool aggregateFunctionPreservesNullPayload(const AggregateFunctionPtr & function return function->getOwnNullAdapter(function, {nullable_argument_type}, function->getParameters(), properties) == function; } +/// A branch of `if` runs only on the rows its condition selects when lazy execution picks it up. +/// A nested short-circuit mask is not assumed to protect the rows the outer condition excluded, so +/// the walk continues through it. +bool branchCanBeSkippedByShortCircuit(const QueryTreeNodePtr & branch, bool every_function_is_lazy) +{ + std::vector nodes_to_process; + nodes_to_process.push_back(branch.get()); + + while (!nodes_to_process.empty()) + { + const auto * node = nodes_to_process.back(); + nodes_to_process.pop_back(); + + /// A constant is already evaluated when the query tree is built, so it cannot throw per row. + if (node->as()) + continue; + + /// A table expression or a subquery in an argument slot is evaluated in its own scope, as a + /// set or a scalar, so lazy execution of this expression never reaches inside it. + switch (node->getNodeType()) + { + case QueryTreeNodeType::TABLE: + case QueryTreeNodeType::TABLE_FUNCTION: + case QueryTreeNodeType::QUERY: + case QueryTreeNodeType::UNION: + continue; + default: + break; + } + + if (const auto * function_node = node->as()) + { + if (auto function_base = function_node->getFunction()) + { + if (every_function_is_lazy) + return true; + + /// getArgumentColumns describes each slot the way the resolver did, so the `in` + /// right-hand side is reported as a set instead of being asked for a result type. + const auto argument_columns = function_node->getArgumentColumns(); + DataTypesWithConstInfo argument_types; + argument_types.reserve(argument_columns.size()); + for (const auto & argument_column : argument_columns) + argument_types.push_back({argument_column.type, argument_column.column != nullptr}); + + if (function_base->isSuitableForShortCircuitArgumentsExecution(argument_types)) + return true; + } + } + + for (const auto & child : node->getChildren()) + if (child) + nodes_to_process.push_back(child.get()); + } + + return false; +} + class RewriteAggregateFunctionWithIfVisitor : public InDepthQueryTreeVisitorWithContext { public: @@ -115,6 +174,18 @@ class RewriteAggregateFunctionWithIfVisitor : public InDepthQueryTreeVisitorWith auto if_arguments_nodes = if_node->getArguments().getNodes(); auto * first_const_node = if_arguments_nodes[1]->as(); auto * second_const_node = if_arguments_nodes[2]->as(); + + const auto short_circuit = getSettings()[Setting::short_circuit_function_evaluation]; + if (short_circuit != ShortCircuitFunctionEvaluation::DISABLE && (second_const_node || first_const_node)) + { + /// The -If combinator evaluates its argument for every row, so lifting a branch that lazy + /// execution would have skipped makes a throwing expression run on the rows the condition + /// excluded. + const auto & lifted_branch = second_const_node ? if_arguments_nodes[1] : if_arguments_nodes[2]; + if (branchCanBeSkippedByShortCircuit(lifted_branch, short_circuit == ShortCircuitFunctionEvaluation::FORCE_ENABLE)) + return; + } + if (second_const_node) { const auto & second_const_value = second_const_node->getValue(); diff --git a/tests/queries/0_stateless/05137_rewrite_aggregate_function_with_if_short_circuit.reference b/tests/queries/0_stateless/05137_rewrite_aggregate_function_with_if_short_circuit.reference new file mode 100644 index 0000000000000..dce6fe983969c --- /dev/null +++ b/tests/queries/0_stateless/05137_rewrite_aggregate_function_with_if_short_circuit.reference @@ -0,0 +1,22 @@ +87 +87 +87 +21.75 +4 +13 +91 +87 +1 +2 +1 +1 +1 +1 +1 +0 +force_enable +1 +0 +1 +disable +1 diff --git a/tests/queries/0_stateless/05137_rewrite_aggregate_function_with_if_short_circuit.sql b/tests/queries/0_stateless/05137_rewrite_aggregate_function_with_if_short_circuit.sql new file mode 100644 index 0000000000000..4892d394b77ee --- /dev/null +++ b/tests/queries/0_stateless/05137_rewrite_aggregate_function_with_if_short_circuit.sql @@ -0,0 +1,60 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/85784 +-- https://github.com/ClickHouse/ClickHouse/issues/118625 + +-- At short_circuit_function_evaluation = 'disable' the un-rewritten queries throw as well, so every +-- expected value below requires short-circuit evaluation to be on. +-- optimize_if_chain_to_multiif is pinned off: it flattens the nested `if` below into `multiIf` before +-- this pass runs, and the pass matches only `if`, so the nested witness would stop exercising it. +SET enable_analyzer = 1, optimize_rewrite_aggregate_function_with_if = 1, + optimize_rewrite_sum_if_to_count_if = 0, short_circuit_function_evaluation = 'enable', + optimize_if_chain_to_multiif = 0; + +SELECT sum(if(number = 0, 0, intDiv(42, number))) FROM numbers(5); +SELECT sum(if(number = 0, 0, intDiv(42, number))) FROM numbers(5) SETTINGS optimize_rewrite_aggregate_function_with_if = 0; +SELECT sum(if(number != 0, intDiv(42, number), 0)) FROM numbers(5); +SELECT avg(if(number = 0, NULL, intDiv(42, number))) FROM numbers(5); +SELECT count(if(number = 0, NULL, intDiv(42, number))) FROM numbers(5); + +SELECT sum(if((number = 0) OR ((number % 6) = 0), 0, intDiv(6, number))) FROM numbers(6); +SELECT sum(if(number = 0, 0, intDiv(42, number) + 1)) FROM numbers(5); +SELECT sum(if(number = 0, 0, if(number > 100, 0, intDiv(42, number)))) FROM numbers(5); +-- A conversion is protected at 'enable' too: the guard asks the generic suitability predicate, not a +-- list of arithmetic functions. +SELECT count(if(s = 'bad', NULL, toDate(s))) FROM (SELECT arrayJoin(['2020-01-01', 'bad']) AS s); + +-- A subquery on the right of IN is not an expression: it has no result type, and laziness does not +-- reach inside it. +SELECT sum(if(number = 0, 0, number IN (SELECT 1 UNION ALL SELECT 2))) FROM numbers(5); +SELECT count() > 0 FROM ( + EXPLAIN QUERY TREE SELECT sum(if(number = 0, 0, number IN (SELECT intDiv(42, number) FROM numbers(1, 5)))) FROM numbers(5) +) WHERE explain LIKE '%sumIf%'; +SELECT count() > 0 FROM ( + EXPLAIN QUERY TREE SELECT sum(if(number = 0, 0, number IN (SELECT intDiv(42, number) FROM numbers(1, 5) UNION ALL SELECT 1))) FROM numbers(5) +) WHERE explain LIKE '%sumIf%'; + +-- The rewrite is still applied when the lifted branch was never evaluated lazily. +SELECT count() > 0 FROM (EXPLAIN QUERY TREE SELECT sum(if(number = 0, 0, number)) FROM numbers(5)) WHERE explain LIKE '%sumIf%'; +SELECT count() > 0 FROM (EXPLAIN QUERY TREE SELECT sum(if(number = 0, number, 0)) FROM numbers(5)) WHERE explain LIKE '%sumIf%'; +SELECT count() > 0 FROM (EXPLAIN QUERY TREE SELECT sum(if(number = 0, 0, number * 2)) FROM numbers(5)) WHERE explain LIKE '%sumIf%'; +SELECT count() FROM (EXPLAIN QUERY TREE SELECT sum(if(number = 0, 0, intDiv(42, number))) FROM numbers(5)) WHERE explain LIKE '%sumIf%'; + +SELECT 'force_enable'; +-- addDate reports no short-circuit suitability, so only forced laziness protects it. +SELECT count(if(s = 'bad', NULL, addDate(s, INTERVAL 1 DAY))) +FROM (SELECT arrayJoin(['2020-01-01', 'bad']) AS s) +SETTINGS short_circuit_function_evaluation = 'force_enable'; +-- The same query throws at 'enable': nothing was protected there, so the rewrite changes nothing. +SELECT count(if(s = 'bad', NULL, addDate(s, INTERVAL 1 DAY))) +FROM (SELECT arrayJoin(['2020-01-01', 'bad']) AS s); -- { serverError CANNOT_PARSE_DATETIME } +SELECT count() FROM (EXPLAIN QUERY TREE SELECT sum(if(number = 0, 0, number * 2)) FROM numbers(5)) +WHERE explain LIKE '%sumIf%' +SETTINGS short_circuit_function_evaluation = 'force_enable'; +-- A bare column is still rewritten at forced laziness: the guard declines only for a function. +SELECT count() > 0 FROM (EXPLAIN QUERY TREE SELECT sum(if(number = 0, 0, number)) FROM numbers(5)) +WHERE explain LIKE '%sumIf%' +SETTINGS short_circuit_function_evaluation = 'force_enable'; + +SELECT 'disable'; +SELECT count() > 0 FROM (EXPLAIN QUERY TREE SELECT sum(if(number = 0, 0, intDiv(42, number))) FROM numbers(5)) +WHERE explain LIKE '%sumIf%' +SETTINGS short_circuit_function_evaluation = 'disable'; From f6349089d23032baf83484a222254f5916f382bf Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 13 Nov 2026 15:38:51 +0000 Subject: [PATCH 180/219] Backport #119192 to 26.8: Analyzer: resolve ambiguous column to alias under prefer_column_name_to_alias --- src/Analyzer/Resolve/IdentifierLookup.h | 23 +++- src/Analyzer/Resolve/IdentifierResolver.cpp | 19 ++++ src/Analyzer/Resolve/QueryAnalyzer.cpp | 29 ++++- src/Core/Settings.cpp | 16 +++ ...umn_name_to_alias_ambiguous_join.reference | 18 ++++ ...er_column_name_to_alias_ambiguous_join.sql | 101 ++++++++++++++++++ 6 files changed, 201 insertions(+), 5 deletions(-) create mode 100644 tests/queries/0_stateless/05153_prefer_column_name_to_alias_ambiguous_join.reference create mode 100644 tests/queries/0_stateless/05153_prefer_column_name_to_alias_ambiguous_join.sql diff --git a/src/Analyzer/Resolve/IdentifierLookup.h b/src/Analyzer/Resolve/IdentifierLookup.h index ebd686782e63d..f2cc7c689f23f 100644 --- a/src/Analyzer/Resolve/IdentifierLookup.h +++ b/src/Analyzer/Resolve/IdentifierLookup.h @@ -56,6 +56,12 @@ struct IdentifierLookup /// lookup must not share an identifier resolve cache entry with an ordinary expression lookup. bool is_matcher_qualifier = false; + /// Join tree resolution of an expression identifier that binds to columns of several joined tables + /// returns a result with `ambiguous_in_join_tree` set instead of throwing `AMBIGUOUS_IDENTIFIER`. + /// The caller decides whether another resolution path (aliases) can take over. + /// Participates in comparison and hashing for the same reason as `is_matcher_qualifier`. + bool allow_ambiguous_join_tree_identifier = false; + bool isExpressionLookup() const { return lookup_context == IdentifierLookupContext::EXPRESSION; @@ -81,7 +87,8 @@ inline bool operator==(const IdentifierLookup & lhs, const IdentifierLookup & rh { return lhs.identifier.getFullName() == rhs.identifier.getFullName() && lhs.lookup_context == rhs.lookup_context - && lhs.is_matcher_qualifier == rhs.is_matcher_qualifier; + && lhs.is_matcher_qualifier == rhs.is_matcher_qualifier + && lhs.allow_ambiguous_join_tree_identifier == rhs.allow_ambiguous_join_tree_identifier; } [[maybe_unused]] inline bool operator!=(const IdentifierLookup & lhs, const IdentifierLookup & rhs) @@ -95,7 +102,8 @@ struct IdentifierLookupHash { return std::hash()(identifier_lookup.identifier.getFullName()) ^ static_cast(identifier_lookup.lookup_context) - ^ (static_cast(identifier_lookup.is_matcher_qualifier) << 8); + ^ (static_cast(identifier_lookup.is_matcher_qualifier) << 8) + ^ (static_cast(identifier_lookup.allow_ambiguous_join_tree_identifier) << 9); } }; @@ -134,6 +142,15 @@ struct IdentifierResolveResult QueryTreeNodePtr resolved_identifier; IdentifierResolvePlace resolve_place = IdentifierResolvePlace::NONE; + /// Only for lookups with `allow_ambiguous_join_tree_identifier`: the identifier binds to columns + /// of several joined tables, so `resolved_identifier` is empty and no join tree column may win. + bool ambiguous_in_join_tree = false; + + static IdentifierResolveResult ambiguousInJoinTree() + { + return { .resolved_identifier = nullptr, .resolve_place = IdentifierResolvePlace::NONE, .ambiguous_in_join_tree = true }; + } + explicit operator bool() const { chassert(check_invariant()); @@ -174,7 +191,7 @@ struct IdentifierResolveResult { if (!resolved_identifier) { - buffer << "unresolved"; + buffer << (ambiguous_in_join_tree ? "ambiguous" : "unresolved"); return; } diff --git a/src/Analyzer/Resolve/IdentifierResolver.cpp b/src/Analyzer/Resolve/IdentifierResolver.cpp index e64cd591cd545..b39959dc9d00c 100644 --- a/src/Analyzer/Resolve/IdentifierResolver.cpp +++ b/src/Analyzer/Resolve/IdentifierResolver.cpp @@ -1148,6 +1148,8 @@ IdentifierResolveResult IdentifierResolver::tryResolveIdentifierFromCrossJoin(co { auto expr = from_cross_join_node.getTableExpressionTypedAt(i); auto identifier = tryResolveIdentifierFromJoinTreeNode(identifier_lookup, expr, scope); + if (identifier.ambiguous_in_join_tree) + return identifier; if (!identifier) continue; @@ -1184,6 +1186,9 @@ IdentifierResolveResult IdentifierResolver::tryResolveIdentifierFromCrossJoin(co } else if (!prefer_left_table) { + if (identifier_lookup.allow_ambiguous_join_tree_identifier) + return IdentifierResolveResult::ambiguousInJoinTree(); + throw Exception(ErrorCodes::AMBIGUOUS_IDENTIFIER, "JOIN {} ambiguous identifier '{}'. In scope {}", table_expression_node->formatASTForErrorMessage(), @@ -1419,6 +1424,7 @@ IdentifierResolveResult IdentifierResolver::tryResolveIdentifierFromJoin(const I } } + bool ambiguous_in_join_tree = false; auto try_resolve_identifier_from_join_tree_node = [&](const TableExpressionNodePtr & join_tree_node, bool may_be_override_by_using_column) { /// scope.join_using_columns holds raw pointers to this stack-local map. The pop must run @@ -1434,6 +1440,7 @@ IdentifierResolveResult IdentifierResolver::tryResolveIdentifierFromJoin(const I SCOPE_EXIT({ if (pushed) scope.join_using_columns.pop_back(); }); auto res = tryResolveIdentifierFromJoinTreeNode(identifier_lookup, join_tree_node, scope); + ambiguous_in_join_tree |= res.ambiguous_in_join_tree; return std::move(res.resolved_identifier); }; @@ -1468,6 +1475,9 @@ IdentifierResolveResult IdentifierResolver::tryResolveIdentifierFromJoin(const I if (!binds_left || binds_right) right_resolved_identifier = try_resolve_identifier_from_join_tree_node(from_join_node.getRightTableExpressionNodeTyped(), join_kind != JoinKind::Right); + if (ambiguous_in_join_tree) + return IdentifierResolveResult::ambiguousInJoinTree(); + /** The alias / table-name qualifier can restrict resolution to one side while the identifier is * actually a database-qualified reference (`db.table.column`) to the pruned side (the same token * is the table name of one side and the database name of the other). The database-qualified @@ -1481,6 +1491,9 @@ IdentifierResolveResult IdentifierResolver::tryResolveIdentifierFromJoin(const I right_resolved_identifier = try_resolve_identifier_from_join_tree_node(from_join_node.getRightTableExpressionNodeTyped(), join_kind != JoinKind::Right); else if (binds_right && qualifierBindsToJoinSubtree(from_join_node.getLeftTableExpressionNodeTyped(), identifier_lookup.identifier, scope, /*database_qualified=*/ true)) left_resolved_identifier = try_resolve_identifier_from_join_tree_node(from_join_node.getLeftTableExpressionNodeTyped(), join_kind == JoinKind::Right); + + if (ambiguous_in_join_tree) + return IdentifierResolveResult::ambiguousInJoinTree(); } if (!identifier_lookup.isExpressionLookup()) @@ -1684,6 +1697,10 @@ IdentifierResolveResult IdentifierResolver::tryResolveIdentifierFromJoin(const I resolved_side = JoinTableSide::Left; resolved_identifier = left_resolved_identifier; } + else if (identifier_lookup.allow_ambiguous_join_tree_identifier) + { + return IdentifierResolveResult::ambiguousInJoinTree(); + } else { throw Exception(ErrorCodes::AMBIGUOUS_IDENTIFIER, @@ -1912,6 +1929,8 @@ IdentifierResolveResult IdentifierResolver::tryResolveIdentifierFromArrayJoin(co { const auto & from_array_join_node = table_expression_node->as(); auto resolve_result = tryResolveIdentifierFromJoinTreeNode(identifier_lookup, from_array_join_node.getTableExpressionNodeTyped(), scope); + if (resolve_result.ambiguous_in_join_tree) + return resolve_result; if (scope.table_expressions_in_resolve_process.contains(table_expression_node.get()) || !identifier_lookup.isExpressionLookup()) return resolve_result; diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index 443e177066048..19b0e5a554eb5 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -1570,15 +1570,40 @@ IdentifierResolveResult QueryAnalyzer::tryResolveIdentifier(const IdentifierLook if (unlikely(prefer_column_name_to_alias)) { + bool can_check_aliases = identifier_resolve_context.allow_to_check_aliases && !already_in_resolve_process; + bool ambiguous_in_join_tree = false; + if (identifier_resolve_context.allow_to_check_join_tree) { - resolve_result = identifier_resolver.tryResolveIdentifierFromJoinTree(identifier_lookup, scope); + /** A column name that is ambiguous between joined tables but also names an alias resolves to the alias, + * as the old analyzer did (`JoinToSubqueryTransformVisitor`, `allow_ambiguous = got_alias`). + * Example: SELECT t1.x AS x FROM t1, t2, t3 WHERE ... ORDER BY x + */ + bool alias_can_take_over = can_check_aliases + && identifier_lookup.isExpressionLookup() + && scope.aliases.find(identifier_lookup, ScopeAliases::FindOption::FIRST_NAME) != nullptr; + + if (alias_can_take_over) + { + auto tolerant_lookup = identifier_lookup; + tolerant_lookup.allow_ambiguous_join_tree_identifier = true; + resolve_result = identifier_resolver.tryResolveIdentifierFromJoinTree(tolerant_lookup, scope); + ambiguous_in_join_tree = resolve_result.ambiguous_in_join_tree; + } + else + { + resolve_result = identifier_resolver.tryResolveIdentifierFromJoinTree(identifier_lookup, scope); + } } - if (identifier_resolve_context.allow_to_check_aliases && !resolve_result.resolved_identifier && !already_in_resolve_process) + if (can_check_aliases && !resolve_result.resolved_identifier) { resolve_result = tryResolveIdentifierFromAliases(identifier_lookup, scope, identifier_resolve_context); } + + /// No alias took over: resolve from the join tree again to throw the original `AMBIGUOUS_IDENTIFIER`. + if (ambiguous_in_join_tree && !resolve_result.resolved_identifier) + resolve_result = identifier_resolver.tryResolveIdentifierFromJoinTree(identifier_lookup, scope); } else { diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 21c96f7a0d4cd..6d3bfc254a2bb 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -3126,6 +3126,22 @@ Possible values: - 0 — The column name is substituted with the alias. - 1 — The column name is not substituted with the alias. +If the column name is ambiguous between joined tables and an alias with the same name exists, the alias is used: + +```sql +SET prefer_column_name_to_alias = 1; +SELECT t1.id + 10 AS id, id AS x +FROM (SELECT 1 AS id) AS t1, (SELECT 1 AS k) AS t2, (SELECT 2 AS id) AS t3; +``` + +```text +┌─id─┬──x─┐ +│ 11 │ 11 │ +└────┴────┘ +``` + +Here `id` in `id AS x` is a column of both `t1` and `t3`, so it resolves to the alias `t1.id + 10`. + **Example** The difference between enabled and disabled: diff --git a/tests/queries/0_stateless/05153_prefer_column_name_to_alias_ambiguous_join.reference b/tests/queries/0_stateless/05153_prefer_column_name_to_alias_ambiguous_join.reference new file mode 100644 index 0000000000000..e063858fc003c --- /dev/null +++ b/tests/queries/0_stateless/05153_prefer_column_name_to_alias_ambiguous_join.reference @@ -0,0 +1,18 @@ +1 1 +2 2 +1 1 +2 2 +11 1 +12 2 +2 200 +2 200 +11 11 +12 12 +12 12 +11 1 +12 2 +12 2 +1 buyer 1 +buyer 2 buyer 1 +100 pubA +200 pubB diff --git a/tests/queries/0_stateless/05153_prefer_column_name_to_alias_ambiguous_join.sql b/tests/queries/0_stateless/05153_prefer_column_name_to_alias_ambiguous_join.sql new file mode 100644 index 0000000000000..2b0defaa2a3c6 --- /dev/null +++ b/tests/queries/0_stateless/05153_prefer_column_name_to_alias_ambiguous_join.sql @@ -0,0 +1,101 @@ +-- An unqualified column name that is ambiguous between three or more joined tables +-- resolves to the alias with the same name under prefer_column_name_to_alias = 1, +-- as the old analyzer did. + +SET enable_analyzer = 1; +SET prefer_column_name_to_alias = 1; + +DROP TABLE IF EXISTS ranged_items; +DROP TABLE IF EXISTS product; +DROP TABLE IF EXISTS store; + +CREATE TABLE ranged_items (SNO Int32, skuID Int32) ENGINE = Memory; +CREATE TABLE product (PIN Int32) ENGINE = Memory; +CREATE TABLE store (SNO Int32) ENGINE = Memory; +INSERT INTO ranged_items VALUES (1, 10), (2, 20), (2, 20); +INSERT INTO product VALUES (10), (20); +INSERT INTO store VALUES (1), (2); + +-- comma join, alias name used in ORDER BY +SELECT ranged_items.SNO AS SNO, count() AS c +FROM ranged_items, product, store +WHERE ranged_items.skuID = product.PIN AND ranged_items.SNO = store.SNO +GROUP BY ranged_items.SNO ORDER BY SNO; + +-- comma join, alias name used in GROUP BY and ORDER BY +SELECT ranged_items.SNO AS SNO, count() AS c +FROM ranged_items, product, store +WHERE ranged_items.skuID = product.PIN AND ranged_items.SNO = store.SNO +GROUP BY SNO ORDER BY SNO; + +-- alias points at the last table +SELECT store.SNO + 10 AS SNO, count() AS c +FROM ranged_items, product, store +WHERE ranged_items.skuID = product.PIN AND ranged_items.SNO = store.SNO +GROUP BY SNO ORDER BY SNO; + +-- alias name used in WHERE and inside a SELECT expression +SELECT ranged_items.SNO AS SNO, SNO * 100 AS SNO100 +FROM ranged_items, product, store +WHERE ranged_items.skuID = product.PIN AND ranged_items.SNO = store.SNO AND SNO = 2 +ORDER BY SNO100; + +-- without an alias the identifier is still ambiguous +SELECT ranged_items.SNO AS s, count() +FROM ranged_items, product, store +WHERE ranged_items.skuID = product.PIN AND ranged_items.SNO = store.SNO +GROUP BY SNO; -- { serverError AMBIGUOUS_IDENTIFIER } + +-- an alias whose expression is the ambiguous identifier itself cannot take over +SELECT SNO AS SNO +FROM ranged_items, product, store +WHERE ranged_items.skuID = product.PIN AND ranged_items.SNO = store.SNO; -- { serverError AMBIGUOUS_IDENTIFIER } + +-- three tables: the alias is used +SELECT ranged_items.SNO + 10 AS SNO, SNO AS x +FROM ranged_items, product, store +WHERE ranged_items.skuID = product.PIN AND ranged_items.SNO = store.SNO +ORDER BY x; + +-- two tables: the left table's column wins, the alias is not used +SELECT ranged_items.SNO + 10 AS SNO, SNO AS x +FROM ranged_items, store +WHERE ranged_items.SNO = store.SNO +ORDER BY x; + +DROP TABLE ranged_items; +DROP TABLE product; +DROP TABLE store; + +DROP TABLE IF EXISTS drl; +DROP TABLE IF EXISTS ap; +DROP TABLE IF EXISTS ab; + +CREATE TABLE drl (BuyerID String, PublisherID Int32) ENGINE = Memory; +CREATE TABLE ap (PublisherID Int32, PublisherName String) ENGINE = Memory; +CREATE TABLE ab (BuyerID Int32, BuyerName String) ENGINE = Memory; +INSERT INTO drl VALUES ('1', 100), ('2', 200); +INSERT INTO ap VALUES (100, 'pubA'), (200, 'pubB'); +INSERT INTO ab VALUES (1, 'buyer 1'), (2, 'buyer 2'); + +-- LEFT JOIN chain, alias name used in WHERE +SELECT drl.BuyerID AS BuyerID, ab.BuyerName AS BuyerName +FROM drl LEFT JOIN ap ON drl.PublisherID = ap.PublisherID + LEFT JOIN ab ON drl.BuyerID = toString(ab.BuyerID) +WHERE BuyerID IN ('1'); + +-- ambiguity in the nested JOIN, the column of the outer right table must not win over the alias +SELECT ab0.BuyerName AS BuyerName, ab2.BuyerName +FROM ab AS ab0 LEFT JOIN ab AS ab1 ON ab0.BuyerID = ab1.BuyerID + LEFT JOIN ab AS ab2 ON ab0.BuyerID = ab2.BuyerID + 1 +WHERE BuyerName = 'buyer 2'; + +-- alias name used in an ON clause resolves to the alias as well +SELECT drl.PublisherID AS PublisherID, ap.PublisherName +FROM drl LEFT JOIN ab ON drl.BuyerID = toString(ab.BuyerID) + LEFT JOIN ap ON PublisherID = ap.PublisherID +ORDER BY PublisherID; + +DROP TABLE drl; +DROP TABLE ap; +DROP TABLE ab; From b5225c762c823e5ba717b56cfd78354848fa3add Mon Sep 17 00:00:00 2001 From: Alexei Fedotov Date: Fri, 13 Nov 2026 18:06:21 +0100 Subject: [PATCH 181/219] Remove an unused repository switch from ReleaseBranchCI Co-Authored-By: Claude Sonnet 5.5 --- ci/workflows/release_branches.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/ci/workflows/release_branches.py b/ci/workflows/release_branches.py index 66711c40744a0..f7879b2048a60 100644 --- a/ci/workflows/release_branches.py +++ b/ci/workflows/release_branches.py @@ -1,5 +1,3 @@ -import os - from praktika import Workflow from ci.defs.defs import ( @@ -25,9 +23,6 @@ artifact = artifact.add_tags({"retention": "long"}) clickhouse_binaries_with_tags.append(artifact) -# Reports and CIDB are public; a build in ClickHouse-internal, a full fork, must not reveal the fix there. -public_build_in_internal = os.getenv("GITHUB_REPOSITORY") == "ClickHouse/ClickHouse-internal" - workflow = Workflow.Config( name="ReleaseBranchCI", event=Workflow.Event.PUSH, @@ -73,10 +68,10 @@ ], dockers=DOCKERS, secrets=SECRETS + LOOM_SECRETS, - enable_job_filtering_by_changes=not public_build_in_internal, + enable_job_filtering_by_changes=True, enable_cache=True, - enable_report=not public_build_in_internal, - enable_cidb=not public_build_in_internal, + enable_report=True, + enable_cidb=True, enable_commit_status_on_failure=True, enable_slack_feed=True, pre_hooks=[ From 61d81603eed4b38fd92a86b96f35b5be502a9b25 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Wed, 11 Nov 2026 13:36:52 +0000 Subject: [PATCH 182/219] Backport #128815 to 26.8: Use Rust `nightly-2026-11-02` --- ci/docker/fasttest/Dockerfile | 4 +- contrib/chdig | 2 +- contrib/corrosion-cmake/CMakeLists.txt | 2 +- contrib/delta-kernel-rs | 2 +- contrib/rust_vendor | 2 +- .../develop-contribute/build/build.mdx | 4 +- rust/vendor.sh | 2 +- rust/workspace/Cargo.lock | 320 +++++++++--------- 8 files changed, 169 insertions(+), 169 deletions(-) diff --git a/ci/docker/fasttest/Dockerfile b/ci/docker/fasttest/Dockerfile index 4021c1a7cc497..b69cfe5d6491e 100644 --- a/ci/docker/fasttest/Dockerfile +++ b/ci/docker/fasttest/Dockerfile @@ -109,8 +109,8 @@ RUN ARCH=$(uname -m) && \ ./rustup-init -y && \ rm rustup-init && \ chmod 777 -R /rust && \ - rustup toolchain install nightly-2026-03-22 --no-self-update && \ - rustup default nightly-2026-03-22 && \ + rustup toolchain install nightly-2026-11-02 --no-self-update && \ + rustup default nightly-2026-11-02 && \ rustup toolchain remove stable && \ rustup component add rust-src && \ rustup target add x86_64-unknown-linux-gnu && \ diff --git a/contrib/chdig b/contrib/chdig index 9302521212c89..a28e1072f6e2a 160000 --- a/contrib/chdig +++ b/contrib/chdig @@ -1 +1 @@ -Subproject commit 9302521212c898fa7d91df827136e74f3a760bb7 +Subproject commit a28e1072f6e2aeb98116ae6f6ecda06d57e877ff diff --git a/contrib/corrosion-cmake/CMakeLists.txt b/contrib/corrosion-cmake/CMakeLists.txt index 1c5c64726b90c..b06b1f9fa819d 100644 --- a/contrib/corrosion-cmake/CMakeLists.txt +++ b/contrib/corrosion-cmake/CMakeLists.txt @@ -37,7 +37,7 @@ endmacro() # Keep in sync with toolchain in docker images # (must be called before find_package(Rust) and inclusion of Corrosion.cmake) -set(Rust_TOOLCHAIN "nightly-2026-03-22") +set(Rust_TOOLCHAIN "nightly-2026-11-02") macro(find_rust) set_rust_target() diff --git a/contrib/delta-kernel-rs b/contrib/delta-kernel-rs index 88ff5a9474782..3ae30627f181f 160000 --- a/contrib/delta-kernel-rs +++ b/contrib/delta-kernel-rs @@ -1 +1 @@ -Subproject commit 88ff5a947478215d285783b42aafe6f3d04aca58 +Subproject commit 3ae30627f181f4445b756d95ac1ad573d1c666a9 diff --git a/contrib/rust_vendor b/contrib/rust_vendor index 8f91308de3d1b..c4fd435abe869 160000 --- a/contrib/rust_vendor +++ b/contrib/rust_vendor @@ -1 +1 @@ -Subproject commit 8f91308de3d1b78a1fc3256a2e64bd91744c3a11 +Subproject commit c4fd435abe8696742d992306a2c6f48a2e57e3ae diff --git a/docs/resources/develop-contribute/build/build.mdx b/docs/resources/develop-contribute/build/build.mdx index 7bfe50b296eb9..5808d06f2c5a8 100644 --- a/docs/resources/develop-contribute/build/build.mdx +++ b/docs/resources/develop-contribute/build/build.mdx @@ -69,8 +69,8 @@ As with C++ dependencies, ClickHouse uses vendoring to control exactly what's in Although in release mode any rust modern rustup toolchain version should work with these dependencies, if you plan to enable sanitizers you must use a version that matches the exact same `std` as the one used in CI (for which we vendor the crates): ```bash -rustup toolchain install nightly-2026-03-22 -rustup default nightly-2026-03-22 +rustup toolchain install nightly-2026-11-02 +rustup default nightly-2026-11-02 rustup component add rust-src ``` ## Build ClickHouse {#build-clickhouse} diff --git a/rust/vendor.sh b/rust/vendor.sh index 84d1accf1ad2d..ca7ca26457e6a 100755 --- a/rust/vendor.sh +++ b/rust/vendor.sh @@ -4,7 +4,7 @@ set -e # std is required for sanitizers builds # and we need to match toolchain version for std (to vendor proper dependencies) -TOOLCHAIN=nightly-2026-03-22 +TOOLCHAIN=nightly-2026-11-02 function cargo() { rustup run "$TOOLCHAIN" cargo "$@"; } function rustc() { rustup run "$TOOLCHAIN" rustc "$@"; } rustup component add --toolchain "$TOOLCHAIN" rust-src diff --git a/rust/workspace/Cargo.lock b/rust/workspace/Cargo.lock index 68d084df90648..7b87c76d377d4 100644 --- a/rust/workspace/Cargo.lock +++ b/rust/workspace/Cargo.lock @@ -151,9 +151,9 @@ checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" [[package]] name = "arbitrary" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +checksum = "3bc62ac97cc33321f50863d514c3bc38a453947a8f9e781137e47c7401020aed" [[package]] name = "ariadne" @@ -179,7 +179,7 @@ checksum = "82f6aeea286b8eb4dd3431a1be1b59d290ace00f5bfd8e2a159bc2a05e2c1667" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.9", ] [[package]] @@ -202,9 +202,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.13.1" +version = "2.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b588b76d00fde79687d7646a9b5bdf3cc0f655e0bbd080335a95d7e96f3587da" +checksum = "3ded4057c258ba199e2d26386d3af3780957ecaee6c4ef4041c6b4b8b97c0b06" [[package]] name = "block-buffer" @@ -237,9 +237,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.4.3" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "509591b7bcd67f4ef775afad7662703b4935daaa6ec0e5605cfb1090b32a2b6d" +checksum = "375989decec1a0fa424442bed520fa12ede43a48475ce8c7417d2c69b32b5f6b" dependencies = [ "find-msvc-tools", "shlex 2.0.1", @@ -247,9 +247,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.4" +version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" +checksum = "4e7648175b45a9a48536d676f68d918270699102aa8dab5496df06904c914600" [[package]] name = "cfg_aliases" @@ -285,9 +285,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.6" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "473c7e07f409a8d772161724aa8db6a765a2532a70f9667eeb7b49d3d02fbdca" +checksum = "aa8876b300ab35ba921adea3dfd70157a46249b33f95c9084ae5709785478946" dependencies = [ "clap_builder", "clap_derive", @@ -295,9 +295,9 @@ dependencies = [ [[package]] name = "clap_builder" -version = "4.6.6" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7b48fea5a88e9ae728a2dcbedbfc0e730f7d60da42e1cb049a83c9fb8b789889" +checksum = "ec0797fb7aeb1406c84efac526901f7ec3ead2124f946b494e72879d4b54704d" dependencies = [ "anstream 1.0.0", "anstyle", @@ -307,32 +307,32 @@ dependencies = [ [[package]] name = "clap_complete" -version = "4.6.9" +version = "4.6.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be2ad0423bdbbb0e25bc89add796f3559706d4a95e1bc98e4d9662a957b6a19" +checksum = "037e2a1a92236d0aff7e845093f64661d6df4c02c9fcc61a60e9e1d736fa392f" dependencies = [ "clap", ] [[package]] name = "clap_derive" -version = "4.6.4" +version = "4.6.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d012d2b9d65aca7f18f4d9878a045bc17899bba951561ba5ec3c2ba1eed9a061" +checksum = "f9c751b79415d4e559e3d1fcf128e09e720eb673a06d26cf6f392d37d75b66e0" dependencies = [ "anstyle", "heck", "proc-macro2", "pulldown-cmark", "quote", - "syn 3.0.3", + "syn 3.0.9", ] [[package]] name = "clap_lex" -version = "1.1.0" +version = "1.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" +checksum = "1c133bc6a41be0d194c306b5506d15e6feeea7b1d6604bd3f8310dfb2ca96486" [[package]] name = "cobs" @@ -340,7 +340,7 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fa961b519f0b462e3a3b4a34b64d119eeaca1d59af726fe450bbba07a9fc0a1" dependencies = [ - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] @@ -351,7 +351,7 @@ checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681" dependencies = [ "serde", "termcolor", - "unicode-width 0.2.2", + "unicode-width 0.2.3", ] [[package]] @@ -502,18 +502,18 @@ version = "0.132.1" [[package]] name = "crc32fast" -version = "1.5.0" +version = "1.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" +checksum = "5f97d3416d5337d15c4b0d78c8c91df922ca143f57ec20e2d1c188333047ef04" dependencies = [ "cfg-if", ] [[package]] name = "crossbeam" -version = "0.8.4" +version = "0.8.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1137cd7e7fc0fb5d3c5a8678be38ec56e819125d8d7907411fe24ccb943faca8" +checksum = "e71406cd8807725f7ac2f999a4cdd32e98f829fdf65f528343cebf945e41df1e" dependencies = [ "crossbeam-channel", "crossbeam-deque", @@ -524,18 +524,18 @@ dependencies = [ [[package]] name = "crossbeam-channel" -version = "0.5.16" +version = "0.5.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d85363c37faeca707aef026efa9f3b34d077bce547e48f770770625c6013679e" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-deque" -version = "0.8.7" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5181e0de7b61eb03a81e347d6dd8797bae9da5146707b51077e2d71a54ec0ceb" +checksum = "622f3fc73690be383c7214310406f28a90e6edeadc3cea882f9d71e495b9711a" dependencies = [ "crossbeam-epoch", "crossbeam-utils", @@ -543,27 +543,27 @@ dependencies = [ [[package]] name = "crossbeam-epoch" -version = "0.9.20" +version = "0.9.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2d6914041f254d6e9176c01941b21115dcfb7089e55135a35411081bd106ef3f" +checksum = "dc74980687109a3b14c72fd458107bf0baa1da1a1a805e178d15501ba9b86d9d" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-queue" -version = "0.3.13" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "803d13fb3b09d88be9f4dbc29062c66b19bf7170867ceb746d2a8689bf6c7a26" +checksum = "03e8bd762f7479489c70ed6c768ddca99d7296857de437a68dcb2a94365b3fae" dependencies = [ "crossbeam-utils", ] [[package]] name = "crossbeam-utils" -version = "0.8.22" +version = "0.8.23" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "61803da095bee82a81bb1a452ecc25d3b2f1416d1897eb86430c6159ef717c17" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" [[package]] name = "crypto-common" @@ -598,9 +598,9 @@ dependencies = [ [[package]] name = "cxx" -version = "1.0.199" +version = "1.0.205" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "824894a4a85dca76d4c95c2b9098c036f5a29f627b30c12780774f6654e60974" +checksum = "c8a02bfeba9074b2bbb39492c9b0908ec854f455a3cccdcde7a1de55e2315e40" dependencies = [ "cc", "cxx-build", @@ -613,9 +613,9 @@ dependencies = [ [[package]] name = "cxx-build" -version = "1.0.199" +version = "1.0.205" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1ae0b651ea5b0000b19513aef5a03f194d7e3486f2d9258b658da8677fe9036" +checksum = "bbc2ac76997619d2d888b58ad79488f1c835de41ca4dd3dd7082acf4ecb0a50f" dependencies = [ "cc", "codespan-reporting", @@ -623,39 +623,39 @@ dependencies = [ "proc-macro2", "quote", "scratch", - "syn 3.0.3", + "syn 3.0.9", ] [[package]] name = "cxxbridge-cmd" -version = "1.0.199" +version = "1.0.205" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fb05f91d3fb8435d9bab6ac5ce6ac1868be774325fb7fb2a91be39393b21388e" +checksum = "54a486ed16a3f2a053fe7f5b98db4c1a59846a23e8cb9e49f9d1bd48830d51a2" dependencies = [ "clap", "codespan-reporting", "indexmap", "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.9", ] [[package]] name = "cxxbridge-flags" -version = "1.0.199" +version = "1.0.205" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bf293202e0e3e98495785745389e8d0755b217e66f19194a5c695c25e03282ef" +checksum = "c72569be94e2283764d334cd285980b747dbe221e27e315484eb6bcd93fa808a" [[package]] name = "cxxbridge-macro" -version = "1.0.199" +version = "1.0.205" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca001d746947c7249ed9d332a10f7a59daedbafeb0ec68c5c18a7db7a93f6ccc" +checksum = "423e795284a7f7b3f843c956a5322c4bc497b60c2dd1a6339da10313cb18d575" dependencies = [ "indexmap", "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.9", ] [[package]] @@ -705,9 +705,9 @@ dependencies = [ [[package]] name = "defmt" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e2953bfe4f93bbd20cc71198842756f77d161884c99ebbabc41d80231ded88d1" +checksum = "e4a0795ac740cb54d7b3775c9c17345f6aaecf884c01733b39d6017738e0e3e1" dependencies = [ "bitflags 1.3.2", "defmt-macros", @@ -715,30 +715,30 @@ dependencies = [ [[package]] name = "defmt-macros" -version = "1.1.1" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bad9c72e7ca2137e0dc3813245a0d282fd6daad32fd800af018306a9169b5fe8" +checksum = "840bb056bd99c495c9188198a258ab787c79f306d864db3e6e8451442d6d8366" dependencies = [ "defmt-parser", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", ] [[package]] name = "defmt-parser" -version = "1.0.0" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "10d60334b3b2e7c9d91ef8150abfb6fa4c1c39ebbcf4a81c2e346aad939fee3e" +checksum = "71958f69fb2de856c5dd51ac9377e986ca1aa5919f931bf5395a807277f889a0" dependencies = [ - "thiserror 2.0.20", + "thiserror 2.0.21", ] [[package]] name = "deranged" -version = "0.5.8" +version = "0.5.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7cd812cc2bc1d69d4764bd80df88b4317eaef9e773c75226407d9bc0876b211c" +checksum = "9e9de72ce2ad1f90dc62fa25f0f430ef85eb4b0d8fa0be4f30373bc40a21d28e" [[package]] name = "derive_builder" @@ -810,9 +810,9 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "either" -version = "1.17.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e5e8f6c15a24b9a3ee5efec809ccd006d3b30e8b3bb63c39af737c7f87daa1d" +checksum = "0e9c71c2167ca323c882b99918929403426e2373ea17242ff5653e0d5e1058be" [[package]] name = "embedded-io" @@ -869,9 +869,9 @@ dependencies = [ [[package]] name = "equivalent" -version = "1.0.2" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" +checksum = "00d174d5400e5e8fd687ad1049e2f578285fa914201b1af7e8b112a4546bd826" [[package]] name = "errno" @@ -885,9 +885,9 @@ dependencies = [ [[package]] name = "find-msvc-tools" -version = "0.1.11" +version = "0.1.15" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d45db016d36b838f563236e9193d0ee6ce38f3f68b6c94e914b4929c96bbb890" +checksum = "1c66915cc5391097de7a2b0b972e8505aa33c0df401fd3a6811dbdb32881ba48" [[package]] name = "fnv" @@ -1051,9 +1051,9 @@ checksum = "b9e0384b61958566e926dc50660321d12159025e767c18e043daf26b70104c39" [[package]] name = "indexmap" -version = "2.14.0" +version = "2.14.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" +checksum = "cc4e190f5d26ca7051642629da2c52fc03bde85a03197c99408dcd291734c855" dependencies = [ "equivalent", "hashbrown 0.17.1", @@ -1093,9 +1093,9 @@ checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" [[package]] name = "jiff" -version = "0.2.35" +version = "0.2.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "668b7183bd07af9a4885f5c35b0cc5c83c4607a913c16b7e17291832910d2dcc" +checksum = "3ac703058abac89e5c39b0b879dce2b6b14a4ba36600c7e7e3d0c60a6883ef3c" dependencies = [ "defmt", "jiff-core", @@ -1108,18 +1108,19 @@ dependencies = [ [[package]] name = "jiff-core" -version = "0.1.0" +version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7feca88439efe53da3754500c1851dedf3cb36c524dd5cf8225cc0794de95d09" +checksum = "aee43309eb53bafb8c36d35d6e444769c8f8e6206a9fbf7546f5db9d7d4a0e93" dependencies = [ "defmt", + "log", ] [[package]] name = "jiff-static" -version = "0.2.35" +version = "0.2.39" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3a69dcb3a21cfb32ce1cd056169337ca284af0766dd766e7878819b251a49204" +checksum = "6dc32470c67eb7d0e9ccb74303fa925313dc04cecb1aa048beaa6f391e36a93d" dependencies = [ "jiff-core", "proc-macro2", @@ -1129,9 +1130,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.104" +version = "0.3.107" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e0c1080212aad755ea003d18543e8768dd432c48819efd73a7bf1e39b7a5a3a" +checksum = "8dc09d2154098bbf4ef54eab6b117a3c24cbbffe94046b16ee6d6d4510584d3a" dependencies = [ "cfg-if", "futures-util", @@ -1140,9 +1141,9 @@ dependencies = [ [[package]] name = "lazy_static" -version = "1.5.0" +version = "1.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" +checksum = "20870f649af7073d53e38067b2a84312175d56ea15217e1b15bc83506ec50afb" [[package]] name = "leb128fmt" @@ -1152,9 +1153,9 @@ checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" [[package]] name = "libc" -version = "0.2.189" +version = "0.2.191" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3eaf3ede3fee6db1a4c2ee091bf8a8b4dccdc6d17f656fb07896ee72867612f2" +checksum = "65cd87c9a6d60536580b316ad985726104730a5151c99845f2490428ef91240a" [[package]] name = "libm" @@ -1164,9 +1165,9 @@ checksum = "b6d2cec3eae94f9f509c767b45932f1ada8350c4bdb85af2fcab4a3c14807981" [[package]] name = "libredox" -version = "0.1.20" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28d0a00925a9f930d679b6789b721e3a7f9ed110f41b86d2497caa780c3a070a" +checksum = "61ff90caf6077a803a240f62fdbe88645a890bbca49ef8174c3cb0404362171d" dependencies = [ "libc", ] @@ -1188,9 +1189,9 @@ checksum = "32a66949e030da00e8c7d4434b251670a91556f4144941d37452769c25d58a53" [[package]] name = "log" -version = "0.4.33" +version = "0.4.35" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ceec5bc11778974d1bcb055b18002eba7f4b3518b6a0081b3af5f21666da9ad" +checksum = "5d15f86d12da44cd8d4ec3c73984537eb7f4cadcb9776a575a0a8a05ca11005c" [[package]] name = "mach2" @@ -1209,9 +1210,9 @@ checksum = "cf8baf1c55e62ffcace7a9f06f4bd9cd3f0c4beb022d3b367256b91b87513d98" [[package]] name = "memfd" -version = "0.6.5" +version = "0.6.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ad38eb12aea514a0466ea40a80fd8cc83637065948eb4a426e4aa46261175227" +checksum = "57804b2c9b69967f1536a56f86297e367a33b19e98852ed624b84551cdbc0d90" dependencies = [ "rustix", ] @@ -1222,7 +1223,7 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "cfg-if", "cfg_aliases", "libc", @@ -1245,9 +1246,9 @@ dependencies = [ [[package]] name = "num_threads" -version = "0.1.7" +version = "0.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c7398b9c8b70908f6371f47ed36737907c87c52af34c268fed0bf0ceb92ead9" +checksum = "cabcc654086861f9e1bc5115258de62ef92c79b20f4615a1b0cc57c4e82027d4" dependencies = [ "libc", ] @@ -1302,9 +1303,9 @@ checksum = "05c8b63e8d9609db387f0324918f81d68fe27748f084ef092fb35954d0539a85" [[package]] name = "portable-atomic-util" -version = "0.2.7" +version = "0.2.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2a106d1259c23fac8e543272398ae0e3c0b8d33c88ed73d0cc71b0f1d902618" +checksum = "10ab3eb7f3becc3a1cbc4f2c6f20267996cfc1a6467a873763411b136a122715" dependencies = [ "portable-atomic", ] @@ -1323,9 +1324,9 @@ dependencies = [ [[package]] name = "powerfmt" -version = "0.2.0" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" +checksum = "4a6394b9e965e73d0a289ee54f589087e2c676aedf60885baf52c76b771e4958" [[package]] name = "ppv-lite86" @@ -1389,11 +1390,11 @@ dependencies = [ [[package]] name = "pulldown-cmark" -version = "0.13.4" +version = "0.13.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e9f068eba8e7071c5f9511831b44f32c740d5adf574e990f946ddb53db2f314e" +checksum = "3db2be46835f013dd3f4fa7f5f7402c537787b54ebb6fe2b94bbdd7e2669bb7b" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "memchr", "unicase", ] @@ -1494,22 +1495,22 @@ dependencies = [ [[package]] name = "ref-cast" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "216e8f773d7923bcba9ceb86a86c93cabb3903a11872fc3f138c49630e50b96d" +checksum = "7e440fb4e4b4147295338efb76001ab9e4efc0e5839df2c47fc5ac2381d365c3" dependencies = [ "ref-cast-impl", ] [[package]] name = "ref-cast-impl" -version = "1.0.26" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2c9283685feec7d69af75fb0e858d5e7378f33fe4fc699383b2916ab9273e03c" +checksum = "92ecd8964f8453721699a1ed72037b0db49ce2f5a5138486ee89bed6f67cdf3a" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.9", ] [[package]] @@ -1569,11 +1570,11 @@ checksum = "6b1e7f9a428571be2dc5bc0505c13fb6bf936822b894ec87abf8a08a4e51742d" [[package]] name = "rustix" -version = "1.1.4" +version = "1.1.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" +checksum = "e5e76742e5294153d4dde1207dbec5bdd81eef6fabdc0d2118babadabfd858aa" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "errno", "libc", "linux-raw-sys", @@ -1594,9 +1595,9 @@ checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" [[package]] name = "schemars" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "687274d293b6cdc6e73e0fee520bf2049650090d7164f87672d212a3c530cf4a" +checksum = "ab508826f74a77ca9d5aba6ff19b522583ee3eaf28a19384ff3d0e5835fadf6e" dependencies = [ "dyn-clone", "ref-cast", @@ -1607,14 +1608,14 @@ dependencies = [ [[package]] name = "schemars_derive" -version = "1.2.2" +version = "1.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d98c67716b46af2f0b8cf752abc930f6f9aecfbf671ecfb531db8a31dbe4e2ba" +checksum = "e1c3a92094fa7d61aa124645844facb6b554dfc797136d0f5fd1f890e2bffc69" dependencies = [ "proc-macro2", "quote", "serde_derive_internals", - "syn 3.0.3", + "syn 3.0.9", ] [[package]] @@ -1660,7 +1661,7 @@ checksum = "e7a5d71263a5a7d47b41f6b3f06ba276f10cc18b0931f1799f710578e2309348" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.9", ] [[package]] @@ -1671,14 +1672,14 @@ checksum = "f852137cce035d6a4df67ccce505ff6b3e9fd3a10e3e52b24dc71e650bb1a9bd" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.9", ] [[package]] name = "serde_json" -version = "1.0.151" +version = "1.0.154" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c841b55ecdae098c80dcae9cf767f6f8a0c2cdb3416bbef72181df4d0fe73f14" +checksum = "e7e9cc8b1b85264074fbcc02a88680c4096b1e47df8f739dceb03bf482f04bd6" dependencies = [ "itoa", "memchr", @@ -1747,7 +1748,7 @@ dependencies = [ "skim-tuikit", "time", "timer", - "unicode-width 0.2.2", + "unicode-width 0.2.3", "vte", "which", ] @@ -1770,7 +1771,7 @@ dependencies = [ "nix", "skim-common", "term 0.7.0", - "unicode-width 0.2.2", + "unicode-width 0.2.3", ] [[package]] @@ -1781,9 +1782,9 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" [[package]] name = "smallvec" -version = "1.15.2" +version = "1.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ed6a63f02c8539c91a8685a86f4099661ba3da017932f6ebbea6de3f0fa7c90" +checksum = "d0d5d4b1981b8bbbb7aed204cae5726d683d5d6ac7e74ec54122ea28f315e7c8" dependencies = [ "serde", ] @@ -1854,9 +1855,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.3" +version = "3.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53e9bae58849f64dfa4f5d5ae372c8341f7305f82a3868709269343628b659a3" +checksum = "d78c8dee4c7bf0e14673097256fed6142ce9d3b85a408189d07482442145823b" dependencies = [ "proc-macro2", "quote", @@ -1909,11 +1910,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.20" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec86235f5fcc2a73650310756d2ac5b138a5780bbbdfae3eeccec992c435ba4f" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" dependencies = [ - "thiserror-impl 2.0.20", + "thiserror-impl 2.0.21", ] [[package]] @@ -1929,13 +1930,13 @@ dependencies = [ [[package]] name = "thiserror-impl" -version = "2.0.20" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bc04cd3e1236dd4a98afca4569f2deb3f120e5422a4023be2cb683f8486292af" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn 3.0.3", + "syn 3.0.9", ] [[package]] @@ -2012,15 +2013,15 @@ checksum = "b6f5e870be6c3b371b77fe0ee0bafb859fa4964b4404c27de1d380043c4dda20" [[package]] name = "unicase" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbc4bc3a9f746d862c45cb89d705aa10f187bb96c76001afab07a0d35ce60142" +checksum = "357cc3acc6a036009fd6c973ed009037c732d60d0b4f6c673e9041497482a28f" [[package]] name = "unicode-ident" -version = "1.0.24" +version = "1.0.27" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6e4313cd5fcd3dad5cafa179702e2b244f760991f45397d14d4ebf38247da75" +checksum = "a2c754d6c33795a1c324727428e5a7dedb5b06195f9890bdbcba760d3e246563" [[package]] name = "unicode-segmentation" @@ -2036,9 +2037,9 @@ checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" [[package]] name = "unicode-width" -version = "0.2.2" +version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" +checksum = "ca718471876992059f251e33aafc334346ada08042b7661dd1fe313a9bced557" [[package]] name = "unicode_categories" @@ -2134,12 +2135,11 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.127" +version = "0.2.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1b70935747edd64d89de3efa29d73789b806c15798f8e7dca4d8ac356b50ce70" +checksum = "a0181c14ddd6c7b8b0b0132a5f1ea2f32be0d893cd3ea97b6e3b28886b8a3d1c" dependencies = [ "cfg-if", - "once_cell", "rustversion", "wasm-bindgen-macro", "wasm-bindgen-shared", @@ -2147,9 +2147,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.127" +version = "0.2.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "77775f8f3f7217702089053b94958f8f54061a3f663417df76e19cbdcca29bc1" +checksum = "6e931a02a39abdbf66966fc35902b5ed58344750a70885bed4ebb3d8eaf28ed9" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -2157,22 +2157,22 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.127" +version = "0.2.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e11d33f857dc2fb11b8bc75aee111aa9cbeb12cd9f25efd3d4c2a3dd4e235284" +checksum = "2ca4b54280e455bd831e33a6b8e734baf432fe63fbc35c42ac14ce67a043a53a" dependencies = [ "bumpalo", "proc-macro2", "quote", - "syn 2.0.119", + "syn 3.0.9", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.127" +version = "0.2.130" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ef64dbcc55df09c7e5a46182d181c2cfa3e925f3da937ea764728b4bbb9dcbf" +checksum = "3b1f9c3798725016c6ee5f2f57869ec0831e8bbc0ab0310a86e787e97f41389e" dependencies = [ "unicode-ident", ] @@ -2189,12 +2189,12 @@ dependencies = [ [[package]] name = "wasm-encoder" -version = "0.256.0" +version = "0.263.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ec1492381bfd5ea51c2a99a919b676662559925cb8d7490547ec2e14c1ad3eb1" +checksum = "0b883a26aea02eaabe2d2c3fba7eb039ef5cdcb5d5201ae5551e73d5e251a81f" dependencies = [ "leb128fmt", - "wasmparser 0.256.0", + "wasmparser 0.263.0", ] [[package]] @@ -2203,7 +2203,7 @@ version = "0.248.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aa4439c5eee9df71ee0c6efb37f63b1fcb1fec38f85f5142c54e7ed05d33091a" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "hashbrown 0.17.1", "indexmap", "semver", @@ -2212,11 +2212,11 @@ dependencies = [ [[package]] name = "wasmparser" -version = "0.256.0" +version = "0.263.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "60bd825ffedc6cba8a642924ba7ae424afbc47811cffbcb7b92031ec24e59b4c" +checksum = "4f5fa0a054ae153606df820eb0a84213e28cda41bbe04ba2628e71b6ba100fcb" dependencies = [ - "bitflags 2.13.1", + "bitflags 2.13.2", "indexmap", "semver", ] @@ -2238,7 +2238,7 @@ version = "45.0.1" dependencies = [ "addr2line", "async-trait", - "bitflags 2.13.1", + "bitflags 2.13.2", "bumpalo", "cc", "cfg-if", @@ -2343,7 +2343,7 @@ dependencies = [ "pulley-interpreter", "smallvec", "target-lexicon", - "thiserror 2.0.20", + "thiserror 2.0.21", "wasmparser 0.248.0", "wasmtime-environ", "wasmtime-internal-core", @@ -2419,22 +2419,22 @@ dependencies = [ [[package]] name = "wast" -version = "256.0.0" +version = "263.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a3ad42723fc9222da007f05812a3249c9a4ee7af79d497fc2a2079579ad7efe6" +checksum = "69d90bcf581b501aa1c358bc5dc0faa979cd375796908860ebcb6d1d70b907c7" dependencies = [ "bumpalo", "leb128fmt", "memchr", - "unicode-width 0.2.2", - "wasm-encoder 0.256.0", + "unicode-width 0.2.3", + "wasm-encoder 0.263.0", ] [[package]] name = "wat" -version = "1.256.0" +version = "1.263.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "37cc86c54d8011b3202e265bfebada440733ea3a788ffaf46d395f7d2fdeacfb" +checksum = "bbc9c5aca783b3f1838be89f3f9937c528b90ec5faa6fb62d809726c16a7d8f7" dependencies = [ "wast", ] @@ -2492,7 +2492,7 @@ dependencies = [ "regalloc2", "smallvec", "target-lexicon", - "thiserror 2.0.20", + "thiserror 2.0.21", "wasmparser 0.248.0", "wasmtime-environ", "wasmtime-internal-core", @@ -2596,18 +2596,18 @@ checksum = "cfe53a6657fd280eaa890a3bc59152892ffa3e30101319d168b781ed6529b049" [[package]] name = "zerocopy" -version = "0.8.56" +version = "0.8.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "556764e583adb45a9f8d413c2a147fa7e8d821e48e12b14fd560b607998b75eb" +checksum = "e5fe1f8f1b06191a00962174c61aa5005e0bb391a6d80d07e24d115c01a92ed8" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.56" +version = "0.8.63" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2ab42fc20575779bd240faa45f94a74256f755c0fa9e89f0ede20d91d0cdfc1" +checksum = "863ad3ac83293fb4d740aedbfdc9240dd8d1a50c1099acd76ce80ce7c7230c7f" dependencies = [ "proc-macro2", "quote", From b7fc485ab0b67074258303b096aedf6d3a313091 Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Wed, 11 Nov 2026 23:37:27 +0100 Subject: [PATCH 183/219] Remove `ppa:ubuntu-toolchain-r/test` from `binary-builder` image The PPA was added in 2019 to get gcc-9. None of the packages we install come from it anymore, but `add-apt-repository` queries the Launchpad API, and a timeout there broke `Dockers Build` on both architectures: `TimeoutError: [Errno 110] Connection timed out`. Also drop `software-properties-common`, which was installed only for `add-apt-repository`. https://s3.amazonaws.com/clickhouse-test-reports/praktika.html?PR=117346&sha=7ef2c32ada54daaef89b1fcb0a7b10c5299f86ca&name_0=PR&name_1=Dockers%20Build%20%28amd%29 https://github.com/ClickHouse/ClickHouse/pull/117346 Co-Authored-By: Claude Opus 5.5 (cherry picked from commit 884f6b35ec13b9347ea67e3f815a3cf92220b7e7) --- ci/docker/binary-builder/Dockerfile | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/ci/docker/binary-builder/Dockerfile b/ci/docker/binary-builder/Dockerfile index e085b9c4c2d3d..2dbb0db9a494a 100644 --- a/ci/docker/binary-builder/Dockerfile +++ b/ci/docker/binary-builder/Dockerfile @@ -19,12 +19,9 @@ COPY --from=clickhouse/cctools:859fb360308eb8ac47ad \ /etc/ssl/ # Refresh the kitware GPG key inherited from fasttest base image, it may be stale -RUN curl -s https://apt.kitware.com/keys/kitware-archive-latest.asc | gpg --dearmor - > /etc/apt/trusted.gpg.d/kitware.gpg \ - && apt-get update \ - && apt-get install software-properties-common --yes --no-install-recommends --verbose-versions +RUN curl -s https://apt.kitware.com/keys/kitware-archive-latest.asc | gpg --dearmor - > /etc/apt/trusted.gpg.d/kitware.gpg -RUN add-apt-repository ppa:ubuntu-toolchain-r/test --yes \ - && apt-get update \ +RUN apt-get update \ && apt-get -o Dpkg::Options::="--force-overwrite" upgrade --yes \ && apt-get -o Dpkg::Options::="--force-overwrite" install --yes \ binutils-dev \ From dc972b538126ef815067700d97c849b980b65654 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Fri, 13 Nov 2026 22:59:20 +0000 Subject: [PATCH 184/219] Backport #131394 to 26.8: Fix a typo in a comment in `HTTPServerResponse` --- src/Server/HTTP/HTTPServerResponse.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Server/HTTP/HTTPServerResponse.cpp b/src/Server/HTTP/HTTPServerResponse.cpp index 4076083cfb822..bf789bb05bda4 100644 --- a/src/Server/HTTP/HTTPServerResponse.cpp +++ b/src/Server/HTTP/HTTPServerResponse.cpp @@ -35,7 +35,7 @@ std::shared_ptr HTTPServerResponse::send() if (request && request->getMethod() == HTTPRequest::HTTP_HEAD) { // HTTP_HEAD is a special case - // client usually reads nothing from socket after headers even when 'Contex-Lenght' is sent + // client usually reads nothing from socket after headers even when 'Content-Length' is sent // if server wrote a message to the connection with enabled 'Connection: Keep-Alive' // the connection would be poisoned. // Next request over that connection reads previously unreaded message as a HTTP status line From c5229eff4fe4ae96ac9fde03f0fd868f137331cd Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 14 Nov 2026 01:24:25 +0000 Subject: [PATCH 185/219] Backport #131343 to 26.8: Reject out-of-range `Decimal` scale in `AggregateFunction` type parameters --- src/IO/WriteHelpers.h | 15 +++++++++++++-- src/Parsers/parseFieldFromCastedLiteral.cpp | 8 ++++++++ ...meter_decimal_scale_out_of_bounds.reference | 1 + ...n_parameter_decimal_scale_out_of_bounds.sql | 18 ++++++++++++++++++ 4 files changed, 40 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/05632_aggregate_function_parameter_decimal_scale_out_of_bounds.reference create mode 100644 tests/queries/0_stateless/05632_aggregate_function_parameter_decimal_scale_out_of_bounds.sql diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index 1afee1a496e0d..9c7106fc03d43 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -35,6 +36,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int ARGUMENT_OUT_OF_BOUND; +} + class Exception; /// Helper functions for formatted and binary output. @@ -1302,8 +1308,13 @@ void writeDecimalFractional(const T & x, UInt32 scale, WriteBuffer & ostr, bool } constexpr size_t max_digits = std::numeric_limits::digits10; - chassert(scale <= max_digits); - chassert(fractional_length <= max_digits); + /// The digits go to a buffer on the stack, so this is checked in the release build as well: the scale + /// of a `DecimalField` is not tied to a data type, and a value that slipped through a boundary without + /// validation must not write past the buffer. + if (scale > max_digits || fractional_length > max_digits) [[unlikely]] + throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, + "Cannot write a decimal with scale {} and {} fractional digits: at most {} digits are supported", + scale, fractional_length, max_digits); /// Rounding to a narrower field is the caller's job, because a carry out of the fractional part /// belongs to the whole part, which has already been written by then. chassert(!fixed_fractional_length || fractional_length >= scale); diff --git a/src/Parsers/parseFieldFromCastedLiteral.cpp b/src/Parsers/parseFieldFromCastedLiteral.cpp index 67e9e05ce6e71..d8105cf6ed786 100644 --- a/src/Parsers/parseFieldFromCastedLiteral.cpp +++ b/src/Parsers/parseFieldFromCastedLiteral.cpp @@ -17,6 +17,7 @@ namespace DB namespace ErrorCodes { + extern const int ARGUMENT_OUT_OF_BOUND; extern const int BAD_ARGUMENTS; } @@ -48,6 +49,13 @@ Field convertFieldTo(const Field & src) template Field convertFieldTo(const Field & src, UInt32 scale) { + /// The scale comes from the type name as text, which is not checked by `DataTypeFactory`. An + /// out-of-range scale later overruns a fixed-size buffer in `writeDecimalFractional`, so reject + /// it here, matching `readDecimalFieldBinary` and `DataTypeDecimalBase`. + if (scale > DecimalUtils::max_precision) + throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Scale {} is out of bounds (max scale: {})", + scale, DecimalUtils::max_precision); + if (src.getType() == Field::TypeToEnum>::value && src.safeGet>().getScale() == scale) return src; diff --git a/tests/queries/0_stateless/05632_aggregate_function_parameter_decimal_scale_out_of_bounds.reference b/tests/queries/0_stateless/05632_aggregate_function_parameter_decimal_scale_out_of_bounds.reference new file mode 100644 index 0000000000000..9972842f9827a --- /dev/null +++ b/tests/queries/0_stateless/05632_aggregate_function_parameter_decimal_scale_out_of_bounds.reference @@ -0,0 +1 @@ +1 1 diff --git a/tests/queries/0_stateless/05632_aggregate_function_parameter_decimal_scale_out_of_bounds.sql b/tests/queries/0_stateless/05632_aggregate_function_parameter_decimal_scale_out_of_bounds.sql new file mode 100644 index 0000000000000..290390b7a34e8 --- /dev/null +++ b/tests/queries/0_stateless/05632_aggregate_function_parameter_decimal_scale_out_of_bounds.sql @@ -0,0 +1,18 @@ +-- The scale of a `Decimal` parameter in the name of an `AggregateFunction` type is read from text, +-- without `DataTypeFactory`. A scale above the maximum precision used to overrun a buffer on the stack +-- when the parameter was written back as text. + +SELECT toTypeName(CAST(1 AS AggregateFunction(quantile(CAST('1e-10', 'Decimal32(10)')), UInt64))); -- { serverError ARGUMENT_OUT_OF_BOUND } +SELECT toTypeName(CAST(1 AS AggregateFunction(quantile(CAST('1e-19', 'Decimal64(19)')), UInt64))); -- { serverError ARGUMENT_OUT_OF_BOUND } +SELECT toTypeName(CAST(1 AS AggregateFunction(quantile(CAST('1e-39', 'Decimal128(39)')), UInt64))); -- { serverError ARGUMENT_OUT_OF_BOUND } +SELECT toTypeName(CAST(1 AS AggregateFunction(quantile(CAST('1e-77', 'Decimal256(77)')), UInt64))); -- { serverError ARGUMENT_OUT_OF_BOUND } +SELECT toTypeName(CAST(1 AS AggregateFunction(quantile(CAST('1e-88', 'Decimal256(88)')), UInt64))); -- { serverError ARGUMENT_OUT_OF_BOUND } +SELECT toTypeName(CAST(1 AS AggregateFunction(quantile(CAST('3.462164989639087034239942739923635180669988563696446547344599548079e-22', 'Decimal256(88)')), UInt64))); -- { serverError ARGUMENT_OUT_OF_BOUND } +SELECT toTypeName(CAST(1 AS AggregateFunction(quantile(CAST('1', 'Decimal256(4294967295)')), UInt64))); -- { serverError ARGUMENT_OUT_OF_BOUND } + +-- The maximum scale is still accepted, and the parameter is written back in full. +DROP TABLE IF EXISTS t_decimal_scale_parameter; +CREATE TABLE t_decimal_scale_parameter (s AggregateFunction(quantile(CAST('1e-76', 'Decimal256(76)')), UInt64)) ENGINE = Memory; +SELECT position(type, '::Decimal256(76)') > 0, position(type, concat('0.', repeat('0', 75), '1')) > 0 +FROM system.columns WHERE database = currentDatabase() AND table = 't_decimal_scale_parameter'; +DROP TABLE t_decimal_scale_parameter; From 75bf6375c594e7016b1360234fb067c2a1840d0a Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 14 Nov 2026 07:22:13 +0000 Subject: [PATCH 186/219] Backport #131029 to 26.8: Fix use-after-free on renaming Delta Lake columns to their physical names --- .github/workflows/backport_branches.yml | 72 +- .github/workflows/release_branches.yml | 72 +- .gitmodules | 3 - .../Foundation/include/Poco/FileChannel.h | 1 + base/poco/Foundation/src/FileChannel.cpp | 40 +- .../Net/include/Poco/Net/HTTPClientSession.h | 4 + base/poco/Net/src/HTTPClientSession.cpp | 30 + base/poco/Net/src/IPAddress.cpp | 18 +- .../include/Poco/Net/DistrustedCertificates.h | 44 + .../src/DistrustedCertificates.cpp | 95 ++ base/poco/NetSSL_OpenSSL/src/SSLManager.cpp | 15 + base/poco/XML/CMakeLists.txt | 1 + base/poco/XML/include/Poco/XML/expat.h | 5 +- .../XML/include/Poco/XML/expat_external.h | 2 +- base/poco/XML/src/ascii.h | 3 +- base/poco/XML/src/entities.h | 99 ++ base/poco/XML/src/internal.h | 8 +- base/poco/XML/src/xcs.h | 2 + base/poco/XML/src/xmlparse.cpp | 871 ++++++++-- base/poco/XML/src/xmltok.h | 4 +- base/poco/XML/src/xmltok_impl.c | 6 +- base/poco/XML/src/xmlxcs.c | 21 + .../integration/dotnet_client/Dockerfile | 11 +- ci/workflows/backport_branches.py | 2 +- ci/workflows/release_branches.py | 2 +- cmake/target.cmake | 2 +- contrib/CMakeLists.txt | 2 +- contrib/arrow-cmake/CMakeLists.txt | 185 +- contrib/arrow-cmake/orc_check.cmake | 14 - contrib/brotli | 2 +- contrib/brotli-cmake/CMakeLists.txt | 51 +- contrib/c-ares | 2 +- contrib/grpc | 2 +- contrib/grpc-cmake/grpc.cmake | 119 +- contrib/hive-metastore-cmake/CMakeLists.txt | 8 +- contrib/jemalloc | 2 +- .../internal/jemalloc_internal_defs.h.in | 9 +- .../internal/jemalloc_internal_defs.h.in | 9 +- .../internal/jemalloc_internal_defs.h.in | 9 +- .../internal/jemalloc_internal_defs.h.in | 9 +- .../internal/jemalloc_internal_defs.h.in | 9 +- .../internal/jemalloc_internal_defs.h.in | 9 +- .../internal/jemalloc_internal_defs.h.in | 8 + .../internal/jemalloc_internal_defs.h.in | 9 +- .../internal/jemalloc_internal_defs.h.in | 9 +- .../internal/jemalloc_internal_defs.h.in | 9 +- .../internal/jemalloc_internal_defs.h.in | 9 +- .../internal/jemalloc_internal_defs.h.in | 9 +- .../internal/jemalloc_internal_defs.h.in | 9 +- contrib/mongo-c-driver | 2 +- contrib/mongo-c-driver-cmake/CMakeLists.txt | 8 +- contrib/mongo-cxx-driver | 2 +- contrib/orc | 1 - contrib/sqlite-amalgamation | 2 +- .../regular-functions/ai-functions.mdx | 79 +- .../_server_settings_outside_source.mdx | 2 +- docs/reference/statements/grant.mdx | 28 + programs/local/LocalServer.cpp | 8 + programs/main.cpp | 2 - programs/server/Server.cpp | 13 + programs/server/config.xml | 4 +- src/Access/AccessControl.cpp | 38 +- src/Access/Common/AccessType.h | 13 +- src/Access/ContextAccess.cpp | 9 +- src/Access/EnabledQuota.cpp | 17 + src/Access/EnabledQuota.h | 9 + src/Access/QuotaCache.cpp | 26 +- src/Access/QuotaCache.h | 7 +- src/Access/SettingsConstraints.cpp | 39 +- src/Access/SettingsConstraints.h | 13 +- .../AggregateFunctionGini.cpp | 22 +- .../AggregateFunctionGroupArray.cpp | 31 +- .../FunctionSecretArgumentsFinderTreeNode.cpp | 6 + .../FunctionSecretArgumentsFinderTreeNode.h | 3 +- .../Passes/FunctionToSubcolumnsPass.cpp | 4 + src/Analyzer/QueryTreeBuilder.cpp | 34 +- src/Backups/BackupEntriesCollector.cpp | 37 +- src/Backups/BackupEntriesCollector.h | 4 +- src/Backups/BackupIO_AzureBlobStorage.cpp | 17 +- src/Backups/BackupMetadataFinder.cpp | 11 + src/Backups/BackupUtils.cpp | 88 + src/Backups/BackupUtils.h | 17 + src/Backups/RestorerFromBackup.cpp | 91 +- src/Backups/RestorerFromBackup.h | 4 + src/CMakeLists.txt | 13 +- .../BuzzHouse/Generator/TableSettings.cpp | 2 +- src/Client/ClientBase.cpp | 6 +- src/Client/ClientBaseHelpers.cpp | 10 +- src/Client/ClientBaseHelpers.h | 4 +- src/Client/ReplxxLineReader.cpp | 21 +- src/Client/ReplxxLineReader.h | 19 +- src/Columns/ColumnArray.cpp | 24 +- src/Columns/ColumnDynamic.cpp | 29 + src/Columns/ColumnDynamic.h | 35 +- src/Columns/ColumnLowCardinality.cpp | 10 +- src/Columns/ColumnLowCardinality.h | 1 + src/Columns/ReverseIndex.h | 99 +- .../gtest_reverse_index_exception_safety.cpp | 126 ++ src/Common/ColumnsHashing.h | 2 +- src/Common/CurrentMetrics.cpp | 3 + src/Common/DNSResolver.cpp | 6 + src/Common/FailPoint.cpp | 4 + src/Common/HashTable/Prefetching.h | 14 + src/Common/HashTable/StringHashTable.h | 5 + src/Common/ProfileEvents.cpp | 3 + src/Common/ProgressIndication.cpp | 5 +- src/Common/ProgressIndication.h | 4 + src/Common/RemoteHostFilter.cpp | 4 + src/Common/Scheduler/CPUSlotsAllocation.cpp | 7 +- src/Common/Scheduler/CPUSlotsAllocation.h | 4 + src/Common/Scheduler/IWorkloadNode.h | 4 + .../Nodes/SpaceShared/AllocationQueue.cpp | 3 + .../Scheduler/Nodes/TimeShared/FifoQueue.h | 3 + src/Common/Scheduler/Nodes/WorkloadNode.h | 29 + .../tests/gtest_workload_resource_manager.cpp | 173 ++ .../Scheduler/WorkloadResourceManager.cpp | 4 + src/Common/ThreadStatus.cpp | 5 +- src/Common/ZooKeeper/ZooKeeperConstants.h | 2 + src/Common/ZooKeeper/ZooKeeperIO.cpp | 23 +- src/Common/maskURIPassword.h | 65 +- src/Common/mysqlxx/Connection.cpp | 8 + src/Common/mysqlxx/Pool.cpp | 53 + src/Common/setThreadName.h | 2 + .../tests/gtest_distrusted_certificates.cpp | 137 ++ .../gtest_file_channel_rotation_failure.cpp | 133 ++ src/Coordination/Changelog.cpp | 2 + src/Coordination/KeeperStorageImpl.cpp | 13 +- src/Coordination/tests/gtest_coordination.cpp | 63 + .../tests/gtest_coordination_changelog.cpp | 40 + src/Core/BackgroundSchedulePool.cpp | 38 +- src/Core/BackgroundSchedulePool.h | 8 +- src/Core/BaseSettings.h | 10 +- src/Core/BaseSettingsProgramOptions.h | 4 +- src/Core/Field.cpp | 4 + src/Core/FormatFactorySettings.h | 7 + src/Core/PostgreSQLProtocol.h | 28 +- src/Core/ServerSettings.cpp | 20 + src/Core/Settings.cpp | 39 +- src/Core/Settings.h | 1 + src/Core/SettingsChangesHistory.cpp | 6 +- src/Core/SettingsFields.cpp | 77 + src/Core/SettingsFields.h | 38 + src/Core/Streaming/CursorTree.cpp | 9 + .../tests/gtest_BackgroundSchedulePool.cpp | 222 +++ src/DataTypes/DataTypeMap.cpp | 2 + src/DataTypes/IDataType.cpp | 1 + src/DataTypes/Serializations/ISerialization.h | 3 + .../SerializationLowCardinality.cpp | 7 + .../Serializations/SerializationObject.cpp | 34 +- .../gtest_low_cardinality_serialization.cpp | 179 ++ .../tests/gtest_DataTypeObject_create.cpp | 129 ++ src/Databases/DataLake/HiveCatalog.cpp | 14 +- src/Databases/DatabaseReplicated.cpp | 134 +- src/Databases/DatabaseReplicated.h | 22 +- src/Databases/DatabaseReplicatedWorker.cpp | 65 +- src/Databases/DatabaseReplicatedWorker.h | 25 + src/Databases/MySQL/DatabaseMySQL.cpp | 11 +- .../ClickHouseDictionarySource.cpp | 9 +- src/Dictionaries/DirectDictionary.cpp | 12 +- src/Dictionaries/MongoDBDictionarySource.cpp | 7 +- src/Dictionaries/MySQLDictionarySource.cpp | 6 +- src/Dictionaries/SSDCacheDictionaryStorage.h | 49 +- .../YAMLRegExpTreeDictionarySource.cpp | 2 +- src/Dictionaries/readInvalidateQuery.cpp | 33 +- src/Dictionaries/readInvalidateQuery.h | 5 + src/Disks/DiskFromAST.cpp | 335 +++- src/Disks/DiskFromAST.h | 13 +- .../DiskObjectStorageTransaction.cpp | 41 +- .../AzureBlobStorageCommon.cpp | 30 + .../AzureBlobStorage/AzureBlobStorageCommon.h | 6 + .../AzureBlobStorage/AzureObjectStorage.cpp | 6 +- src/Disks/IDisk.h | 9 +- .../IO/AsynchronousBoundedReadBuffer.cpp | 1 + .../IO/ReadBufferFromAzureBlobStorage.cpp | 2 +- src/Disks/IStoragePolicy.h | 3 + src/Disks/StoragePolicy.cpp | 29 + src/Disks/StoragePolicy.h | 6 + src/Disks/getDiskConfigurationFromAST.cpp | 40 +- src/Disks/getDiskConfigurationFromAST.h | 9 + .../gtest_azure_read_buffer_missing_etag.cpp | 106 ++ src/Formats/FormatFactory.cpp | 1 + src/Formats/FormatSchemaInfo.cpp | 5 +- src/Formats/FormatSettings.h | 8 + src/Functions/AI/AIQuotaTracker.cpp | 33 + src/Functions/AI/AIQuotaTracker.h | 24 +- src/Functions/AI/AIRequestExecutor.cpp | 269 +++ src/Functions/AI/AIRequestExecutor.h | 41 + src/Functions/AI/AnthropicProvider.cpp | 2 +- src/Functions/AI/AnthropicProvider.h | 2 +- src/Functions/AI/IAIProvider.cpp | 6 +- src/Functions/AI/IAIProvider.h | 38 +- src/Functions/AI/OpenAIProvider.cpp | 23 +- src/Functions/AI/OpenAIProvider.h | 2 +- src/Functions/FunctionBaseAI.cpp | 481 +++--- src/Functions/FunctionBaseAI.h | 49 +- src/Functions/FunctionDynamicAdaptor.h | 2 + src/Functions/FunctionVariantAdaptor.h | 2 + src/Functions/FunctionsJSON.cpp | 5 +- src/Functions/FunctionsMiscellaneous.h | 14 + src/Functions/IFunction.h | 11 + src/Functions/IFunctionAdaptors.h | 3 + src/Functions/aiClassify.cpp | 7 +- src/Functions/aiEmbed.cpp | 46 +- src/Functions/aiExtract.cpp | 7 +- src/Functions/aiFilter.cpp | 7 +- src/Functions/aiGenerate.cpp | 7 +- src/Functions/aiRedact.cpp | 7 +- src/Functions/aiSimilarity.cpp | 48 +- src/Functions/aiTranslate.cpp | 7 +- src/Functions/parseDateTime.cpp | 41 +- src/IO/CachedInMemoryReadBufferFromFile.cpp | 2 + src/IO/PipelineReadBuffer.cpp | 1 + src/IO/ReadPipeline.h | 4 +- src/IO/S3/Client.cpp | 25 +- src/IO/S3/Client.h | 2 + src/IO/S3/PocoHTTPClient.cpp | 23 +- src/IO/SharedThreadPools.h | 4 +- src/IO/SnappyFramedReadBuffer.cpp | 4 +- src/IO/SnappyFramedReadBuffer.h | 3 +- src/IO/tests/gtest_brotli_decoder.cpp | 58 + src/Interpreters/Aggregator.cpp | 2 +- src/Interpreters/BloomFilter.cpp | 2 - src/Interpreters/BloomFilter.h | 7 +- src/Interpreters/Context.cpp | 55 +- src/Interpreters/Context.h | 11 + src/Interpreters/DatabaseCatalog.cpp | 16 + src/Interpreters/InterpreterAlterQuery.cpp | 143 +- src/Interpreters/InterpreterCreateQuery.cpp | 120 +- src/Interpreters/InterpreterCreateQuery.h | 4 + src/Interpreters/InterpreterDeleteQuery.cpp | 4 +- src/Interpreters/InterpreterInsertQuery.cpp | 8 +- src/Interpreters/InterpreterSystemQuery.cpp | 3 +- src/Interpreters/JIT/CHJIT.cpp | 9 +- src/Interpreters/JIT/CHJIT.h | 6 +- src/Interpreters/convertColumnToType.cpp | 102 +- src/Interpreters/convertColumnToType.h | 6 +- .../tests/gtest_convert_column_to_type.cpp | 18 + src/Loggers/Loggers.cpp | 9 + src/Parsers/FunctionSecretArgumentsFinder.cpp | 217 ++- src/Parsers/FunctionSecretArgumentsFinder.h | 25 + src/Planner/Planner.cpp | 4 +- src/Processors/Chunk.cpp | 12 +- src/Processors/Executors/ExecutorTasks.cpp | 105 +- src/Processors/Executors/ExecutorTasks.h | 42 +- src/Processors/Executors/PipelineExecutor.cpp | 99 +- src/Processors/Executors/PipelineExecutor.h | 20 +- src/Processors/Formats/IInputFormat.h | 2 + .../Formats/Impl/ArrowBatchRowLimit.cpp | 319 ++++ .../Formats/Impl/ArrowBatchRowLimit.h | 47 + .../Formats/Impl/ArrowBufferedStreams.cpp | 2 +- .../Formats/Impl/ArrowBufferedStreams.h | 2 +- .../Formats/Impl/ArrowColumnToCHColumn.cpp | 2 +- .../Formats/Impl/ArrowColumnToCHColumn.h | 2 +- .../ArrowIPC/ArrowIPCBlockOutputFormat.cpp | 40 +- .../Impl/ArrowIPC/ArrowIPCBlockOutputFormat.h | 4 + .../Impl/ArrowIPC/RecordBatchDecoder.cpp | 85 +- .../Impl/ArrowIPC/RecordBatchDecoder.h | 10 +- .../Impl/ArrowIPC/RecordBatchEncoder.cpp | 285 ++-- .../Impl/ArrowIPC/RecordBatchEncoder.h | 18 +- .../Formats/Impl/AvroRowInputFormat.cpp | 4 +- .../Formats/Impl/CHColumnToArrowColumn.cpp | 125 +- .../Formats/Impl/CHColumnToArrowColumn.h | 3 + .../Formats/Impl/MsgPackRowInputFormat.cpp | 4 +- .../Formats/Impl/MsgPackRowInputFormat.h | 3 +- .../Formats/Impl/MySQLDumpRowInputFormat.cpp | 9 +- .../Impl/NativeORCBlockInputFormat.cpp | 1062 ++++++------ .../Formats/Impl/NativeORCBlockInputFormat.h | 55 +- .../Formats/Impl/ORC/ColumnReader.cpp | 1218 ++++++++++++++ .../Formats/Impl/ORC/ColumnReader.h | 63 + .../Formats/Impl/ORC/Compression.cpp | 437 +++++ src/Processors/Formats/Impl/ORC/Compression.h | 42 + src/Processors/Formats/Impl/ORC/Proto.cpp | 944 +++++++++++ src/Processors/Formats/Impl/ORC/Proto.h | 343 ++++ src/Processors/Formats/Impl/ORC/RLE.cpp | 1129 +++++++++++++ src/Processors/Formats/Impl/ORC/RLE.h | 151 ++ src/Processors/Formats/Impl/ORC/Reader.cpp | 842 ++++++++++ src/Processors/Formats/Impl/ORC/Reader.h | 188 +++ .../Formats/Impl/ORC/SearchArgument.cpp | 1075 ++++++++++++ .../Formats/Impl/ORC/SearchArgument.h | 198 +++ .../Formats/Impl/ORC/Statistics.cpp | 462 +++++ src/Processors/Formats/Impl/ORC/Statistics.h | 71 + src/Processors/Formats/Impl/ORC/Stream.cpp | 248 +++ src/Processors/Formats/Impl/ORC/Stream.h | 140 ++ src/Processors/Formats/Impl/ORC/Timezone.cpp | 90 + src/Processors/Formats/Impl/ORC/Timezone.h | 25 + src/Processors/Formats/Impl/ORC/Type.cpp | 380 +++++ src/Processors/Formats/Impl/ORC/Type.h | 114 ++ src/Processors/Formats/Impl/ORC/Vector.cpp | 143 ++ src/Processors/Formats/Impl/ORC/Vector.h | 174 ++ src/Processors/Formats/Impl/ORC/Writer.cpp | 1493 +++++++++++++++++ src/Processors/Formats/Impl/ORC/Writer.h | 78 + .../Formats/Impl/ORCBlockOutputFormat.cpp | 490 +----- .../Formats/Impl/ORCBlockOutputFormat.h | 40 +- .../Formats/Impl/Parquet/Decoding.cpp | 21 +- .../Formats/Impl/Parquet/Decoding.h | 3 + .../Formats/Impl/Parquet/SchemaConverter.cpp | 4 +- .../Formats/Impl/ParquetBlockOutputFormat.cpp | 49 +- .../Formats/Impl/ParquetBlockOutputFormat.h | 12 +- .../Impl/ParquetV3BlockInputFormat.cpp | 32 +- .../Formats/Impl/ParquetV3BlockInputFormat.h | 11 +- .../Formats/Impl/RawBLOBRowOutputFormat.cpp | 20 +- .../Formats/Impl/RawBLOBRowOutputFormat.h | 8 +- .../QueryPlanOptimizationSettings.cpp | 5 +- .../QueryPlanOptimizationSettings.h | 3 +- .../Optimizations/SipHashingWriteBuffer.h | 36 + .../calculateHashTableCacheKeys.cpp | 10 +- .../QueryPlan/Optimizations/optimizeJoin.cpp | 52 +- .../Optimizations/optimizeReadInOrder.cpp | 42 +- .../QueryPlan/Optimizations/optimizeTree.cpp | 4 +- src/Processors/QueryPlan/PartsSplitter.cpp | 1 + src/Processors/QueryPlan/PartsSplitter.h | 3 + src/Processors/QueryPlan/ReadFromLoopStep.cpp | 4 +- .../QueryPlan/ReadFromMergeTree.cpp | 282 +++- src/Processors/QueryPlan/ReadFromMergeTree.h | 12 + ...t_siphashing_write_buffer_key_identity.cpp | 98 ++ .../Transforms/AddingDefaultsTransform.cpp | 3 +- ...umn_to_arrow_column_record_batch_split.cpp | 127 ++ ...gtest_pipeline_executor_late_cpu_slots.cpp | 241 +++ .../gtest_write_orc_iceberg_required.cpp | 16 +- src/Server/ArrowFlight/ArrowFlightServer.cpp | 136 +- src/Server/PostgreSQLHandler.cpp | 4 +- src/Storages/AlterCommands.cpp | 64 + src/Storages/AlterCommands.h | 3 + .../DistributedAsyncInsertBatch.cpp | 18 + .../DistributedAsyncInsertDirectoryQueue.cpp | 6 + .../DistributedAsyncInsertHelpers.cpp | 8 + .../DistributedAsyncInsertHelpers.h | 9 + src/Storages/Distributed/DistributedSink.cpp | 18 +- src/Storages/Hive/HiveFile.cpp | 93 +- src/Storages/Hive/HiveFile.h | 8 +- src/Storages/Hive/StorageHive.cpp | 4 + src/Storages/IStorage.h | 3 + src/Storages/IStorageCluster.cpp | 2 +- src/Storages/IStorageCluster.h | 11 +- src/Storages/IndicesDescription.cpp | 122 +- src/Storages/IndicesDescription.h | 15 + .../MaterializedView/RefreshSchedule.cpp | 7 +- .../MergeTree/DataPartStorageOnDiskBase.cpp | 31 +- .../MergeTree/DataPartStorageOnDiskBase.h | 10 + .../MergeTree/DataPartStorageOnDiskPacked.cpp | 8 +- src/Storages/MergeTree/KeyCondition.cpp | 70 +- .../MergeTree/MergeTreeBlockReadUtils.cpp | 3 +- src/Storages/MergeTree/MergeTreeData.cpp | 90 +- src/Storages/MergeTree/MergeTreeData.h | 12 + .../MergeTree/MergeTreeDataSelectExecutor.cpp | 18 + .../MergeTree/MergeTreeDataSelectExecutor.h | 13 + .../MergeTree/MergeTreeDataWriter.cpp | 5 + .../MergeTree/MergeTreeIndexBloomFilter.cpp | 19 +- .../MergeTreeIndexBloomFilterText.cpp | 30 + src/Storages/MergeTree/MergeTreeIndices.h | 5 + .../MergeTreeReaderCompactSingleBuffer.cpp | 4 +- .../MergeTree/MergeTreeReaderWide.cpp | 6 +- .../MergeTree/MergeTreeSelectProcessor.cpp | 11 +- .../MergeTree/MergeTreeSelectProcessor.h | 6 +- .../MergeTree/MergeTreeSequentialSource.cpp | 2 +- src/Storages/MergeTree/MergeTreeSettings.cpp | 25 +- src/Storages/MergeTree/MergeTreeSettings.h | 12 +- .../MergeTreeSplitPrewhereIntoReadSteps.cpp | 55 +- .../MergeTreeSplitPrewhereIntoReadSteps.h | 5 +- .../MergeTree/MergeTreeWhereOptimizer.cpp | 70 +- .../MergeTree/MergeTreeWhereOptimizer.h | 12 +- .../MergeTree/MutateFromLogEntryTask.cpp | 2 +- src/Storages/MergeTree/MutateTask.cpp | 4 +- .../ReplicatedMergeTreeCleanupThread.cpp | 2 +- src/Storages/MergeTree/checkDataPart.cpp | 27 + src/Storages/MergeTree/checkDataPart.h | 3 + .../MergeTree/registerStorageMergeTree.cpp | 15 +- .../gtest_merge_tree_index_bloom_filter.cpp | 74 + .../ObjectStorage/Azure/Configuration.cpp | 118 +- .../ObjectStorage/Azure/Configuration.h | 2 +- .../Common/AvroForIcebergDeserializer.cpp | 1 + .../Common/AvroForIcebergDeserializer.h | 5 + .../DataLakes/DeltaLake/PartitionPruner.cpp | 27 +- .../DataLakes/DeltaLake/PartitionPruner.h | 3 + .../DataLakes/DeltaLake/TableSnapshot.cpp | 36 +- .../DataLakes/DeltaLake/WriteTransaction.cpp | 78 +- .../DeltaLakeMetadataDeltaKernel.cpp | 15 +- .../DataLakes/Iceberg/IcebergIterator.cpp | 3 +- .../DataLakes/Iceberg/IcebergMetadata.cpp | 9 + .../Iceberg/IcebergMetadataFilesCache.h | 2 +- .../DataLakes/Iceberg/MetadataGenerator.cpp | 13 +- .../DataLakes/Iceberg/MultipleFileWriter.cpp | 3 + .../DataLakes/Iceberg/Mutations.cpp | 4 + .../DataLakes/Iceberg/SchemaProcessor.cpp | 50 + .../DataLakes/Iceberg/SchemaProcessor.h | 3 + .../Iceberg/StatelessMetadataFileGetter.cpp | 9 +- .../Iceberg/StatelessMetadataFileGetter.h | 3 +- .../StorageObjectStorageCluster.cpp | 9 +- .../StorageObjectStorageCluster.h | 3 +- .../StorageObjectStorageSource.cpp | 22 +- .../StorageObjectStorageSource.h | 5 +- .../ObjectStorageQueue/AzureQueue_fwd.h | 3 +- ...bjectStorageQueueMoveDestinationAccess.cpp | 119 ++ .../ObjectStorageQueueMoveDestinationAccess.h | 37 + .../ObjectStorageQueuePostProcessor.cpp | 22 + .../ObjectStorageQueueSettings.cpp | 31 +- .../ObjectStorageQueueSettings.h | 3 +- .../StorageObjectStorageQueue.cpp | 66 +- .../registerQueueStorage.cpp | 9 + .../StorageMaterializedPostgreSQL.cpp | 8 + .../StorageMaterializedPostgreSQL.h | 1 + src/Storages/StorageAlias.cpp | 22 + src/Storages/StorageAlias.h | 7 +- src/Storages/StorageBuffer.cpp | 7 + src/Storages/StorageBuffer.h | 1 + src/Storages/StorageDistributed.cpp | 22 +- src/Storages/StorageExecutable.cpp | 6 +- src/Storages/StorageFactory.cpp | 53 + src/Storages/StorageFactory.h | 24 + src/Storages/StorageFileCluster.cpp | 16 +- src/Storages/StorageFileCluster.h | 3 +- src/Storages/StorageInMemoryMetadata.cpp | 37 + src/Storages/StorageJoin.cpp | 2 +- src/Storages/StorageLog.cpp | 2 +- src/Storages/StorageMaterializedView.cpp | 14 +- src/Storages/StorageMaterializedView.h | 1 + src/Storages/StorageMerge.cpp | 17 +- src/Storages/StorageMerge.h | 1 + src/Storages/StorageMergeTree.cpp | 5 + src/Storages/StorageMongoDB.cpp | 58 + src/Storages/StorageMongoDB.h | 3 + src/Storages/StorageMySQL.cpp | 2 +- src/Storages/StorageProxy.h | 1 + src/Storages/StorageReplicatedMergeTree.cpp | 49 +- src/Storages/StorageSet.cpp | 2 +- src/Storages/StorageStripeLog.cpp | 2 +- src/Storages/StorageURLCluster.cpp | 16 +- src/Storages/StorageURLCluster.h | 3 +- src/Storages/StorageView.cpp | 3 +- .../System/StorageSystemDashboards.cpp | 11 +- .../System/StorageSystemMutations.cpp | 115 +- src/Storages/System/StorageSystemMutations.h | 3 +- ...torageSystemObjectStorageQueueSettings.cpp | 4 +- .../tests/gtest_azure_configuration.cpp | 34 + src/TableFunctions/ITableFunction.cpp | 16 +- src/TableFunctions/ITableFunction.h | 9 +- src/TableFunctions/TableFunctionMySQL.cpp | 2 +- .../TableFunctionViewIfPermitted.cpp | 3 + src/configure_config.cmake | 3 +- tests/integration/conftest.py | 30 + .../helpers/keeper_snapshot_utils.py | 33 +- .../integration/helpers/s3_mocks/broken_s3.py | 34 + .../test_ai_functions/mock_ai_server.py | 88 + tests/integration/test_ai_functions/test.py | 505 +++++- .../configs/config_with_hosts.xml | 1 + .../test_allowed_url_from_config/test.py | 116 ++ .../test_arrowflight_interface/test.py | 37 + .../test.py | 67 +- .../configs/disk_s3.xml | 8 + .../test_backup_restore_s3/test.py | 181 ++ .../test_backup_metadata_only.py | 134 ++ .../test_checking_s3_blobs_paranoid/test.py | 48 + .../test.py | 69 + tests/integration/test_crash_log/test.py | 5 + .../configs/prepare_on_startup.xml | 4 + .../test.py | 24 + tests/integration/test_dotnet_client/test.py | 9 +- tests/integration/test_git_import/test.py | 29 +- .../test_jemalloc_percpu_arena/test.py | 67 +- .../test_keeper_persistent_watches/test.py | 55 + .../test_mask_sensitive_info/test.py | 24 +- .../configs/remote_servers.xml | 13 + .../test.py | 120 ++ .../__init__.py | 0 .../configs/remote_servers.xml | 11 + .../test.py | 99 ++ .../test.py | 73 + .../test_scheduler_cpu_preemptive/test.py | 77 + .../test_manifest_length_cache_weight.py | 99 ++ .../test_async_metadata_refresh.py | 46 +- .../configs/small_message_broker_pool.xml | 3 + .../test_storage_kafka/test_batch_fast.py | 13 +- .../test_message_broker_pool_resize.py | 117 ++ .../test_storage_nats/test_nats_core.py | 38 +- tests/integration/test_storage_s3/test.py | 35 + tests/integration/test_storage_url/test.py | 3 +- .../__init__.py | 0 .../configs/rabbitmq.xml | 6 + .../configs/remote_servers.xml | 14 + .../configs/users.xml | 23 + .../test.py | 296 ++++ .../__init__.py | 0 .../configs/logger.xml | 9 + .../test_text_log_level_after_reload/test.py | 65 + tests/integration/test_ttl_move/test.py | 9 +- .../performance/final_read_in_order_limit.xml | 12 +- .../group_by_two_level_few_keys.xml | 42 + .../json_typed_path_subcolumn_lookup.xml | 223 +++ .../map_bucketed_serialization.xml | 8 +- tests/performance/map_subcolumns_prewhere.xml | 10 +- .../prewhere_subcolumn_read_steps.xml | 19 + .../small_block_contention_workload.xml | 13 + .../subcolumn_deserialization_tuple_map.xml | 8 +- .../0_stateless/00900_long_parquet_load_2.sh | 4 + tests/queries/0_stateless/01035_avg.reference | 8 +- tests/queries/0_stateless/01035_avg.sql | 2 +- .../01271_show_privileges.reference | 9 + .../01308_orc_output_format_arrays.reference | 2 +- .../01502_jemalloc_percpu_arena.sh | 10 +- .../01505_pipeline_executor_UAF.sh | 3 - ...01514_distributed_cancel_query_on_error.sh | 47 +- .../01721_join_implicit_cast_long.sql.j2 | 6 +- ...302_defaults_in_columnar_formats.reference | 2 + .../02302_defaults_in_columnar_formats.sql | 10 + ...dd_share_nested_offsets_disabled.reference | 4 + ...ndex_add_share_nested_offsets_disabled.sql | 16 + ...x_add_with_invalid_alter_command.reference | 1 + ...t_index_add_with_invalid_alter_command.sql | 24 + ...2346_text_index_transform_access.reference | 24 + .../02346_text_index_transform_access.sql | 167 ++ ...sform_access_replicated_database.reference | 4 + ...x_transform_access_replicated_database.sql | 54 + ...t_index_transform_alter_segments.reference | 4 + ...346_text_index_transform_alter_segments.sh | 37 + ...text_index_transform_data_access.reference | 6 + ...02346_text_index_transform_data_access.sql | 68 + ..._text_index_transform_on_cluster.reference | 14 + .../02346_text_index_transform_on_cluster.sh | 77 + .../0_stateless/02372_analyzer_join.sql.j2 | 2 +- .../0_stateless/02426_orc_bug.reference | 2 +- .../02884_authentication_quota.reference | 12 +- .../0_stateless/02884_authentication_quota.sh | 6 +- ...runing_empty_partition_zookeeper.reference | 6 + ...wu_58_pruning_empty_partition_zookeeper.sh | 140 ++ .../0_stateless/03300_ai_functions.reference | 2 +- .../0_stateless/03300_ai_functions.sql | 3 + .../0_stateless/03784_bad_base_backup.sh | 12 + .../03821_drop_workload_during_query_race.sh | 9 +- .../03822_file_function_read_on_file_grant.sh | 2 +- .../03923_kafka2_keeper_offsets.sh | 2 +- ...d_filters_after_final_in_explain.reference | 8 + ...eferred_filters_after_final_in_explain.sql | 1 + ...subcolumn_function_to_subcolumns.reference | 2 +- ...9_map_subcolumn_function_to_subcolumns.sql | 11 +- .../04000_map_subcolumns_prewhere.reference | 2 +- .../04000_map_subcolumns_prewhere.sql | 7 +- .../04039_text_index_map_values_subcolumn.sql | 5 +- ...0_commit_order_projection_merge_levels.sql | 15 +- ...42_ai_functions_named_collection_access.sh | 5 +- ...04207_alias_map_arrayelement_subcolumn.sql | 25 +- ...337_orc_schema_inference_deep_recursion.sh | 23 +- ..._virtual_row_disabled_with_final.reference | 7 - .../04489_virtual_row_disabled_with_final.sql | 46 - ...00_group_by_limit_pushdown_no_order_by.sql | 8 +- ...01_group_by_limit_pushdown_heap_stress.sql | 8 +- ...04513_prewhere_map_subcolumn_read_cost.sql | 9 +- ...map_element_duplicate_keys_first_match.sql | 19 +- ...functions_compatibility_defaults.reference | 4 +- ...28_ai_functions_compatibility_defaults.sql | 13 +- .../04648_kafka2_race_creation_removal.sh | 2 +- .../04707_tuple_subcolumn_pushdown_orc.sh | 2 +- ...shable_mv_randomize_for_overflow.reference | 1 + ...7_refreshable_mv_randomize_for_overflow.sh | 25 + .../04856_orc_corrupt_stripe_footer.reference | 1 - .../04856_orc_corrupt_stripe_footer.sh | 10 +- ...orc_unimplemented_codec_no_abort.reference | 14 + .../05036_orc_unimplemented_codec_no_abort.sh | 59 + ..._schema_cache_leftover_temp_file.reference | 2 + ..._format_schema_cache_leftover_temp_file.sh | 45 + ...replicated_max_logs_to_keep_zero.reference | 4 + .../05053_replicated_max_logs_to_keep_zero.sh | 42 + ...postgresql_protocol_message_size.reference | 4 + .../05054_postgresql_protocol_message_size.sh | 66 + ...5_arrow_split_large_record_batch.reference | 10 + .../05055_arrow_split_large_record_batch.sh | 56 + ...t_schema_source_query_background_access.sh | 7 +- ...ow_record_batch_split_boundaries.reference | 4 + ...056_arrow_record_batch_split_boundaries.sh | 52 + ...ert_column_to_type_bool_identity.reference | 16 + ...9_convert_column_to_type_bool_identity.sql | 26 + ...raints_nested_settings_effective.reference | 35 + ...s_constraints_nested_settings_effective.sh | 146 ++ ...ttings_equal_value_marks_changed.reference | 18 + ...sted_settings_equal_value_marks_changed.sh | 48 + ...ndex_map_lowcardinality_subcolumn_cast.sql | 7 +- ...ble_from_path_read_on_file_grant.reference | 7 + ...tach_table_from_path_read_on_file_grant.sh | 67 + ...8_quota_client_key_stale_context.reference | 12 + .../05218_quota_client_key_stale_context.sh | 54 + ...l_nullable_partition_key_pruning.reference | 43 + ...ot_null_nullable_partition_key_pruning.sql | 115 ++ ...d_drop_memory_reservation_in_use.reference | 1 + ...workload_drop_memory_reservation_in_use.sh | 35 + ...ssd_cache_dictionaries_same_path.reference | 7 + .../05258_ssd_cache_dictionaries_same_path.sh | 64 + ...age_partition_strategy_key_value.reference | 11 + ...b_storage_partition_strategy_key_value.sql | 106 ++ ...91_system_mutations_hide_secrets.reference | 5 + .../05291_system_mutations_hide_secrets.sql | 47 + ...16_ai_functions_last_in_prewhere.reference | 15 + .../05316_ai_functions_last_in_prewhere.sh | 92 + ...rupt_stripe_footer_stream_column.reference | 3 + ...orc_corrupt_stripe_footer_stream_column.sh | 49 + ..._read_in_order_final_virtual_row.reference | 144 ++ .../05325_read_in_order_final_virtual_row.sql | 162 ++ ..._inference_deep_named_type_chain.reference | 2 + ..._schema_inference_deep_named_type_chain.sh | 55 + ...le_function_table_without_source.reference | 3 + ...kup_table_function_table_without_source.sh | 19 + ...39_prewhere_subcolumn_read_steps.reference | 7 + .../05339_prewhere_subcolumn_read_steps.sql | 79 + ...ement_to_subcolumn_bucketed_only.reference | 30 + ...map_element_to_subcolumn_bucketed_only.sql | 76 + ...ions_allow_sql_named_collections.reference | 34 + ...i_functions_allow_sql_named_collections.sh | 80 + ...85_ipv6_all_zero_literal_host_ip.reference | 3 + .../05385_ipv6_all_zero_literal_host_ip.sql | 13 + ...i_functions_retry_setting_limits.reference | 2 + ...5394_ai_functions_retry_setting_limits.sql | 31 + ...ubcolumn_size_without_reading_prefixes.sql | 2 +- ...rite_many_partitions_thread_pool.reference | 3 + ...eberg_write_many_partitions_thread_pool.sh | 39 + ...06_join_swap_array_join_estimate.reference | 13 + .../05406_join_swap_array_join_estimate.sql | 84 + ...407_bool_force_setting_shorthand.reference | 4 + .../05407_bool_force_setting_shorthand.sh | 28 + ...mic_deep_value_column_operations.reference | 4 + ...07_dynamic_deep_value_column_operations.sh | 50 + ...407_orc_timezone_traversal_abort.reference | 5 + .../05407_orc_timezone_traversal_abort.sh | 137 ++ ...e_date_time_daylight_saving_time.reference | 65 + ...0_parse_date_time_daylight_saving_time.sql | 51 + ...oom_filter_text_index_total_size.reference | 2 + ...421_bloom_filter_text_index_total_size.sql | 32 + ...ter_text_index_total_size_attach.reference | 4 + ...oom_filter_text_index_total_size_attach.sh | 20 + .../05427_azure_sas_url_parsing.reference | 0 .../05427_azure_sas_url_parsing.sql | 9 + ..._dictionary_read_truncate_source.reference | 4 + ..._direct_dictionary_read_truncate_source.sh | 45 + ...mysqldump_create_without_columns.reference | 51 + .../05428_mysqldump_create_without_columns.sh | 38 + .../05441_ai_functions_privileges.reference | 4 + .../05441_ai_functions_privileges.sql.j2 | 70 + .../05441_deep_refresh_cursor.reference | 1 + .../0_stateless/05441_deep_refresh_cursor.sh | 10 + ...05441_rawblob_aggregate_function.reference | 5 + .../05441_rawblob_aggregate_function.sh | 18 + ...ing_table_unknown_storage_policy.reference | 4 + ...o_existing_table_unknown_storage_policy.sh | 53 + ..._s3queue_move_destination_access.reference | 35 + .../05441_s3queue_move_destination_access.sh | 374 +++++ ...ns_privileges_stored_expressions.reference | 1 + ...unctions_privileges_stored_expressions.sql | 56 + ...ll_as_default_non_nullable_field.reference | 33 + ...rrow_null_as_default_non_nullable_field.sh | 118 ++ ...t_mv_dedup_empty_partition_block.reference | 3 + ..._insert_mv_dedup_empty_partition_block.sql | 39 + ...converted_outer_join_type_change.reference | 12 + ...order_converted_outer_join_type_change.sql | 71 + ...uted_insert_select_source_access.reference | 9 + ...distributed_insert_select_source_access.sh | 87 + ...nt96_date_time_overflow_behavior.reference | 13 + ...rquet_int96_date_time_overflow_behavior.sh | 19 + .../05444_ai_functions_no_user.reference | 9 + .../05444_ai_functions_no_user.sql | 98 ++ ...444_parquet_int96_lower_boundary.reference | 2 + .../05444_parquet_int96_lower_boundary.sh | 16 + ...uted_async_insert_batch_per_user.reference | 1 + ...distributed_async_insert_batch_per_user.sh | 41 + ...rminal_concurrent_sessions_input.reference | 2 + ...5_webterminal_concurrent_sessions_input.sh | 110 ++ ...cluster_function_column_defaults.reference | 18 + ...select_cluster_function_column_defaults.sh | 113 ++ ...sas_url_hidden_in_table_function.reference | 13 + ..._azure_sas_url_hidden_in_table_function.sh | 43 + .../05447_orc_native_reader_writer.reference | 14 + .../05447_orc_native_reader_writer.sh | 46 + .../05448_orc_date_after_2059.reference | 12 + .../0_stateless/05448_orc_date_after_2059.sh | 20 + ...mats_read_padding_object_storage.reference | 18 + ...ext_formats_read_padding_object_storage.sh | 46 + .../05453_orc_bad_bloom_filter.reference | 8 + .../0_stateless/05453_orc_bad_bloom_filter.sh | 27 + ...mote_clickhouse_invalidate_query.reference | 4 + ...nary_remote_clickhouse_invalidate_query.sh | 52 + ...n_extract_datetime_compatibility.reference | 5 + ...57_json_extract_datetime_compatibility.sql | 19 + ...check_ignores_query_cancellation.reference | 4 + ...7_part_check_ignores_query_cancellation.sh | 144 ++ ...entials_hidden_in_logged_queries.reference | 39 + ...re_credentials_hidden_in_logged_queries.sh | 62 + ...ai_functions_background_defaults.reference | 3 + ...05459_ai_functions_background_defaults.sql | 28 + .../05473_orc_footer_content_length.reference | 2 + .../05473_orc_footer_content_length.sh | 53 + .../05474_orc_decimal64_v2.reference | 20 + .../0_stateless/05474_orc_decimal64_v2.sh | 13 + ..._too_small_last_column_id_128455.reference | 10 + ..._column_too_small_last_column_id_128455.sh | 130 ++ ...05480_snappy_framed_read_padding.reference | 4 + .../05480_snappy_framed_read_padding.sh | 32 + ..._storage_parquet_footer_prefetch.reference | 4 + ..._object_storage_parquet_footer_prefetch.sh | 38 + .../05481_view_with_two_array_joins.reference | 38 + .../05481_view_with_two_array_joins.sql | 47 + ..._record_batch_split_many_batches.reference | 2 + ...3_arrow_record_batch_split_many_batches.sh | 41 + ...abase_ddl_during_restore_replica.reference | 1 + ...ted_database_ddl_during_restore_replica.sh | 65 + ..._replica_waits_for_initial_query.reference | 5 + ...restore_replica_waits_for_initial_query.sh | 58 + ...ictionary_cache_file_cannot_grow.reference | 6 + ...cache_dictionary_cache_file_cannot_grow.sh | 57 + ...on_create_temporary_table_access.reference | 31 + ..._function_create_temporary_table_access.sh | 110 ++ ...async_insert_after_invalid_entry.reference | 6 + ...sgpack_async_insert_after_invalid_entry.sh | 36 + ...5493_mysql_host_connector_syntax.reference | 13 + .../05493_mysql_host_connector_syntax.sh | 77 + ...ttings_system_table_hide_secrets.reference | 4 + ...eue_settings_system_table_hide_secrets.sql | 22 + ...dex_archive_newer_format_version.reference | 7 + ...skip_index_archive_newer_format_version.sh | 53 + ...ntity_without_credentials_marker.reference | 25 + ...isk_identity_without_credentials_marker.sh | 85 + ...apper_of_server_credentials_disk.reference | 21 + ...disk_wrapper_of_server_credentials_disk.sh | 74 + ...p_by_two_level_few_keys_prefetch.reference | 115 ++ ...8_group_by_two_level_few_keys_prefetch.sql | 62 + ...498_s3_disk_sse_c_key_validation.reference | 1 + .../05498_s3_disk_sse_c_key_validation.sql | 31 + ...urce_merge_loop_executable_crash.reference | 9 + ...7_mv_source_merge_loop_executable_crash.sh | 133 ++ .../05507_orc_malformed_timestamps.reference | 16 + .../05507_orc_malformed_timestamps.sh | 19 + ...8_orc_writer_out_of_range_values.reference | 7 + .../05508_orc_writer_out_of_range_values.sh | 26 + ..._not_grow_with_distinct_literals.reference | 1 + ...xt_does_not_grow_with_distinct_literals.sh | 31 + ...05509_orc_malformed_dictionaries.reference | 16 + .../05509_orc_malformed_dictionaries.sh | 21 + ...510_orc_allocation_amplification.reference | 32 + .../05510_orc_allocation_amplification.sh | 27 + ...rc_malformed_stripes_and_streams.reference | 16 + ...05511_orc_malformed_stripes_and_streams.sh | 19 + .../05512_orc_malformed_row_groups.reference | 32 + .../05512_orc_malformed_row_groups.sh | 26 + ...c_metadata_repeated_field_limits.reference | 16 + ...5513_orc_metadata_repeated_field_limits.sh | 17 + ...imestamp_statistics_before_epoch.reference | 21 + ...4_orc_timestamp_statistics_before_epoch.sh | 45 + .../05515_orc_nullable_floats.reference | 2 + .../0_stateless/05515_orc_nullable_floats.sh | 30 + ..._replica_waits_for_initial_alter.reference | 6 + ...restore_replica_waits_for_initial_alter.sh | 58 + ...rquet_encoder_shared_thread_pool.reference | 2 + ...5565_parquet_encoder_shared_thread_pool.sh | 40 + ...up_array_sample_counter_overflow.reference | 3 + ...67_group_array_sample_counter_overflow.sql | 28 + ...05567_mongodb_server_credentials.reference | 27 + .../05567_mongodb_server_credentials.sh | 95 ++ ...db_dictionary_server_credentials.reference | 12 + ...8_mongodb_dictionary_server_credentials.sh | 85 + ...c_requested_type_of_another_kind.reference | 18 + ...5574_orc_requested_type_of_another_kind.sh | 56 + ...ke_partition_pruning_null_in_set.reference | 15 + ...elta_lake_partition_pruning_null_in_set.sh | 81 + ...ing_result_memory_limit_in_local.reference | 6 + ...cumulating_result_memory_limit_in_local.sh | 18 + ...imestamp_before_first_transition.reference | 7 + ...6_orc_timestamp_before_first_transition.sh | 40 + ...5617_gini_deserialize_validation.reference | 3 + .../05617_gini_deserialize_validation.sql | 15 + ..._struct_without_requested_fields.reference | 159 ++ ...nested_struct_without_requested_fields.sql | 101 ++ ...lumn_mapping_swapped_names_order.reference | 10 + ...lake_column_mapping_swapped_names_order.sh | 36 + ...21_readonly_named_collection_ddl.reference | 13 + .../05621_readonly_named_collection_ddl.sh | 25 + .../data_orc/bad_bloom_filter_1.6.0.orc | Bin 0 -> 709 bytes .../data_orc/bad_bloom_filter_1.6.11.orc | Bin 0 -> 717 bytes .../0_stateless/data_orc/decimal64_v2.orc | Bin 0 -> 738 bytes .../malformed_compression_block_1g.orc | Bin 0 -> 414 bytes .../data_orc/malformed_dictionary_empty.orc | Bin 0 -> 170 bytes .../malformed_dictionary_size_max.orc | Bin 0 -> 174 bytes .../malformed_footer_too_many_statistics.orc | Bin 0 -> 353 bytes .../malformed_metadata_too_many_stripes.orc | Bin 0 -> 673 bytes ...med_patched_base_terminal_continuation.orc | Bin 0 -> 83 bytes .../malformed_row_group_count_overflow.orc | Bin 0 -> 237 bytes .../malformed_row_index_seek_past_stream.orc | Bin 0 -> 213 bytes .../data_orc/malformed_snappy_chunk_size.orc | Bin 0 -> 31 bytes .../malformed_stream_offset_overflow.orc | Bin 0 -> 301 bytes .../malformed_stripe_offset_overflow.orc | Bin 0 -> 301 bytes .../malformed_timestamp_nanoseconds.orc | Bin 0 -> 238 bytes ...ormed_timestamp_statistics_nanoseconds.orc | Bin 0 -> 264 bytes .../data_orc/malformed_zstd_chunk_size.orc | Bin 0 -> 38 bytes .../data_orc/row_index_seek_control.orc | Bin 0 -> 213 bytes .../timestamps_before_epoch_arrow.orc | Bin 0 -> 294 bytes ...amps_before_epoch_pyarrow_bloom_filter.orc | Bin 0 -> 517 bytes .../05443_int96_out_of_range.parquet | Bin 0 -> 700 bytes .../05444_int96_lower_boundary.parquet | Bin 0 -> 535 bytes tests/queries/0_stateless/orc_malformed.lib | 55 + 793 files changed, 32784 insertions(+), 3558 deletions(-) create mode 100644 base/poco/NetSSL_OpenSSL/include/Poco/Net/DistrustedCertificates.h create mode 100644 base/poco/NetSSL_OpenSSL/src/DistrustedCertificates.cpp create mode 100644 base/poco/XML/src/entities.h delete mode 100644 contrib/arrow-cmake/orc_check.cmake delete mode 160000 contrib/orc create mode 100644 src/Columns/tests/gtest_reverse_index_exception_safety.cpp create mode 100644 src/Common/tests/gtest_distrusted_certificates.cpp create mode 100644 src/Common/tests/gtest_file_channel_rotation_failure.cpp create mode 100644 src/DataTypes/Serializations/tests/gtest_low_cardinality_serialization.cpp create mode 100644 src/Disks/tests/gtest_azure_read_buffer_missing_etag.cpp create mode 100644 src/Functions/AI/AIRequestExecutor.cpp create mode 100644 src/Functions/AI/AIRequestExecutor.h create mode 100644 src/IO/tests/gtest_brotli_decoder.cpp create mode 100644 src/Processors/Formats/Impl/ArrowBatchRowLimit.cpp create mode 100644 src/Processors/Formats/Impl/ArrowBatchRowLimit.h create mode 100644 src/Processors/Formats/Impl/ORC/ColumnReader.cpp create mode 100644 src/Processors/Formats/Impl/ORC/ColumnReader.h create mode 100644 src/Processors/Formats/Impl/ORC/Compression.cpp create mode 100644 src/Processors/Formats/Impl/ORC/Compression.h create mode 100644 src/Processors/Formats/Impl/ORC/Proto.cpp create mode 100644 src/Processors/Formats/Impl/ORC/Proto.h create mode 100644 src/Processors/Formats/Impl/ORC/RLE.cpp create mode 100644 src/Processors/Formats/Impl/ORC/RLE.h create mode 100644 src/Processors/Formats/Impl/ORC/Reader.cpp create mode 100644 src/Processors/Formats/Impl/ORC/Reader.h create mode 100644 src/Processors/Formats/Impl/ORC/SearchArgument.cpp create mode 100644 src/Processors/Formats/Impl/ORC/SearchArgument.h create mode 100644 src/Processors/Formats/Impl/ORC/Statistics.cpp create mode 100644 src/Processors/Formats/Impl/ORC/Statistics.h create mode 100644 src/Processors/Formats/Impl/ORC/Stream.cpp create mode 100644 src/Processors/Formats/Impl/ORC/Stream.h create mode 100644 src/Processors/Formats/Impl/ORC/Timezone.cpp create mode 100644 src/Processors/Formats/Impl/ORC/Timezone.h create mode 100644 src/Processors/Formats/Impl/ORC/Type.cpp create mode 100644 src/Processors/Formats/Impl/ORC/Type.h create mode 100644 src/Processors/Formats/Impl/ORC/Vector.cpp create mode 100644 src/Processors/Formats/Impl/ORC/Vector.h create mode 100644 src/Processors/Formats/Impl/ORC/Writer.cpp create mode 100644 src/Processors/Formats/Impl/ORC/Writer.h create mode 100644 src/Processors/QueryPlan/Optimizations/SipHashingWriteBuffer.h create mode 100644 src/Processors/QueryPlan/tests/gtest_siphashing_write_buffer_key_identity.cpp create mode 100644 src/Processors/tests/gtest_ch_column_to_arrow_column_record_batch_split.cpp create mode 100644 src/Processors/tests/gtest_pipeline_executor_late_cpu_slots.cpp create mode 100644 src/Storages/MergeTree/tests/gtest_merge_tree_index_bloom_filter.cpp create mode 100644 src/Storages/ObjectStorageQueue/ObjectStorageQueueMoveDestinationAccess.cpp create mode 100644 src/Storages/ObjectStorageQueue/ObjectStorageQueueMoveDestinationAccess.h create mode 100644 tests/integration/test_backup_restore_s3/test_backup_metadata_only.py create mode 100644 tests/integration/test_create_union_system_log_tables/configs/prepare_on_startup.xml create mode 100644 tests/integration/test_refreshable_mv_definer_interserver_secret/__init__.py create mode 100644 tests/integration/test_refreshable_mv_definer_interserver_secret/configs/remote_servers.xml create mode 100644 tests/integration/test_refreshable_mv_definer_interserver_secret/test.py create mode 100644 tests/integration/test_storage_iceberg_no_spark/test_manifest_length_cache_weight.py create mode 100644 tests/integration/test_storage_kafka/configs/small_message_broker_pool.xml create mode 100644 tests/integration/test_storage_kafka/test_message_broker_pool_resize.py create mode 100644 tests/integration/test_streaming_engines_definer_interserver_secret/__init__.py create mode 100644 tests/integration/test_streaming_engines_definer_interserver_secret/configs/rabbitmq.xml create mode 100644 tests/integration/test_streaming_engines_definer_interserver_secret/configs/remote_servers.xml create mode 100644 tests/integration/test_streaming_engines_definer_interserver_secret/configs/users.xml create mode 100644 tests/integration/test_streaming_engines_definer_interserver_secret/test.py create mode 100644 tests/integration/test_text_log_level_after_reload/__init__.py create mode 100644 tests/integration/test_text_log_level_after_reload/configs/logger.xml create mode 100644 tests/integration/test_text_log_level_after_reload/test.py create mode 100644 tests/performance/group_by_two_level_few_keys.xml create mode 100644 tests/performance/json_typed_path_subcolumn_lookup.xml create mode 100644 tests/performance/prewhere_subcolumn_read_steps.xml create mode 100644 tests/performance/small_block_contention_workload.xml create mode 100644 tests/queries/0_stateless/02346_text_index_add_share_nested_offsets_disabled.reference create mode 100644 tests/queries/0_stateless/02346_text_index_add_share_nested_offsets_disabled.sql create mode 100644 tests/queries/0_stateless/02346_text_index_add_with_invalid_alter_command.reference create mode 100644 tests/queries/0_stateless/02346_text_index_add_with_invalid_alter_command.sql create mode 100644 tests/queries/0_stateless/02346_text_index_transform_access.reference create mode 100644 tests/queries/0_stateless/02346_text_index_transform_access.sql create mode 100644 tests/queries/0_stateless/02346_text_index_transform_access_replicated_database.reference create mode 100644 tests/queries/0_stateless/02346_text_index_transform_access_replicated_database.sql create mode 100644 tests/queries/0_stateless/02346_text_index_transform_alter_segments.reference create mode 100755 tests/queries/0_stateless/02346_text_index_transform_alter_segments.sh create mode 100644 tests/queries/0_stateless/02346_text_index_transform_data_access.reference create mode 100644 tests/queries/0_stateless/02346_text_index_transform_data_access.sql create mode 100644 tests/queries/0_stateless/02346_text_index_transform_on_cluster.reference create mode 100755 tests/queries/0_stateless/02346_text_index_transform_on_cluster.sh create mode 100644 tests/queries/0_stateless/03100_lwu_58_pruning_empty_partition_zookeeper.reference create mode 100755 tests/queries/0_stateless/03100_lwu_58_pruning_empty_partition_zookeeper.sh delete mode 100644 tests/queries/0_stateless/04489_virtual_row_disabled_with_final.reference delete mode 100644 tests/queries/0_stateless/04489_virtual_row_disabled_with_final.sql create mode 100644 tests/queries/0_stateless/04757_refreshable_mv_randomize_for_overflow.reference create mode 100755 tests/queries/0_stateless/04757_refreshable_mv_randomize_for_overflow.sh create mode 100644 tests/queries/0_stateless/05036_orc_unimplemented_codec_no_abort.reference create mode 100755 tests/queries/0_stateless/05036_orc_unimplemented_codec_no_abort.sh create mode 100644 tests/queries/0_stateless/05045_format_schema_cache_leftover_temp_file.reference create mode 100755 tests/queries/0_stateless/05045_format_schema_cache_leftover_temp_file.sh create mode 100644 tests/queries/0_stateless/05053_replicated_max_logs_to_keep_zero.reference create mode 100755 tests/queries/0_stateless/05053_replicated_max_logs_to_keep_zero.sh create mode 100644 tests/queries/0_stateless/05055_arrow_split_large_record_batch.reference create mode 100755 tests/queries/0_stateless/05055_arrow_split_large_record_batch.sh create mode 100644 tests/queries/0_stateless/05056_arrow_record_batch_split_boundaries.reference create mode 100755 tests/queries/0_stateless/05056_arrow_record_batch_split_boundaries.sh create mode 100644 tests/queries/0_stateless/05059_convert_column_to_type_bool_identity.reference create mode 100644 tests/queries/0_stateless/05059_convert_column_to_type_bool_identity.sql create mode 100644 tests/queries/0_stateless/05141_settings_constraints_nested_settings_effective.reference create mode 100755 tests/queries/0_stateless/05141_settings_constraints_nested_settings_effective.sh create mode 100644 tests/queries/0_stateless/05175_nested_settings_equal_value_marks_changed.reference create mode 100755 tests/queries/0_stateless/05175_nested_settings_equal_value_marks_changed.sh create mode 100644 tests/queries/0_stateless/05218_attach_table_from_path_read_on_file_grant.reference create mode 100755 tests/queries/0_stateless/05218_attach_table_from_path_read_on_file_grant.sh create mode 100644 tests/queries/0_stateless/05218_quota_client_key_stale_context.reference create mode 100755 tests/queries/0_stateless/05218_quota_client_key_stale_context.sh create mode 100644 tests/queries/0_stateless/05233_assume_not_null_nullable_partition_key_pruning.reference create mode 100644 tests/queries/0_stateless/05233_assume_not_null_nullable_partition_key_pruning.sql create mode 100644 tests/queries/0_stateless/05239_workload_drop_memory_reservation_in_use.reference create mode 100755 tests/queries/0_stateless/05239_workload_drop_memory_reservation_in_use.sh create mode 100644 tests/queries/0_stateless/05258_ssd_cache_dictionaries_same_path.reference create mode 100755 tests/queries/0_stateless/05258_ssd_cache_dictionaries_same_path.sh create mode 100644 tests/queries/0_stateless/05291_azure_blob_storage_partition_strategy_key_value.reference create mode 100644 tests/queries/0_stateless/05291_azure_blob_storage_partition_strategy_key_value.sql create mode 100644 tests/queries/0_stateless/05291_system_mutations_hide_secrets.reference create mode 100644 tests/queries/0_stateless/05291_system_mutations_hide_secrets.sql create mode 100644 tests/queries/0_stateless/05316_ai_functions_last_in_prewhere.reference create mode 100755 tests/queries/0_stateless/05316_ai_functions_last_in_prewhere.sh create mode 100644 tests/queries/0_stateless/05317_orc_corrupt_stripe_footer_stream_column.reference create mode 100755 tests/queries/0_stateless/05317_orc_corrupt_stripe_footer_stream_column.sh create mode 100644 tests/queries/0_stateless/05325_read_in_order_final_virtual_row.reference create mode 100644 tests/queries/0_stateless/05325_read_in_order_final_virtual_row.sql create mode 100644 tests/queries/0_stateless/05339_avro_schema_inference_deep_named_type_chain.reference create mode 100755 tests/queries/0_stateless/05339_avro_schema_inference_deep_named_type_chain.sh create mode 100644 tests/queries/0_stateless/05339_backup_table_function_table_without_source.reference create mode 100755 tests/queries/0_stateless/05339_backup_table_function_table_without_source.sh create mode 100644 tests/queries/0_stateless/05339_prewhere_subcolumn_read_steps.reference create mode 100644 tests/queries/0_stateless/05339_prewhere_subcolumn_read_steps.sql create mode 100644 tests/queries/0_stateless/05340_map_element_to_subcolumn_bucketed_only.reference create mode 100644 tests/queries/0_stateless/05340_map_element_to_subcolumn_bucketed_only.sql create mode 100644 tests/queries/0_stateless/05353_ai_functions_allow_sql_named_collections.reference create mode 100755 tests/queries/0_stateless/05353_ai_functions_allow_sql_named_collections.sh create mode 100644 tests/queries/0_stateless/05385_ipv6_all_zero_literal_host_ip.reference create mode 100644 tests/queries/0_stateless/05385_ipv6_all_zero_literal_host_ip.sql create mode 100644 tests/queries/0_stateless/05394_ai_functions_retry_setting_limits.reference create mode 100644 tests/queries/0_stateless/05394_ai_functions_retry_setting_limits.sql create mode 100644 tests/queries/0_stateless/05406_iceberg_write_many_partitions_thread_pool.reference create mode 100755 tests/queries/0_stateless/05406_iceberg_write_many_partitions_thread_pool.sh create mode 100644 tests/queries/0_stateless/05406_join_swap_array_join_estimate.reference create mode 100644 tests/queries/0_stateless/05406_join_swap_array_join_estimate.sql create mode 100644 tests/queries/0_stateless/05407_bool_force_setting_shorthand.reference create mode 100755 tests/queries/0_stateless/05407_bool_force_setting_shorthand.sh create mode 100644 tests/queries/0_stateless/05407_dynamic_deep_value_column_operations.reference create mode 100755 tests/queries/0_stateless/05407_dynamic_deep_value_column_operations.sh create mode 100644 tests/queries/0_stateless/05407_orc_timezone_traversal_abort.reference create mode 100755 tests/queries/0_stateless/05407_orc_timezone_traversal_abort.sh create mode 100644 tests/queries/0_stateless/05410_parse_date_time_daylight_saving_time.reference create mode 100644 tests/queries/0_stateless/05410_parse_date_time_daylight_saving_time.sql create mode 100644 tests/queries/0_stateless/05421_bloom_filter_text_index_total_size.reference create mode 100644 tests/queries/0_stateless/05421_bloom_filter_text_index_total_size.sql create mode 100644 tests/queries/0_stateless/05422_bloom_filter_text_index_total_size_attach.reference create mode 100755 tests/queries/0_stateless/05422_bloom_filter_text_index_total_size_attach.sh create mode 100644 tests/queries/0_stateless/05427_azure_sas_url_parsing.reference create mode 100644 tests/queries/0_stateless/05427_azure_sas_url_parsing.sql create mode 100644 tests/queries/0_stateless/05427_direct_dictionary_read_truncate_source.reference create mode 100755 tests/queries/0_stateless/05427_direct_dictionary_read_truncate_source.sh create mode 100644 tests/queries/0_stateless/05428_mysqldump_create_without_columns.reference create mode 100755 tests/queries/0_stateless/05428_mysqldump_create_without_columns.sh create mode 100644 tests/queries/0_stateless/05441_ai_functions_privileges.reference create mode 100644 tests/queries/0_stateless/05441_ai_functions_privileges.sql.j2 create mode 100644 tests/queries/0_stateless/05441_deep_refresh_cursor.reference create mode 100755 tests/queries/0_stateless/05441_deep_refresh_cursor.sh create mode 100644 tests/queries/0_stateless/05441_rawblob_aggregate_function.reference create mode 100755 tests/queries/0_stateless/05441_rawblob_aggregate_function.sh create mode 100644 tests/queries/0_stateless/05441_restore_into_existing_table_unknown_storage_policy.reference create mode 100755 tests/queries/0_stateless/05441_restore_into_existing_table_unknown_storage_policy.sh create mode 100644 tests/queries/0_stateless/05441_s3queue_move_destination_access.reference create mode 100755 tests/queries/0_stateless/05441_s3queue_move_destination_access.sh create mode 100644 tests/queries/0_stateless/05442_ai_functions_privileges_stored_expressions.reference create mode 100644 tests/queries/0_stateless/05442_ai_functions_privileges_stored_expressions.sql create mode 100644 tests/queries/0_stateless/05443_arrow_null_as_default_non_nullable_field.reference create mode 100755 tests/queries/0_stateless/05443_arrow_null_as_default_non_nullable_field.sh create mode 100644 tests/queries/0_stateless/05443_async_insert_mv_dedup_empty_partition_block.reference create mode 100644 tests/queries/0_stateless/05443_async_insert_mv_dedup_empty_partition_block.sql create mode 100644 tests/queries/0_stateless/05443_join_reorder_converted_outer_join_type_change.reference create mode 100644 tests/queries/0_stateless/05443_join_reorder_converted_outer_join_type_change.sql create mode 100644 tests/queries/0_stateless/05443_parallel_distributed_insert_select_source_access.reference create mode 100755 tests/queries/0_stateless/05443_parallel_distributed_insert_select_source_access.sh create mode 100644 tests/queries/0_stateless/05443_parquet_int96_date_time_overflow_behavior.reference create mode 100755 tests/queries/0_stateless/05443_parquet_int96_date_time_overflow_behavior.sh create mode 100644 tests/queries/0_stateless/05444_ai_functions_no_user.reference create mode 100644 tests/queries/0_stateless/05444_ai_functions_no_user.sql create mode 100644 tests/queries/0_stateless/05444_parquet_int96_lower_boundary.reference create mode 100755 tests/queries/0_stateless/05444_parquet_int96_lower_boundary.sh create mode 100644 tests/queries/0_stateless/05445_distributed_async_insert_batch_per_user.reference create mode 100755 tests/queries/0_stateless/05445_distributed_async_insert_batch_per_user.sh create mode 100644 tests/queries/0_stateless/05445_webterminal_concurrent_sessions_input.reference create mode 100755 tests/queries/0_stateless/05445_webterminal_concurrent_sessions_input.sh create mode 100644 tests/queries/0_stateless/05446_insert_select_cluster_function_column_defaults.reference create mode 100755 tests/queries/0_stateless/05446_insert_select_cluster_function_column_defaults.sh create mode 100644 tests/queries/0_stateless/05447_azure_sas_url_hidden_in_table_function.reference create mode 100755 tests/queries/0_stateless/05447_azure_sas_url_hidden_in_table_function.sh create mode 100644 tests/queries/0_stateless/05447_orc_native_reader_writer.reference create mode 100755 tests/queries/0_stateless/05447_orc_native_reader_writer.sh create mode 100644 tests/queries/0_stateless/05448_orc_date_after_2059.reference create mode 100755 tests/queries/0_stateless/05448_orc_date_after_2059.sh create mode 100644 tests/queries/0_stateless/05448_text_formats_read_padding_object_storage.reference create mode 100755 tests/queries/0_stateless/05448_text_formats_read_padding_object_storage.sh create mode 100644 tests/queries/0_stateless/05453_orc_bad_bloom_filter.reference create mode 100755 tests/queries/0_stateless/05453_orc_bad_bloom_filter.sh create mode 100644 tests/queries/0_stateless/05457_dictionary_remote_clickhouse_invalidate_query.reference create mode 100755 tests/queries/0_stateless/05457_dictionary_remote_clickhouse_invalidate_query.sh create mode 100644 tests/queries/0_stateless/05457_json_extract_datetime_compatibility.reference create mode 100644 tests/queries/0_stateless/05457_json_extract_datetime_compatibility.sql create mode 100644 tests/queries/0_stateless/05457_part_check_ignores_query_cancellation.reference create mode 100755 tests/queries/0_stateless/05457_part_check_ignores_query_cancellation.sh create mode 100644 tests/queries/0_stateless/05458_azure_credentials_hidden_in_logged_queries.reference create mode 100755 tests/queries/0_stateless/05458_azure_credentials_hidden_in_logged_queries.sh create mode 100644 tests/queries/0_stateless/05459_ai_functions_background_defaults.reference create mode 100644 tests/queries/0_stateless/05459_ai_functions_background_defaults.sql create mode 100644 tests/queries/0_stateless/05473_orc_footer_content_length.reference create mode 100755 tests/queries/0_stateless/05473_orc_footer_content_length.sh create mode 100644 tests/queries/0_stateless/05474_orc_decimal64_v2.reference create mode 100755 tests/queries/0_stateless/05474_orc_decimal64_v2.sh create mode 100644 tests/queries/0_stateless/05480_iceberg_add_column_too_small_last_column_id_128455.reference create mode 100755 tests/queries/0_stateless/05480_iceberg_add_column_too_small_last_column_id_128455.sh create mode 100644 tests/queries/0_stateless/05480_snappy_framed_read_padding.reference create mode 100755 tests/queries/0_stateless/05480_snappy_framed_read_padding.sh create mode 100644 tests/queries/0_stateless/05481_object_storage_parquet_footer_prefetch.reference create mode 100755 tests/queries/0_stateless/05481_object_storage_parquet_footer_prefetch.sh create mode 100644 tests/queries/0_stateless/05481_view_with_two_array_joins.reference create mode 100644 tests/queries/0_stateless/05481_view_with_two_array_joins.sql create mode 100644 tests/queries/0_stateless/05483_arrow_record_batch_split_many_batches.reference create mode 100755 tests/queries/0_stateless/05483_arrow_record_batch_split_many_batches.sh create mode 100644 tests/queries/0_stateless/05483_replicated_database_ddl_during_restore_replica.reference create mode 100755 tests/queries/0_stateless/05483_replicated_database_ddl_during_restore_replica.sh create mode 100644 tests/queries/0_stateless/05484_replicated_database_restore_replica_waits_for_initial_query.reference create mode 100755 tests/queries/0_stateless/05484_replicated_database_restore_replica_waits_for_initial_query.sh create mode 100644 tests/queries/0_stateless/05488_ssd_cache_dictionary_cache_file_cannot_grow.reference create mode 100755 tests/queries/0_stateless/05488_ssd_cache_dictionary_cache_file_cannot_grow.sh create mode 100644 tests/queries/0_stateless/05493_describe_table_function_create_temporary_table_access.reference create mode 100755 tests/queries/0_stateless/05493_describe_table_function_create_temporary_table_access.sh create mode 100644 tests/queries/0_stateless/05493_msgpack_async_insert_after_invalid_entry.reference create mode 100755 tests/queries/0_stateless/05493_msgpack_async_insert_after_invalid_entry.sh create mode 100644 tests/queries/0_stateless/05493_mysql_host_connector_syntax.reference create mode 100755 tests/queries/0_stateless/05493_mysql_host_connector_syntax.sh create mode 100644 tests/queries/0_stateless/05493_s3queue_settings_system_table_hide_secrets.reference create mode 100644 tests/queries/0_stateless/05493_s3queue_settings_system_table_hide_secrets.sql create mode 100644 tests/queries/0_stateless/05493_skip_index_archive_newer_format_version.reference create mode 100755 tests/queries/0_stateless/05493_skip_index_archive_newer_format_version.sh create mode 100644 tests/queries/0_stateless/05494_custom_disk_identity_without_credentials_marker.reference create mode 100755 tests/queries/0_stateless/05494_custom_disk_identity_without_credentials_marker.sh create mode 100644 tests/queries/0_stateless/05496_dynamic_disk_wrapper_of_server_credentials_disk.reference create mode 100755 tests/queries/0_stateless/05496_dynamic_disk_wrapper_of_server_credentials_disk.sh create mode 100644 tests/queries/0_stateless/05498_group_by_two_level_few_keys_prefetch.reference create mode 100644 tests/queries/0_stateless/05498_group_by_two_level_few_keys_prefetch.sql create mode 100644 tests/queries/0_stateless/05498_s3_disk_sse_c_key_validation.reference create mode 100644 tests/queries/0_stateless/05498_s3_disk_sse_c_key_validation.sql create mode 100644 tests/queries/0_stateless/05507_mv_source_merge_loop_executable_crash.reference create mode 100755 tests/queries/0_stateless/05507_mv_source_merge_loop_executable_crash.sh create mode 100644 tests/queries/0_stateless/05507_orc_malformed_timestamps.reference create mode 100755 tests/queries/0_stateless/05507_orc_malformed_timestamps.sh create mode 100644 tests/queries/0_stateless/05508_orc_writer_out_of_range_values.reference create mode 100755 tests/queries/0_stateless/05508_orc_writer_out_of_range_values.sh create mode 100644 tests/queries/0_stateless/05509_jit_llvm_context_does_not_grow_with_distinct_literals.reference create mode 100755 tests/queries/0_stateless/05509_jit_llvm_context_does_not_grow_with_distinct_literals.sh create mode 100644 tests/queries/0_stateless/05509_orc_malformed_dictionaries.reference create mode 100755 tests/queries/0_stateless/05509_orc_malformed_dictionaries.sh create mode 100644 tests/queries/0_stateless/05510_orc_allocation_amplification.reference create mode 100755 tests/queries/0_stateless/05510_orc_allocation_amplification.sh create mode 100644 tests/queries/0_stateless/05511_orc_malformed_stripes_and_streams.reference create mode 100755 tests/queries/0_stateless/05511_orc_malformed_stripes_and_streams.sh create mode 100644 tests/queries/0_stateless/05512_orc_malformed_row_groups.reference create mode 100755 tests/queries/0_stateless/05512_orc_malformed_row_groups.sh create mode 100644 tests/queries/0_stateless/05513_orc_metadata_repeated_field_limits.reference create mode 100755 tests/queries/0_stateless/05513_orc_metadata_repeated_field_limits.sh create mode 100644 tests/queries/0_stateless/05514_orc_timestamp_statistics_before_epoch.reference create mode 100755 tests/queries/0_stateless/05514_orc_timestamp_statistics_before_epoch.sh create mode 100644 tests/queries/0_stateless/05515_orc_nullable_floats.reference create mode 100755 tests/queries/0_stateless/05515_orc_nullable_floats.sh create mode 100644 tests/queries/0_stateless/05546_replicated_database_restore_replica_waits_for_initial_alter.reference create mode 100755 tests/queries/0_stateless/05546_replicated_database_restore_replica_waits_for_initial_alter.sh create mode 100644 tests/queries/0_stateless/05565_parquet_encoder_shared_thread_pool.reference create mode 100755 tests/queries/0_stateless/05565_parquet_encoder_shared_thread_pool.sh create mode 100644 tests/queries/0_stateless/05567_group_array_sample_counter_overflow.reference create mode 100644 tests/queries/0_stateless/05567_group_array_sample_counter_overflow.sql create mode 100644 tests/queries/0_stateless/05567_mongodb_server_credentials.reference create mode 100755 tests/queries/0_stateless/05567_mongodb_server_credentials.sh create mode 100644 tests/queries/0_stateless/05568_mongodb_dictionary_server_credentials.reference create mode 100755 tests/queries/0_stateless/05568_mongodb_dictionary_server_credentials.sh create mode 100644 tests/queries/0_stateless/05574_orc_requested_type_of_another_kind.reference create mode 100755 tests/queries/0_stateless/05574_orc_requested_type_of_another_kind.sh create mode 100644 tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.reference create mode 100755 tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.sh create mode 100644 tests/queries/0_stateless/05615_formats_accumulating_result_memory_limit_in_local.reference create mode 100755 tests/queries/0_stateless/05615_formats_accumulating_result_memory_limit_in_local.sh create mode 100644 tests/queries/0_stateless/05616_orc_timestamp_before_first_transition.reference create mode 100755 tests/queries/0_stateless/05616_orc_timestamp_before_first_transition.sh create mode 100644 tests/queries/0_stateless/05617_gini_deserialize_validation.reference create mode 100644 tests/queries/0_stateless/05617_gini_deserialize_validation.sql create mode 100644 tests/queries/0_stateless/05617_orc_nested_struct_without_requested_fields.reference create mode 100644 tests/queries/0_stateless/05617_orc_nested_struct_without_requested_fields.sql create mode 100644 tests/queries/0_stateless/05621_delta_lake_column_mapping_swapped_names_order.reference create mode 100755 tests/queries/0_stateless/05621_delta_lake_column_mapping_swapped_names_order.sh create mode 100644 tests/queries/0_stateless/05621_readonly_named_collection_ddl.reference create mode 100755 tests/queries/0_stateless/05621_readonly_named_collection_ddl.sh create mode 100644 tests/queries/0_stateless/data_orc/bad_bloom_filter_1.6.0.orc create mode 100644 tests/queries/0_stateless/data_orc/bad_bloom_filter_1.6.11.orc create mode 100644 tests/queries/0_stateless/data_orc/decimal64_v2.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_compression_block_1g.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_dictionary_empty.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_dictionary_size_max.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_footer_too_many_statistics.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_metadata_too_many_stripes.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_patched_base_terminal_continuation.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_row_group_count_overflow.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_row_index_seek_past_stream.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_snappy_chunk_size.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_stream_offset_overflow.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_stripe_offset_overflow.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_timestamp_nanoseconds.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_timestamp_statistics_nanoseconds.orc create mode 100644 tests/queries/0_stateless/data_orc/malformed_zstd_chunk_size.orc create mode 100644 tests/queries/0_stateless/data_orc/row_index_seek_control.orc create mode 100644 tests/queries/0_stateless/data_orc/timestamps_before_epoch_arrow.orc create mode 100644 tests/queries/0_stateless/data_orc/timestamps_before_epoch_pyarrow_bloom_filter.orc create mode 100644 tests/queries/0_stateless/data_parquet/05443_int96_out_of_range.parquet create mode 100644 tests/queries/0_stateless/data_parquet/05444_int96_lower_boundary.parquet create mode 100644 tests/queries/0_stateless/orc_malformed.lib diff --git a/.github/workflows/backport_branches.yml b/.github/workflows/backport_branches.yml index 5087d25f38766..35dd0cae11954 100644 --- a/.github/workflows/backport_branches.yml +++ b/.github/workflows/backport_branches.yml @@ -367,6 +367,76 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_darwin)' --workflow "BackportPR" --ci --timestamp + build_arm_v80compat: + runs-on: [self-hosted, arm-large] + needs: [config_workflow, dockers_build_amd, dockers_build_arm] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV92ODBjb21wYXQp') }} + name: "Build (arm_v80compat)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_v80compat)' --workflow "BackportPR" --ci --timestamp + + build_amd_compat: + runs-on: [self-hosted, arm-large] + needs: [config_workflow, dockers_build_amd, dockers_build_arm] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9jb21wYXQp') }} + name: "Build (amd_compat)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_compat)' --workflow "BackportPR" --ci --timestamp + docker_server_image: runs-on: [self-hosted, style-checker] needs: [build_amd_release, build_arm_release, config_workflow, dockers_build_amd, dockers_build_arm] @@ -1384,7 +1454,7 @@ jobs: finish_workflow: runs-on: [self-hosted, style-checker-aarch64] - needs: [build_amd_asan_ubsan, build_amd_darwin, build_amd_debug, build_amd_release, build_amd_tsan, build_arm_darwin, build_arm_release, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_3_3, stateless_tests_amd_asan_ubsan_distributed_plan_parallel, stress_test_amd_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] + needs: [build_amd_asan_ubsan, build_amd_compat, build_amd_darwin, build_amd_debug, build_amd_release, build_amd_tsan, build_arm_darwin, build_arm_release, build_arm_v80compat, compatibility_check_amd_release, compatibility_check_arm_release, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_3_3, stateless_tests_amd_asan_ubsan_distributed_plan_parallel, stress_test_amd_tsan, unit_tests_asan_ubsan, unit_tests_asan_ubsan_function_prop_fuzzer, unit_tests_tsan, unit_tests_tsan_function_prop_fuzzer] if: ${{ always() }} name: "Finish Workflow" outputs: diff --git a/.github/workflows/release_branches.yml b/.github/workflows/release_branches.yml index 7800092f13d84..db5db0386c040 100644 --- a/.github/workflows/release_branches.yml +++ b/.github/workflows/release_branches.yml @@ -540,6 +540,76 @@ jobs: . ./ci/tmp/praktika_setup_env.sh PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_darwin)' --workflow "ReleaseBranchCI" --ci --timestamp + build_arm_v80compat: + runs-on: [self-hosted, arm-large] + needs: [config_workflow, dockers_build_amd, dockers_build_arm] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFybV92ODBjb21wYXQp') }} + name: "Build (arm_v80compat)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (arm_v80compat)' --workflow "ReleaseBranchCI" --ci --timestamp + + build_amd_compat: + runs-on: [self-hosted, arm-large] + needs: [config_workflow, dockers_build_amd, dockers_build_arm] + if: ${{ !cancelled() && !contains(needs.*.outputs.pipeline_status, 'failure') && !contains(needs.*.outputs.pipeline_status, 'undefined') && !contains(fromJson(needs.config_workflow.outputs.data).workflow_config.cache_success_base64, 'QnVpbGQgKGFtZF9jb21wYXQp') }} + name: "Build (amd_compat)" + outputs: + data: ${{ steps.run.outputs.DATA }} + pipeline_status: ${{ steps.run.outputs.pipeline_status || 'undefined' }} + steps: + - name: Checkout code + uses: actions/checkout@v6 + with: + ref: ${{ env.CHECKOUT_REF }} + + - name: Prepare env script + run: | + rm -rf ./ci/tmp + mkdir -p ./ci/tmp + cat > ./ci/tmp/praktika_setup_env.sh << 'ENV_SETUP_SCRIPT_EOF' + export PYTHONPATH=./ci:.: + + cat > ./ci/tmp/workflow_job.json << 'EOF' + ${{ toJson(job) }} + EOF + cat > ./ci/tmp/workflow_status.json << 'EOF' + ${{ toJson(needs) }} + EOF + ENV_SETUP_SCRIPT_EOF + + - name: Run + id: run + run: | + . ./ci/tmp/praktika_setup_env.sh + PYTHONUNBUFFERED=1 python3 -m praktika run 'Build (amd_compat)' --workflow "ReleaseBranchCI" --ci --timestamp + sign_macos_binary_amd_darwin: runs-on: [self-hosted, release-runner] needs: [build_amd_darwin, config_workflow, dockers_build_amd, dockers_build_arm] @@ -2117,7 +2187,7 @@ jobs: finish_workflow: runs-on: [self-hosted, style-checker-aarch64] - needs: [build_amd_asan_ubsan, build_amd_darwin, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_darwin, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_1_8, integration_tests_amd_asan_ubsan_db_disk_2_8, integration_tests_amd_asan_ubsan_db_disk_3_8, integration_tests_amd_asan_ubsan_db_disk_4_8, integration_tests_amd_asan_ubsan_db_disk_5_8, integration_tests_amd_asan_ubsan_db_disk_6_8, integration_tests_amd_asan_ubsan_db_disk_7_8, integration_tests_amd_asan_ubsan_db_disk_8_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, sign_macos_binary_amd_darwin, sign_macos_binary_arm_darwin, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_3_3, stateless_tests_amd_asan_ubsan_distributed_plan_parallel, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, unit_tests_asan_ubsan, unit_tests_msan, unit_tests_tsan] + needs: [build_amd_asan_ubsan, build_amd_compat, build_amd_darwin, build_amd_debug, build_amd_msan, build_amd_release, build_amd_tsan, build_arm_asan_ubsan, build_arm_darwin, build_arm_debug, build_arm_msan, build_arm_release, build_arm_tsan, build_arm_v80compat, config_workflow, docker_keeper_image, docker_server_image, dockers_build_amd, dockers_build_arm, install_packages_amd_release, install_packages_arm_release, integration_tests_amd_asan_ubsan_db_disk_1_8, integration_tests_amd_asan_ubsan_db_disk_2_8, integration_tests_amd_asan_ubsan_db_disk_3_8, integration_tests_amd_asan_ubsan_db_disk_4_8, integration_tests_amd_asan_ubsan_db_disk_5_8, integration_tests_amd_asan_ubsan_db_disk_6_8, integration_tests_amd_asan_ubsan_db_disk_7_8, integration_tests_amd_asan_ubsan_db_disk_8_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_1_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_2_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_3_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_4_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_5_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_6_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_7_8, integration_tests_amd_asan_ubsan_db_disk_old_analyzer_8_8, integration_tests_amd_tsan_1_6, integration_tests_amd_tsan_2_6, integration_tests_amd_tsan_3_6, integration_tests_amd_tsan_4_6, integration_tests_amd_tsan_5_6, integration_tests_amd_tsan_6_6, sign_macos_binary_amd_darwin, sign_macos_binary_arm_darwin, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_1_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_2_3, stateless_tests_amd_asan_ubsan_db_disk_distributed_plan_sequential_3_3, stateless_tests_amd_asan_ubsan_distributed_plan_parallel, stress_test_amd_asan_ubsan, stress_test_amd_debug, stress_test_amd_msan, stress_test_amd_tsan, stress_test_arm_asan_ubsan, stress_test_arm_asan_ubsan_s3, stress_test_arm_debug, stress_test_arm_msan, stress_test_arm_release, stress_test_arm_tsan, unit_tests_asan_ubsan, unit_tests_msan, unit_tests_tsan] if: ${{ always() }} name: "Finish Workflow" outputs: diff --git a/.gitmodules b/.gitmodules index 26fec9523a636..8644bac77ee9d 100644 --- a/.gitmodules +++ b/.gitmodules @@ -76,9 +76,6 @@ [submodule "contrib/fastops"] path = contrib/fastops url = https://github.com/ClickHouse/fastops -[submodule "contrib/orc"] - path = contrib/orc - url = https://github.com/ClickHouse/orc [submodule "contrib/sparsehash-c11"] path = contrib/sparsehash-c11 url = https://github.com/sparsehash/sparsehash-c11 diff --git a/base/poco/Foundation/include/Poco/FileChannel.h b/base/poco/Foundation/include/Poco/FileChannel.h index 7db27772c0a26..028f1766aaa95 100644 --- a/base/poco/Foundation/include/Poco/FileChannel.h +++ b/base/poco/Foundation/include/Poco/FileChannel.h @@ -246,6 +246,7 @@ class Foundation_API FileChannel : public Channel void unsafeOpen(); private: + void unsafeRotate(); LogFile * newLogFile(); void archiveCorrupted(const std::string & path); bool setNoPurge(const std::string & value); diff --git a/base/poco/Foundation/src/FileChannel.cpp b/base/poco/Foundation/src/FileChannel.cpp index 7b3423a646afe..c473de5200fa9 100644 --- a/base/poco/Foundation/src/FileChannel.cpp +++ b/base/poco/Foundation/src/FileChannel.cpp @@ -25,6 +25,7 @@ #include "Poco/Exception.h" #include "Poco/Ascii.h" #include "Poco/CombinedRotateStrategy.h" +#include namespace Poco { @@ -111,15 +112,7 @@ void FileChannel::log(const Message& msg) if (_pRotateStrategy && _pArchiveStrategy && _pRotateStrategy->mustRotate(_pFile)) { - try - { - _pFile = _pArchiveStrategy->archive(_pFile, _streamCompress); - purge(); - } - catch (...) - { - _pFile = newLogFile(); - } + unsafeRotate(); // we must call mustRotate() again to give the // RotateByIntervalStrategy a chance to write its timestamp // to the new file. @@ -340,17 +333,24 @@ void FileChannel::unsafeOpen() { _pFile = newLogFile(); if (_rotateOnOpen && _pFile->size() > 0) - { - try - { - _pFile = _pArchiveStrategy->archive(_pFile, _streamCompress); - purge(); - } - catch (...) - { - _pFile = newLogFile(); - } - } + unsafeRotate(); + } +} + + +void FileChannel::unsafeRotate() +{ + /// `archive` deletes the current file, so `_pFile` must not dangle if opening the new one fails. + /// The raw pointer is intentional: `archive` owns `pFile` and deletes it, so it must not be deleted here. + LogFile * pFile = std::exchange(_pFile, nullptr); + try + { + _pFile = _pArchiveStrategy->archive(pFile, _streamCompress); + purge(); + } + catch (...) + { + _pFile = newLogFile(); } } diff --git a/base/poco/Net/include/Poco/Net/HTTPClientSession.h b/base/poco/Net/include/Poco/Net/HTTPClientSession.h index 8ab1c0f19bf24..c8dd216b070f2 100644 --- a/base/poco/Net/include/Poco/Net/HTTPClientSession.h +++ b/base/poco/Net/include/Poco/Net/HTTPClientSession.h @@ -290,6 +290,10 @@ namespace Net /// This method should only be called if the request contains /// a "Expect: 100-continue" header. + bool receiveEarlyResponse(HTTPResponse & response); + /// After sending the request failed, drops the rest of it and reads the response header the + /// server has already sent. If true, receiveResponse() must be called with the same response. + virtual void flushRequest(); /// Flushes the request stream. /// diff --git a/base/poco/Net/src/HTTPClientSession.cpp b/base/poco/Net/src/HTTPClientSession.cpp index 1754f59d8e5c2..6a183c563b647 100644 --- a/base/poco/Net/src/HTTPClientSession.cpp +++ b/base/poco/Net/src/HTTPClientSession.cpp @@ -427,6 +427,36 @@ bool HTTPClientSession::peekResponse(HTTPResponse& response) } +bool HTTPClientSession::receiveEarlyResponse(HTTPResponse& response) +{ + poco_assert (!_responseReceived); + + /// Drops the unsent rest of the request, which rethrows the send error. + try + { + flushRequest(); + } + catch (...) + { + } + clearException(); + + response.clear(); + HTTPHeaderInputStream his(*this); + try + { + response.read(his); + } + catch (Exception&) + { + close(); + return false; + } + _responseReceived = response.getStatus() != HTTPResponse::HTTP_CONTINUE; + return _responseReceived; +} + + void HTTPClientSession::reset() { close(); diff --git a/base/poco/Net/src/IPAddress.cpp b/base/poco/Net/src/IPAddress.cpp index 5a7281e4be477..fdd86c30df0f1 100644 --- a/base/poco/Net/src/IPAddress.cpp +++ b/base/poco/Net/src/IPAddress.cpp @@ -38,6 +38,20 @@ using Poco::Net::Impl::IPv6AddressImpl; #endif +namespace +{ +#if defined(POCO_HAVE_IPv6) + /// `IPv6AddressImpl::parse` returns the all-zero address both on failure and for a valid unscoped all-zero + /// literal; a valid scoped result has a non-zero scope. macOS `inet_pton` accepts and ignores a `%scope` suffix. + bool isUnscopedIPv6Literal(const std::string& addr) + { + struct in6_addr ia; + return addr.find('%') == std::string::npos && inet_pton(AF_INET6, addr.c_str(), &ia) == 1; + } +#endif +} + + namespace Poco { namespace Net { @@ -104,7 +118,7 @@ IPAddress::IPAddress(const std::string& addr) } IPv6AddressImpl addr6(IPv6AddressImpl::parse(addr)); - if (addr6 != IPv6AddressImpl()) + if (addr6 != IPv6AddressImpl() || isUnscopedIPv6Literal(addr)) { newIPv6(addr6.addr(), addr6.scope()); return; @@ -518,7 +532,7 @@ bool IPAddress::tryParse(const std::string& addr, IPAddress& result) } #if defined(POCO_HAVE_IPv6) IPv6AddressImpl impl6(IPv6AddressImpl::parse(addr)); - if (impl6 != IPv6AddressImpl()) + if (impl6 != IPv6AddressImpl() || isUnscopedIPv6Literal(addr)) { result.newIPv6(impl6.addr(), impl6.scope()); return true; diff --git a/base/poco/NetSSL_OpenSSL/include/Poco/Net/DistrustedCertificates.h b/base/poco/NetSSL_OpenSSL/include/Poco/Net/DistrustedCertificates.h new file mode 100644 index 0000000000000..5aca47d6be881 --- /dev/null +++ b/base/poco/NetSSL_OpenSSL/include/Poco/Net/DistrustedCertificates.h @@ -0,0 +1,44 @@ +// +// DistrustedCertificates.h +// +// Library: NetSSL_OpenSSL +// Package: SSLCore +// Module: DistrustedCertificates +// +// Copyright (c) 2026, ClickHouse, Inc. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#ifndef NetSSL_DistrustedCertificates_INCLUDED +#define NetSSL_DistrustedCertificates_INCLUDED + + +#include +#include "Poco/Net/NetSSL.h" + + +namespace Poco +{ +namespace Net +{ + + + NetSSL_API int checkDistrustedCertificate(X509 * certificate); + /// Checks the certificate against the list of distrusted public keys built into + /// the binary, and returns `X509_V_OK` if it is not on the list, `X509_V_ERR_CERT_REVOKED` + /// if it is, or another `X509_V_ERR_*` code if the check itself failed. + /// + /// The list contains the keys of CA certificates that must not be trusted even if + /// the trust store of the operating system still contains them, e.g. because their + /// private keys are compromised. A key is identified by the SHA-256 of its DER-encoded + /// `SubjectPublicKeyInfo`, so that every certificate of the same key is rejected, + /// including cross-signed ones. + + +} +} // namespace Poco::Net + + +#endif // NetSSL_DistrustedCertificates_INCLUDED diff --git a/base/poco/NetSSL_OpenSSL/src/DistrustedCertificates.cpp b/base/poco/NetSSL_OpenSSL/src/DistrustedCertificates.cpp new file mode 100644 index 0000000000000..448d4f601d9d0 --- /dev/null +++ b/base/poco/NetSSL_OpenSSL/src/DistrustedCertificates.cpp @@ -0,0 +1,95 @@ +// +// DistrustedCertificates.cpp +// +// Library: NetSSL_OpenSSL +// Package: SSLCore +// Module: DistrustedCertificates +// +// Copyright (c) 2026, ClickHouse, Inc. +// +// SPDX-License-Identifier: BSL-1.0 +// + + +#include "Poco/Net/DistrustedCertificates.h" +#include +#include +#include +#include + + +namespace +{ + using SPKIHash = std::array; + + consteval unsigned char hexDigit(char c) + { + if (c >= '0' && c <= '9') + return c - '0'; + if (c >= 'A' && c <= 'F') + return c - 'A' + 10; + throw "Invalid hex digit"; + } + + consteval SPKIHash fromHex(const char (&hex)[65]) + { + SPKIHash result{}; + for (size_t i = 0; i < result.size(); ++i) + result[i] = static_cast(hexDigit(hex[2 * i]) << 4 | hexDigit(hex[2 * i + 1])); + return result; + } + + /// SHA-256 of the DER-encoded `SubjectPublicKeyInfo` of every distrusted key. + constexpr SPKIHash distrusted_keys[] = + { + /// eMudhra: the private keys of the intermediate CAs below, issued in 2025, can be recovered + /// from their certificates. Each of them is issued by `emSign Root TLS CA - G3` or `emSign Root CS CA - G3` + /// and cross-signed by Sectigo `AAA Certificate Services`; the cross-signed certificates have the same keys. + /// The roots that issued or cross-signed them are distrusted as well. + /// https://emudhra.we-have-a-problem.com/ + fromHex("EBED6475C2C2D80FCB199F3FE8DD6E6B3CE60B77EB15D4E42C3DEC0C6F5B5105"), /// EM DV TLS CA - G3B + fromHex("43BFFA19D6EF651451F6841E019931E96F6F2EAEFF28AB4B6F1CE4576DE2717B"), /// EM OV TLS CA - G3B + fromHex("E0254C7FCDC265E2F0F8A428DABB75AB4AE039B4872D62F5EE57F0877D5A8A41"), /// EM EV TLS CA - G3B + fromHex("49DD353DC2B919A3770645117D0E60C16115E7D1BC259EEFE734B00EA434FE5F"), /// EM DV TLS CA - G3C + fromHex("63ECEFEE431D49523BC1136C4E72D0206A477086CAF606CD5CF025686C109CF8"), /// EM OV TLS CA - G3C + fromHex("5C0660A2B164910D8FF589C18E14552E9E0EC237DCEFE2CB736D41A4811F81A0"), /// EM EV TLS CA - G3C + fromHex("EC4FFA1AD0AC88081314B7AAFFEF6A91F97CD57291BB5BA686C5415F5EB4F6B6"), /// EM OV CS CA - G3B + fromHex("92C063D67E8F14DB71D288E11266FFDAF69CBF115F7BAB14F3BC2F68D25E0E30"), /// EM OV CS CA - G3C + fromHex("C619E4780ED8E0ABEDA24AD732567DE4383A7EA941338F6D1A7D1B49AC792FE9"), /// emSign Root TLS CA - G3 + fromHex("0A6C2B2CA5B364207954171987AC9D2178561569D8205457E77CA90C32EDE282"), /// emSign Root CS CA - G3 + fromHex("8D417DB2DD8BF5E3084D1E3F196D583849D81BDD4C00C70B9D39369E96B8C782"), /// emSign ECC Root CA - G3 + }; +} + + +namespace Poco { +namespace Net { + + +int checkDistrustedCertificate(X509 * certificate) +{ + X509_PUBKEY * key = X509_get_X509_PUBKEY(certificate); + if (key == nullptr) + return X509_V_ERR_UNSPECIFIED; + + /// Not `X509_pubkey_digest`: it hashes only the key bits, without the algorithm identifier. + unsigned char * der = nullptr; + int der_size = i2d_X509_PUBKEY(key, &der); + if (der_size <= 0) + return X509_V_ERR_OUT_OF_MEM; + + SPKIHash hash; + unsigned int hash_size = 0; + int ok = EVP_Digest(der, der_size, hash.data(), &hash_size, EVP_sha256(), nullptr); + OPENSSL_free(der); + if (ok != 1 || hash_size != hash.size()) + return X509_V_ERR_UNSPECIFIED; + + if (std::find(std::begin(distrusted_keys), std::end(distrusted_keys), hash) != std::end(distrusted_keys)) + return X509_V_ERR_CERT_REVOKED; + + return X509_V_OK; +} + + +} } // namespace Poco::Net diff --git a/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp b/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp index 94b66cdcf5f1e..fd4e31f53d258 100644 --- a/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp +++ b/base/poco/NetSSL_OpenSSL/src/SSLManager.cpp @@ -14,6 +14,7 @@ #include "Poco/Net/SSLManager.h" #include "Poco/Net/Context.h" +#include "Poco/Net/DistrustedCertificates.h" #include "Poco/Net/Utility.h" #include "Poco/Net/PrivateKeyPassphraseHandler.h" #include "Poco/Net/RejectCertificateHandler.h" @@ -196,6 +197,20 @@ SSLManager::InvalidCertificateHandlerPtr SSLManager::clientCertificateHandler() int SSLManager::verifyCallback(bool server, int ok, X509_STORE_CTX* pStore) { + /// OpenSSL calls this for every certificate of the chain, with `ok` set to 1 if it has + /// no other errors, so this rejects a chain that contains a distrusted certificate at any + /// depth, whichever trust store (the system one, the configured one or the embedded one) + /// it was built from. The rejection is reported like any other verification error. + if (X509 * cert = X509_STORE_CTX_get_current_cert(pStore)) + { + int distrusted = checkDistrustedCertificate(cert); + if (distrusted != X509_V_OK) + { + X509_STORE_CTX_set_error(pStore, distrusted); + ok = 0; + } + } + if (!ok) { int depth = X509_STORE_CTX_get_error_depth(pStore); diff --git a/base/poco/XML/CMakeLists.txt b/base/poco/XML/CMakeLists.txt index 1d3c8d332f1b1..d0d9f26fc2684 100644 --- a/base/poco/XML/CMakeLists.txt +++ b/base/poco/XML/CMakeLists.txt @@ -9,6 +9,7 @@ add_library (Poco::XML::Expat ALIAS _poco_xml_expat) target_compile_options (_poco_xml_expat PRIVATE -Wno-cast-qual + -Wno-declaration-after-statement -Wno-disabled-macro-expansion -Wno-empty-translation-unit -Wno-extra-semi-stmt diff --git a/base/poco/XML/include/Poco/XML/expat.h b/base/poco/XML/include/Poco/XML/expat.h index 02a3fe98b2ac5..9e57096d428f7 100644 --- a/base/poco/XML/include/Poco/XML/expat.h +++ b/base/poco/XML/include/Poco/XML/expat.h @@ -21,6 +21,7 @@ Copyright (c) 2024 Taichi Haradaguchi <20001722@ymail.ne.jp> Copyright (c) 2025 Matthew Fernandez Copyright (c) 2026 Braian Plaku + Copyright (c) 2026 Xisco Fauli Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -1094,7 +1095,7 @@ enum XML_Parser_Property { # if defined(XML_TESTING) XML_PROP_INVALID = 0, # endif -# if XML_GE == 1 +# if defined(XML_GE) && XML_GE == 1 XML_PROP_ALLOC_TRACKER_ACTIVATION_THRESHOLD = 1, // of type `uint64_t` XML_PROP_ALLOC_TRACKER_MAXIMUM_AMPLIFICATION = 2, // of type `double` XML_PROP_BILLION_LAUGHS_ACTIVATION_THRESHOLD = 3, // of type `uint64_t` @@ -1156,7 +1157,7 @@ XML_SetReparseDeferralEnabled(XML_Parser parser, XML_Bool enabled); See https://semver.org */ # define XML_MAJOR_VERSION 2 -# define XML_MINOR_VERSION 9 +# define XML_MINOR_VERSION 10 # define XML_MICRO_VERSION 0 # ifdef __cplusplus diff --git a/base/poco/XML/include/Poco/XML/expat_external.h b/base/poco/XML/include/Poco/XML/expat_external.h index 9fb80149acd3a..f34750afbddca 100644 --- a/base/poco/XML/include/Poco/XML/expat_external.h +++ b/base/poco/XML/include/Poco/XML/expat_external.h @@ -12,7 +12,7 @@ Copyright (c) 2001-2002 Greg Stein Copyright (c) 2002-2006 Karl Waclawek Copyright (c) 2016 Cristian Rodríguez - Copyright (c) 2016-2025 Sebastian Pipping + Copyright (c) 2016-2026 Sebastian Pipping Copyright (c) 2017 Rhodri James Copyright (c) 2018 Yury Gribov Copyright (c) 2026 Matthew Fernandez diff --git a/base/poco/XML/src/ascii.h b/base/poco/XML/src/ascii.h index 56c8807aa5746..656c95e28ddb3 100644 --- a/base/poco/XML/src/ascii.h +++ b/base/poco/XML/src/ascii.h @@ -10,7 +10,7 @@ Copyright (c) 2000 Clark Cooper Copyright (c) 2002 Fred L. Drake, Jr. Copyright (c) 2007 Karl Waclawek - Copyright (c) 2017 Sebastian Pipping + Copyright (c) 2017-2026 Sebastian Pipping Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -113,6 +113,7 @@ #define ASCII_LT 0x3C #define ASCII_EQUALS 0x3D #define ASCII_GT 0x3E +#define ASCII_QUEST 0x3F #define ASCII_LSQB 0x5B #define ASCII_RSQB 0x5D #define ASCII_UNDERSCORE 0x5F diff --git a/base/poco/XML/src/entities.h b/base/poco/XML/src/entities.h new file mode 100644 index 0000000000000..d0cd309c1111f --- /dev/null +++ b/base/poco/XML/src/entities.h @@ -0,0 +1,99 @@ +/* Internal helpers related to entities + __ __ _ + ___\ \/ /_ __ __ _| |_ + / _ \\ /| '_ \ / _` | __| + | __// \| |_) | (_| | |_ + \___/_/\_\ .__/ \__,_|\__| + |_| XML parser + + Copyright (c) 2026 Sebastian Pipping + Licensed under the MIT license: + + Permission is hereby granted, free of charge, to any person obtaining + a copy of this software and associated documentation files (the + "Software"), to deal in the Software without restriction, including + without limitation the rights to use, copy, modify, merge, publish, + distribute, sublicense, and/or sell copies of the Software, and to permit + persons to whom the Software is furnished to do so, subject to the + following conditions: + + The above copyright notice and this permission notice shall be included + in all copies or substantial portions of the Software. + + THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, + EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF + MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN + NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, + DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR + OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE + USE OR OTHER DEALINGS IN THE SOFTWARE. + + SPDX-License-Identifier: MIT +*/ + +#if ! defined(EXPAT_ENTITIES_H) +# define EXPAT_ENTITIES_H 1 + +# include "expat_config.h" // for XML_GE +# include "Poco/XML/expat_external.h" // for XML_Char +# include "internal.h" // for XML_NONTESTING_STATIC +# include // for size_t +# include // for size_t + +typedef struct { + const XML_Char *name; + const XML_Char *textPtr; + size_t textLen; /* length in XML_Chars */ + size_t processed; /* # of processed bytes - when suspended */ + const XML_Char *systemId; + const XML_Char *base; + const XML_Char *publicId; + const XML_Char *notation; + bool open; + bool hasMore; /* true if entity has not been completely processed */ + /* An entity can be open while being already completely processed (!hasMore). + The reason is the delayed closing of entities until their inner + entities are processed and closed */ + bool is_param; + bool is_internal; /* true if declared in internal subset outside PE */ +# if XML_GE == 1 + /* Indirect accounting bytes charged when this entity is expanded: 1 per text + character, plus each reference's "&name;"/"%name;" token length and its own + expansion. Matches XML_ACCOUNT_ENTITY_EXPANSION, so it is not the + replacement-text size. 0 means "not computed yet" (an empty entity is 0 + too, told apart by textLen == 0). */ + XmlBigCount expandedSize; + bool beingAnalyzed; /* on the resolveEntityClosureExpandedSize depth-first + search stack; used for cycle detection */ + const XML_Char + *scanPtr; /* cursor position into the entity's replacement text; NULL when + `textPtr` is NULL, else a pointer into the entity value, + ranging from `.textPtr` to `.textPtr + .textLen`. */ +# endif /* XML_GE == 1 */ +} ENTITY; + +# if XML_GE == 1 +/* How a scanEntityText call ended. */ +enum EntityScanResult { + ENTITY_SCAN_COMPLETE, /* reached end of text; *expandedSize is final */ + ENTITY_SCAN_BLOCKED, /* stopped at a defined entity whose expanded + size has not been computed yet; *blockerOut + names the entity whose size to compute + first */ + ENTITY_SCAN_OVERFLOW /* the expanded size would exceed XmlBigCount; + callers must abort parsing */ +}; + +XML_NONTESTING_STATIC int isCDataCommentOrPIStart(const XML_Char *pos, + size_t maxLen, + size_t *openTextLen); + +XML_NONTESTING_STATIC int isPredefinedEntityName(const XML_Char *name, + size_t nameLen); + +XML_NONTESTING_STATIC enum EntityScanResult +scanEntityText(XML_Parser parser, ENTITY *entity, const XML_Char **posPtr, + XmlBigCount *expandedSize, ENTITY **blockerOut); +# endif /* XML_GE == 1 */ + +#endif // ! defined(EXPAT_ENTITIES_H) diff --git a/base/poco/XML/src/internal.h b/base/poco/XML/src/internal.h index d0465449a3991..58ea8e2b4dc21 100644 --- a/base/poco/XML/src/internal.h +++ b/base/poco/XML/src/internal.h @@ -29,6 +29,7 @@ Copyright (c) 2026 Matthew Wozniczka Copyright (c) 2026 Braian Plaku Copyright (c) 2026 Florian Schmaus + Copyright (c) 2026 Brooks Davis Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -90,11 +91,10 @@ # define EXPAT_FMT_SIZE_T(midpart) "%" midpart "u" # endif # else -# include // PRIdPTR, PRIuPTR # define EXPAT_FMT_LLX(midpart) "%" midpart "llx" # define EXPAT_FMT_ULL(midpart) "%" midpart "llu" -# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart PRIdPTR -# define EXPAT_FMT_SIZE_T(midpart) "%" midpart PRIuPTR +# define EXPAT_FMT_PTRDIFF_T(midpart) "%" midpart "td" +# define EXPAT_FMT_SIZE_T(midpart) "%" midpart "zu" # endif # ifndef UNUSED_P @@ -131,6 +131,8 @@ union expat_align { extern "C" { #endif +typedef unsigned long long XmlBigCount; + void _INTERNAL_trim_to_complete_utf8_characters(const char *from, const char **fromLimRef); diff --git a/base/poco/XML/src/xcs.h b/base/poco/XML/src/xcs.h index 1e63a9b1de2f6..2b4268716cced 100644 --- a/base/poco/XML/src/xcs.h +++ b/base/poco/XML/src/xcs.h @@ -44,6 +44,8 @@ extern size_t xcslen(const XML_Char *s); extern int xcscmp(const XML_Char *a, const XML_Char *b); extern int xcsncmp(const XML_Char *a, const XML_Char *b, size_t len); +extern const XML_Char *xcsnstr(const XML_Char *haystack, const XML_Char *needle, + size_t maxLen); #ifdef __cplusplus } diff --git a/base/poco/XML/src/xmlparse.cpp b/base/poco/XML/src/xmlparse.cpp index ac428105490cc..40f3093fd68ec 100644 --- a/base/poco/XML/src/xmlparse.cpp +++ b/base/poco/XML/src/xmlparse.cpp @@ -1,4 +1,4 @@ -/* e3ca845466942140c9af531ac57808c4650116df5d6dbeda3b6546d2548a586e (2.9.0+) +/* d0712c5efef760b5355ed61473a927256c6474aecb853016d2e335e571c64b0c (2.10.0+) __ __ _ ___\ \/ /_ __ __ _| |_ / _ \\ /| '_ \ / _` | __| @@ -58,6 +58,8 @@ Copyright (c) 2026 Braian Plaku Copyright (c) 2026 Filippo Tedeschi Copyright (c) 2026 Junki Lee + Copyright (c) 2026 Shani Yosef + Copyright (c) 2026 Pu Cheng Licensed under the MIT license: Permission is hereby granted, free of charge, to any person obtaining @@ -233,11 +235,13 @@ typedef char ICHAR; /* Do safe (NULL-aware) pointer arithmetic */ #define EXPAT_SAFE_PTR_DIFF(p, q) (((p) && (q)) ? ((p) - (q)) : 0) +#define EXPAT_SAFE_PTR_ADD(p, a) ((p) ? ((p) + (a)) : NULL) #define EXPAT_MIN(a, b) (((a) < (b)) ? (a) : (b)) #include "internal.h" #include "hash_table.h" +#include "entities.h" #include "xmltok.h" #include "xmlrole.h" @@ -315,24 +319,6 @@ typedef struct tag { BINDING *bindings; } TAG; -typedef struct { - const XML_Char *name; - const XML_Char *textPtr; - int textLen; /* length in XML_Chars */ - int processed; /* # of processed bytes - when suspended */ - const XML_Char *systemId; - const XML_Char *base; - const XML_Char *publicId; - const XML_Char *notation; - bool open; - bool hasMore; /* true if entity has not been completely processed */ - /* An entity can be open while being already completely processed (!hasMore). - The reason is the delayed closing of entities until their inner - entities are processed and closed */ - bool is_param; - bool is_internal; /* true if declared in internal subset outside PE */ -} ENTITY; - typedef struct { enum XML_Content_Type type; enum XML_Content_Quant quant; @@ -371,7 +357,7 @@ typedef struct attribute_id { typedef struct { const ATTRIBUTE_ID *id; - XML_Bool isCdata; + bool isCdata; const XML_Char *value; } DEFAULT_ATTRIBUTE; @@ -461,7 +447,6 @@ enum XML_Account { }; #if XML_GE == 1 -typedef unsigned long long XmlBigCount; typedef struct accounting { XmlBigCount countBytesDirect; XmlBigCount countBytesIndirect; @@ -473,6 +458,7 @@ typedef struct accounting { typedef struct MALLOC_TRACKER { size_t bytesAllocated; size_t peakBytesAllocated; // updated live only for debug level >=2 + size_t bytesAllocatedWhenLastReset; unsigned long debugLevel; float maximumAmplificationFactor; // >=1.0 XmlBigCount activationThresholdBytes; @@ -484,6 +470,13 @@ typedef struct entity_stats { unsigned int maximumDepthSeen; unsigned long debugLevel; } ENTITY_STATS; + +/* One pending entity-analysis task on the explicit stack used by + resolveEntityClosureExpandedSize's depth-first search over the entity + reference graph. */ +typedef struct { + ENTITY *entity; /* entity currently being scanned */ +} ENTITY_ANALYSIS_TASK; #endif /* XML_GE == 1 */ typedef enum XML_Error Processor(XML_Parser parser, const char *start, @@ -543,15 +536,15 @@ static enum XML_Error addBinding(XML_Parser parser, PREFIX *prefix, const ATTRIBUTE_ID *attId, const XML_Char *uri, BINDING **bindingsPtr); static int defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, - XML_Bool isCdata, XML_Bool isId, - const XML_Char *value, XML_Parser parser); + bool isCdata, bool isId, const XML_Char *value, + XML_Parser parser); static enum XML_Error storeAttributeValue(XML_Parser parser, - const ENCODING *enc, XML_Bool isCdata, + const ENCODING *enc, bool isCdata, const char *ptr, const char *end, STRING_POOL *pool, enum XML_Account account); static enum XML_Error -appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, +appendAttributeValue(XML_Parser parser, const ENCODING *enc, bool isCdata, const char *ptr, const char *end, STRING_POOL *pool, enum XML_Account account, const char **nextPtr); static ATTRIBUTE_ID *getAttributeId(XML_Parser parser, const ENCODING *enc, @@ -566,6 +559,9 @@ static enum XML_Error callStoreEntityValue(XML_Parser parser, const ENCODING *enc, const char *start, const char *end, enum XML_Account account); +static enum XML_Error resolveEntityClosureExpandedSize(XML_Parser parser, + HASH_TABLE *table, + ENTITY *root); #else static enum XML_Error storeSelfEntityValue(XML_Parser parser, ENTITY *entity); #endif @@ -603,7 +599,7 @@ static const XML_Char *poolCopyString(STRING_POOL *pool, const XML_Char *s); static const XML_Char *poolCopyStringNoFinish(STRING_POOL *pool, const XML_Char *s); static const XML_Char *poolCopyStringN(STRING_POOL *pool, const XML_Char *s, - int n); + size_t n); static const XML_Char *poolAppendString(STRING_POOL *pool, const XML_Char *s); static int nextScaffoldPart(XML_Parser parser); @@ -621,12 +617,17 @@ static XML_Parser parserCreate(const XML_Char *encodingName, const XML_Char *nameSep, DTD *dtd, XML_Parser parentParser); -static void parserInit(XML_Parser parser, const XML_Char *encodingName); +static void parserInit(XML_Parser parser, const XML_Char *encodingName, + bool isReset); #if XML_GE == 1 -static float accountingGetCurrentAmplification(XML_Parser rootParser); -static void accountingReportStats(XML_Parser originParser, const char *epilog); -static void accountingOnAbort(XML_Parser originParser); +static float accountingGetCurrentAmplification(XML_Parser rootParser, + XmlBigCount extraIndirectBytes); +static void accountingReportStats(XML_Parser originParser, const char *epilog, + XmlBigCount extraIndirectBytes, + int sourceLine); +static void accountingOnAbort(XML_Parser originParser, + XmlBigCount extraIndirectBytes, int sourceLine); static void accountingReportDiff(XML_Parser rootParser, unsigned int levelsAwayFromRootParser, const char *before, const char *after, @@ -636,6 +637,9 @@ static XML_Bool accountingDiffTolerated(XML_Parser originParser, int tok, const char *before, const char *after, int source_line, enum XML_Account account); +static XML_Bool +accountingCurrentAmplificationTolerated(XML_Parser rootParser, + XmlBigCount extraIndirectBytes); static void entityTrackingReportStats(XML_Parser parser, ENTITY *entity, const char *action, int sourceLine); @@ -643,6 +647,10 @@ static void entityTrackingOnOpen(XML_Parser parser, ENTITY *entity, int sourceLine); static void entityTrackingOnClose(XML_Parser parser, ENTITY *entity, int sourceLine); +static void entityTrackingOnScanStart(XML_Parser parser, ENTITY *entity, + int sourceLine); +static void entityTrackingOnScanStop(XML_Parser parser, ENTITY *entity, + int sourceLine); #endif /* XML_GE == 1 */ static XML_Parser getRootParserOf(XML_Parser parser, @@ -798,8 +806,8 @@ struct XML_ParserStruct { const XML_Char *m_declNotationPublicId; ELEMENT_TYPE *m_declElementType; ATTRIBUTE_ID *m_declAttributeId; - XML_Bool m_declAttributeIsCdata; - XML_Bool m_declAttributeIsId; + bool m_declAttributeIsCdata; + bool m_declAttributeIsId; DTD *m_dtd; const XML_Char *m_curBase; TAG *m_tagStack; @@ -871,11 +879,22 @@ expat_heap_increase_tolerable(XML_Parser rootParser, size_t increase, size_t newTotal = 0; bool tolerable = true; + // The idea is to subtract the amount of memory that was already + // allocated the last time that `XML_ParserReset` was called for + // this parser, the memory use of one document could skew the + // memory statso of the next document. + const size_t bytesAllocated + = (rootParser->m_alloc_tracker.bytesAllocatedWhenLastReset + > rootParser->m_alloc_tracker.bytesAllocated) + ? 0 + : (rootParser->m_alloc_tracker.bytesAllocated + - rootParser->m_alloc_tracker.bytesAllocatedWhenLastReset); + // Detect integer overflow - if (SIZE_MAX - rootParser->m_alloc_tracker.bytesAllocated < increase) { + if (SIZE_MAX - bytesAllocated < increase) { tolerable = false; } else { - newTotal = rootParser->m_alloc_tracker.bytesAllocated + increase; + newTotal = bytesAllocated + increase; if (newTotal >= rootParser->m_alloc_tracker.activationThresholdBytes) { assert(newTotal > 0); @@ -1472,7 +1491,7 @@ parserCreate(const XML_Char *encodingName, poolInit(&parser->m_tempPool, parser); poolInit(&parser->m_temp2Pool, parser); - parserInit(parser, encodingName); + parserInit(parser, encodingName, /*isReset=*/false); if (encodingName && ! parser->m_protocolEncodingName) { if (dtd) { @@ -1499,7 +1518,7 @@ parserCreate(const XML_Char *encodingName, } static void -parserInit(XML_Parser parser, const XML_Char *encodingName) { +parserInit(XML_Parser parser, const XML_Char *encodingName, bool isReset) { parser->m_processor = prologInitProcessor; XmlPrologStateInit(&parser->m_prologState); if (encodingName != NULL) { @@ -1547,8 +1566,8 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_declAttributeType = NULL; parser->m_declNotationName = NULL; parser->m_declNotationPublicId = NULL; - parser->m_declAttributeIsCdata = XML_FALSE; - parser->m_declAttributeIsId = XML_FALSE; + parser->m_declAttributeIsCdata = false; + parser->m_declAttributeIsId = false; memset(&parser->m_position, 0, sizeof(POSITION)); parser->m_errorCode = XML_ERROR_NONE; parser->m_eventPtr = NULL; @@ -1567,6 +1586,7 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_unknownEncodingRelease = NULL; parser->m_unknownEncodingData = NULL; parser->m_parsingStatus.parsing = XML_INITIALIZED; + parser->m_parsingStatus.finalBuffer = XML_FALSE; // Reentry can only be triggered inside m_processor calls parser->m_reenter = XML_FALSE; parser->m_handlerCallDepth = 0; @@ -1580,15 +1600,24 @@ parserInit(XML_Parser parser, const XML_Char *encodingName) { parser->m_hash_secret_salt_set = XML_FALSE; #if XML_GE == 1 - memset(&parser->m_accounting, 0, sizeof(ACCOUNTING)); - parser->m_accounting.debugLevel = getDebugLevel("EXPAT_ACCOUNTING_DEBUG", 0u); - parser->m_accounting.maximumAmplificationFactor - = EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT; - parser->m_accounting.activationThresholdBytes - = EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT; - - memset(&parser->m_entity_stats, 0, sizeof(ENTITY_STATS)); - parser->m_entity_stats.debugLevel = getDebugLevel("EXPAT_ENTITY_DEBUG", 0u); + parser->m_accounting.countBytesDirect = 0; + parser->m_accounting.countBytesIndirect = 0; + if (! isReset) { + parser->m_accounting.debugLevel + = getDebugLevel("EXPAT_ACCOUNTING_DEBUG", 0u); + parser->m_accounting.maximumAmplificationFactor + = EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_MAXIMUM_AMPLIFICATION_DEFAULT; + parser->m_accounting.activationThresholdBytes + = EXPAT_BILLION_LAUGHS_ATTACK_PROTECTION_ACTIVATION_THRESHOLD_DEFAULT; + } + + parser->m_entity_stats.countEverOpened = 0; + parser->m_entity_stats.currentDepth = 0; + parser->m_entity_stats.maximumDepthSeen = 0; + if (! isReset) + parser->m_entity_stats.debugLevel = getDebugLevel("EXPAT_ENTITY_DEBUG", 0u); +#else + UNUSED_P(isReset); #endif } @@ -1669,8 +1698,16 @@ XML_ParserReset(XML_Parser parser, const XML_Char *encodingName) { poolClear(&parser->m_temp2Pool); FREE(parser, (void *)parser->m_protocolEncodingName); parser->m_protocolEncodingName = NULL; - parserInit(parser, encodingName); + parserInit(parser, encodingName, /*isReset=*/true); dtdReset(parser->m_dtd, parser); + +#if XML_GE == 1 + parser->m_alloc_tracker.bytesAllocatedWhenLastReset + = parser->m_alloc_tracker.bytesAllocated; +#endif + + // NOTE: This effectively checks for `copyString` having failed during the + // call to `parserInit` above if (encodingName && ! parser->m_protocolEncodingName) { parser->m_errorCode = XML_ERROR_NO_MEMORY; return XML_FALSE; @@ -2340,13 +2377,12 @@ XML_SetHashSalt16Bytes(XML_Parser parser, const uint8_t entropy[16]) { enum XML_Status XMLCALL XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { - if ((parser == NULL) || (len < 0) || ((s == NULL) && (len != 0))) { - if (parser != NULL) - parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT; + if (parser == NULL || isCalledFromInsideHandler(parser)) return XML_STATUS_ERROR; - } - if (isCalledFromInsideHandler(parser)) + if ((len < 0) || ((s == NULL) && (len != 0))) { + parser->m_errorCode = XML_ERROR_INVALID_ARGUMENT; return XML_STATUS_ERROR; + } switch (parser->m_parsingStatus.parsing) { case XML_SUSPENDED: parser->m_errorCode = XML_ERROR_SUSPENDED; @@ -2366,8 +2402,15 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { #if XML_CONTEXT_BYTES == 0 if (parser->m_bufferPtr == parser->m_bufferEnd) { - const char *end; - int nLeftOver; + // This null-object trick allows us to stop worrying about case `s == NULL` + // within this branch, e.g. with `s + len - end`. + const char *const empty = ""; + if (s == NULL) { + assert(len == 0); + s = empty; + } + const char *end = s + len; + size_t nLeftOver; enum XML_Status result; /* Detect overflow (a+b > MAX <==> b > MAX-a) */ if ((uint64_t)len > UINT64_MAX - parser->m_parseEndByteIndex) { @@ -2384,7 +2427,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { parser->m_parsingStatus.finalBuffer = (XML_Bool)isFinal; parser->m_errorCode - = callProcessor(parser, s, parser->m_parseEndPtr = s + len, &end); + = callProcessor(parser, s, parser->m_parseEndPtr = end, &end); if (parser->m_errorCode != XML_ERROR_NONE) { parser->m_eventEndPtr = parser->m_eventPtr; @@ -2414,8 +2457,14 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { // Back up and restore the parsing status to avoid XML_ERROR_SUSPENDED // (and XML_ERROR_FINISHED) from XML_GetBuffer. const enum XML_Parsing originalStatus = parser->m_parsingStatus.parsing; + if (nLeftOver > INT_MAX) { + parser->m_errorCode = XML_ERROR_NO_MEMORY; + parser->m_eventPtr = parser->m_eventEndPtr = NULL; + parser->m_processor = errorProcessor; + return XML_STATUS_ERROR; + } parser->m_parsingStatus.parsing = XML_PARSING; - void *const temp = XML_GetBuffer(parser, nLeftOver); + void *const temp = XML_GetBuffer(parser, (int)nLeftOver); parser->m_parsingStatus.parsing = originalStatus; // GetBuffer may have overwritten this, but we want to remember what the // app requested, not how many bytes were left over after parsing. @@ -2433,7 +2482,7 @@ XML_Parse(XML_Parser parser, const char *s, int len, int isFinal) { memcpy(parser->m_buffer, end, nLeftOver); } parser->m_bufferPtr = parser->m_buffer; - parser->m_bufferEnd = parser->m_buffer + nLeftOver; + parser->m_bufferEnd = EXPAT_SAFE_PTR_ADD(parser->m_buffer, nLeftOver); parser->m_positionPtr = parser->m_bufferPtr; parser->m_parseEndPtr = parser->m_bufferEnd; parser->m_eventPtr = parser->m_bufferPtr; @@ -3515,7 +3564,7 @@ externalEntityInitProcessor2(XML_Parser parser, const char *start, #if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, start, next, __LINE__, XML_ACCOUNT_DIRECT)) { - accountingOnAbort(parser); + accountingOnAbort(parser, 0, __LINE__); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; } #endif /* XML_GE == 1 */ @@ -3545,6 +3594,11 @@ externalEntityInitProcessor2(XML_Parser parser, const char *start, } parser->m_eventPtr = start; return XML_ERROR_PARTIAL_CHAR; + case XML_TOK_NONE: + if (! parser->m_parsingStatus.finalBuffer) { + *endPtr = start; + } + return XML_ERROR_NONE; } parser->m_processor = externalEntityInitProcessor3; return externalEntityInitProcessor3(parser, start, end, endPtr); @@ -3644,7 +3698,7 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, : next; if (! accountingDiffTolerated(parser, tok, s, accountAfter, __LINE__, account)) { - accountingOnAbort(parser); + accountingOnAbort(parser, 0, __LINE__); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; } #endif @@ -4076,10 +4130,12 @@ doContent(XML_Parser parser, int startTagLevel, const ENCODING *enc, const enum XML_Convert_Result convert_res = XmlConvert( enc, &s, next, &dataPtr, (ICHAR *)parser->m_dataBufEnd); *eventEndPP = s; - beforeHandler(parser); - charDataHandler(parser->m_handlerArg, parser->m_dataBuf, - (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); - afterHandler(parser); + if (parser->m_characterDataHandler) { + beforeHandler(parser); + charDataHandler(parser->m_handlerArg, parser->m_dataBuf, + (int)(dataPtr - (ICHAR *)parser->m_dataBuf)); + afterHandler(parser); + } if ((convert_res == XML_CONVERT_COMPLETED) || (convert_res == XML_CONVERT_INPUT_INCOMPLETE)) break; @@ -4176,9 +4232,9 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, TAG_NAME *tagNamePtr, BINDING **bindingsPtr, enum XML_Account account) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ - int attIndex = 0; + size_t attIndex = 0; XML_Char *uri; - int nPrefixes = 0; + size_t nPrefixes = 0; BINDING *binding; const XML_Char *localPart; @@ -4205,8 +4261,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, return XML_ERROR_NO_MEMORY; /* get the attributes from the tokenizer */ - size_t n = (size_t)XmlGetAttributes(enc, attStr, (int)parser->m_attsSize, - parser->m_atts); + size_t n = XmlGetAttributes(enc, attStr, parser->m_attsSize, parser->m_atts); /* Detect and prevent integer overflow */ if (n > SIZE_MAX - nDefaultAtts) { @@ -4241,11 +4296,22 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* Detect and prevent integer overflow. */ if (n > (size_t)INT_MAX) return XML_ERROR_NO_MEMORY; - XmlGetAttributes(enc, attStr, (int)n, parser->m_atts); + XmlGetAttributes(enc, attStr, n, parser->m_atts); } } /* the attribute list for the application */ + // NOTE: `parser->m_atts` is used to read `ATTRIBUTE` elements + // from start to end while at the same time writing `XML_Char *` + // elements from start to end *to the same block of memory*. + // That black magic only works because the first member `.name` in + // `ATTRIBUTE` is no longer being read after `appAtts[attIndex]` + // has been written, which for the first element is the very same + // address in memory. + // And because `attIndex` is increased twice for default attributes + // further down, struct `ATTRIBUTE` must be at least twice the size + // of a pointer, or the allocation size math for `m_attsSize` starts + // to underallocate. const XML_Char **const appAtts = (const XML_Char **)parser->m_atts; for (size_t i = 0; i < n; i++) { ATTRIBUTE *currAtt = &parser->m_atts[i]; @@ -4265,9 +4331,11 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, return XML_ERROR_DUPLICATE_ATTRIBUTE; } (attId->name)[-1] = 1; + // NOTE: The next call overwrites `parser->m_atts`, so we cannot use + // e.g. `parser->m_atts[i].name;` after this line! appAtts[attIndex++] = attId->name; if (! parser->m_atts[i].normalized) { - XML_Bool isCdata = XML_TRUE; + bool isCdata = true; /* figure out whether declared as other than CDATA */ if (attId->maybeTokenized) { @@ -4319,11 +4387,16 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, } /* set-up for XML_GetSpecifiedAttributeCount and XML_GetIdAttributeIndex */ - parser->m_nSpecifiedAtts = attIndex; + // Detect and reject integer overflow + if (attIndex > INT_MAX) + return XML_ERROR_NO_MEMORY; + parser->m_nSpecifiedAtts = (int)attIndex; if (elementType->idAtt && (elementType->idAtt->name)[-1]) { - for (int i = 0; i < attIndex; i += 2) + for (size_t i = 0; i < attIndex; i += 2) if (appAtts[i] == elementType->idAtt->name) { - parser->m_idAttIndex = i; + if (i > INT_MAX) + return XML_ERROR_NO_MEMORY; + parser->m_idAttIndex = (int)i; break; } } else @@ -4356,7 +4429,7 @@ storeAtts(XML_Parser parser, const ENCODING *enc, const char *attStr, /* expand prefixed attribute names, check for duplicates, and clear flags that say whether attributes were specified */ - int i = 0; + size_t i = 0; if (nPrefixes) { unsigned int j; /* hash table index */ unsigned long version = parser->m_nsAttsVersion; @@ -4912,7 +4985,7 @@ doCdataSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, int tok = XmlCdataSectionTok(enc, s, end, &next); #if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, account)) { - accountingOnAbort(parser); + accountingOnAbort(parser, 0, __LINE__); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; } #else @@ -5084,7 +5157,7 @@ doIgnoreSection(XML_Parser parser, const ENCODING *enc, const char **startPtr, # if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, XML_ACCOUNT_DIRECT)) { - accountingOnAbort(parser); + accountingOnAbort(parser, 0, __LINE__); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; } # endif @@ -5176,7 +5249,7 @@ processXmlDecl(XML_Parser parser, int isGeneralTextEntity, const char *s, #if XML_GE == 1 if (! accountingDiffTolerated(parser, XML_TOK_XML_DECL, s, next, __LINE__, XML_ACCOUNT_DIRECT)) { - accountingOnAbort(parser); + accountingOnAbort(parser, 0, __LINE__); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; } #endif @@ -5397,7 +5470,7 @@ entityValueInitProcessor(XML_Parser parser, const char *s, const char *end, # if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, XML_ACCOUNT_DIRECT)) { - accountingOnAbort(parser); + accountingOnAbort(parser, 0, __LINE__); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; } # endif @@ -5450,7 +5523,7 @@ externalParEntProcessor(XML_Parser parser, const char *s, const char *end, else if (tok == XML_TOK_BOM) { if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, XML_ACCOUNT_DIRECT)) { - accountingOnAbort(parser); + accountingOnAbort(parser, 0, __LINE__); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; } @@ -5629,7 +5702,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, break; default: if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, account)) { - accountingOnAbort(parser); + accountingOnAbort(parser, 0, __LINE__); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; } } @@ -5726,7 +5799,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, return XML_ERROR_INVALID_TOKEN; } - if (parser->m_doctypeName) { + if (parser->m_doctypeName && parser->m_startDoctypeDeclHandler) { beforeHandler(parser); parser->m_startDoctypeDeclHandler( parser->m_handlerArg, parser->m_doctypeName, parser->m_doctypeSysid, @@ -5792,7 +5865,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, handleDefault = XML_FALSE; } break; - case XML_ROLE_INSTANCE_START: + case XML_ROLE_INSTANCE_START: { #ifdef XML_DTD /* if there is no DOCTYPE declaration then now is the last chance to read the foreign DTD @@ -5836,6 +5909,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, #endif /* XML_DTD */ parser->m_processor = contentProcessor; return contentProcessor(parser, s, end, nextPtr); + } case XML_ROLE_ATTLIST_ELEMENT_NAME: parser->m_declElementType = getElementType(parser, enc, s, next); if (! parser->m_declElementType) @@ -5845,16 +5919,16 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, parser->m_declAttributeId = getAttributeId(parser, enc, s, next); if (! parser->m_declAttributeId) return XML_ERROR_NO_MEMORY; - parser->m_declAttributeIsCdata = XML_FALSE; + parser->m_declAttributeIsCdata = false; parser->m_declAttributeType = NULL; - parser->m_declAttributeIsId = XML_FALSE; + parser->m_declAttributeIsId = false; goto checkAttListDeclHandler; case XML_ROLE_ATTRIBUTE_TYPE_CDATA: - parser->m_declAttributeIsCdata = XML_TRUE; + parser->m_declAttributeIsCdata = true; parser->m_declAttributeType = atypeCDATA; goto checkAttListDeclHandler; case XML_ROLE_ATTRIBUTE_TYPE_ID: - parser->m_declAttributeIsId = XML_TRUE; + parser->m_declAttributeIsId = true; parser->m_declAttributeType = atypeID; goto checkAttListDeclHandler; case XML_ROLE_ATTRIBUTE_TYPE_IDREF: @@ -5942,7 +6016,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, /* ID attributes aren't allowed to have a default */ if (! defineAttribute( parser->m_declElementType, parser->m_declAttributeId, - parser->m_declAttributeIsCdata, XML_FALSE, attVal, parser)) + parser->m_declAttributeIsCdata, false, attVal, parser)) return XML_ERROR_NO_MEMORY; if (parser->m_attlistDeclHandler && parser->m_declAttributeType) { if (*parser->m_declAttributeType == XML_T(ASCII_LPAREN) @@ -5972,17 +6046,23 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, #if XML_GE == 1 // This will store the given replacement text in // parser->m_declEntity->textPtr. - enum XML_Error result = callStoreEntityValue( + const enum XML_Error result = callStoreEntityValue( parser, enc, s + enc->minBytesPerChar, next - enc->minBytesPerChar, XML_ACCOUNT_NONE); + if (result != XML_ERROR_NONE) { + poolDiscard(&dtd->entityValuePool); + return result; + } if (parser->m_declEntity) { /* Detect and prevent signed integer overflow */ if ((size_t)poolLength(&dtd->entityValuePool) > (size_t)INT_MAX) { + poolDiscard(&dtd->entityValuePool); return XML_ERROR_NO_MEMORY; } - parser->m_declEntity->textPtr = poolStart(&dtd->entityValuePool); - parser->m_declEntity->textLen - = (int)(poolLength(&dtd->entityValuePool)); + const XML_Char *const valueStart = poolStart(&dtd->entityValuePool); + parser->m_declEntity->textPtr = valueStart; + parser->m_declEntity->textLen = poolLength(&dtd->entityValuePool); + parser->m_declEntity->scanPtr = valueStart; poolFinish(&dtd->entityValuePool); if (parser->m_entityDeclHandler) { *eventEndPP = s; @@ -5990,14 +6070,12 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->is_param, parser->m_declEntity->textPtr, - parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0); + (int)parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0); afterHandler(parser); handleDefault = XML_FALSE; } } else poolDiscard(&dtd->entityValuePool); - if (result != XML_ERROR_NONE) - return result; #else // This will store "&entity123;" in parser->m_declEntity->textPtr // to end up as "&entity123;" in the handler. @@ -6013,7 +6091,7 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, parser->m_entityDeclHandler( parser->m_handlerArg, parser->m_declEntity->name, parser->m_declEntity->is_param, parser->m_declEntity->textPtr, - parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0); + (int)parser->m_declEntity->textLen, parser->m_curBase, 0, 0, 0); afterHandler(parser); handleDefault = XML_FALSE; } @@ -6132,6 +6210,17 @@ doProlog(XML_Parser parser, const ENCODING *enc, const char *s, const char *end, } break; case XML_ROLE_GENERAL_ENTITY_NAME: { + // NOTE: XML 1.0 fourth edition demands in section + // "4.6 Predefined Entities" that bad redefinitions of predefined + // general entities must be detected and rejected. Expat has been + // ignoring both conforming and non-conforming redefinitions and + // thus has been failing to reject bad redefinitions, for years. + // As a result, users learned that they can get away with + // `` in practice. We cannot start conforming to + // that part of the spec now, or we risk rejecting many documents + // that parsed fine for years and that may not be fixable now or + // in time. So Expat continues to discard redefinitions of + // predefined entities altogether. if (XmlPredefinedEntityName(enc, s, next)) { parser->m_declEntity = NULL; break; @@ -6658,7 +6747,7 @@ epilogProcessor(XML_Parser parser, const char *s, const char *end, #if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, s, next, __LINE__, XML_ACCOUNT_DIRECT)) { - accountingOnAbort(parser); + accountingOnAbort(parser, 0, __LINE__); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; } #endif @@ -6725,6 +6814,415 @@ epilogProcessor(XML_Parser parser, const char *s, const char *end, } } +#if XML_GE == 1 +// Checks `name` for being a predefined entity (i.e. one of "amp", "apos", "gt", +// "lt", "quot"). Similar in implementation to function +// `PREFIX(predefinedEntityName)` but operating on an `XML_Char` string with an +// explicit length, instead. Returns +// ASCII_AMP/ASCII_APOS/ASCII_GT/ASCII_LT/ASCII_QUOT for a match or 0 for no +// match. +XML_NONTESTING_STATIC int +isPredefinedEntityName(const XML_Char *name, size_t nameLen) { + switch (nameLen) { + case 2: + // Wanted: "lt" or "gt" + if (name[1] != XML_T(ASCII_t)) + break; + switch (name[0]) { + case XML_T(ASCII_l): + return ASCII_LT; + case XML_T(ASCII_g): + return ASCII_GT; + } + break; + case 3: + // Wanted: "amp" + if (name[0] == XML_T(ASCII_a) && name[1] == XML_T(ASCII_m) + && name[2] == XML_T(ASCII_p)) + return ASCII_AMP; + break; + case 4: + // Wanted: "apos" or "quot" + switch (name[0]) { + case XML_T(ASCII_a): + if (name[1] == XML_T(ASCII_p) && name[2] == XML_T(ASCII_o) + && name[3] == XML_T(ASCII_s)) + return ASCII_APOS; + break; + case XML_T(ASCII_q): + if (name[1] == XML_T(ASCII_u) && name[2] == XML_T(ASCII_o) + && name[3] == XML_T(ASCII_t)) + return ASCII_QUOT; + break; + } + } + return 0; +} + +static const XML_Char cdataOpen[] + = {ASCII_LT, ASCII_EXCL, ASCII_LSQB, ASCII_C, ASCII_D, + ASCII_A, ASCII_T, ASCII_A, ASCII_LSQB, '\0'}; // i.e. "" +static const XML_Char commentOpen[] + = {ASCII_LT, ASCII_EXCL, ASCII_MINUS, ASCII_MINUS, '\0'}; // i.e. "" +static const XML_Char piClose[] = {ASCII_QUEST, ASCII_GT, '\0'}; // i.e. "?>" + +static const size_t cdataOpenLen = sizeof(cdataOpen) / sizeof(cdataOpen[0]) - 1; +static const size_t cdataCloseLen + = sizeof(cdataClose) / sizeof(cdataClose[0]) - 1; +static const size_t commentOpenLen + = sizeof(commentOpen) / sizeof(commentOpen[0]) - 1; +static const size_t commentCloseLen + = sizeof(commentClose) / sizeof(commentClose[0]) - 1; +static const size_t piCloseLen = sizeof(piClose) / sizeof(piClose[0]) - 1; + +// Checks the string starting at `pos` for starting a CDATA section, a comment +// or processing instructions. Returns +// XML_TOK_CDATA_SECT_OPEN/XML_TOK_COMMENT/XML_TOK_PI for a match or +// XML_TOK_INVALID (=0) for no match. Only in the success case, the length +// of the token is written to `*openTextLen`. +XML_NONTESTING_STATIC int +isCDataCommentOrPIStart(const XML_Char *pos, size_t maxLen, + size_t *openTextLen) { + assert(pos != NULL); + assert(openTextLen != NULL); + + if (maxLen < 2) + return XML_TOK_INVALID; + + if (pos[0] != XML_T(ASCII_LT)) + return XML_TOK_INVALID; + + if (pos[1] == XML_T(ASCII_QUEST)) { + *openTextLen = 2; + return XML_TOK_PI; + } + + if (pos[1] != XML_T(ASCII_EXCL)) + return XML_TOK_INVALID; + + if (maxLen >= cdataOpenLen && xcsncmp(pos, cdataOpen, cdataOpenLen) == 0) { + *openTextLen = cdataOpenLen; + return XML_TOK_CDATA_SECT_OPEN; + } + + if (maxLen >= commentOpenLen + && xcsncmp(pos, commentOpen, commentOpenLen) == 0) { + *openTextLen = commentOpenLen; + return XML_TOK_COMMENT; + } + + return XML_TOK_INVALID; +} + +/* Scan `entity`'s body, starting at `*posPtr`, accumulating the expanded + size into `*expandedSize`. + + Return value: how the scan ended, see enum EntityScanResult. + + *blockerOut (out, set only for ENTITY_SCAN_BLOCKED): + The non-empty entity the scan stopped at because its expanded size + has not been computed yet. The caller computes that size first and + calls again with *posPtr unchanged to resume. + The pointer is borrowed from the entity hash table (via lookup), not + a new allocation, so it must not be freed. + + *posPtr (in/out, scan cursor): + Advanced to end-of-text on a complete scan, or parked at the + offending reference otherwise, so a follow-up call resumes from + the same position. Not meaningful after ENTITY_SCAN_OVERFLOW. + + *expandedSize (in/out, expanded-byte accumulator): + The expanded size of this one `entity` (its text plus the expandedSize + of entities it references), not a total across all entities. Not + meaningful after ENTITY_SCAN_OVERFLOW; the overflowing value is never + stored anywhere. */ +XML_NONTESTING_STATIC enum EntityScanResult +scanEntityText(XML_Parser parser, ENTITY *entity, const XML_Char **posPtr, + XmlBigCount *expandedSize, ENTITY **blockerOut) { + if (blockerOut != NULL) + *blockerOut = NULL; + const XML_Char refChar = entity->is_param ? XML_T('%') : XML_T('&'); + HASH_TABLE *const refTable +# ifdef XML_DTD + = entity->is_param ? &parser->m_dtd->paramEntities + : &parser->m_dtd->generalEntities; +# else + = &parser->m_dtd->generalEntities; +# endif + + const XML_Char *pos = *posPtr; + const XML_Char *const end = entity->textPtr + entity->textLen; + + while (pos < end) { + // Detect and skip CDATA/comment/PI sections + // because entities have no effect in these places + if (*pos == XML_T('<')) { + // Detect CDATA/comment/PI section start + size_t openTextLen = 0; + const int tok = isCDataCommentOrPIStart(pos, end - pos, &openTextLen); + if (tok != XML_TOK_INVALID) { + assert(openTextLen != 0); + + // Detect and avoid integer overflow + if (openTextLen > (XmlBigCount)(-1) - *expandedSize) + return ENTITY_SCAN_OVERFLOW; + + *expandedSize += openTextLen; + pos += openTextLen; + + // Detect CDATA/comment/PI section end + const XML_Char *closeText = NULL; + size_t closeLen = 0; + + switch (tok) { + case XML_TOK_CDATA_SECT_OPEN: + closeText = cdataClose; + closeLen = cdataCloseLen; + break; + case XML_TOK_COMMENT: + closeText = commentClose; + closeLen = commentCloseLen; + break; + case XML_TOK_PI: + closeText = piClose; + closeLen = piCloseLen; + break; + default: + assert(0 && "never gets here"); + break; + } + + const size_t remainderLen = end - pos; + const XML_Char *const closeStart + = xcsnstr(pos, closeText, remainderLen); + if (closeStart == NULL) { + // The CDATA/comment/PI section is NOT terminated + if (remainderLen > (XmlBigCount)(-1) - *expandedSize) + return ENTITY_SCAN_OVERFLOW; + *expandedSize += remainderLen; + *posPtr = end; + return ENTITY_SCAN_COMPLETE; + } + + // The CDATA/comment/PI section is NOT terminated + const size_t skippedLen = closeStart + closeLen - pos; + + // Detect and avoid integer overflow + if (skippedLen > (XmlBigCount)(-1) - *expandedSize) + return ENTITY_SCAN_OVERFLOW; + + *expandedSize += skippedLen; + pos += skippedLen; + + continue; + } + } + + // Detect an entity reference + if (*pos != refChar) { + // Detect and avoid integer overflow + if (*expandedSize == (XmlBigCount)(-1)) + return ENTITY_SCAN_OVERFLOW; + (*expandedSize)++; + pos++; + continue; + } + + const XML_Char *nameStart = pos + 1; + const XML_Char *nameEnd = nameStart; + while (nameEnd < end && *nameEnd != XML_T(';')) + nameEnd++; + + if (nameEnd >= end) { + /* Unterminated reference at end of text: treat the trailing + bytes as literal characters. */ + XmlBigCount addend = (XmlBigCount)(end - pos); + // Detect and avoid integer overflow + if (*expandedSize > (XmlBigCount)(-1) - addend) + return ENTITY_SCAN_OVERFLOW; + *expandedSize += addend; + *posPtr = end; + return ENTITY_SCAN_COMPLETE; + } + + const size_t nameLen = (size_t)(nameEnd - nameStart); + + /* At runtime the whole "&name;"/"%name;" token is re-scanned and charged to + the indirect accounting, then its expansion is charged separately; so + every reference below counts this token length plus its expansion. */ + const XmlBigCount refTokenLen + = (XmlBigCount)(/*sigil*/ 1 + nameLen + /*semicolon*/ 1); + + ENTITY *refEntity + = (ENTITY *)lookupWithLength(parser, refTable, nameStart, nameLen, 0); + + XmlBigCount addend = 0; + if (! refEntity) { + /* Predefined entities (amp/lt/gt/quot/apos) are general-only, hence the + `! entity->is_param` guard, and expand to a single character. */ + if (! entity->is_param && isPredefinedEntityName(nameStart, nameLen)) { + addend = refTokenLen + 1; + } else { + /* Undefined entity: expands to nothing, so only the token. */ + addend = refTokenLen; + } + } else if (refEntity->beingAnalyzed) { + /* Cycle: refEntity is already on the depth-first search stack, so its + expandedSize is not final yet; charge just the token. Do not fail the + parse here: per XML 1.0r4, + "A parsed entity MUST NOT contain a recursive reference to + itself, either directly or indirectly", but also "Only parsed + entities that are referenced directly or indirectly within the + document are required to be well-formed" - so a declared-but- + never-expanded cycle must be accepted; if the cycle is actually + expanded, the runtime recursive-ref check reports + XML_ERROR_RECURSIVE_ENTITY_REF. */ + addend = refTokenLen; + } else if (refEntity->textLen > 0 && refEntity->expandedSize == 0) { + /* Non-empty entity whose expanded size has not been computed yet; + report refEntity as a blocker so the caller can compute its size + first. */ + *posPtr = pos; + if (blockerOut != NULL) + *blockerOut = refEntity; + return ENTITY_SCAN_BLOCKED; + } else if (refEntity->expandedSize > 0) { + /* refEntity is fully resolved: expandedSize holds its final expanded + size (a non-zero value is never a partial here - the only entity that + briefly holds a partial is one currently on the resolve stack, which + is caught by the beingAnalyzed branch above). Add the token plus that + expansion. */ + if (refEntity->expandedSize > (XmlBigCount)(-1) - refTokenLen) + return ENTITY_SCAN_OVERFLOW; + addend = refTokenLen + refEntity->expandedSize; + } else { + /* Empty entity (textLen == 0): expands to nothing, so only the token. */ + addend = refTokenLen; + } + + // Detect and avoid integer overflow + if (*expandedSize > (XmlBigCount)(-1) - addend) + return ENTITY_SCAN_OVERFLOW; + *expandedSize += addend; + pos = nameEnd + 1; + } + + /* Loop only exits here with pos == end (a complete scan). */ + assert(pos == end); + *posPtr = end; + return ENTITY_SCAN_COMPLETE; +} + +/* Resolve expandedSize using an iterative depth-first search over the + reference graph. Each entity is scanned at most twice (once on push, + once after its blocker resolves), so the total work is O(N + T) where + N is the number of entities and T is the sum of all entities' textLen. + `entity->beingAnalyzed` marks the entities currently on the stack for + cycle detection; it is cleared again on pop. + Returns XML_ERROR_NO_MEMORY if the stack cannot be allocated, and + XML_ERROR_AMPLIFICATION_LIMIT_BREACH if an entity's expanded size + overflows XmlBigCount. */ +static enum XML_Error +resolveEntityClosureExpandedSize(XML_Parser parser, HASH_TABLE *table, + ENTITY *root) { + if (root == NULL || root->expandedSize != 0 || root->textPtr == NULL + || root->textLen == 0) + return XML_ERROR_NONE; + + if (table->used == 0) + return XML_ERROR_NONE; + + // Detect and avoid integer overflow + if (table->used > SIZE_MAX / sizeof(ENTITY_ANALYSIS_TASK)) + return XML_ERROR_NO_MEMORY; + + ENTITY_ANALYSIS_TASK *stack + = MALLOC(parser, sizeof(ENTITY_ANALYSIS_TASK) * table->used); + if (stack == NULL) + return XML_ERROR_NO_MEMORY; + + const XML_Parser rootParser = getRootParserOf(parser, NULL); + assert(! rootParser->m_parentParser); + + size_t top = 0; + stack[top].entity = root; + root->beingAnalyzed = XML_TRUE; + top++; + rootParser->m_entity_stats.currentDepth++; + if (rootParser->m_entity_stats.currentDepth + > rootParser->m_entity_stats.maximumDepthSeen) { + rootParser->m_entity_stats.maximumDepthSeen++; + } + + while (top > 0) { + ENTITY_ANALYSIS_TASK *task = &stack[top - 1]; + ENTITY *e = task->entity; + XmlBigCount sizeSoFar = e->expandedSize; + ENTITY *blocker = NULL; + + entityTrackingOnScanStart(parser, e, __LINE__); + + const enum EntityScanResult scanResult + = scanEntityText(parser, e, &e->scanPtr, &sizeSoFar, &blocker); + switch (scanResult) { + case ENTITY_SCAN_COMPLETE: + case ENTITY_SCAN_BLOCKED: + /* Save the size accumulated so far; together with task->pos it is + the state that lets the scan resume once the blocker's size has + been computed. This is the one place where expandedSize briefly + holds neither 0 nor the final size - only while the entity is + on the stack, where the beingAnalyzed check in scanEntityText + keeps every other reader away from the intermediate value. */ + e->expandedSize = sizeSoFar; + break; + default: + break; + } + + entityTrackingOnScanStop(parser, e, __LINE__); + + switch (scanResult) { + case ENTITY_SCAN_OVERFLOW: + /* Abort parsing right away; the beingAnalyzed marks still set on + the stacked entities do not matter once the parse is dead. */ + FREE(parser, stack); + /* expandedSize overflowed; pass SIZE_MAX (unrepresentable size). */ + accountingOnAbort(parser, SIZE_MAX, __LINE__); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + + case ENTITY_SCAN_COMPLETE: + e->beingAnalyzed = XML_FALSE; + top--; + rootParser->m_entity_stats.currentDepth--; + break; + + case ENTITY_SCAN_BLOCKED: + /* Push the blocker. A blocker's expanded size is always still + uncomputed and it is never already on the stack, so each entity + is pushed at most once and top <= table->used. */ + assert(! blocker->beingAnalyzed); + assert(top < table->used); + stack[top].entity = blocker; + blocker->beingAnalyzed = XML_TRUE; + top++; + rootParser->m_entity_stats.currentDepth++; + if (rootParser->m_entity_stats.currentDepth + > rootParser->m_entity_stats.maximumDepthSeen) { + rootParser->m_entity_stats.maximumDepthSeen++; + } + break; + } + } + + FREE(parser, stack); + return XML_ERROR_NONE; +} +#endif /* XML_GE == 1 */ + static enum XML_Error processEntity(XML_Parser parser, ENTITY *entity, bool betweenDecl, enum EntityType type) { @@ -6753,6 +7251,37 @@ processEntity(XML_Parser parser, ENTITY *entity, bool betweenDecl, /* LCOV_EXCL_STOP */ } +#if XML_GE == 1 + // Compute entity size assuming immediate access + if (entity->textLen > 0 && entity->expandedSize == 0) { + HASH_TABLE *const table +# ifdef XML_DTD + = entity->is_param ? &parser->m_dtd->paramEntities + : &parser->m_dtd->generalEntities; +# else + = &parser->m_dtd->generalEntities; +# endif + const enum XML_Error result + = resolveEntityClosureExpandedSize(parser, table, entity); + if (result != XML_ERROR_NONE) + return result; + } + + /* Reject an amplification attack before acquiring openEntity, so a rejected + entity leaks nothing and leaves entity state untouched. */ + if (entity->expandedSize > 0) { + const XML_Parser rootParser = getRootParserOf(parser, NULL); + /* Overflow of expandedSize aborts parsing during resolution + (on first expansion), so the stored value here is always exact. */ + const XmlBigCount expandedBytes = entity->expandedSize; + if (accountingCurrentAmplificationTolerated(rootParser, expandedBytes) + != XML_TRUE) { + accountingOnAbort(rootParser, expandedBytes, __LINE__); + return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; + } + } +#endif /* XML_GE == 1 */ + if (*freeEntityList) { openEntity = *freeEntityList; *freeEntityList = openEntity->next; @@ -6765,7 +7294,7 @@ processEntity(XML_Parser parser, ENTITY *entity, bool betweenDecl, entity->hasMore = true; #if XML_GE == 1 entityTrackingOnOpen(parser, entity, __LINE__); -#endif +#endif /* XML_GE == 1 */ entity->processed = 0; openEntity->next = *openEntityList; *openEntityList = openEntity; @@ -6827,7 +7356,7 @@ internalEntityProcessor(XML_Parser parser, const char *s, const char *end, && (parser->m_parsingStatus.parsing == XML_SUSPENDED || (parser->m_parsingStatus.parsing == XML_PARSING && parser->m_reenter))) { - entity->processed = (int)(next - (const char *)entity->textPtr); + entity->processed = next - (const char *)entity->textPtr; return result; } @@ -6876,7 +7405,7 @@ errorProcessor(XML_Parser parser, const char *s, const char *end, } static enum XML_Error -storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, +storeAttributeValue(XML_Parser parser, const ENCODING *enc, bool isCdata, const char *ptr, const char *end, STRING_POOL *pool, enum XML_Account account) { const char *next = ptr; @@ -6908,8 +7437,7 @@ storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, // processed. A XML_SUSPENDED check here is not required as // appendAttributeValue will never suspend the parser. if (nextInEntity < textEnd) { - entity->processed - = (int)(nextInEntity - (const char *)entity->textPtr); + entity->processed = nextInEntity - (const char *)entity->textPtr; continue; } assert(nextInEntity == textEnd); @@ -6954,7 +7482,7 @@ storeAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, } static enum XML_Error -appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, +appendAttributeValue(XML_Parser parser, const ENCODING *enc, bool isCdata, const char *ptr, const char *end, STRING_POOL *pool, enum XML_Account account, const char **nextPtr) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ @@ -6968,7 +7496,7 @@ appendAttributeValue(XML_Parser parser, const ENCODING *enc, XML_Bool isCdata, int tok = XmlAttributeValueTok(enc, ptr, end, &next); #if XML_GE == 1 if (! accountingDiffTolerated(parser, tok, ptr, next, __LINE__, account)) { - accountingOnAbort(parser); + accountingOnAbort(parser, 0, __LINE__); return XML_ERROR_AMPLIFICATION_LIMIT_BREACH; } #endif @@ -7184,7 +7712,7 @@ storeEntityValue(XML_Parser parser, const ENCODING *enc, if (! accountingDiffTolerated(parser, tok, entityTextPtr, next, __LINE__, account)) { - accountingOnAbort(parser); + accountingOnAbort(parser, 0, __LINE__); result = XML_ERROR_AMPLIFICATION_LIMIT_BREACH; goto endEntityValue; } @@ -7369,8 +7897,7 @@ callStoreEntityValue(XML_Parser parser, const ENCODING *enc, // processed. A XML_SUSPENDED check here is not required as // appendAttributeValue will never suspend the parser. if (textEnd != nextInEntity) { - entity->processed - = (int)(nextInEntity - (const char *)entity->textPtr); + entity->processed = nextInEntity - (const char *)entity->textPtr; continue; } @@ -7430,8 +7957,9 @@ storeSelfEntityValue(XML_Parser parser, ENTITY *entity) { poolDiscard(pool); return XML_ERROR_NO_MEMORY; } - entity->textPtr = poolStart(pool); - entity->textLen = (int)(poolLength(pool)); + const XML_Char *const valueStart = poolStart(pool); + entity->textPtr = valueStart; + entity->textLen = poolLength(pool); poolFinish(pool); return XML_ERROR_NONE; @@ -7564,8 +8092,8 @@ reportDefault(XML_Parser parser, const ENCODING *enc, const char *s, } static int -defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, XML_Bool isCdata, - XML_Bool isId, const XML_Char *value, XML_Parser parser) { +defineAttribute(ELEMENT_TYPE *type, ATTRIBUTE_ID *attId, bool isCdata, + bool isId, const XML_Char *value, XML_Parser parser) { DEFAULT_ATTRIBUTE *att; if (value || isId) { /* The handling of default attributes gets messed up if we have @@ -7726,7 +8254,7 @@ static const XML_Char * getContext(XML_Parser parser) { DTD *const dtd = parser->m_dtd; /* save one level of indirection */ HASH_TABLE_ITER iter; - XML_Bool needSep = XML_FALSE; + bool needSep = false; if (dtd->defaultPrefix.binding) { if (! poolAppendChar(&parser->m_tempPool, XML_T(ASCII_EQUALS))) @@ -7757,7 +8285,7 @@ getContext(XML_Parser parser) { */ return NULL; /* LCOV_EXCL_LINE */ } - needSep = XML_TRUE; + needSep = true; } hashTableIterInit(&iter, &(dtd->prefixes)); @@ -7786,7 +8314,7 @@ getContext(XML_Parser parser) { len--; if (! poolAppendChars(&parser->m_tempPool, prefix->binding->uri, len)) return NULL; - needSep = XML_TRUE; + needSep = true; } hashTableIterInit(&iter, &(dtd->generalEntities)); @@ -7800,7 +8328,7 @@ getContext(XML_Parser parser) { return NULL; if (! poolAppendChars(&parser->m_tempPool, e->name, xcslen(e->name))) return NULL; - needSep = XML_TRUE; + needSep = true; } if (! poolAppendChar(&parser->m_tempPool, XML_T('\0'))) @@ -8208,6 +8736,9 @@ copyEntityTable(XML_Parser oldParser, HASH_TABLE *newTable, return 0; newE->textPtr = tem; newE->textLen = oldE->textLen; +#if XML_GE == 1 + newE->scanPtr = tem; +#endif /* XML_GE == 1 */ } if (oldE->notation) { const XML_Char *tem = poolCopyString(newPool, oldE->notation); @@ -8530,7 +9061,7 @@ poolCopyStringNoFinish(STRING_POOL *pool, const XML_Char *s) { } static const XML_Char * -poolCopyStringN(STRING_POOL *pool, const XML_Char *s, int n) { +poolCopyStringN(STRING_POOL *pool, const XML_Char *s, size_t n) { if (! pool->ptr && ! poolGrow(pool)) { /* The following line is unreachable given the current usage of * poolCopyStringN(). Currently it is called from exactly one @@ -8975,25 +9506,27 @@ copyString(const XML_Char *s, XML_Parser parser) { #if XML_GE == 1 static float -accountingGetCurrentAmplification(XML_Parser rootParser) { +accountingGetCurrentAmplification(XML_Parser rootParser, + XmlBigCount extraIndirectBytes) { // 1.........1.........12 => 22 const size_t lenOfShortestInclude = sizeof("") - 1; - const XmlBigCount countBytesOutput - = rootParser->m_accounting.countBytesDirect - + rootParser->m_accounting.countBytesIndirect; + /* Add as floats: u64 counts (max ~1.8e19) are far below FLT_MAX, so the sum + is always finite and cannot wrap the way an unsigned integer add could. */ + const float indirect = (float)rootParser->m_accounting.countBytesIndirect + + (float)extraIndirectBytes; const float amplificationFactor = rootParser->m_accounting.countBytesDirect - ? ((float)countBytesOutput + ? (((float)rootParser->m_accounting.countBytesDirect + indirect) / (float)(rootParser->m_accounting.countBytesDirect)) - : ((float)(lenOfShortestInclude - + rootParser->m_accounting.countBytesIndirect) + : (((float)lenOfShortestInclude + indirect) / (float)lenOfShortestInclude); assert(! rootParser->m_parentParser); return amplificationFactor; } static void -accountingReportStats(XML_Parser originParser, const char *epilog) { +accountingReportStats(XML_Parser originParser, const char *epilog, + XmlBigCount extraIndirectBytes, int sourceLine) { const XML_Parser rootParser = getRootParserOf(originParser, NULL); assert(! rootParser->m_parentParser); @@ -9001,19 +9534,33 @@ accountingReportStats(XML_Parser originParser, const char *epilog) { return; } + /* On an early abort the amplifying bytes have not been charged to the + indirect counter yet; fold extraIndirectBytes in (with overflow detection) + so the reported byte count and amplification reflect what was detected. */ + const XmlBigCount indirect = rootParser->m_accounting.countBytesIndirect; + const XmlBigCount indirectWithExtra + = (extraIndirectBytes > (XmlBigCount)(-1) - indirect) + ? (XmlBigCount)(-1) + : indirect + extraIndirectBytes; const float amplificationFactor - = accountingGetCurrentAmplification(rootParser); + = accountingGetCurrentAmplification(rootParser, extraIndirectBytes); fprintf(stderr, "expat: Accounting(%p): Direct " EXPAT_FMT_ULL( "10") ", indirect " EXPAT_FMT_ULL("10") ", amplification %8.2f%s", (void *)rootParser, rootParser->m_accounting.countBytesDirect, - rootParser->m_accounting.countBytesIndirect, - (double)amplificationFactor, epilog); + indirectWithExtra, (double)amplificationFactor, epilog); + /* An abort has no accountingReportDiff before it, so append the originating + source line here to show which rejection site these stats belong to. */ + if (sourceLine > 0) { + fprintf(stderr, " (xmlparse.c:%d)\n", sourceLine); + } } static void -accountingOnAbort(XML_Parser originParser) { - accountingReportStats(originParser, " ABORTING\n"); +accountingOnAbort(XML_Parser originParser, XmlBigCount extraIndirectBytes, + int sourceLine) { + accountingReportStats(originParser, " ABORTING", extraIndirectBytes, + sourceLine); } static void @@ -9053,6 +9600,32 @@ accountingReportDiff(XML_Parser rootParser, fprintf(stderr, "\"\n"); } +static XML_Bool +accountingCurrentAmplificationTolerated(XML_Parser rootParser, + XmlBigCount extraIndirectBytes) { + assert(! rootParser->m_parentParser); + + const XmlBigCount countBytesOutput + = rootParser->m_accounting.countBytesDirect + + rootParser->m_accounting.countBytesIndirect; + /* Overflow check: extraIndirectBytes can be SIZE_MAX, which would otherwise + wrap the unsigned total around to a small value and wrongly report the + output as tolerated. */ + const XmlBigCount countBytesOutputWithExtra + = (extraIndirectBytes > (XmlBigCount)(-1) - countBytesOutput) + ? (XmlBigCount)(-1) + : countBytesOutput + extraIndirectBytes; + const float amplificationFactor + = accountingGetCurrentAmplification(rootParser, extraIndirectBytes); + const XML_Bool tolerated + = (countBytesOutputWithExtra + < rootParser->m_accounting.activationThresholdBytes) + || (amplificationFactor + <= rootParser->m_accounting.maximumAmplificationFactor); + + return tolerated; +} + static XML_Bool accountingDiffTolerated(XML_Parser originParser, int tok, const char *before, const char *after, int source_line, @@ -9089,18 +9662,11 @@ accountingDiffTolerated(XML_Parser originParser, int tok, const char *before, return XML_FALSE; *additionTarget += bytesMore; - const XmlBigCount countBytesOutput - = rootParser->m_accounting.countBytesDirect - + rootParser->m_accounting.countBytesIndirect; - const float amplificationFactor - = accountingGetCurrentAmplification(rootParser); const XML_Bool tolerated - = (countBytesOutput < rootParser->m_accounting.activationThresholdBytes) - || (amplificationFactor - <= rootParser->m_accounting.maximumAmplificationFactor); + = accountingCurrentAmplificationTolerated(rootParser, 0); if (rootParser->m_accounting.debugLevel >= 2u) { - accountingReportStats(rootParser, ""); + accountingReportStats(rootParser, "", 0, 0); accountingReportDiff(rootParser, levelsAwayFromRootParser, before, after, bytesMore, source_line, account); } @@ -9146,14 +9712,21 @@ entityTrackingReportStats(XML_Parser rootParser, ENTITY *entity, = limitingNeeded ? (maxLimitedDepth - /* make space for ellipis */ 2) : candidateIndentDepth; + const ptrdiff_t scannedLength + = EXPAT_SAFE_PTR_DIFF(entity->scanPtr, entity->textPtr); + const char *const confidenceMarker + = ((size_t)scannedLength == entity->textLen) ? "!" : "?"; + fprintf( stderr, - "expat: Entities(%p): Count %9u, depth %2u/%2u %*s%s%s%s; %s length %d (xmlparse.c:%d)\n", + "expat: Entities(%p): Count %9u, depth %2u/%2u %*s%s%s%s; %s scanned " EXPAT_FMT_PTRDIFF_T( + "") "/" EXPAT_FMT_SIZE_T("") ", expanded " EXPAT_FMT_ULL("") "%s (xmlparse.c:%d)\n", (void *)rootParser, rootParser->m_entity_stats.countEverOpened, rootParser->m_entity_stats.currentDepth, rootParser->m_entity_stats.maximumDepthSeen, indentDepth * 2, "", ellipisOrEmpty, entity->is_param ? "%" : "&", entityName, action, - entity->textLen, sourceLine); + scannedLength, entity->textLen, entity->expandedSize, confidenceMarker, + sourceLine); } static void @@ -9168,6 +9741,7 @@ entityTrackingOnOpen(XML_Parser originParser, ENTITY *entity, int sourceLine) { rootParser->m_entity_stats.maximumDepthSeen++; } + // NOTE: The space at the end of "OPEN " is for length alignment with "CLOSE" entityTrackingReportStats(rootParser, entity, "OPEN ", sourceLine); } @@ -9180,6 +9754,25 @@ entityTrackingOnClose(XML_Parser originParser, ENTITY *entity, int sourceLine) { rootParser->m_entity_stats.currentDepth--; } +static void +entityTrackingOnScanStart(XML_Parser originParser, ENTITY *entity, + int sourceLine) { + const XML_Parser rootParser = getRootParserOf(originParser, NULL); + assert(! rootParser->m_parentParser); + + entityTrackingReportStats(rootParser, entity, "START", sourceLine); +} + +static void +entityTrackingOnScanStop(XML_Parser originParser, ENTITY *entity, + int sourceLine) { + const XML_Parser rootParser = getRootParserOf(originParser, NULL); + assert(! rootParser->m_parentParser); + + // NOTE: The space at the end of "STOP " is for length alignment with "START" + entityTrackingReportStats(rootParser, entity, "STOP ", sourceLine); +} + #endif /* XML_GE == 1 */ static XML_Parser diff --git a/base/poco/XML/src/xmltok.h b/base/poco/XML/src/xmltok.h index 4f6fb3850cafc..fb8329e06394a 100644 --- a/base/poco/XML/src/xmltok.h +++ b/base/poco/XML/src/xmltok.h @@ -180,8 +180,8 @@ struct encoding { const char *); int (*nameLength)(const ENCODING *, const char *); const char *(*skipS)(const ENCODING *, const char *); - int (*getAtts)(const ENCODING *enc, const char *ptr, int attsMax, - ATTRIBUTE *atts); + size_t (*getAtts)(const ENCODING *enc, const char *ptr, size_t attsMax, + ATTRIBUTE *atts); int (*charRefNumber)(const ENCODING *enc, const char *ptr); int (*predefinedEntityName)(const ENCODING *, const char *, const char *); void (*updatePosition)(const ENCODING *, const char *ptr, const char *end, diff --git a/base/poco/XML/src/xmltok_impl.c b/base/poco/XML/src/xmltok_impl.c index 571af3698af7a..67fececb1f61c 100644 --- a/base/poco/XML/src/xmltok_impl.c +++ b/base/poco/XML/src/xmltok_impl.c @@ -1505,11 +1505,11 @@ PREFIX(isPublicId)(const ENCODING *enc, const char *ptr, const char *end, first attsMax attributes are stored in atts. */ -static int -PREFIX(getAtts)(const ENCODING *enc, const char *ptr, int attsMax, +static size_t +PREFIX(getAtts)(const ENCODING *enc, const char *ptr, size_t attsMax, ATTRIBUTE *atts) { enum { other, inName, inValue } state = inName; - int nAtts = 0; + size_t nAtts = 0; int open = 0; /* defined when state == inValue; initialization just to shut up compilers */ diff --git a/base/poco/XML/src/xmlxcs.c b/base/poco/XML/src/xmlxcs.c index ae5e042d778ff..a97647fa9d69e 100644 --- a/base/poco/XML/src/xmlxcs.c +++ b/base/poco/XML/src/xmlxcs.c @@ -88,3 +88,24 @@ xcsncmp(const XML_Char *a, const XML_Char *b, size_t len) { return strncmp(a, b, len); #endif } + +// A port of `strnstr(3bsd)` to `XML_Char` +const XML_Char * +xcsnstr(const XML_Char *haystack, const XML_Char *needle, size_t maxLen) { + // NOTE: We are not calling `xcsnlen(haystack, maxLen)` because + // `haystack` could be long and `xcsnlen` could be used in a loop + // and we need to avoid quadratic runtime behavior. + if (! *needle) + return haystack; // i.e. empty needle is always a match + const size_t needleLen = xcslen(needle); + if (needleLen > maxLen) + return NULL; + const size_t maxTries = (maxLen - needleLen) + 1; + for (size_t i = 0; i < maxTries && haystack[i]; i++) { + if (haystack[i] == needle[0] + && xcsncmp(haystack + i, needle, needleLen) == 0) { + return haystack + i; + } + } + return NULL; +} diff --git a/ci/docker/integration/dotnet_client/Dockerfile b/ci/docker/integration/dotnet_client/Dockerfile index 3fe1ba7dafc50..82bf1d6867186 100644 --- a/ci/docker/integration/dotnet_client/Dockerfile +++ b/ci/docker/integration/dotnet_client/Dockerfile @@ -1,5 +1,5 @@ # docker build . -# docker run -it --rm --network=host 14f23e59669c dotnet run --host localhost --port 8123 --user default --database default +# docker run -it --rm --network=host 14f23e59669c dotnet /client/out/clickhouse.test.dll --host localhost --port 8123 --user default --database default # Rebuild to enable SBOM/provenance attestations (see #97903, #98511) FROM mcr.microsoft.com/dotnet/sdk:8.0 @@ -9,3 +9,12 @@ COPY *.cs *.csproj /client/ ARG VERSION=4.1.0 RUN dotnet add package ClickHouse.Client -v ${VERSION} + +# Build here, not in the test. `dotnet run` restores, builds and runs, which starts an MSBuild node +# and a Roslyn compiler server alongside the application: three .NET processes where the test needs +# one, and the two extra ones are not what this test covers. They are already a known source of +# noise in this suite - the analyzer crash that +# https://github.com/ClickHouse/ClickHouse/pull/107375 had to work around came from the in-test +# build. Publishing here leaves the test with a single `dotnet ` process. +ENV DOTNET_CLI_TELEMETRY_OPTOUT=1 DOTNET_NOLOGO=1 MSBUILDDISABLENODEREUSE=1 +RUN dotnet publish -c Release -o /client/out && rm -rf /client/obj /client/bin diff --git a/ci/workflows/backport_branches.py b/ci/workflows/backport_branches.py index 92a5c210d533b..5bde2757897b3 100644 --- a/ci/workflows/backport_branches.py +++ b/ci/workflows/backport_branches.py @@ -18,7 +18,7 @@ *[ job for job in JobConfigs.special_build_jobs - if any(t in job.name for t in ("darwin",)) + if any(t in job.name for t in ("darwin", "compat")) ], JobConfigs.docker_server, JobConfigs.docker_keeper, diff --git a/ci/workflows/release_branches.py b/ci/workflows/release_branches.py index 5e738ce647786..f7879b2048a60 100644 --- a/ci/workflows/release_branches.py +++ b/ci/workflows/release_branches.py @@ -32,7 +32,7 @@ *[ job for job in JobConfigs.special_build_jobs - if any(t in job.name for t in ("darwin",)) + if any(t in job.name for t in ("darwin", "compat")) ], *JobConfigs.sign_macos_binary_jobs, JobConfigs.docker_server, diff --git a/cmake/target.cmake b/cmake/target.cmake index 84f1aaaf4a447..53a8ce14b4a44 100644 --- a/cmake/target.cmake +++ b/cmake/target.cmake @@ -56,7 +56,7 @@ if (OS_WASM) set (ENABLE_GRPC OFF CACHE INTERNAL "") # Protobuf needs a `protoc` that runs on the host, and the nested native configure at the # bottom of the top-level `CMakeLists.txt` would be handed `emcc` as its host compiler. - # Arrow, Parquet and ORC hard-depend on it, so they go with it. + # Arrow and Parquet hard-depend on it, so they go with it. set (ENABLE_PROTOBUF OFF CACHE INTERNAL "") # Its `kj` library uses `fallocate` and friends unconditionally in its POSIX branch, # which the Emscripten libc does not provide. diff --git a/contrib/CMakeLists.txt b/contrib/CMakeLists.txt index 537a9f057984d..0a2ee36dfce6b 100644 --- a/contrib/CMakeLists.txt +++ b/contrib/CMakeLists.txt @@ -103,7 +103,7 @@ add_contrib (minizip-ng-cmake minizip-ng) add_contrib (snappy-cmake snappy) add_contrib (thrift-cmake thrift) add_contrib (StringZilla-cmake StringZilla) -# parquet/arrow/orc +# parquet/arrow add_contrib (xsimd-cmake xsimd) # required by arrow's SIMD codepaths add_contrib (arrow-cmake arrow) # requires: snappy, thrift, double-conversion, xsimd add_contrib (avro-cmake avro) # requires: snappy diff --git a/contrib/arrow-cmake/CMakeLists.txt b/contrib/arrow-cmake/CMakeLists.txt index 52dd396142ac0..37c97f40e8090 100644 --- a/contrib/arrow-cmake/CMakeLists.txt +++ b/contrib/arrow-cmake/CMakeLists.txt @@ -24,14 +24,6 @@ if (NOT ENABLE_PARQUET) return() endif() -# ORC lives here because Arrow has an adapter for it, but nothing about reading Parquet needs it. -# It is also the only thing in the whole tree that runs `protoc`, and on a cross-compile that means -# bootstrapping a native protobuf first, so turning it off is what makes a Parquet-only build cheap. -option (ENABLE_ORC "Enable ORC (pulls in protobuf and, when cross-compiling, a native protoc)" ON) -if (NOT ENABLE_ORC) - message(STATUS "Not using ORC") -endif() - # Support C11 if(NOT DEFINED CMAKE_C_STANDARD) set(CMAKE_C_STANDARD 11) @@ -65,38 +57,8 @@ else() endif() -# === orc -set(ORC_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/orc/c++") -set(ORC_INCLUDE_DIR "${ORC_SOURCE_DIR}/include") -set(ORC_SOURCE_SRC_DIR "${ORC_SOURCE_DIR}/src") -# set(ORC_SOURCE_WRAP_DIR "${ORC_SOURCE_DIR}/wrap") - -set(ORC_BUILD_SRC_DIR "${CMAKE_CURRENT_BINARY_DIR}/../orc/c++/src") -set(ORC_BUILD_INCLUDE_DIR "${CMAKE_CURRENT_BINARY_DIR}/../orc/c++/include") - -set(ORC_ADDITION_SOURCE_DIR ${CMAKE_CURRENT_BINARY_DIR}) set(ARROW_SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/arrow/cpp/src") -if (NOT CMAKE_HOST_SYSTEM_NAME STREQUAL CMAKE_SYSTEM_NAME OR NOT CMAKE_HOST_SYSTEM_PROCESSOR STREQUAL CMAKE_SYSTEM_PROCESSOR OR USE_MUSL) - set(PROTOBUF_EXECUTABLE "${PROJECT_BINARY_DIR}/native/contrib/google-protobuf-cmake/protoc") - set(GRPC_EXECUTABLE "${PROJECT_BINARY_DIR}/native/contrib/grpc-cmake/grpc_cpp_plugin") -else () - set(PROTOBUF_EXECUTABLE $) - set(GRPC_EXECUTABLE $) -endif() -set(PROTO_DIR "${ORC_SOURCE_DIR}/../proto") - -if (ENABLE_ORC) - add_custom_command(OUTPUT orc_proto.pb.h orc_proto.pb.cc - COMMAND ${PROTOBUF_EXECUTABLE} - -I ${PROTO_DIR} - --cpp_out="${CMAKE_CURRENT_BINARY_DIR}" - "${PROTO_DIR}/orc_proto.proto" - # Without these, the generated sources are not regenerated when the schema or - # `protoc` itself changes, and a stale `orc_proto.pb.h` is compiled instead. - DEPENDS "${PROTO_DIR}/orc_proto.proto" ${PROTOBUF_EXECUTABLE}) -endif() - # === flatbuffers set(FLATBUFFERS_SRC_DIR "${ClickHouse_SOURCE_DIR}/contrib/flatbuffers") set(FLATBUFFERS_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/flatbuffers") @@ -130,134 +92,6 @@ add_library(ch_contrib::arrow_ipc_metadata ALIAS _arrow_ipc_metadata) # NOTE: cannot use ch_contrib::hdfs since it's INCLUDE_DIRECTORIES does not includes trailing "hdfs/" set(HDFS_INCLUDE_DIR "${ClickHouse_SOURCE_DIR}/contrib/libhdfs3/include/hdfs/") -# Everything down to the end of the `_orc` target below is ORC. Left un-indented inside the -# condition to keep it comparable with the Arrow build description it was transcribed from. -if (ENABLE_ORC) - -# arrow-cmake cmake file calling orc cmake subroutine which detects certain compiler features. -# Apple Clang compiler failed to compile this code without specifying c++11 standard. -# As result these compiler features detected as absent. In result it failed to compile orc itself. -# In orc makefile there is code that sets flags, but arrow-cmake ignores these flags. -if (CMAKE_CXX_COMPILER_ID STREQUAL "AppleClang") - set(CXX11_FLAGS "-std=c++0x") -endif () - -set (ORC_CXX_HAS_INITIALIZER_LIST 1) -set (ORC_CXX_HAS_NOEXCEPT 1) -set (ORC_CXX_HAS_NULLPTR 1) -set (ORC_CXX_HAS_OVERRIDE 1) -set (ORC_CXX_HAS_UNIQUE_PTR 1) -set (ORC_CXX_HAS_CSTDINT 1) -set (ORC_CXX_HAS_THREAD_LOCAL 1) - -include(orc_check.cmake) -configure_file("${ORC_SOURCE_SRC_DIR}/Adaptor.hh.in" "${ORC_BUILD_INCLUDE_DIR}/Adaptor.hh") - - -# ARROW_ORC + adapters/orc/CMakefiles -set(ORC_SRCS - "${CMAKE_CURRENT_BINARY_DIR}/orc_proto.pb.h" - "${ORC_ADDITION_SOURCE_DIR}/orc_proto.pb.cc" - "${ORC_SOURCE_SRC_DIR}/Adaptor.cc" - "${ORC_SOURCE_SRC_DIR}/Adaptor.hh.in" - "${ORC_SOURCE_SRC_DIR}/BlockBuffer.cc" - "${ORC_SOURCE_SRC_DIR}/BlockBuffer.hh" - "${ORC_SOURCE_SRC_DIR}/BloomFilter.cc" - "${ORC_SOURCE_SRC_DIR}/BloomFilter.hh" - "${ORC_SOURCE_SRC_DIR}/Bpacking.hh" - "${ORC_SOURCE_SRC_DIR}/BpackingDefault.cc" - "${ORC_SOURCE_SRC_DIR}/BpackingDefault.hh" - "${ORC_SOURCE_SRC_DIR}/ByteRLE.cc" - "${ORC_SOURCE_SRC_DIR}/ByteRLE.hh" - "${ORC_SOURCE_SRC_DIR}/ColumnPrinter.cc" - "${ORC_SOURCE_SRC_DIR}/ColumnReader.cc" - "${ORC_SOURCE_SRC_DIR}/ColumnReader.hh" - "${ORC_SOURCE_SRC_DIR}/ColumnWriter.cc" - "${ORC_SOURCE_SRC_DIR}/ColumnWriter.hh" - "${ORC_SOURCE_SRC_DIR}/Common.cc" - "${ORC_SOURCE_SRC_DIR}/Dictionary.cc" - "${ORC_SOURCE_SRC_DIR}/DictionaryLoader.cc" - "${ORC_SOURCE_SRC_DIR}/Compression.cc" - "${ORC_SOURCE_SRC_DIR}/Compression.hh" - "${ORC_SOURCE_SRC_DIR}/ConvertColumnReader.cc" - "${ORC_SOURCE_SRC_DIR}/ConvertColumnReader.hh" - "${ORC_SOURCE_SRC_DIR}/CpuInfoUtil.cc" - "${ORC_SOURCE_SRC_DIR}/CpuInfoUtil.hh" - "${ORC_SOURCE_SRC_DIR}/Dispatch.hh" - "${ORC_SOURCE_SRC_DIR}/Exceptions.cc" - "${ORC_SOURCE_SRC_DIR}/Geospatial.cc" - "${ORC_SOURCE_SRC_DIR}/Int128.cc" - "${ORC_SOURCE_SRC_DIR}/LzoDecompressor.cc" - "${ORC_SOURCE_SRC_DIR}/LzoDecompressor.hh" - "${ORC_SOURCE_SRC_DIR}/MemoryPool.cc" - "${ORC_SOURCE_SRC_DIR}/Murmur3.cc" - "${ORC_SOURCE_SRC_DIR}/Murmur3.hh" - "${ORC_SOURCE_SRC_DIR}/Options.hh" - "${ORC_SOURCE_SRC_DIR}/OrcFile.cc" - "${ORC_SOURCE_SRC_DIR}/RLE.cc" - "${ORC_SOURCE_SRC_DIR}/RLE.hh" - "${ORC_SOURCE_SRC_DIR}/RLEV2Util.cc" - "${ORC_SOURCE_SRC_DIR}/RLEV2Util.hh" - "${ORC_SOURCE_SRC_DIR}/RLEv1.cc" - "${ORC_SOURCE_SRC_DIR}/RLEv1.hh" - "${ORC_SOURCE_SRC_DIR}/RLEv2.hh" - "${ORC_SOURCE_SRC_DIR}/Reader.cc" - "${ORC_SOURCE_SRC_DIR}/Reader.hh" - "${ORC_SOURCE_SRC_DIR}/RleDecoderV2.cc" - "${ORC_SOURCE_SRC_DIR}/RleEncoderV2.cc" - "${ORC_SOURCE_SRC_DIR}/SchemaEvolution.cc" - "${ORC_SOURCE_SRC_DIR}/SchemaEvolution.hh" - "${ORC_SOURCE_SRC_DIR}/Statistics.cc" - "${ORC_SOURCE_SRC_DIR}/Statistics.hh" - "${ORC_SOURCE_SRC_DIR}/StripeStream.cc" - "${ORC_SOURCE_SRC_DIR}/StripeStream.hh" - "${ORC_SOURCE_SRC_DIR}/Timezone.cc" - "${ORC_SOURCE_SRC_DIR}/Timezone.hh" - "${ORC_SOURCE_SRC_DIR}/TypeImpl.cc" - "${ORC_SOURCE_SRC_DIR}/TypeImpl.hh" - "${ORC_SOURCE_SRC_DIR}/Utils.hh" - "${ORC_SOURCE_SRC_DIR}/Vector.cc" - "${ORC_SOURCE_SRC_DIR}/Writer.cc" - "${ORC_SOURCE_SRC_DIR}/io/InputStream.cc" - "${ORC_SOURCE_SRC_DIR}/io/InputStream.hh" - "${ORC_SOURCE_SRC_DIR}/io/OutputStream.cc" - "${ORC_SOURCE_SRC_DIR}/io/OutputStream.hh" - "${ORC_SOURCE_SRC_DIR}/io/Cache.cc" - "${ORC_SOURCE_SRC_DIR}/io/Cache.hh" - "${ORC_SOURCE_SRC_DIR}/sargs/ExpressionTree.cc" - "${ORC_SOURCE_SRC_DIR}/sargs/ExpressionTree.hh" - "${ORC_SOURCE_SRC_DIR}/sargs/Literal.cc" - "${ORC_SOURCE_SRC_DIR}/sargs/PredicateLeaf.cc" - "${ORC_SOURCE_SRC_DIR}/sargs/PredicateLeaf.hh" - "${ORC_SOURCE_SRC_DIR}/sargs/SargsApplier.cc" - "${ORC_SOURCE_SRC_DIR}/sargs/SargsApplier.hh" - "${ORC_SOURCE_SRC_DIR}/sargs/SearchArgument.cc" - "${ORC_SOURCE_SRC_DIR}/sargs/SearchArgument.hh" - "${ORC_SOURCE_SRC_DIR}/sargs/TruthValue.cc" - ) - -add_library(_orc ${ORC_SRCS}) -target_link_libraries(_orc PRIVATE - ch_contrib::protobuf - ch_contrib::lz4 - ch_contrib::snappy - ch_contrib::zlib - ch_contrib::zstd) -target_include_directories(_orc SYSTEM BEFORE PUBLIC - ${ORC_INCLUDE_DIR} - "${ClickHouse_SOURCE_DIR}/contrib/arrow-cmake/cpp/src/orc/c++/include") -target_compile_options(_orc PRIVATE - -include "${ClickHouse_SOURCE_DIR}/contrib/google-protobuf/src/google/protobuf/stubs/port.h" -) -target_include_directories(_orc SYSTEM BEFORE PUBLIC ${ORC_BUILD_INCLUDE_DIR}) -target_include_directories(_orc SYSTEM PRIVATE - ${ORC_SOURCE_SRC_DIR} - ${ORC_SOURCE_WRAP_DIR} - ${ORC_BUILD_SRC_DIR} - ${ORC_ADDITION_SOURCE_DIR} - ${ARROW_SRC_DIR}) - -endif() # ENABLE_ORC # === arrow @@ -269,11 +103,8 @@ configure_file("${LIBRARY_DIR}/util/config_internal.h.cmake" "${ARROW_GENERATED_ # arrow/cpp/src/arrow/CMakeLists.txt (ARROW_SRCS + ARROW_COMPUTE + ARROW_IPC) # find . \( -iname \*.cc -o -iname \*.cpp -o -iname \*.c \) | sort | awk '{print "\"${LIBRARY_DIR}" substr($1,2) "\"" }' | grep -v 'test.cc' | grep -v 'json' | grep -v 'flight' \| -# grep -v 'csv' | grep -v 'acero' | grep -v 'dataset' | grep -v 'testing' | grep -v 'gpu' | grep -v 'engine' | grep -v 'filesystem' | grep -v 'benchmark.cc' +# grep -v 'csv' | grep -v 'acero' | grep -v 'dataset' | grep -v 'testing' | grep -v 'gpu' | grep -v 'engine' | grep -v 'filesystem' | grep -v 'benchmark.cc' | grep -v 'adapters/orc' set(ARROW_SRCS - "${LIBRARY_DIR}/adapters/orc/adapter.cc" - "${LIBRARY_DIR}/adapters/orc/options.cc" - "${LIBRARY_DIR}/adapters/orc/util.cc" "${LIBRARY_DIR}/array/array_base.cc" "${LIBRARY_DIR}/array/array_binary.cc" "${LIBRARY_DIR}/array/array_decimal.cc" @@ -495,16 +326,8 @@ set(ARROW_SRCS "${LIBRARY_DIR}/vendored/uriparser/UriResolve.c" "${LIBRARY_DIR}/vendored/uriparser/UriShorten.c" "${LIBRARY_DIR}/visitor.cc" - - "${ARROW_SRC_DIR}/arrow/adapters/orc/adapter.cc" - "${ARROW_SRC_DIR}/arrow/adapters/orc/util.cc" - "${ARROW_SRC_DIR}/arrow/adapters/orc/options.cc" ) -if (NOT ENABLE_ORC) - # The Arrow-to-ORC adapter is the one part of Arrow that includes `orc/OrcFile.hh`. - list (FILTER ARROW_SRCS EXCLUDE REGEX "/adapters/orc/") -endif() add_definitions(-DARROW_WITH_BROTLI) SET(ARROW_SRCS "${LIBRARY_DIR}/util/compression_brotli.cc" ${ARROW_SRCS}) @@ -575,12 +398,6 @@ target_link_libraries(_arrow PRIVATE # PUBLIC so the xsimd headers are visible to _parquet too (it links _arrow PUBLIC). target_link_libraries(_arrow PUBLIC ch_contrib::xsimd) -if (ENABLE_ORC) - target_link_libraries(_arrow PUBLIC _orc) - # ORC is the only thing here with generated protobuf sources. - add_dependencies(_arrow protoc) -endif() - target_include_directories(_arrow SYSTEM BEFORE PUBLIC ${ARROW_SRC_DIR}) target_include_directories(_arrow SYSTEM BEFORE PUBLIC "${ClickHouse_SOURCE_DIR}/contrib/arrow-cmake/cpp/src") diff --git a/contrib/arrow-cmake/orc_check.cmake b/contrib/arrow-cmake/orc_check.cmake deleted file mode 100644 index 523e1cf1d86bc..0000000000000 --- a/contrib/arrow-cmake/orc_check.cmake +++ /dev/null @@ -1,14 +0,0 @@ -set (HAS_PREAD 1) -set (HAS_STRPTIME 1) -set (HAS_STOLL 1) -set (INT64_IS_LL 1) -set (HAS_DIAGNOSTIC_PUSH 1) -set (HAS_STD_ISNAN 1) -set (HAS_STD_MUTEX 1) -set (NEEDS_REDUNDANT_MOVE 1) -set (HAS_PRE_1970 1) -set (HAS_POST_2038 1) -set (NEEDS_Z_PREFIX 0) - -set(CMAKE_REQUIRED_INCLUDES) -set(CMAKE_REQUIRED_LIBRARIES) diff --git a/contrib/brotli b/contrib/brotli index d224526f8fa73..2d9c9b61f4e7d 160000 --- a/contrib/brotli +++ b/contrib/brotli @@ -1 +1 @@ -Subproject commit d224526f8fa736f24b2339e6f421a142de6586c8 +Subproject commit 2d9c9b61f4e7d3523c9dfb62e48fe91aaf86e812 diff --git a/contrib/brotli-cmake/CMakeLists.txt b/contrib/brotli-cmake/CMakeLists.txt index b89e81ecda19e..8c2b41721dc9a 100644 --- a/contrib/brotli-cmake/CMakeLists.txt +++ b/contrib/brotli-cmake/CMakeLists.txt @@ -9,35 +9,41 @@ set(BROTLI_SOURCE_DIR "${ClickHouse_SOURCE_DIR}/contrib/brotli/c") set(BROTLI_BINARY_DIR "${ClickHouse_BINARY_DIR}/contrib/brotli/c") set(SRCS - "${BROTLI_SOURCE_DIR}/enc/command.c" - "${BROTLI_SOURCE_DIR}/enc/fast_log.c" + "${BROTLI_SOURCE_DIR}/common/constants.c" + "${BROTLI_SOURCE_DIR}/common/context.c" + "${BROTLI_SOURCE_DIR}/common/dictionary.c" + "${BROTLI_SOURCE_DIR}/common/platform.c" + "${BROTLI_SOURCE_DIR}/common/shared_dictionary.c" + "${BROTLI_SOURCE_DIR}/common/transform.c" "${BROTLI_SOURCE_DIR}/dec/bit_reader.c" - "${BROTLI_SOURCE_DIR}/dec/state.c" - "${BROTLI_SOURCE_DIR}/dec/huffman.c" "${BROTLI_SOURCE_DIR}/dec/decode.c" - "${BROTLI_SOURCE_DIR}/enc/encode.c" - "${BROTLI_SOURCE_DIR}/enc/dictionary_hash.c" + "${BROTLI_SOURCE_DIR}/dec/huffman.c" + "${BROTLI_SOURCE_DIR}/dec/prefix.c" + "${BROTLI_SOURCE_DIR}/dec/state.c" + "${BROTLI_SOURCE_DIR}/dec/static_init.c" + "${BROTLI_SOURCE_DIR}/enc/backward_references.c" + "${BROTLI_SOURCE_DIR}/enc/backward_references_hq.c" + "${BROTLI_SOURCE_DIR}/enc/bit_cost.c" + "${BROTLI_SOURCE_DIR}/enc/block_splitter.c" + "${BROTLI_SOURCE_DIR}/enc/brotli_bit_stream.c" "${BROTLI_SOURCE_DIR}/enc/cluster.c" + "${BROTLI_SOURCE_DIR}/enc/command.c" + "${BROTLI_SOURCE_DIR}/enc/compound_dictionary.c" + "${BROTLI_SOURCE_DIR}/enc/compress_fragment.c" + "${BROTLI_SOURCE_DIR}/enc/compress_fragment_two_pass.c" + "${BROTLI_SOURCE_DIR}/enc/dictionary_hash.c" + "${BROTLI_SOURCE_DIR}/enc/encode.c" + "${BROTLI_SOURCE_DIR}/enc/encoder_dict.c" "${BROTLI_SOURCE_DIR}/enc/entropy_encode.c" + "${BROTLI_SOURCE_DIR}/enc/fast_log.c" + "${BROTLI_SOURCE_DIR}/enc/histogram.c" "${BROTLI_SOURCE_DIR}/enc/literal_cost.c" - "${BROTLI_SOURCE_DIR}/enc/compress_fragment_two_pass.c" + "${BROTLI_SOURCE_DIR}/enc/memory.c" + "${BROTLI_SOURCE_DIR}/enc/metablock.c" "${BROTLI_SOURCE_DIR}/enc/static_dict.c" - "${BROTLI_SOURCE_DIR}/enc/compress_fragment.c" - "${BROTLI_SOURCE_DIR}/enc/block_splitter.c" - "${BROTLI_SOURCE_DIR}/enc/backward_references_hq.c" - "${BROTLI_SOURCE_DIR}/enc/histogram.c" - "${BROTLI_SOURCE_DIR}/enc/brotli_bit_stream.c" + "${BROTLI_SOURCE_DIR}/enc/static_dict_lut.c" + "${BROTLI_SOURCE_DIR}/enc/static_init.c" "${BROTLI_SOURCE_DIR}/enc/utf8_util.c" - "${BROTLI_SOURCE_DIR}/enc/encoder_dict.c" - "${BROTLI_SOURCE_DIR}/enc/backward_references.c" - "${BROTLI_SOURCE_DIR}/enc/bit_cost.c" - "${BROTLI_SOURCE_DIR}/enc/metablock.c" - "${BROTLI_SOURCE_DIR}/enc/memory.c" - "${BROTLI_SOURCE_DIR}/common/dictionary.c" - "${BROTLI_SOURCE_DIR}/common/transform.c" - "${BROTLI_SOURCE_DIR}/common/platform.c" - "${BROTLI_SOURCE_DIR}/common/context.c" - "${BROTLI_SOURCE_DIR}/common/constants.c" ) add_library(_brotli ${SRCS}) @@ -45,4 +51,3 @@ add_library(ch_contrib::brotli ALIAS _brotli) target_include_directories(_brotli SYSTEM BEFORE PUBLIC "${BROTLI_SOURCE_DIR}/include") -target_compile_definitions(_brotli PRIVATE BROTLI_BUILD_PORTABLE=1) diff --git a/contrib/c-ares b/contrib/c-ares index c7a3138dcfe3b..bc91829cce983 160000 --- a/contrib/c-ares +++ b/contrib/c-ares @@ -1 +1 @@ -Subproject commit c7a3138dcfe3bb0eaaf10c0c24c36dc66dc790ab +Subproject commit bc91829cce983a35ec01f5dbc05465858a3df469 diff --git a/contrib/grpc b/contrib/grpc index a703d1f9448ff..f205cf1fc40ff 160000 --- a/contrib/grpc +++ b/contrib/grpc @@ -1 +1 @@ -Subproject commit a703d1f9448ff77deca3beb36dfc7d337a209d3e +Subproject commit f205cf1fc40ffe44a9517529ec0cf32112c99f0d diff --git a/contrib/grpc-cmake/grpc.cmake b/contrib/grpc-cmake/grpc.cmake index 419802aeb70b2..8393bacbb8d7a 100644 --- a/contrib/grpc-cmake/grpc.cmake +++ b/contrib/grpc-cmake/grpc.cmake @@ -180,6 +180,7 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/call/call_spine.cc ${_gRPC_SOURCE_DIR}/src/core/call/call_state.cc ${_gRPC_SOURCE_DIR}/src/core/call/client_call.cc + ${_gRPC_SOURCE_DIR}/src/core/call/evaluate_args.cc ${_gRPC_SOURCE_DIR}/src/core/call/interception_chain.cc ${_gRPC_SOURCE_DIR}/src/core/call/message.cc ${_gRPC_SOURCE_DIR}/src/core/call/metadata.cc @@ -217,6 +218,7 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/client_channel/retry_service_config.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/retry_throttle.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel.cc + ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel_metrics.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel_pool_interface.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel_stream_client.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel_stream_limiter.cc @@ -231,6 +233,7 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/credentials/call/external/file_external_account_credentials.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/call/external/url_external_account_credentials.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/call/gcp_service_account_identity/gcp_service_account_identity_credentials.cc + ${_gRPC_SOURCE_DIR}/src/core/credentials/call/gdch_service_account/gdch_service_account_credentials.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/call/iam/iam_credentials.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/call/json_util.cc ${_gRPC_SOURCE_DIR}/src/core/credentials/call/jwt/json_token.cc @@ -301,7 +304,6 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/chttp2_plugin.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/chttp2_connector.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/server/chttp2_server.cc - ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/bin_decoder.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/bin_encoder.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/call_tracer_wrapper.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -334,6 +336,7 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_callbacks.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_promise.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_rate_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/reclaimer.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/stream_lists.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/transport_common.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/varint.cc @@ -346,10 +349,20 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/cel/expr/syntax.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/config/common/mutation_rules/v3/mutation_rules.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/config/core/v3/cel.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/extensions/filters/http/ext_proc/v3/ext_proc.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/extensions/filters/http/ext_proc/v3/processing_mode.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/extensions/grpc_service/call_credentials/access_token/v3/access_token_credentials.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/extensions/grpc_service/channel_credentials/tls/v3/tls_credentials.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/extensions/grpc_service/channel_credentials/xds/v3/xds_credentials.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/channelz/channelz.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/envoy/service/ext_proc/v3/external_processor.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/channelz/v1/channelz.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/gcp/altscontext.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/gcp/handshaker.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/gcp/transport_security_common.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/health/v1/health.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/lb/v1/load_balancer.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/lookup/v1/rls.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/lookup/v1/rls_config.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/channelz/v2/channelz.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/channelz/v2/service.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/cel/expr/checked.upbdefs.c @@ -485,20 +498,12 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/api/http.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/api/httpbody.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/any.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/duration.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/empty.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/struct.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/timestamp.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/wrappers.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/rpc/status.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/gcp/altscontext.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/gcp/handshaker.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/gcp/transport_security_common.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/health/v1/health.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/lb/v1/load_balancer.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/lookup/v1/rls.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/lookup/v1/rls_config.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/udpa/annotations/migrate.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/udpa/annotations/security.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/udpa/annotations/sensitive.upb_minitable.c @@ -598,6 +603,8 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/envoy/data/accesslog/v3/accesslog.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/envoy/extensions/clusters/aggregate/v3/cluster.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/envoy/extensions/filters/common/fault/v3/fault.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/ext_proc/v3/ext_proc.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/ext_proc/v3/processing_mode.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/fault/v3/fault.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/gcp_authn/v3/gcp_authn.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/envoy/extensions/filters/http/rbac/v3/rbac.upbdefs.c @@ -654,12 +661,12 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/google/protobuf/timestamp.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/google/protobuf/wrappers.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/google/rpc/status.upbdefs.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/src/proto/grpc/channelz/channelz.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/grpc/channelz/v1/channelz.upbdefs.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/grpc/lookup/v1/rls_config.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/src/proto/grpc/channelz/v2/channelz.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/src/proto/grpc/channelz/v2/promise.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/src/proto/grpc/channelz/v2/property_list.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/src/proto/grpc/channelz/v2/service.upbdefs.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/src/proto/grpc/lookup/v1/rls_config.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/udpa/annotations/migrate.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/udpa/annotations/security.upbdefs.c ${_gRPC_SOURCE_DIR}/src/core/ext/upbdefs-gen/udpa/annotations/sensitive.upbdefs.c @@ -693,6 +700,8 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/filter/auth/client_auth_filter.cc ${_gRPC_SOURCE_DIR}/src/core/filter/auth/server_auth_filter.cc ${_gRPC_SOURCE_DIR}/src/core/filter/composite/composite_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/filter/ext_proc/ext_proc_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/filter/ext_proc/ext_proc_messages.cc ${_gRPC_SOURCE_DIR}/src/core/filter/fused_filters.cc ${_gRPC_SOURCE_DIR}/src/core/handshaker/http_connect/http_connect_client_handshaker.cc ${_gRPC_SOURCE_DIR}/src/core/handshaker/security/pipelined_secure_endpoint.cc @@ -708,13 +717,18 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/telemetry.cc ${_gRPC_SOURCE_DIR}/src/core/lib/resource_tracker/resource_tracker.cc ${_gRPC_SOURCE_DIR}/src/core/lib/transport/promise_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/load_balancing/autosharding/autosharding.cc ${_gRPC_SOURCE_DIR}/src/core/net/socket_mutator.cc + ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/dns_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/service_config_helper.cc ${_gRPC_SOURCE_DIR}/src/core/server/xds_server_config_fetcher_legacy.cc ${_gRPC_SOURCE_DIR}/src/core/telemetry/context_list_entry.cc ${_gRPC_SOURCE_DIR}/src/core/telemetry/instrument.cc ${_gRPC_SOURCE_DIR}/src/core/transport/message_size_service_config.cc ${_gRPC_SOURCE_DIR}/src/core/transport/session_endpoint.cc ${_gRPC_SOURCE_DIR}/src/core/tsi/ssl_telemetry_utils.cc + ${_gRPC_SOURCE_DIR}/src/core/tsi/tls_telemetry.cc + ${_gRPC_SOURCE_DIR}/src/core/util/address_sorting_init.cc ${_gRPC_SOURCE_DIR}/src/core/util/grpc_check.cc ${_gRPC_SOURCE_DIR}/src/core/util/postmortem_emit.cc ${_gRPC_SOURCE_DIR}/src/core/util/wait_for_single_owner.cc @@ -825,9 +839,6 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set.cc ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set_windows.cc ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_windows.cc - ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address.cc - ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_posix.cc - ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_windows.cc ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/sockaddr_utils_posix.cc ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_factory_posix.cc ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_mutator.cc @@ -872,7 +883,6 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/thread_quota.cc ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/audit_logging.cc ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/authorization_policy_provider_vtable.cc - ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/evaluate_args.cc ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/grpc_authorization_engine.cc ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/grpc_server_authz_filter.cc ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/matchers.cc @@ -941,16 +951,6 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/load_balancing/xds/xds_wrr_locality.cc ${_gRPC_SOURCE_DIR}/src/core/plugin_registry/grpc_plugin_registry.cc ${_gRPC_SOURCE_DIR}/src/core/plugin_registry/grpc_plugin_registry_extra.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/c_ares/dns_resolver_ares.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/c_ares/grpc_ares_wrapper.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/dns_resolver_plugin.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/event_engine/event_engine_client_channel_resolver.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/event_engine/service_config_helper.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/native/dns_resolver.cc ${_gRPC_SOURCE_DIR}/src/core/resolver/endpoint_addresses.cc ${_gRPC_SOURCE_DIR}/src/core/resolver/fake/fake_resolver.cc ${_gRPC_SOURCE_DIR}/src/core/resolver/google_c2p/google_c2p_resolver.cc @@ -1024,7 +1024,6 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/util/http_client/parser.cc ${_gRPC_SOURCE_DIR}/src/core/util/json/json_object_loader.cc ${_gRPC_SOURCE_DIR}/src/core/util/json/json_reader.cc - ${_gRPC_SOURCE_DIR}/src/core/util/json/json_util.cc ${_gRPC_SOURCE_DIR}/src/core/util/json/json_writer.cc ${_gRPC_SOURCE_DIR}/src/core/util/latent_see.cc ${_gRPC_SOURCE_DIR}/src/core/util/load_file.cc @@ -1044,8 +1043,10 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/util/work_serializer.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/certificate_provider_store.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/file_watcher_certificate_provider_factory.cc + ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/streaming_call_promise_wrapper.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_audit_logger_registry.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_bootstrap_grpc.cc + ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_bootstrap_grpc_builder.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_certificate_provider.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_client_grpc.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_cluster.cc @@ -1055,8 +1056,10 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_common_types_parser.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_endpoint.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_endpoint_parser.cc + ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_grpc_service_parser.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_health_status.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_http_composite_filter.cc + ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_http_ext_proc_filter.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_http_fault_filter.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_http_filter.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_http_filter_registry.cc @@ -1077,6 +1080,8 @@ add_library(grpc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_route_config_parser.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_routing.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_server_grpc.cc + ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_tls_context.cc + ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_tls_context_parser.cc ${_gRPC_SOURCE_DIR}/src/core/xds/grpc/xds_transport_grpc.cc ${_gRPC_SOURCE_DIR}/src/core/xds/xds_client/lrs_client.cc ${_gRPC_SOURCE_DIR}/src/core/xds/xds_client/xds_api.cc @@ -1135,6 +1140,7 @@ add_library(grpc_unsecure ${_gRPC_SOURCE_DIR}/src/core/call/call_spine.cc ${_gRPC_SOURCE_DIR}/src/core/call/call_state.cc ${_gRPC_SOURCE_DIR}/src/core/call/client_call.cc + ${_gRPC_SOURCE_DIR}/src/core/call/evaluate_args.cc ${_gRPC_SOURCE_DIR}/src/core/call/interception_chain.cc ${_gRPC_SOURCE_DIR}/src/core/call/message.cc ${_gRPC_SOURCE_DIR}/src/core/call/metadata.cc @@ -1169,6 +1175,7 @@ add_library(grpc_unsecure ${_gRPC_SOURCE_DIR}/src/core/client_channel/retry_service_config.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/retry_throttle.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel.cc + ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel_metrics.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel_pool_interface.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel_stream_client.cc ${_gRPC_SOURCE_DIR}/src/core/client_channel/subchannel_stream_limiter.cc @@ -1211,7 +1218,6 @@ add_library(grpc_unsecure ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/chttp2_plugin.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/client/chttp2_connector.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/server/chttp2_server.cc - ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/bin_decoder.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/bin_encoder.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/call_tracer_wrapper.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/chttp2_transport.cc @@ -1244,6 +1250,7 @@ add_library(grpc_unsecure ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_callbacks.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_promise.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/ping_rate_policy.cc + ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/reclaimer.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/stream_lists.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/transport_common.cc ${_gRPC_SOURCE_DIR}/src/core/ext/transport/chttp2/transport/varint.cc @@ -1255,23 +1262,22 @@ add_library(grpc_unsecure ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/api/annotations.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/api/http.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/any.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/duration.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/empty.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/struct.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/timestamp.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/wrappers.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/rpc/status.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/gcp/altscontext.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/gcp/handshaker.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/gcp/transport_security_common.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/health/v1/health.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/lb/v1/load_balancer.upb_minitable.c + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/grpc/lookup/v1/rls.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/channelz/v2/channelz.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/channelz/v2/promise.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/channelz/v2/property_list.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/channelz/v2/service.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/gcp/altscontext.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/gcp/handshaker.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/gcp/transport_security_common.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/health/v1/health.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/lb/v1/load_balancer.upb_minitable.c - ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/src/proto/grpc/lookup/v1/rls.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/validate/validate.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/xds/data/orca/v3/orca_load_report.upb_minitable.c ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/xds/service/orca/v3/orca.upb_minitable.c @@ -1300,11 +1306,16 @@ add_library(grpc_unsecure ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/telemetry.cc ${_gRPC_SOURCE_DIR}/src/core/lib/resource_tracker/resource_tracker.cc ${_gRPC_SOURCE_DIR}/src/core/lib/transport/promise_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/load_balancing/autosharding/autosharding.cc ${_gRPC_SOURCE_DIR}/src/core/net/socket_mutator.cc + ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/dns_resolver.cc + ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/service_config_helper.cc + ${_gRPC_SOURCE_DIR}/src/core/server/server_config_selector_filter.cc ${_gRPC_SOURCE_DIR}/src/core/telemetry/context_list_entry.cc ${_gRPC_SOURCE_DIR}/src/core/telemetry/instrument.cc ${_gRPC_SOURCE_DIR}/src/core/transport/message_size_service_config.cc ${_gRPC_SOURCE_DIR}/src/core/transport/session_endpoint.cc + ${_gRPC_SOURCE_DIR}/src/core/util/address_sorting_init.cc ${_gRPC_SOURCE_DIR}/src/core/util/grpc_check.cc ${_gRPC_SOURCE_DIR}/src/core/util/postmortem_emit.cc ${_gRPC_SOURCE_DIR}/src/core/util/wait_for_single_owner.cc @@ -1414,9 +1425,6 @@ add_library(grpc_unsecure ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set.cc ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_set_windows.cc ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/pollset_windows.cc - ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address.cc - ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_posix.cc - ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/resolve_address_windows.cc ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/sockaddr_utils_posix.cc ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_factory_posix.cc ${_gRPC_SOURCE_DIR}/src/core/lib/iomgr/socket_mutator.cc @@ -1460,7 +1468,6 @@ add_library(grpc_unsecure ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/resource_quota.cc ${_gRPC_SOURCE_DIR}/src/core/lib/resource_quota/thread_quota.cc ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/authorization_policy_provider_vtable.cc - ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/evaluate_args.cc ${_gRPC_SOURCE_DIR}/src/core/lib/security/authorization/grpc_server_authz_filter.cc ${_gRPC_SOURCE_DIR}/src/core/lib/slice/percent_encoding.cc ${_gRPC_SOURCE_DIR}/src/core/lib/slice/slice.cc @@ -1520,16 +1527,6 @@ add_library(grpc_unsecure ${_gRPC_SOURCE_DIR}/src/core/load_balancing/weighted_target/weighted_target.cc ${_gRPC_SOURCE_DIR}/src/core/plugin_registry/grpc_plugin_registry.cc ${_gRPC_SOURCE_DIR}/src/core/plugin_registry/grpc_plugin_registry_noextra.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/c_ares/dns_resolver_ares.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/c_ares/grpc_ares_ev_driver_posix.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/c_ares/grpc_ares_ev_driver_windows.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/c_ares/grpc_ares_wrapper.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/c_ares/grpc_ares_wrapper_posix.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/c_ares/grpc_ares_wrapper_windows.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/dns_resolver_plugin.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/event_engine/event_engine_client_channel_resolver.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/event_engine/service_config_helper.cc - ${_gRPC_SOURCE_DIR}/src/core/resolver/dns/native/dns_resolver.cc ${_gRPC_SOURCE_DIR}/src/core/resolver/endpoint_addresses.cc ${_gRPC_SOURCE_DIR}/src/core/resolver/fake/fake_resolver.cc ${_gRPC_SOURCE_DIR}/src/core/resolver/polling_resolver.cc @@ -1704,6 +1701,29 @@ target_link_libraries(upb_mini_descriptor_lib upb_mini_table_lib ) +add_library(upb_descriptor_lib + ${_gRPC_SOURCE_DIR}/src/core/ext/upb-gen/google/protobuf/descriptor.upb_minitable.c +) + +target_compile_features(upb_descriptor_lib PUBLIC cxx_std_17) + +target_include_directories(upb_descriptor_lib + PUBLIC ${_gRPC_SOURCE_DIR}/include + PRIVATE + ${_gRPC_SOURCE_DIR} + ${_gRPC_ADDRESS_SORTING_INCLUDE_DIR} + ${_gRPC_RE2_INCLUDE_DIR} + ${_gRPC_SSL_INCLUDE_DIR} + ${_gRPC_UPB_GENERATED_DIR} + ${_gRPC_UPB_GRPC_GENERATED_DIR} + ${_gRPC_UPB_INCLUDE_DIR} + ${_gRPC_XXHASH_INCLUDE_DIR} + ${_gRPC_ZLIB_INCLUDE_DIR} +) +target_link_libraries(upb_descriptor_lib + ${_gRPC_ALLTARGETS_LIBRARIES} +) + add_library(upb_reflection_lib ${_gRPC_SOURCE_DIR}/third_party/upb/upb/message/copy.c ${_gRPC_SOURCE_DIR}/third_party/upb/upb/message/merge.c @@ -1743,6 +1763,7 @@ target_include_directories(upb_reflection_lib ) target_link_libraries(upb_reflection_lib ${_gRPC_ALLTARGETS_LIBRARIES} + upb_descriptor_lib upb_mini_descriptor_lib upb_wire_lib ) diff --git a/contrib/hive-metastore-cmake/CMakeLists.txt b/contrib/hive-metastore-cmake/CMakeLists.txt index c9d698264c4a6..bf92db62d1b92 100644 --- a/contrib/hive-metastore-cmake/CMakeLists.txt +++ b/contrib/hive-metastore-cmake/CMakeLists.txt @@ -2,11 +2,11 @@ if (NOT TARGET ch_contrib::hdfs) if (ENABLE_HIVE) message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use Hive without HDFS") endif() -# `Storages/Hive/HiveFile.h` includes `orc/OrcFile.hh` and `parquet/arrow/reader.h` outright, -# so Hive needs both readers, not just the metastore client. -elseif (NOT TARGET _orc) +# `Storages/Hive/HiveFile.h` uses the ORC reader and `parquet/arrow/reader.h` outright, +# so Hive needs both readers, not just the metastore client. The ORC reader needs only snappy. +elseif (NOT TARGET ch_contrib::snappy OR NOT TARGET _parquet) if (ENABLE_HIVE) - message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use Hive without ORC") + message (${RECONFIGURE_MESSAGE_LEVEL} "Cannot use Hive without ORC and Parquet") endif() else() option(ENABLE_HIVE "Enable Hive" ${ENABLE_LIBRARIES}) diff --git a/contrib/jemalloc b/contrib/jemalloc index 5f864d8bd4c4f..9e5a679b18b9c 160000 --- a/contrib/jemalloc +++ b/contrib/jemalloc @@ -1 +1 @@ -Subproject commit 5f864d8bd4c4fa7a99cb47c374d5c6465a160578 +Subproject commit 9e5a679b18b9c8d6b7f6fc319cb63808e88bc5ba diff --git a/contrib/jemalloc-cmake/include_darwin_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_darwin_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in index 3f3b872e850f5..e71ff144fb27d 100644 --- a/contrib/jemalloc-cmake/include_darwin_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_darwin_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -163,11 +163,12 @@ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) /* - * Defined when the TSD thread-locals use the initial-exec (static) TLS model, - * i.e. live at a fixed offset from the thread pointer. Gates the fast path of - * JEMALLOC_TLS_ADDR (see tsd_internals.h). + * Route TSD thread-local address computation through a noinline + * optimization-barrier accessor so the compiler cannot cache it across a + * user-space context switch (fibers migrating between OS threads). See + * tsd_tls_addr.h. */ -#define JEMALLOC_TLS_MODEL_INITIAL_EXEC +#define JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables diff --git a/contrib/jemalloc-cmake/include_darwin_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_darwin_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in index e00bb848e7b09..a8cd1ce994698 100644 --- a/contrib/jemalloc-cmake/include_darwin_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_darwin_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -163,11 +163,12 @@ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) /* - * Defined when the TSD thread-locals use the initial-exec (static) TLS model, - * i.e. live at a fixed offset from the thread pointer. Gates the fast path of - * JEMALLOC_TLS_ADDR (see tsd_internals.h). + * Route TSD thread-local address computation through a noinline + * optimization-barrier accessor so the compiler cannot cache it across a + * user-space context switch (fibers migrating between OS threads). See + * tsd_tls_addr.h. */ -#define JEMALLOC_TLS_MODEL_INITIAL_EXEC +#define JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables diff --git a/contrib/jemalloc-cmake/include_freebsd_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_freebsd_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in index 83b88f0fd9d46..9086baf5c2fe8 100644 --- a/contrib/jemalloc-cmake/include_freebsd_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_freebsd_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -142,11 +142,12 @@ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) /* - * Defined when the TSD thread-locals use the initial-exec (static) TLS model, - * i.e. live at a fixed offset from the thread pointer. Gates the fast path of - * JEMALLOC_TLS_ADDR (see tsd_internals.h). + * Route TSD thread-local address computation through a noinline + * optimization-barrier accessor so the compiler cannot cache it across a + * user-space context switch (fibers migrating between OS threads). See + * tsd_tls_addr.h. */ -#define JEMALLOC_TLS_MODEL_INITIAL_EXEC +#define JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables diff --git a/contrib/jemalloc-cmake/include_freebsd_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_freebsd_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in index 86f1f3ceca3d0..56d2ce53f4567 100644 --- a/contrib/jemalloc-cmake/include_freebsd_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_freebsd_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in @@ -140,11 +140,12 @@ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) /* - * Defined when the TSD thread-locals use the initial-exec (static) TLS model, - * i.e. live at a fixed offset from the thread pointer. Gates the fast path of - * JEMALLOC_TLS_ADDR (see tsd_internals.h). + * Route TSD thread-local address computation through a noinline + * optimization-barrier accessor so the compiler cannot cache it across a + * user-space context switch (fibers migrating between OS threads). See + * tsd_tls_addr.h. */ -#define JEMALLOC_TLS_MODEL_INITIAL_EXEC +#define JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables diff --git a/contrib/jemalloc-cmake/include_freebsd_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_freebsd_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in index 672f828f3ca45..229cc9593830b 100644 --- a/contrib/jemalloc-cmake/include_freebsd_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_freebsd_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -142,11 +142,12 @@ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) /* - * Defined when the TSD thread-locals use the initial-exec (static) TLS model, - * i.e. live at a fixed offset from the thread pointer. Gates the fast path of - * JEMALLOC_TLS_ADDR (see tsd_internals.h). + * Route TSD thread-local address computation through a noinline + * optimization-barrier accessor so the compiler cannot cache it across a + * user-space context switch (fibers migrating between OS threads). See + * tsd_tls_addr.h. */ -#define JEMALLOC_TLS_MODEL_INITIAL_EXEC +#define JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables diff --git a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in index 9a98fbd3810a0..d8ba561bd2075 100644 --- a/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_aarch64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -140,11 +140,12 @@ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) /* - * Defined when the TSD thread-locals use the initial-exec (static) TLS model, - * i.e. live at a fixed offset from the thread pointer. Gates the fast path of - * JEMALLOC_TLS_ADDR (see tsd_internals.h). + * Route TSD thread-local address computation through a noinline + * optimization-barrier accessor so the compiler cannot cache it across a + * user-space context switch (fibers migrating between OS threads). See + * tsd_tls_addr.h. */ -#define JEMALLOC_TLS_MODEL_INITIAL_EXEC +#define JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables diff --git a/contrib/jemalloc-cmake/include_linux_aarch64_musl/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_aarch64_musl/jemalloc/internal/jemalloc_internal_defs.h.in index 0ac9fb9c4526d..a19d65dccf5fe 100644 --- a/contrib/jemalloc-cmake/include_linux_aarch64_musl/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_aarch64_musl/jemalloc/internal/jemalloc_internal_defs.h.in @@ -142,6 +142,14 @@ /* Non-empty if the tls_model attribute is supported. */ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) +/* + * Route TSD thread-local address computation through a noinline + * optimization-barrier accessor so the compiler cannot cache it across a + * user-space context switch (fibers migrating between OS threads). See + * tsd_tls_addr.h. + */ +#define JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS + /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables * inline functions. diff --git a/contrib/jemalloc-cmake/include_linux_e2k/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_e2k/jemalloc/internal/jemalloc_internal_defs.h.in index 94f65fa0ce076..971ab7bd66d0c 100644 --- a/contrib/jemalloc-cmake/include_linux_e2k/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_e2k/jemalloc/internal/jemalloc_internal_defs.h.in @@ -127,11 +127,12 @@ #undef JEMALLOC_TLS_MODEL /* - * Defined when the TSD thread-locals use the initial-exec (static) TLS model, - * i.e. live at a fixed offset from the thread pointer. Gates the fast path of - * JEMALLOC_TLS_ADDR (see tsd_internals.h). + * Route TSD thread-local address computation through a noinline + * optimization-barrier accessor so the compiler cannot cache it across a + * user-space context switch (fibers migrating between OS threads). See + * tsd_tls_addr.h. */ -#undef JEMALLOC_TLS_MODEL_INITIAL_EXEC +#define JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables diff --git a/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in index 9f6bce289868e..39a69c4a449d8 100644 --- a/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_ppc64le/jemalloc/internal/jemalloc_internal_defs.h.in @@ -140,11 +140,12 @@ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) /* - * Defined when the TSD thread-locals use the initial-exec (static) TLS model, - * i.e. live at a fixed offset from the thread pointer. Gates the fast path of - * JEMALLOC_TLS_ADDR (see tsd_internals.h). + * Route TSD thread-local address computation through a noinline + * optimization-barrier accessor so the compiler cannot cache it across a + * user-space context switch (fibers migrating between OS threads). See + * tsd_tls_addr.h. */ -#define JEMALLOC_TLS_MODEL_INITIAL_EXEC +#define JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables diff --git a/contrib/jemalloc-cmake/include_linux_riscv64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_riscv64/jemalloc/internal/jemalloc_internal_defs.h.in index 9e495d40d94aa..64c97e6d9340d 100644 --- a/contrib/jemalloc-cmake/include_linux_riscv64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_riscv64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -140,11 +140,12 @@ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) /* - * Defined when the TSD thread-locals use the initial-exec (static) TLS model, - * i.e. live at a fixed offset from the thread pointer. Gates the fast path of - * JEMALLOC_TLS_ADDR (see tsd_internals.h). + * Route TSD thread-local address computation through a noinline + * optimization-barrier accessor so the compiler cannot cache it across a + * user-space context switch (fibers migrating between OS threads). See + * tsd_tls_addr.h. */ -#define JEMALLOC_TLS_MODEL_INITIAL_EXEC +#define JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables diff --git a/contrib/jemalloc-cmake/include_linux_s390x/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_s390x/jemalloc/internal/jemalloc_internal_defs.h.in index 8373a973f759e..867b247302e27 100644 --- a/contrib/jemalloc-cmake/include_linux_s390x/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_s390x/jemalloc/internal/jemalloc_internal_defs.h.in @@ -140,11 +140,12 @@ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) /* - * Defined when the TSD thread-locals use the initial-exec (static) TLS model, - * i.e. live at a fixed offset from the thread pointer. Gates the fast path of - * JEMALLOC_TLS_ADDR (see tsd_internals.h). + * Route TSD thread-local address computation through a noinline + * optimization-barrier accessor so the compiler cannot cache it across a + * user-space context switch (fibers migrating between OS threads). See + * tsd_tls_addr.h. */ -#define JEMALLOC_TLS_MODEL_INITIAL_EXEC +#define JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables diff --git a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in index b1267a4dfa56d..e8983ec778d03 100644 --- a/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_x86_64/jemalloc/internal/jemalloc_internal_defs.h.in @@ -140,11 +140,12 @@ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) /* - * Defined when the TSD thread-locals use the initial-exec (static) TLS model, - * i.e. live at a fixed offset from the thread pointer. Gates the fast path of - * JEMALLOC_TLS_ADDR (see tsd_internals.h). + * Route TSD thread-local address computation through a noinline + * optimization-barrier accessor so the compiler cannot cache it across a + * user-space context switch (fibers migrating between OS threads). See + * tsd_tls_addr.h. */ -#define JEMALLOC_TLS_MODEL_INITIAL_EXEC +#define JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables diff --git a/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in b/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in index 70e304e1629ef..894d9b6d63a21 100644 --- a/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in +++ b/contrib/jemalloc-cmake/include_linux_x86_64_musl/jemalloc/internal/jemalloc_internal_defs.h.in @@ -141,11 +141,12 @@ #define JEMALLOC_TLS_MODEL __attribute__((tls_model("initial-exec"))) /* - * Defined when the TSD thread-locals use the initial-exec (static) TLS model, - * i.e. live at a fixed offset from the thread pointer. Gates the fast path of - * JEMALLOC_TLS_ADDR (see tsd_internals.h). + * Route TSD thread-local address computation through a noinline + * optimization-barrier accessor so the compiler cannot cache it across a + * user-space context switch (fibers migrating between OS threads). See + * tsd_tls_addr.h. */ -#define JEMALLOC_TLS_MODEL_INITIAL_EXEC +#define JEMALLOC_EXPERIMENTAL_FIBER_SAFE_TLS /* * JEMALLOC_DEBUG enables assertions and other sanity checks, and disables diff --git a/contrib/mongo-c-driver b/contrib/mongo-c-driver index 70de9e880072b..58b4e1f39c6f8 160000 --- a/contrib/mongo-c-driver +++ b/contrib/mongo-c-driver @@ -1 +1 @@ -Subproject commit 70de9e880072b120ccbb48c77de334775747d7a9 +Subproject commit 58b4e1f39c6f8bcf4da4204deb84f9b3527388d4 diff --git a/contrib/mongo-c-driver-cmake/CMakeLists.txt b/contrib/mongo-c-driver-cmake/CMakeLists.txt index 43ac3f5cd0283..6d013bc450fa9 100644 --- a/contrib/mongo-c-driver-cmake/CMakeLists.txt +++ b/contrib/mongo-c-driver-cmake/CMakeLists.txt @@ -6,12 +6,12 @@ endif() set(libbson_VERSION_MAJOR 2) set(libbson_VERSION_MINOR 5) -set(libbson_VERSION_PATCH 5) -set(libbson_VERSION 2.5.5) +set(libbson_VERSION_PATCH 8) +set(libbson_VERSION 2.5.8) set(libmongoc_VERSION_MAJOR 2) set(libmongoc_VERSION_MINOR 5) -set(libmongoc_VERSION_PATCH 5) -set(libmongoc_VERSION 2.5.5) +set(libmongoc_VERSION_PATCH 8) +set(libmongoc_VERSION 2.5.8) set(LIBBSON_SOURCES_ROOT "${ClickHouse_SOURCE_DIR}/contrib/mongo-c-driver/src") set(LIBBSON_SOURCE_DIR "${LIBBSON_SOURCES_ROOT}/libbson/src") diff --git a/contrib/mongo-cxx-driver b/contrib/mongo-cxx-driver index fa9ee6ed008e8..35424a5cc5973 160000 --- a/contrib/mongo-cxx-driver +++ b/contrib/mongo-cxx-driver @@ -1 +1 @@ -Subproject commit fa9ee6ed008e84b9562a7a55d70681de8a6bc1c6 +Subproject commit 35424a5cc597391d3737ae5e00f029cd2afc03c4 diff --git a/contrib/orc b/contrib/orc deleted file mode 160000 index 6a2fe65eb16ce..0000000000000 --- a/contrib/orc +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 6a2fe65eb16ce4c760964dca619260226c1b418e diff --git a/contrib/sqlite-amalgamation b/contrib/sqlite-amalgamation index 23c7b76929611..f48768e9089bf 160000 --- a/contrib/sqlite-amalgamation +++ b/contrib/sqlite-amalgamation @@ -1 +1 @@ -Subproject commit 23c7b76929611997c5f0315b3149372b1bee1d41 +Subproject commit f48768e9089bf66717e8c393dce5dd80c86761f5 diff --git a/docs/reference/functions/regular-functions/ai-functions.mdx b/docs/reference/functions/regular-functions/ai-functions.mdx index c842b0943e6f5..2cb7851fbf60b 100644 --- a/docs/reference/functions/regular-functions/ai-functions.mdx +++ b/docs/reference/functions/regular-functions/ai-functions.mdx @@ -28,10 +28,11 @@ All functions are sharing a common infrastructure that provides: - **Quota enforcement**: Per-query limits on tokens ([`ai_function_max_input_tokens_per_query`](/reference/settings/session-settings/ai-function#ai_function_max_input_tokens_per_query), [`ai_function_max_output_tokens_per_query`](/reference/settings/session-settings/ai-function#ai_function_max_output_tokens_per_query)) and API calls ([`ai_function_max_api_calls_per_query`](/reference/settings/session-settings/ai-function#ai_function_max_api_calls_per_query)). - **Retry with backoff**: Transient failures are retried ([`ai_function_max_retries`](/reference/settings/session-settings/ai-function#ai_function_max_retries)) with exponential backoff ([`ai_function_retry_initial_delay_ms`](/reference/settings/session-settings/ai-function#ai_function_retry_initial_delay_ms)). +- **Concurrent requests**: Requests are issued from a server-wide thread pool, several at a time (controlled by [`ai_function_max_concurrent_requests_per_thread`](/reference/settings/session-settings/ai-function#ai_function_max_concurrent_requests_per_thread)). ## Configuration {#configuration} -AI functions reference a **named collection** that stores provider credentials and configuration. Different named collections can be created and used for different functions or functions calls. For example you may want to define a different named collection to use with the text functions (`aiGenerate`, `aiClassify`, `aiFilter`, `aiExtract`, `aiTranslate`, `aiRedact`) vs the embedding functions (`aiEmbed`, `aiSimilarity`), which require different endpoints and usually use different models. +AI functions require the `AI FUNCTIONS` grant (see [Access control](#access-control)) and reference a **named collection** that stores provider credentials and configuration. Different named collections can be created and used for different functions or functions calls. For example you may want to define a different named collection to use with the text functions (`aiGenerate`, `aiClassify`, `aiFilter`, `aiExtract`, `aiTranslate`, `aiRedact`) vs the embedding functions (`aiEmbed`, `aiSimilarity`), which require different endpoints and usually use different models. Example statement to create a named collection with provider credentials, one with a chat endpoint and another with an embedding endpoint: ```sql @@ -50,6 +51,35 @@ CREATE NAMED COLLECTION ai_embedding_credentials AS api_key = 'sk-...'; ``` +### Access control {#access-control} + +Calling an AI function requires two privileges: + +- the function's privilege: either the per-function privilege (e.g. `aiGenerate`) or the `AI FUNCTIONS` group, see [`GRANT`](/reference/statements/grant#ai-functions); +- the `NAMED COLLECTION` privilege on the selected credentials. + +```sql +GRANT AI FUNCTIONS ON *.* TO ai_user_role; +GRANT aiGenerate, aiClassify ON *.* TO ai_analyst_role; +GRANT NAMED COLLECTION ON ai_text_credentials TO ai_user_role, ai_analyst_role; +``` + +AI functions are executed only in queries run by a user. They are rejected in mutations (`ALTER TABLE ... UPDATE`, `ALTER TABLE ... DELETE`) and in the keys, TTL, skip indexes and projections of a table, which are executed in the background, and `Buffer` flushes throw an exception when they evaluate an AI function. `DEFAULT` and `MATERIALIZED` columns are evaluated by the insert; when such a column is added with `ALTER TABLE`, the existing parts compute it in the background, where AI functions are not executed. + +To fill a column of existing rows, use a lightweight [`UPDATE`](/reference/statements/update), which is executed by the query and writes only the updated column: + +```sql +UPDATE t SET y = aiEmbed(x, 'model') WHERE empty(y) SETTINGS enable_lightweight_update = 1; +``` + +For a table that does not meet the [lightweight update requirements](/reference/statements/update#lightweight-update-requirements), use `INSERT ... SELECT`, for example one partition at a time: + +```sql +CREATE TABLE t_tmp AS t; +INSERT INTO t_tmp SELECT x, aiEmbed(x, 'model') AS y FROM t WHERE _partition_id = '202610'; +ALTER TABLE t REPLACE PARTITION ID '202610' FROM t_tmp; +``` + ### Named collection parameters {#named-collection-parameters} | Parameter | Type | Default | Description | @@ -114,6 +144,24 @@ SELECT aiGenerate(body, map('temperature', '0.2', 'system_prompt', 'You are ters All AI-related settings are listed in [Settings](/reference/settings/session-settings) under the `ai_function_` prefix. +### Server configuration {#server-configuration} + +The server setting `ai_function_allow_sql_named_collections` (default `true`) controls which named collections AI functions accept. When set to `false`, AI functions accept only named collections defined in the server configuration file, and using a collection created with `CREATE NAMED COLLECTION` throws an exception. This applies both to the `credentials` parameter and to the default-credentials settings. Changing the setting requires a server restart. + +```xml + + false + + + openai + https://api.openai.com/v1/chat/completions + gpt-4o-mini + sk-... + + + +``` + ### Restricting endpoint hosts {#restricting-endpoint-hosts} The `endpoint` URL in an AI named collection is an outbound destination the server connects to under its own identity, potentially carrying (if specified) the named collection's `api_key` in the request headers. By default, ClickHouse permits any host. To restrict functions to a specific set of providers, configure [`remote_url_allow_hosts`](/reference/settings/server-settings/settings/remote#remote_url_allow_hosts) in the server config, e.g.: @@ -138,6 +186,35 @@ By default, AI functions reject an `endpoint` that would send data in cleartext Note that in either case the provider receives the input data in cleartext after TLS termination; TLS protects the data only on the network path between the server and the provider. +### Throughput {#throughput} + +The AI functions differ from most traditional ClickHouse functions in that their throughput is almost entirely bound by the remote endpoint they send requests to rather than local CPU or I/O. Because of this they benefit significantly from increased concurrency above the standard `max_threads` used throughout ClickHouse. For this reason the requests are issued from a dedicated server-wide thread pool. Its size (`ai_function_max_thread_pool_size`, a server setting) caps how many AI provider requests the whole server has in flight at once, across every query. + +[`ai_function_max_concurrent_requests_per_thread`](/reference/settings/session-settings/ai-function#ai_function_max_concurrent_requests_per_thread) bounds each query thread's share of that pool, so a query running on several threads (at most `max_threads`) can have up to that many requests in flight per thread. To stop queries from raising it, put a ceiling on the setting with a [settings profile constraint](/concepts/features/configuration/settings/constraints-on-settings). The ceiling is still per thread, so a query can have up to the ceiling times its number of threads in flight. For example: + +```xml + + + + + 16 + + + + +``` + +The default is `8`. A value above `1` loosens two guarantees: + +- A request's token usage is only known once its response arrives, so [`ai_function_max_input_tokens_per_query`](/reference/settings/session-settings/ai-function#ai_function_max_input_tokens_per_query) and [`ai_function_max_output_tokens_per_query`](/reference/settings/session-settings/ai-function#ai_function_max_output_tokens_per_query) may overshoot by one request's worth per request in flight, rather than by one request's worth in total. +- A wave of requests is dispatched before any of their errors is seen, so a query that fails on its first row may still have issued, and been billed for, as many requests as the concurrency allows. + +[`ai_function_max_api_calls_per_query`](/reference/settings/session-settings/ai-function#ai_function_max_api_calls_per_query) is unaffected and stays an exact cap, because a slot is reserved before each request is dispatched. + + +For a materialized view, the settings must go on the `INSERT` that feeds the view's source table, not on the `CREATE MATERIALIZED VIEW` statement: the quotas are read from the top-level query, and a `SETTINGS` clause in a view definition is ignored. + + ## Supported providers {#supported-providers} | Provider | `provider` value | Chat functions | Notes | diff --git a/docs/reference/settings/server-settings/_server_settings_outside_source.mdx b/docs/reference/settings/server-settings/_server_settings_outside_source.mdx index 23af0e0fa6040..f72dee5492310 100644 --- a/docs/reference/settings/server-settings/_server_settings_outside_source.mdx +++ b/docs/reference/settings/server-settings/_server_settings_outside_source.mdx @@ -1444,7 +1444,7 @@ Additionally: | Setting | Description | Default Value | |---------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------------------| -| `level` | Maximum Message Level (by default `Trace`) which will be stored in a table. | `Trace` | +| `level` | Maximum message level stored in the table. When set explicitly, the root logger level is raised to at least this value, so `text_log` can be more verbose than the server log file (`logger.level`). When not set, messages up to the most verbose level of the configured log outputs (`logger.level`, `console_log_level`, `errorlog_level`, `syslog_level`) are stored. | `Trace` (limited by the other log outputs when not set) | **Example** diff --git a/docs/reference/statements/grant.mdx b/docs/reference/statements/grant.mdx index de01a6ecc3a9e..645098c3c0785 100644 --- a/docs/reference/statements/grant.mdx +++ b/docs/reference/statements/grant.mdx @@ -160,6 +160,15 @@ The hierarchy of privileges in ClickHouse is shown below: - `SHOW ROW POLICIES` - `SHOW SETTINGS PROFILES` - `SHOW USERS` + - [`AI FUNCTIONS`](#ai-functions) + - `aiClassify` + - `aiEmbed` + - `aiExtract` + - `aiFilter` + - `aiGenerate` + - `aiRedact` + - `aiSimilarity` + - `aiTranslate` - [`ALTER`](#alter) - `ALTER DATABASE` - `ALTER DATABASE SETTINGS` @@ -626,6 +635,25 @@ Allows using [introspection](/concepts/features/performance/troubleshoot/samplin - `addressToSymbol`. Level: `GLOBAL` - `demangle`. Level: `GLOBAL` +### AI FUNCTIONS {#ai-functions} + +Allows using [AI functions](/reference/functions/regular-functions/ai-functions). Calling an AI function also requires the `NAMED COLLECTION` privilege on the selected credentials. + +- `AI FUNCTIONS`. Level: `GROUP` + - `aiClassify`. Level: `GLOBAL` + - `aiEmbed`. Level: `GLOBAL` + - `aiExtract`. Level: `GLOBAL` + - `aiFilter`. Level: `GLOBAL` + - `aiGenerate`. Level: `GLOBAL` + - `aiRedact`. Level: `GLOBAL` + - `aiSimilarity`. Level: `GLOBAL` + - `aiTranslate`. Level: `GLOBAL` + +Examples: + +- `GRANT AI FUNCTIONS ON *.* TO john` +- `GRANT aiGenerate, aiClassify ON *.* TO john` + ### SOURCES {#sources} Allows using external data sources. Applies to [table engines](/reference/engines/table-engines/index) and [table functions](/reference/functions/table-functions/index). diff --git a/programs/local/LocalServer.cpp b/programs/local/LocalServer.cpp index 99ee12067d987..42e7a57dd6a86 100644 --- a/programs/local/LocalServer.cpp +++ b/programs/local/LocalServer.cpp @@ -215,6 +215,9 @@ namespace ServerSetting extern const ServerSettingsUInt64 max_iceberg_manifest_decode_thread_pool_size; extern const ServerSettingsUInt64 max_iceberg_manifest_decode_thread_pool_free_size; extern const ServerSettingsUInt64 iceberg_manifest_decode_thread_pool_queue_size; + extern const ServerSettingsNonZeroUInt64 ai_function_max_thread_pool_size; + extern const ServerSettingsUInt64 ai_function_max_thread_pool_free_size; + extern const ServerSettingsUInt64 ai_function_thread_pool_queue_size; extern const ServerSettingsUInt64 page_cache_history_window_ms; extern const ServerSettingsString page_cache_policy; extern const ServerSettingsDouble page_cache_size_ratio; @@ -470,6 +473,11 @@ void LocalServer::initialize(Poco::Util::Application & self) server_settings[ServerSetting::max_iceberg_manifest_decode_thread_pool_size], server_settings[ServerSetting::max_iceberg_manifest_decode_thread_pool_free_size], server_settings[ServerSetting::iceberg_manifest_decode_thread_pool_queue_size]); + + getAIRequestThreadPool().initialize( + server_settings[ServerSetting::ai_function_max_thread_pool_size], + server_settings[ServerSetting::ai_function_max_thread_pool_free_size], + server_settings[ServerSetting::ai_function_thread_pool_queue_size]); } diff --git a/programs/main.cpp b/programs/main.cpp index 74e1b893ba321..18ea7ad034189 100644 --- a/programs/main.cpp +++ b/programs/main.cpp @@ -296,8 +296,6 @@ static __attribute__((constructor(0))) void init_je_malloc_message() std::string_view message_view{str}; if (message_view == ": background thread creation failed (11)\n") return; - if (message_view == ": Number of CPUs detected is not deterministic. Per-CPU arena disabled.\n") - return; # if defined(SYS_write) syscall(SYS_write, STDERR_FILENO, message_view.data(), message_view.size()); diff --git a/programs/server/Server.cpp b/programs/server/Server.cpp index 9884fc325c09d..c57083a0bb06e 100644 --- a/programs/server/Server.cpp +++ b/programs/server/Server.cpp @@ -428,6 +428,9 @@ namespace ServerSetting extern const ServerSettingsUInt64 max_iceberg_manifest_decode_thread_pool_size; extern const ServerSettingsUInt64 max_iceberg_manifest_decode_thread_pool_free_size; extern const ServerSettingsUInt64 iceberg_manifest_decode_thread_pool_queue_size; + extern const ServerSettingsNonZeroUInt64 ai_function_max_thread_pool_size; + extern const ServerSettingsUInt64 ai_function_max_thread_pool_free_size; + extern const ServerSettingsUInt64 ai_function_thread_pool_queue_size; extern const ServerSettingsUInt64 page_cache_history_window_ms; extern const ServerSettingsString page_cache_policy; extern const ServerSettingsDouble page_cache_size_ratio; @@ -2001,6 +2004,11 @@ try server_settings[ServerSetting::max_iceberg_manifest_decode_thread_pool_free_size], server_settings[ServerSetting::iceberg_manifest_decode_thread_pool_queue_size]); + getAIRequestThreadPool().initialize( + server_settings[ServerSetting::ai_function_max_thread_pool_size], + server_settings[ServerSetting::ai_function_max_thread_pool_free_size], + server_settings[ServerSetting::ai_function_thread_pool_queue_size]); + std::string path_str = getCanonicalPath(String(server_settings[ServerSetting::path]), original_working_directory); fs::path path = path_str; @@ -3053,6 +3061,11 @@ try new_server_settings[ServerSetting::max_iceberg_manifest_decode_thread_pool_free_size], new_server_settings[ServerSetting::iceberg_manifest_decode_thread_pool_queue_size]); + getAIRequestThreadPool().reloadConfiguration( + new_server_settings[ServerSetting::ai_function_max_thread_pool_size], + new_server_settings[ServerSetting::ai_function_max_thread_pool_free_size], + new_server_settings[ServerSetting::ai_function_thread_pool_queue_size]); + global_context->setMergeWorkload(new_server_settings[ServerSetting::merge_workload]); global_context->setMutationWorkload(new_server_settings[ServerSetting::mutation_workload]); global_context->setThrowOnUnknownWorkload(new_server_settings[ServerSetting::throw_on_unknown_workload]); diff --git a/programs/server/config.xml b/programs/server/config.xml index 5fe5bd70ca412..36eb35e48440b 100644 --- a/programs/server/config.xml +++ b/programs/server/config.xml @@ -1354,7 +1354,9 @@ system diff --git a/src/Access/AccessControl.cpp b/src/Access/AccessControl.cpp index c869445e76824..bcca86281d53f 100644 --- a/src/Access/AccessControl.cpp +++ b/src/Access/AccessControl.cpp @@ -602,22 +602,25 @@ AuthResult AccessControl::authenticate(const Credentials & credentials, const Po // NOTE: In the case where the user has never been logged in using LDAP, // Then user_id is not generated, and the authentication quota will always be nullptr. auto authentication_quota = getAuthenticationQuota(credentials.getUserName(), address, client_info.getLastForwardedForHost()); - if (authentication_quota) - { - /// Reserve a single try from the quota to check whether we have another authentication try. - /// This is required for correct behavior in this situation: - /// User has 1 login failures quota. - /// * At the first login with an invalid password: Increase the quota counter. 1 (used) > 1 (max) is false. - /// Then try to authenticate the user and throw an AUTHENTICATION_FAILED error. - /// * In case of the second try: increase quota counter, 2 (used) > 1 (max), then throw QUOTA_EXCEED - /// and don't let the user authenticate. - /// - /// The authentication failures counter will be reset after successful authentication. - authentication_quota->used(QuotaType::FAILED_SEQUENTIAL_AUTHENTICATIONS, 1); - } try { + if (authentication_quota) + { + /// Reserve a single try from the quota to check whether we have another authentication try. + /// This is required for correct behavior in this situation: + /// User has 1 login failures quota. + /// * At the first login with an invalid password: Increase the quota counter. 1 (used) > 1 (max) is false. + /// Then try to authenticate the user and throw an AUTHENTICATION_FAILED error. + /// * In case of the second try: increase quota counter, 2 (used) > 1 (max), then throw QUOTA_EXCEED + /// and don't let the user authenticate. + /// + /// The authentication failures counter will be reset after successful authentication. + /// Inside the try block, so that an exceeded quota is reported like any other authentication failure + /// and doesn't reveal whether the user exists. + authentication_quota->used(QuotaType::FAILED_SEQUENTIAL_AUTHENTICATIONS, 1); + } + const auto auth_result = MultipleAccessStorage::authenticate(credentials, address, *external_authenticators, client_info, allow_no_password, allow_plaintext_password); if (authentication_quota) @@ -859,7 +862,7 @@ std::shared_ptr AccessControl::getEnabledQuota( const String & forwarded_address, const String & custom_quota_key) const { - return quota_cache->getEnabledQuota(user_id, user_name, enabled_roles, address, forwarded_address, custom_quota_key, true); + return quota_cache->getEnabledQuota(user_id, user_name, enabled_roles, address, forwarded_address, custom_quota_key); } std::shared_ptr AccessControl::getAuthenticationQuota( @@ -872,18 +875,13 @@ std::shared_ptr AccessControl::getAuthenticationQuota( const auto new_current_roles = user->granted_roles.findGranted(user->default_roles); const auto roles_info = getEnabledRolesInfo(new_current_roles, {}); - // client_key is not received at the moment of authentication during TCP connection - // if key type is set to QuotaKeyType::CLIENT_KEY - // QuotaCache::QuotaInfo::calculateKey will throw exception without throw_if_client_key_empty = false String quota_key; - bool throw_if_client_key_empty = false; return quota_cache->getEnabledQuota(*user_id, user->getName(), roles_info->enabled_roles, std::make_shared(address), forwarded_address, - quota_key, - throw_if_client_key_empty); + quota_key); } return nullptr; } diff --git a/src/Access/Common/AccessType.h b/src/Access/Common/AccessType.h index c1cbb1aed671e..ba14d5e582375 100644 --- a/src/Access/Common/AccessType.h +++ b/src/Access/Common/AccessType.h @@ -150,7 +150,7 @@ ENUM_ACCESS_OBJECT(Source, APPLY_FOR_SOURCE) /// Represents an access type which can be granted on databases, tables, columns, etc. -enum class AccessType : uint8_t +enum class AccessType : uint16_t { /// Macro M should be defined as M(name, aliases, node_type, parent_group_name) /// where name is identifier with underscores (instead of spaces); @@ -446,6 +446,17 @@ enum class AccessType : uint8_t M(DISK, "", GLOBAL, ALL) \ M(SOURCES, "", GLOBAL, ALL) \ \ + /* AI functions */ \ + M(aiClassify, "", GLOBAL, AI_FUNCTIONS) /* allows to execute function aiClassify() */\ + M(aiEmbed, "", GLOBAL, AI_FUNCTIONS) /* allows to execute function aiEmbed() */\ + M(aiExtract, "", GLOBAL, AI_FUNCTIONS) /* allows to execute function aiExtract() */\ + M(aiFilter, "", GLOBAL, AI_FUNCTIONS) /* allows to execute function aiFilter() */\ + M(aiGenerate, "", GLOBAL, AI_FUNCTIONS) /* allows to execute function aiGenerate() */\ + M(aiRedact, "", GLOBAL, AI_FUNCTIONS) /* allows to execute function aiRedact() */\ + M(aiSimilarity, "", GLOBAL, AI_FUNCTIONS) /* allows to execute function aiSimilarity() */\ + M(aiTranslate, "", GLOBAL, AI_FUNCTIONS) /* allows to execute function aiTranslate() */\ + M(AI_FUNCTIONS, "", GROUP, ALL) /* allows to execute all AI functions */\ + \ /* Consts */ \ M(ALL, "ALL PRIVILEGES", GROUP, NONE) /* full access */ \ M(NONE, "USAGE, NO PRIVILEGES", GROUP, NONE) /* no access */ diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index c014ea12a7a46..81ed1e44fa2ba 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -614,6 +614,10 @@ std::shared_ptr ContextAccess::getQuota() const } } + /// Outside the branch above on purpose: `QuotaCache` refreshes the quota set in place, so an already + /// memoised object has to be re-checked on every call. + enabled_quota->checkClientKeySupplied(); + return enabled_quota; } @@ -809,16 +813,17 @@ bool ContextAccess::checkAccessImplHelper(const ContextPtr & context, AccessFlag const AccessFlags function_ddl = AccessType::CREATE_FUNCTION | AccessType::DROP_FUNCTION; const AccessFlags workload_ddl = AccessType::CREATE_WORKLOAD | AccessType::DROP_WORKLOAD; const AccessFlags resource_ddl = AccessType::CREATE_RESOURCE | AccessType::DROP_RESOURCE; + const AccessFlags named_collection_ddl = AccessType::CREATE_NAMED_COLLECTION | AccessType::DROP_NAMED_COLLECTION | AccessType::ALTER_NAMED_COLLECTION; const AccessFlags handler_ddl = AccessType::CREATE_HANDLER | AccessType::ALTER_HANDLER | AccessType::DROP_HANDLER; const AccessFlags table_and_dictionary_ddl = table_ddl | dictionary_ddl; const AccessFlags table_and_dictionary_and_function_ddl = table_ddl | dictionary_ddl | function_ddl; const AccessFlags write_table_access = AccessType::INSERT | AccessType::OPTIMIZE; const AccessFlags write_dcl_access = AccessType::ACCESS_MANAGEMENT - AccessType::SHOW_ACCESS; - const AccessFlags not_readonly_flags = write_table_access | table_and_dictionary_and_function_ddl | workload_ddl | resource_ddl | handler_ddl | write_dcl_access | AccessType::SYSTEM | AccessType::KILL_QUERY; + const AccessFlags not_readonly_flags = write_table_access | table_and_dictionary_and_function_ddl | workload_ddl | resource_ddl | named_collection_ddl | handler_ddl | write_dcl_access | AccessType::SYSTEM | AccessType::KILL_QUERY; const AccessFlags not_readonly_1_flags = AccessType::CREATE_TEMPORARY_TABLE; - const AccessFlags ddl_flags = table_ddl | dictionary_ddl | function_ddl | workload_ddl | resource_ddl | handler_ddl; + const AccessFlags ddl_flags = table_ddl | dictionary_ddl | function_ddl | workload_ddl | resource_ddl | named_collection_ddl | handler_ddl; const AccessFlags introspection_flags = AccessType::INTROSPECTION; }; static const PrecalculatedFlags precalc; diff --git a/src/Access/EnabledQuota.cpp b/src/Access/EnabledQuota.cpp index 8fa463a890766..d44357445b872 100644 --- a/src/Access/EnabledQuota.cpp +++ b/src/Access/EnabledQuota.cpp @@ -14,6 +14,7 @@ namespace DB namespace ErrorCodes { extern const int QUOTA_EXCEEDED; + extern const int QUOTA_REQUIRES_CLIENT_KEY; } @@ -451,6 +452,22 @@ void EnabledQuota::checkExceeded(QuotaType quota_type) const } +void EnabledQuota::checkClientKeySupplied() const +{ + auto loaded = quotas.load(); + for (const auto & quota : *loaded) + { + if (!quota->requires_client_key) + continue; + throw Exception( + ErrorCodes::QUOTA_REQUIRES_CLIENT_KEY, + "Quota {} (for user {}) requires a client supplied key.", + quota->intervals->quota_name, + getUserName()); + } +} + + void EnabledQuota::checkExceededForQuery(UInt64 normalized_query_hash, QuotaType quota_type) const { if (empty) diff --git a/src/Access/EnabledQuota.h b/src/Access/EnabledQuota.h index 431afed6ad616..ccf68d6c2a625 100644 --- a/src/Access/EnabledQuota.h +++ b/src/Access/EnabledQuota.h @@ -73,6 +73,10 @@ class EnabledQuota : public boost::noncopyable void checkExceeded() const; void checkExceeded(QuotaType quota_type) const; + /// Throws if a governing quota is keyed by `client_key` and this context supplied none. Authentication + /// is metered before a client key can exist, so only the query path calls this. + void checkClientKeySupplied() const; + /// Same as `checkExceeded(quota_type)`, but for `NORMALIZED_QUERY_HASH` quotas the check is /// performed against the intervals resolved for `normalized_query_hash`. void checkExceededForQuery(UInt64 normalized_query_hash, QuotaType quota_type) const; @@ -139,6 +143,11 @@ class EnabledQuota : public boost::noncopyable /// Non-null only for `NORMALIZED_QUERY_HASH` quotas: resolves intervals per query hash. IntervalResolver interval_resolver; + /// This quota is keyed by `client_key` and the context supplied none, so it cannot be metered. + /// Published with the set, so it is refreshed in place whenever quota definitions change; + /// see `checkClientKeySupplied`. + bool requires_client_key = false; + /// Cache of resolved intervals per normalized query hash. mutable std::mutex resolved_intervals_mutex; mutable HashMap> resolved_intervals_cache; diff --git a/src/Access/QuotaCache.cpp b/src/Access/QuotaCache.cpp index 70d878f2bbc29..4a1a84a0b5215 100644 --- a/src/Access/QuotaCache.cpp +++ b/src/Access/QuotaCache.cpp @@ -26,7 +26,6 @@ namespace DB { namespace ErrorCodes { - extern const int QUOTA_REQUIRES_CLIENT_KEY; extern const int LOGICAL_ERROR; } @@ -40,7 +39,7 @@ void QuotaCache::QuotaInfo::setQuota(const QuotaPtr & quota_, const UUID & quota } -String QuotaCache::QuotaInfo::calculateKey(const EnabledQuota & enabled, bool throw_if_client_key_empty) const +String QuotaCache::QuotaInfo::calculateKey(const EnabledQuota & enabled) const { const auto & params = enabled.params; auto mask_address = [this](const Poco::Net::IPAddress & addr) -> String @@ -128,14 +127,9 @@ String QuotaCache::QuotaInfo::calculateKey(const EnabledQuota & enabled, bool th { if (!params.client_key.empty()) return params.client_key; - - if (throw_if_client_key_empty) - throw Exception( - ErrorCodes::QUOTA_REQUIRES_CLIENT_KEY, - "Quota {} (for user {}) requires a client supplied key.", - quota->getName(), - params.user_name); - return ""; // Authentication quota has no client key at time of authentication. + /// An empty key is a legitimate state: at the time of authentication no client key exists yet. + /// The requirement is published as `SingleQuota::requires_client_key` and enforced on the query path. + return ""; } case QuotaKeyType::CLIENT_KEY_OR_USER_NAME: { @@ -256,8 +250,7 @@ std::shared_ptr QuotaCache::getEnabledQuota( const boost::container::flat_set & enabled_roles, const std::shared_ptr & client_address, const String & forwarded_address, - const String & client_key, - bool throw_if_client_key_empty) + const String & client_key) { std::lock_guard lock{mutex}; ensureAllQuotasRead(); @@ -280,7 +273,7 @@ std::shared_ptr QuotaCache::getEnabledQuota( auto res = std::shared_ptr(new EnabledQuota(params)); enabled_quotas.emplace(std::move(params), res); - chooseQuotaToConsumeFor(*res, throw_if_client_key_empty); + chooseQuotaToConsumeFor(*res); return res; } @@ -370,7 +363,7 @@ void QuotaCache::chooseQuotaToConsume() i = enabled_quotas.erase(i); else { - chooseQuotaToConsumeFor(*elem, true); + chooseQuotaToConsumeFor(*elem); ++i; } } @@ -384,7 +377,7 @@ void QuotaCache::chooseQuotaToConsume() LOG_TRACE(getLogger("QuotaCache"), "Re-chose quotas for {} enabled set(s) over {} quotas in {} ms", enabled_quotas.size(), all_quotas.size(), elapsed_ms); } -void QuotaCache::chooseQuotaToConsumeFor(EnabledQuota & enabled, bool throw_if_client_key_empty) +void QuotaCache::chooseQuotaToConsumeFor(EnabledQuota & enabled) { /// `mutex` is already locked. @@ -396,9 +389,10 @@ void QuotaCache::chooseQuotaToConsumeFor(EnabledQuota & enabled, bool throw_if_c if (!info.roles->match(enabled.params.user_id, enabled.params.enabled_roles)) continue; - String key = info.calculateKey(enabled, throw_if_client_key_empty); + String key = info.calculateKey(enabled); auto single = std::make_unique(); single->intervals = info.getOrBuildIntervals(key); + single->requires_client_key = (info.quota->key_type == QuotaKeyType::CLIENT_KEY) && enabled.params.client_key.empty(); /// For NORMALIZED_QUERY_HASH keyed quotas, set up a resolver callback /// so that EnabledQuota can lazily resolve intervals per query hash. diff --git a/src/Access/QuotaCache.h b/src/Access/QuotaCache.h index 7880049c13064..086b9e44d318d 100644 --- a/src/Access/QuotaCache.h +++ b/src/Access/QuotaCache.h @@ -28,8 +28,7 @@ class QuotaCache const boost::container::flat_set & enabled_roles, const std::shared_ptr & address, const String & forwarded_address, - const String & client_key, - bool throw_if_client_key_empty); + const String & client_key); std::vector getAllQuotasUsage() const; @@ -44,7 +43,7 @@ class QuotaCache QuotaInfo(const QuotaPtr & quota_, const UUID & quota_id_) { setQuota(quota_, quota_id_); } void setQuota(const QuotaPtr & quota_, const UUID & quota_id_); - String calculateKey(const EnabledQuota & enabled_quota, bool throw_if_client_key_empty) const; + String calculateKey(const EnabledQuota & enabled_quota) const; boost::shared_ptr getOrBuildIntervals(const String & key); boost::shared_ptr rebuildIntervals(const String & key, std::chrono::system_clock::time_point current_time); void rebuildAllIntervals(); @@ -60,7 +59,7 @@ class QuotaCache void quotaRemoved(const UUID & quota_id) TSA_REQUIRES(mutex); void chooseQuotaToConsumeIfNeeded() TSA_REQUIRES(mutex); void chooseQuotaToConsume() TSA_REQUIRES(mutex); - void chooseQuotaToConsumeFor(EnabledQuota & enabled_quota, bool throw_if_client_key_empty) TSA_REQUIRES(mutex); + void chooseQuotaToConsumeFor(EnabledQuota & enabled_quota) TSA_REQUIRES(mutex); const AccessControl & access_control; mutable std::mutex mutex; diff --git a/src/Access/SettingsConstraints.cpp b/src/Access/SettingsConstraints.cpp index fcf380632efef..9bfcb7946495b 100644 --- a/src/Access/SettingsConstraints.cpp +++ b/src/Access/SettingsConstraints.cpp @@ -311,14 +311,22 @@ void SettingsConstraints::clamp(const Settings & current_settings, SettingsChang checkOrClamp(current_settings, changes, CLAMP_ON_VIOLATION, source); } +void SettingsConstraints::clampRejectingInvalidChanges(const Settings & current_settings, SettingsChanges & changes, SettingSource source) const +{ + checkOrClamp(current_settings, changes, CLAMP_ON_VIOLATION_THROW_ON_INVALID, source); +} + void SettingsConstraints::checkOrClamp(const Settings & current_settings, SettingsChanges & changes, ReactionOnViolation reaction, SettingSource source) const { /// If we filter out settings that match the current default here, `compatibility` will silently override them. /// So when `compatibility` is present, we keep unchanged settings so they are applied after `compatibility`. bool has_compatibility_setting = changes.tryGet("compatibility") != nullptr; + /// With `CLAMP_ON_VIOLATION_THROW_ON_INVALID` the surviving list is stored on the query node as the settings + /// set explicitly on that node, so a change equal to the current value must stay in the list. + bool ignore_unchanged_settings = has_compatibility_setting || reaction == CLAMP_ON_VIOLATION_THROW_ON_INVALID; std::erase_if(changes, [&](SettingChange & change) { - return !checkImpl(current_settings, change, reaction, source, /*ignore_unchanged_settings=*/has_compatibility_setting); + return !checkImpl(current_settings, change, reaction, source, ignore_unchanged_settings); }); } @@ -415,7 +423,10 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, if (setting_name == "profile") return true; - if (reaction == THROW_ON_VIOLATION) + /// Invalid changes (an unknown or disallowed name, an uncastable value) throw in every mode but `CLAMP_ON_VIOLATION`. + const bool throw_on_invalid_change = reaction != CLAMP_ON_VIOLATION; + + if (throw_on_invalid_change) { try { @@ -436,7 +447,8 @@ bool SettingsConstraints::checkImpl(const Settings & current_settings, else if (!access_control->isSettingNameAllowed(setting_name)) return false; - Field new_value = getNewValueToCheck(current_settings, change, ignore_unchanged_settings, reaction == THROW_ON_VIOLATION); + Field new_value = getNewValueToCheck( + current_settings, change, ignore_unchanged_settings, /*throw_on_failure=*/ throw_on_invalid_change); if (new_value.isNull()) return false; @@ -479,9 +491,12 @@ bool SettingsConstraints::Checker::check(SettingChange & change, ReactionOnViolation reaction, SettingSource source) const { + /// Every reaction other than `THROW_ON_VIOLATION` clamps or drops a violating change. + const bool throw_on_violation = reaction == THROW_ON_VIOLATION; + if (!explain.text.empty()) { - if (reaction == THROW_ON_VIOLATION) + if (throw_on_violation) throw Exception(explain, code); return false; } @@ -490,7 +505,7 @@ bool SettingsConstraints::Checker::check(SettingChange & change, auto less_or_cannot_compare = [=](const Field & left, const Field & right) { - if (reaction == THROW_ON_VIOLATION) + if (throw_on_violation) return accurateLess(left, right); try { @@ -504,7 +519,7 @@ bool SettingsConstraints::Checker::check(SettingChange & change, auto equals_or_cannot_compare = [=](const Field & left, const Field & right) { - if (reaction == THROW_ON_VIOLATION) + if (throw_on_violation) return accurateEquals(left, right); try { @@ -519,7 +534,7 @@ bool SettingsConstraints::Checker::check(SettingChange & change, if (constraint.writability == SettingConstraintWritability::CONST) { - if (reaction == THROW_ON_VIOLATION) + if (throw_on_violation) throw Exception(ErrorCodes::SETTING_CONSTRAINT_VIOLATION, "Setting {} should not be changed", setting_name); return false; } @@ -530,7 +545,7 @@ bool SettingsConstraints::Checker::check(SettingChange & change, if (!min_value.isNull() && !max_value.isNull() && less_or_cannot_compare(max_value, min_value)) { - if (reaction == THROW_ON_VIOLATION) + if (throw_on_violation) throw Exception( ErrorCodes::SETTING_CONSTRAINT_VIOLATION, "The maximum ({}) value is less than the minimum ({}) value for setting {}", @@ -547,7 +562,7 @@ bool SettingsConstraints::Checker::check(SettingChange & change, if (!min_value.isNull() && less_or_cannot_compare(effective_value, min_value)) { - if (reaction == THROW_ON_VIOLATION) + if (throw_on_violation) { throw Exception(ErrorCodes::SETTING_CONSTRAINT_VIOLATION, "Setting {} shouldn't be less than {}", setting_name, applyVisitor(FieldVisitorToString(), min_value)); @@ -558,7 +573,7 @@ bool SettingsConstraints::Checker::check(SettingChange & change, if (!max_value.isNull() && less_or_cannot_compare(max_value, effective_value)) { - if (reaction == THROW_ON_VIOLATION) + if (throw_on_violation) { throw Exception(ErrorCodes::SETTING_CONSTRAINT_VIOLATION, "Setting {} shouldn't be greater than {}", setting_name, applyVisitor(FieldVisitorToString(), max_value)); @@ -572,7 +587,7 @@ bool SettingsConstraints::Checker::check(SettingChange & change, bool equals = equals_or_cannot_compare(value, effective_value); if (equals) { - if (reaction == THROW_ON_VIOLATION) + if (throw_on_violation) throw Exception(ErrorCodes::SETTING_CONSTRAINT_VIOLATION, "Setting {} shouldn't be {}", setting_name, applyVisitor(FieldVisitorToString(), value)); /// On clamp paths there is no sensible value to clamp to — disallowed entries are a @@ -584,7 +599,7 @@ bool SettingsConstraints::Checker::check(SettingChange & change, if (!getSettingSourceRestrictions(setting_name).isSourceAllowed(source)) { - if (reaction == THROW_ON_VIOLATION) + if (throw_on_violation) throw Exception(ErrorCodes::READONLY, "Setting {} is not allowed to be set by {}", setting_name, toString(source)); return false; } diff --git a/src/Access/SettingsConstraints.h b/src/Access/SettingsConstraints.h index 1e7470566281f..d522b3971622c 100644 --- a/src/Access/SettingsConstraints.h +++ b/src/Access/SettingsConstraints.h @@ -80,6 +80,7 @@ class SettingsConstraints void merge(const SettingsConstraints & other); /// Checks whether `change` violates these constraints and throws an exception if so. + /// The non-const `SettingsChanges &` overload also drops a change equal to the current value from `changes` (unless `compatibility` is present). void check(const Settings & current_settings, const SettingChange & change, SettingSource source) const; void check(const Settings & current_settings, const SettingsChanges & changes, SettingSource source) const; void check(const Settings & current_settings, SettingsChanges & changes, SettingSource source) const; @@ -95,8 +96,12 @@ class SettingsConstraints void check(const MergeTreeSettings & current_settings, const SettingsChanges & changes) const; /// Checks whether `change` violates these and clamps the `change` if so. + /// A change equal to the current value is dropped from `changes` (unless `compatibility` is present). void clamp(const Settings & current_settings, SettingsChanges & changes, SettingSource source) const; + /// Same as `clamp`, but an unknown or disallowed setting name and an uncastable value throw, as in `check`, + /// and a change equal to the current value is kept in `changes`. + void clampRejectingInvalidChanges(const Settings & current_settings, SettingsChanges & changes, SettingSource source) const; friend bool operator ==(const SettingsConstraints & left, const SettingsConstraints & right); friend bool operator !=(const SettingsConstraints & left, const SettingsConstraints & right) { return !(left == right); } @@ -104,8 +109,12 @@ class SettingsConstraints private: enum ReactionOnViolation { + /// Any violation, an unknown name or an uncastable value throws. THROW_ON_VIOLATION, + /// Violations are clamped or dropped; an unknown name or an uncastable value is dropped. CLAMP_ON_VIOLATION, + /// Violations are clamped or dropped; an unknown name or an uncastable value throws. + CLAMP_ON_VIOLATION_THROW_ON_INVALID, }; struct Constraint @@ -162,8 +171,8 @@ class SettingsConstraints } }; - /// Common logic for `check(Settings, SettingsChanges&)` and `clamp`. Both filter out unchanged settings - /// (unless `compatibility` is present) and differ only in whether violations throw or get clamped to the nearest bound. + /// Common logic for `check`, `clamp` and `clampRejectingInvalidChanges`. The modes differ in whether violations throw or get clamped + /// to the nearest bound and whether an invalid change throws or is dropped. void checkOrClamp(const Settings & current_settings, SettingsChanges & changes, ReactionOnViolation reaction, SettingSource source) const; diff --git a/src/AggregateFunctions/AggregateFunctionGini.cpp b/src/AggregateFunctions/AggregateFunctionGini.cpp index 336519528c92e..1994d725b86b3 100644 --- a/src/AggregateFunctions/AggregateFunctionGini.cpp +++ b/src/AggregateFunctions/AggregateFunctionGini.cpp @@ -25,6 +25,7 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int INCORRECT_DATA; extern const int TOO_LARGE_ARRAY_SIZE; } @@ -67,11 +68,12 @@ struct AggregateFunctionGiniData return static_cast(value); } - void add(const Value & x) + /// Returns false for NaN, which is skipped. Throws for unsupported values. + static bool checkValue(const Value & x) { - /// Skip NaNs as they are not compatible with comparison sorting. + /// NaNs are not compatible with comparison sorting. if (isNaN(x)) - return; + return false; if (!isFinite(x)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Aggregate function `gini` does not support infinite values"); @@ -79,6 +81,14 @@ struct AggregateFunctionGiniData if (x < Value{}) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Aggregate function `gini` does not support negative values"); + return true; + } + + void add(const Value & x) + { + if (!checkValue(x)) + return; + if (unlikely(array.size() >= GINI_MAX_ARRAY_SIZE)) throwTooLargeArraySize(); @@ -107,6 +117,12 @@ struct AggregateFunctionGiniData checkArraySize(size); array.resize(size); buf.readStrict(reinterpret_cast(array.data()), size * sizeof(array[0])); + + for (const auto & value : array) + { + if (!checkValue(value)) + throw Exception(ErrorCodes::INCORRECT_DATA, "NaN in serialized state of aggregate function `gini`"); + } } /** Gini coefficient of the collected values. For sorted values x, the sum of diff --git a/src/AggregateFunctions/AggregateFunctionGroupArray.cpp b/src/AggregateFunctions/AggregateFunctionGroupArray.cpp index 89c46451c9272..51b83e5fd0733 100644 --- a/src/AggregateFunctions/AggregateFunctionGroupArray.cpp +++ b/src/AggregateFunctions/AggregateFunctionGroupArray.cpp @@ -24,6 +24,8 @@ #include +#include + #include constexpr size_t AGGREGATE_FUNCTION_GROUP_ARRAY_MAX_ELEMENT_SIZE = 0xFFFFFF; @@ -90,6 +92,17 @@ struct GroupArraySamplerData size_t total_values = 0; pcg32_fast rng; + /// The counter may come from a deserialized state, so it can be arbitrarily large, + /// and wrapping it around would make `genRandom` divide by zero. + void addTotalValues(UInt64 count) + { + UInt64 new_total_values = 0; + if (common::addOverflow(static_cast(total_values), count, new_total_values)) + throw Exception(ErrorCodes::INCORRECT_DATA, + "The number of values in groupArraySample state overflows: {} + {}", total_values, count); + total_values = new_total_values; + } + UInt64 genRandom(size_t lim) { chassert(lim != 0); @@ -160,7 +173,7 @@ class GroupArrayNumericImpl final void insertWithSampler(Data & a, const T & v, Arena * arena) const { - ++a.total_values; + a.addTotalValues(1); if (a.value.size() < max_elems) a.value.push_back(v, arena); else @@ -183,7 +196,10 @@ class GroupArrayNumericImpl final const auto & row_value = assert_cast &>(*columns[0]).getData()[row_num]; auto & cur_elems = this->data(place); - ++cur_elems.total_values; + if constexpr (Trait::sampler == Sampler::NONE) + ++cur_elems.total_values; + else + cur_elems.addTotalValues(1); if constexpr (Trait::sampler == Sampler::NONE) { @@ -275,8 +291,8 @@ class GroupArrayNumericImpl final } else { + cur_elems.addTotalValues(rhs_elems.total_values); cur_elems.randomShuffle(); - cur_elems.total_values += rhs_elems.total_values; for (size_t i = 0; i < max_elems; ++i) { UInt64 rnd = cur_elems.genRandom(cur_elems.total_values); @@ -525,7 +541,7 @@ class GroupArrayGeneralImpl final void insertWithSampler(Data & a, const Node * v, Arena * arena) const { - ++a.total_values; + a.addTotalValues(1); if (a.value.size() < max_elems) a.value.push_back(v->clone(arena), arena); else @@ -547,7 +563,10 @@ class GroupArrayGeneralImpl final { auto & cur_elems = data(place); - ++cur_elems.total_values; + if constexpr (Trait::sampler == Sampler::NONE) + ++cur_elems.total_values; + else + cur_elems.addTotalValues(1); if constexpr (Trait::sampler == Sampler::NONE) { @@ -642,8 +661,8 @@ class GroupArrayGeneralImpl final } else { + cur_elems.addTotalValues(rhs_elems.total_values); cur_elems.randomShuffle(); - cur_elems.total_values += rhs_elems.total_values; for (size_t i = 0; i < max_elems; ++i) { UInt64 rnd = cur_elems.genRandom(cur_elems.total_values); diff --git a/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.cpp b/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.cpp index 345fbe80a8659..cc0b3797077ee 100644 --- a/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.cpp +++ b/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.cpp @@ -61,6 +61,12 @@ void forEachSecretArgumentNode( continue; } + if (const auto masked = secret_arguments.masked_arguments.find(n); masked != secret_arguments.masked_arguments.end()) + { + on_secret(n, masked->second ? secretValueSlot(arguments[n]) : arguments[n]); + continue; + } + if (secret_arguments.isSecretArgument(n)) on_secret(n, secretValueSlot(arguments[n])); } diff --git a/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.h b/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.h index 6afdd5d3a952b..ecd7322afc29b 100644 --- a/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.h +++ b/src/Analyzer/FunctionSecretArgumentsFinderTreeNode.h @@ -122,7 +122,8 @@ using FunctionSecretArgumentsFinderTreeNode = FunctionSecretArgumentsFinderTreeN using TableFunctionSecretArgumentsFinderTreeNode = FunctionSecretArgumentsFinderTreeNodeImpl; /// Visits the secret value slots selected by a finder result in a resolved argument list, for the -/// query-tree surfaces (`EXPLAIN QUERY TREE`, projection names): the span members and the arguments +/// query-tree surfaces (`EXPLAIN QUERY TREE`, projection names): the span members, the individually +/// masked arguments (whole, unless masked as a named `key = value`) and the arguments /// with a partial replacement (for a `key = value` argument, its value; a tree dump cannot represent /// partial masking, so the whole value is masked: fail closed), and the values of the nested secret /// maps (`headers(..)` / `extra_credentials(..)`; a malformed child is visited whole). The callback diff --git a/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp b/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp index 4ba9480c79bee..4f0c70b6cfdd1 100644 --- a/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp +++ b/src/Analyzer/Passes/FunctionToSubcolumnsPass.cpp @@ -839,6 +839,10 @@ ColumnNode * resolveTrivialAliasChain(ColumnNode * column_node) /// getTypedNodesForOptimization, so their decisions cannot diverge. bool storageAllowsTransformer(const IStorage & storage, TypeIndex type_id, const String & function_name) { + /// Without buckets the key subcolumn `m.key_` reads the whole Map, so `m['key']` is not rewritten to it. + if (function_name == "arrayElement" && type_id == TypeIndex::Map && !storage.hasBucketedMapSerialization()) + return false; + if (storage.supportsOptimizationToSubcolumns()) return true; return storage.supportsOptimizationToTupleElementSubcolumns() && type_id == TypeIndex::Tuple && function_name == "tupleElement"; diff --git a/src/Analyzer/QueryTreeBuilder.cpp b/src/Analyzer/QueryTreeBuilder.cpp index 0b5e861df1473..126a792506caf 100644 --- a/src/Analyzer/QueryTreeBuilder.cpp +++ b/src/Analyzer/QueryTreeBuilder.cpp @@ -311,22 +311,28 @@ QueryTreeNodePtr QueryTreeBuilder::buildSelectExpression( set_query.changes.removeSetting("limit"); set_query.changes.removeSetting("offset"); - /// A nested `SETTINGS` clause (a subquery, a CTE, a view's inner query) used to be applied to - /// the per-node context unchecked, letting a user override `readonly`, `CONST` and `MIN`/`MAX` - /// constraints - e.g. `additional_table_filters`, which the Planner reads from this context. - /// Clamp it instead of throwing, as done for other settings crossing execution contexts - /// (`getSQLSecurityOverriddenContext`, secondary queries, DDL replay): violating changes are - /// dropped, out-of-bounds values are clamped, so a view whose inner clause violates the - /// reader's constraints keeps working. A top-level clause still throws - /// (`applySettingsFromQuery`). Clamp a copy: the `QueryNode` keeps the clause as written, so - /// the tree's AST and hash are unchanged, and every node executing the subquery clamps it - /// against its own constraints. `SETTINGS name = DEFAULT` stays ignored here - see #115415. + /// A nested `SETTINGS` clause (a subquery, a CTE, a view's inner query) is clamped to the session's + /// settings constraints, as done for other settings crossing execution contexts + /// (`getSQLSecurityOverriddenContext`, secondary queries, DDL replay): violating changes are dropped, + /// out-of-bounds values are clamped, so a view whose inner clause violates the reader's constraints + /// keeps working. A top-level clause still throws (`applySettingsFromQuery`). An unknown name or an + /// uncastable value throws as for a top-level clause, on every server that executes the query. The + /// node records the effective `changes`: the ones that survive, with their clamped values and the + /// literals as written, so `toAST`, the tree hash, the `Planner` and the shards receiving the + /// subquery see what was applied. `SETTINGS name = DEFAULT` stays ignored here - see #115415. if (!set_query.changes.empty()) { - auto checked_changes = set_query.changes; - updated_context->clampToSettingsConstraints(checked_changes, SettingSource::QUERY); - updated_context->applySettingsChanges(checked_changes); - settings_changes = set_query.changes; + /// One change at a time, so that each is clamped against the context as the preceding changes + /// left it. Clamping the whole clause as a batch checks every change against the context as it + /// was before the clause - so `SETTINGS use_query_cache = 1, use_query_cache = 0` could resolve + /// differently from the last-wins reading the clause has everywhere else. + for (const auto & change : set_query.changes) + { + SettingsChanges single_change{change}; + updated_context->clampToSettingsConstraintsRejectingInvalidChanges(single_change, SettingSource::QUERY); + updated_context->applySettingsChanges(single_change); + settings_changes.insert(settings_changes.end(), single_change.begin(), single_change.end()); + } } } diff --git a/src/Backups/BackupEntriesCollector.cpp b/src/Backups/BackupEntriesCollector.cpp index 763799a598bc0..24b622386ffb2 100644 --- a/src/Backups/BackupEntriesCollector.cpp +++ b/src/Backups/BackupEntriesCollector.cpp @@ -587,16 +587,17 @@ void BackupEntriesCollector::gatherTablesMetadata() /// partition-related constraints. for (auto & [qualified_name, res_table_info] : table_infos) { - res_table_info.should_backup_data = shouldBackupTableData(qualified_name, res_table_info.storage, rmv_replace_target_ids); - - if (!res_table_info.should_backup_data) - continue; + res_table_info.should_backup_data = shouldBackupTableData(qualified_name, res_table_info.storage, res_table_info.create_table_query, rmv_replace_target_ids); const auto & database_info = database_infos.at(qualified_name.database); auto it = database_info.tables.find(qualified_name.table); if (it == database_info.tables.end()) continue; + /// Validated before `should_backup_data` is acted on, so it holds however the data came to be left out - + /// `structure_only`, a refreshable materialized view target, or a `backup_metadata_only` storage policy. + /// Naming a partition is a user error in each of those cases, and answering it by silently ignoring the + /// clause is worse than refusing the query. const auto & partitions = it->second.partitions; if (partitions && res_table_info.storage && !res_table_info.storage->supportsBackupPartition()) { @@ -606,6 +607,23 @@ void BackupEntriesCollector::gatherTablesMetadata() res_table_info.storage->getName(), tableNameWithTypeToString(qualified_name.database, qualified_name.table, false)); } + + /// The data of a table on a storage policy marked `backup_metadata_only` is never backed up, the storage provides + /// it. A backup of some of its partitions would be restored as the whole table, so refuse it. + if (partitions + && BackupUtils::isDataOnBackupMetadataOnlyStoragePolicy(res_table_info.storage, res_table_info.create_table_query, context)) + { + throw Exception( + ErrorCodes::CANNOT_BACKUP_TABLE, + "Cannot backup specific partitions of {}: it is on a storage policy marked backup_metadata_only, so only its " + "definition is backed up, and it is restored with all the data of its storage", + tableNameWithTypeToString(qualified_name.database, qualified_name.table, false)); + } + + /// An excluded table contributes no partitions, so the scope below is left unset for it. + if (!res_table_info.should_backup_data) + continue; + res_table_info.partitions = partitions; } } @@ -887,8 +905,8 @@ void BackupEntriesCollector::makeBackupEntriesForTableData(const QualifiedTableN bool BackupEntriesCollector::shouldBackupTableData( const QualifiedTableName & table_name, - /// Used in the Cloud build. - [[maybe_unused]] const StoragePtr & storage, + const StoragePtr & storage, + const ASTPtr & create_table_query, const std::unordered_set & rmv_replace_target_ids) const { if (backup_settings.structure_only) @@ -900,6 +918,13 @@ bool BackupEntriesCollector::shouldBackupTableData( LOG_TRACE(log, "Skipping table data for {} (a target of a refreshable materialized view)", table_name.getFullName()); return false; } + + if (BackupUtils::isDataOnBackupMetadataOnlyStoragePolicy(storage, create_table_query, context)) + { + LOG_INFO(log, "Skipping table data for {} (on a storage policy marked backup_metadata_only)", table_name.getFullName()); + return false; + } + return true; } diff --git a/src/Backups/BackupEntriesCollector.h b/src/Backups/BackupEntriesCollector.h index 84c9819aa71e7..928f23abfbdbe 100644 --- a/src/Backups/BackupEntriesCollector.h +++ b/src/Backups/BackupEntriesCollector.h @@ -99,8 +99,8 @@ class BackupEntriesCollector : private boost::noncopyable void makeBackupEntriesForTableData(const QualifiedTableName & table_name); bool shouldBackupTableData( const QualifiedTableName & table_name, - /// Used in the Cloud build. - [[maybe_unused]] const StoragePtr & storage, + const StoragePtr & storage, + const ASTPtr & create_table_query, const std::unordered_set & rmv_replace_target_ids) const; void addBackupEntryUnlocked(const String & file_name, BackupEntryPtr backup_entry); diff --git a/src/Backups/BackupIO_AzureBlobStorage.cpp b/src/Backups/BackupIO_AzureBlobStorage.cpp index 5b6112e9f80f7..49d1ce41f4bbc 100644 --- a/src/Backups/BackupIO_AzureBlobStorage.cpp +++ b/src/Backups/BackupIO_AzureBlobStorage.cpp @@ -2,6 +2,7 @@ #include #if USE_AZURE_BLOB_STORAGE +#include #include #include #include @@ -66,6 +67,14 @@ namespace }; } + /// An Azure disk configured with `connection_string` is described by that whole string. + String maskedDescription(const DataSourceDescription & description) + { + String result = description.description; + maskAzureConnectionStringSecrets(result); + return result; + } + #if CLICKHOUSE_CLOUD /// The recorded endpoint of a snapshot's source disk as a service URL: scheme, host, port and path, without /// query parameters and trailing slashes. A source disk that authenticates with a SAS records it in the @@ -149,7 +158,7 @@ BackupReaderAzureBlobStorage::BackupReaderAzureBlobStorage( const WriteSettings & write_settings_, const ContextPtr & context_) : BackupReaderDefault(read_settings_, write_settings_, getLogger("BackupReaderAzureBlobStorage")) - , data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::Azure, MetadataStorageType::None, connection_params_.getConnectionURL(), false, false, ""} + , data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::Azure, MetadataStorageType::None, connection_params_.getConnectionURLWithoutQuery(), false, false, ""} , connection_params(connection_params_) , blob_path(blob_path_) { @@ -202,7 +211,7 @@ void BackupReaderAzureBlobStorage::copyFileToDisk(const String & path_in_backup, DiskPtr destination_disk, const String & destination_path, WriteMode write_mode) { auto destination_data_source_description = destination_disk->getDataSourceDescription(); - LOG_TRACE(log, "Source description {}, destination description {}", data_source_description.description, destination_data_source_description.description); + LOG_TRACE(log, "Source description {}, destination description {}", data_source_description.description, maskedDescription(destination_data_source_description)); if (destination_data_source_description.object_storage_type == ObjectStorageType::Azure && destination_data_source_description.is_encrypted == encrypted_in_backup) { @@ -249,7 +258,7 @@ BackupWriterAzureBlobStorage::BackupWriterAzureBlobStorage( const ContextPtr & context_, bool attempt_to_create_container) : BackupWriterDefault(read_settings_, write_settings_, getLogger("BackupWriterAzureBlobStorage")) - , data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::Azure, MetadataStorageType::None, connection_params_.getConnectionURL(), false, false, ""} + , data_source_description{DataSourceType::ObjectStorage, ObjectStorageType::Azure, MetadataStorageType::None, connection_params_.getConnectionURLWithoutQuery(), false, false, ""} , connection_params(connection_params_) , blob_path(blob_path_) { @@ -279,7 +288,7 @@ void BackupWriterAzureBlobStorage::copyFileFromDisk( { /// Use the native copy as a more optimal way to copy a file from AzureBlobStorage to AzureBlobStorage if it's possible. auto source_data_source_description = src_disk->getDataSourceDescription(); - LOG_TRACE(log, "Source description {}, destination description {}", source_data_source_description.description, data_source_description.description); + LOG_TRACE(log, "Source description {}, destination description {}", maskedDescription(source_data_source_description), data_source_description.description); if (source_data_source_description.object_storage_type == ObjectStorageType::Azure && source_data_source_description.is_encrypted == copy_encrypted) { diff --git a/src/Backups/BackupMetadataFinder.cpp b/src/Backups/BackupMetadataFinder.cpp index cfc3c257784de..803a89bbe2b65 100644 --- a/src/Backups/BackupMetadataFinder.cpp +++ b/src/Backups/BackupMetadataFinder.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -285,6 +286,16 @@ void BackupMetadataFinder::findTableInBackupImpl( ParserCreateQuery create_parser; ASTPtr create_table_query = parseQuery(create_parser, create_query_str, 0, DBMS_DEFAULT_MAX_PARSER_DEPTH, DBMS_DEFAULT_MAX_PARSER_BACKTRACKS); + + /// `BACKUP` writes the full definition, without `AS` (see `cleanupObjectDefinitionFromTemporaryFlags`). A definition that + /// inherits its engine with `AS` would let the access check of the restore see no engine, so it is rejected. + if (const auto & create = create_table_query->as(); !create.as_table.empty()) + throw Exception( + ErrorCodes::CANNOT_RESTORE_TABLE, + "The definition of {} in the backup uses AS {}, which BACKUP never writes", + tableNameWithTypeToString(table_name_in_backup.database, table_name_in_backup.table, false), + backQuoteIfNeed(create.as_table)); + applyCustomStoragePolicy(create_table_query); renameDatabaseAndTableNameInCreateQuery(create_table_query, renaming_map, context->getGlobalContext()); String create_table_query_str = create_table_query->formatWithSecretsOneLine(); diff --git a/src/Backups/BackupUtils.cpp b/src/Backups/BackupUtils.cpp index 879e8716a447e..71be481e1bca6 100644 --- a/src/Backups/BackupUtils.cpp +++ b/src/Backups/BackupUtils.cpp @@ -3,8 +3,18 @@ #include #include #include +#include +#include #include #include +#include +#include +#include +#include +#include +#include +#include +#include #include #include @@ -149,4 +159,82 @@ bool isInnerTable(const String & /* database_name */, const String & table_name) return table_name.starts_with(".inner.") || table_name.starts_with(".inner_id.") || table_name.starts_with(".tmp.inner.") || table_name.starts_with(".tmp.inner_id."); } +namespace +{ + +/// The `MergeTreeData` behind `storage` if it is on a storage policy marked `backup_metadata_only`. Only a `MergeTreeData` +/// keeps its data on a storage policy: `Alias`, `Distributed` and `StorageProxy` report one but keep no data of their own. +/// A proxy is not resolved here, unlike on master, where `resolveStorageProxyLoading` loads a lazily loaded table: the +/// helpers it needs are not on this branch, so a table behind a proxy keeps its data backed up as usual. +std::shared_ptr getMergeTreeOnBackupMetadataOnlyStoragePolicy(const StoragePtr & storage) +{ + auto merge_tree = std::dynamic_pointer_cast(storage); + if (!merge_tree) + return nullptr; + auto policy = merge_tree->getStoragePolicy(); + if (!policy || !policy->isBackupMetadataOnly()) + return nullptr; + return merge_tree; +} + +} + +/// `context` is unused here: only the `TimeSeries` targets left out on this branch needed it. +bool isDataOnBackupMetadataOnlyStoragePolicy(const StoragePtr & storage, [[maybe_unused]] const ContextPtr & context) +{ + /// Only a table of the plain `MergeTree` family reopens the parts it finds on the storage. `RESTORE` creates a + /// `ReplicatedMergeTree` table in a mode that refuses them, a `SharedMergeTree` table sees only the parts in Keeper, + /// and a view looks for the data of its inner tables in the backup. For the others the flag is ignored. + /// + /// Master warns when it ignores the flag, naming the inner table that carries it. The warning is left out here: + /// it needs the inner tables of a `TimeSeries` table through `getTargetKinds`, which is not static on this + /// branch, and the answer below does not depend on it. + if (typeid_cast(storage.get()) || typeid_cast(storage.get())) + return false; + + auto merge_tree = getMergeTreeOnBackupMetadataOnlyStoragePolicy(storage); + if (!merge_tree) + return false; + + return std::dynamic_pointer_cast(merge_tree) != nullptr; +} + +bool isDataOnBackupMetadataOnlyStoragePolicy(const StoragePtr & storage, const ASTPtr & create_table_query, const ContextPtr & context) +{ + if (storage) + return isDataOnBackupMetadataOnlyStoragePolicy(storage, context); + + const auto * create_query = create_table_query ? create_table_query->as() : nullptr; + return create_query && isCreatedOnBackupMetadataOnlyStoragePolicy(*create_query, context); +} + +bool isCreatedOnBackupMetadataOnlyStoragePolicy(const ASTCreateQuery & create_query, const ContextPtr & context) +{ + const auto * storage = create_query.storage; + if (!storage || !storage->engine || !storage->settings) + return false; + + /// Only an explicit `storage_policy` matters: a read-only policy can't be the default one. + const auto * policy_name = storage->settings->changes.tryGet("storage_policy"); + if (!policy_name) + return false; + + /// Only an engine of the `MergeTree` family keeps its data on a storage policy, see + /// `isDataOnBackupMetadataOnlyStoragePolicy`. `storage_policy` is a builtin setting of the `Log` family too, so what + /// identifies the family is the settings the engine declares, not that it declares this one. An unknown engine is + /// reported by `CREATE`. + const auto & storages = StorageFactory::instance().getAllStorages(); + auto it = storages.find(storage->engine->name); + if (it == storages.end() || it->second.features.has_builtin_setting_fn != MergeTreeSettings::hasBuiltin) + return false; + + /// Only the plain family reopens the parts it finds on the storage: the replicated and shared engines are + /// registered with `supports_replication`. + if (it->second.features.supports_replication) + return false; + + auto policy = context->tryGetStoragePolicy(policy_name->safeGet()); + return policy && policy->isBackupMetadataOnly(); +} + } diff --git a/src/Backups/BackupUtils.h b/src/Backups/BackupUtils.h index ba889eccc48e9..b17a14c365869 100644 --- a/src/Backups/BackupUtils.h +++ b/src/Backups/BackupUtils.h @@ -2,12 +2,14 @@ #include #include +#include namespace DB { class IBackup; class AccessRightsElements; +class ASTCreateQuery; class DDLRenamingMap; struct QualifiedTableName; @@ -28,6 +30,21 @@ bool compareRestoredDatabaseDef(const IAST & restored_database_create_query, con bool isInnerTable(const QualifiedTableName & table_name); bool isInnerTable(const String & database_name, const String & table_name); +/// A table on a storage policy marked `backup_metadata_only` gets its data from the storage itself, e.g. a shared +/// read-only dataset: `BACKUP` stores only the definition of the table, and `RESTORE` recreates the table with its UUID +/// from the backup, because the data is on the storage under a path derived from the UUID, and doesn't restore data. + +/// Whether `storage` keeps its data on a storage policy marked `backup_metadata_only`. Only for the plain `MergeTree` family: +/// for a replicated or shared `MergeTree` table, or a view with an inner table on such a policy, it warns and returns false. +bool isDataOnBackupMetadataOnlyStoragePolicy(const StoragePtr & storage, const ContextPtr & context); + +/// The same for a table that may not exist on this host: a `Replicated` database gives only the definition of a table +/// not created on this replica yet, and then the definition decides. +bool isDataOnBackupMetadataOnlyStoragePolicy(const StoragePtr & storage, const ASTPtr & create_table_query, const ContextPtr & context); + +/// Whether a table created by `create_query` is on a storage policy marked `backup_metadata_only`. +bool isCreatedOnBackupMetadataOnlyStoragePolicy(const ASTCreateQuery & create_query, const ContextPtr & context); + } } diff --git a/src/Backups/RestorerFromBackup.cpp b/src/Backups/RestorerFromBackup.cpp index 56e63dee650f6..a047b66a4ff59 100644 --- a/src/Backups/RestorerFromBackup.cpp +++ b/src/Backups/RestorerFromBackup.cpp @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -331,7 +332,17 @@ void RestorerFromBackup::checkAccessForObjectsFoundInBackup() const continue; } - if (table_name.database == DatabaseCatalog::TEMPORARY_DATABASE) + const ASTCreateQuery & create = table_info.create_table_query->as(); + const bool is_temporary = (table_name.database == DatabaseCatalog::TEMPORARY_DATABASE); + + /// The definition comes from the backup, and the table is created by an internal query that checks + /// nothing, so settings that send the data elsewhere are authorized here, as for `CREATE`. + /// An external engine replaced by `Null` sends nothing, but temporary tables keep their engine. + if (restore_settings.create_table != RestoreTableCreationMode::kMustExist + && (is_temporary || !context->getSettingsRef()[Setting::restore_replace_external_engines_to_null])) + required_access.append_range(StorageFactory::instance().getExportAccess(create)); + + if (is_temporary) { if (restore_settings.create_table != RestoreTableCreationMode::kMustExist) required_access.emplace_back(AccessType::CREATE_TEMPORARY_TABLE); @@ -339,7 +350,6 @@ void RestorerFromBackup::checkAccessForObjectsFoundInBackup() const } AccessFlags flags; - const ASTCreateQuery & create = table_info.create_table_query->as(); if (restore_settings.create_table != RestoreTableCreationMode::kMustExist) { @@ -814,9 +824,32 @@ void RestorerFromBackup::createTable(const QualifiedTableName & table_name) "as a mount, or restore the selected partitions without mount_readonly_parts_from_snapshot."); #endif - /// Generate a new UUID for a table (the same table on different hosts must use the same UUID, `restore_coordination` will make it so). - /// The generated UUID will be ignored if the database does not support UUIDs. - restore_coordination->generateUUIDForTable(*create_table_query); + /// Without the data the table has nothing to find on the storage, so it is restored as an ordinary empty table, + /// with a new UUID: keeping the UUID would show all the data of the storage in a `structure_only` restore. + if (create_table_query->uuid != UUIDHelpers::Nil && restore_settings.shouldRestoreTableData() + && BackupUtils::isCreatedOnBackupMetadataOnlyStoragePolicy(*create_table_query, context)) + { + /// The table finds its data on the storage under a path derived from its UUID, so it keeps its UUID from the backup. + /// The other table can be in any database, so its name is not shown: the user may not be allowed to see it. + auto existing_table = DatabaseCatalog::instance().tryGetByUUID(create_table_query->uuid).second; + if (existing_table) + { + auto existing_table_id = existing_table->getStorageID(); + if (existing_table_id.database_name != table_name.database || existing_table_id.table_name != table_name.table) + throw Exception( + ErrorCodes::CANNOT_RESTORE_TABLE, + "Cannot restore {}: it is on a storage policy marked backup_metadata_only, so it keeps its UUID {} from " + "the backup to find its data on the storage, but another table already has this UUID", + tableNameWithTypeToString(table_name.database, table_name.table, false), + toString(create_table_query->uuid)); + } + } + else + { + /// Generate a new UUID for a table (the same table on different hosts must use the same UUID, `restore_coordination` will make it so). + /// The generated UUID will be ignored if the database does not support UUIDs. + restore_coordination->generateUUIDForTable(*create_table_query); + } /// Add the clause `IF NOT EXISTS` if that is specified in the restore settings. create_table_query->if_not_exists = (restore_settings.create_table == RestoreTableCreationMode::kCreateIfNotExists); @@ -991,12 +1024,20 @@ void RestorerFromBackup::insertDataToTable(const QualifiedTableName & table_name StoragePtr storage; String data_path_in_backup; std::optional partitions; + ASTPtr create_table_query_from_backup; { std::lock_guard lock{mutex}; auto & table_info = table_infos.at(table_name); storage = table_info.storage; data_path_in_backup = table_info.data_path_in_backup; partitions = table_info.partitions; + create_table_query_from_backup = table_info.create_table_query; + } + + if (BackupUtils::isDataOnBackupMetadataOnlyStoragePolicy(storage, context)) + { + skipTableDataOnBackupMetadataOnlyStoragePolicy(table_name, storage, partitions, create_table_query_from_backup); + return; } schedule( @@ -1004,6 +1045,46 @@ void RestorerFromBackup::insertDataToTable(const QualifiedTableName & table_name ThreadName::RESTORE_TABLE_DATA); } +void RestorerFromBackup::skipTableDataOnBackupMetadataOnlyStoragePolicy( + const QualifiedTableName & table_name, const StoragePtr & storage, const std::optional & partitions, const ASTPtr & create_table_query_from_backup) const +{ + if (partitions) + throw Exception( + ErrorCodes::CANNOT_RESTORE_TABLE, + "Cannot restore specific partitions of {}: it is on a storage policy marked backup_metadata_only, so it shows " + "all the data of its storage and the partition filter cannot be applied", + tableNameWithTypeToString(table_name.database, table_name.table, false)); + + /// The table must have its UUID from the backup, under which the storage keeps the data. This catches what `createTable` + /// doesn't, e.g. an existing table with `create_table = 'must-exist'`, or a database without UUIDs. + const auto * create_query = create_table_query_from_backup ? create_table_query_from_backup->as() : nullptr; + if (!create_query) + throw Exception(ErrorCodes::LOGICAL_ERROR, "The backup has no definition of {}", tableNameWithTypeToString(table_name.database, table_name.table, false)); + + const UUID uuid = storage->getStorageID().uuid; + const UUID uuid_in_backup = create_query->uuid; + if (uuid == UUIDHelpers::Nil || uuid_in_backup == UUIDHelpers::Nil) + throw Exception( + ErrorCodes::CANNOT_RESTORE_TABLE, + "Cannot restore {}: it is on a storage policy marked backup_metadata_only, so its data is not restored and it " + "must have its UUID from the backup to find the data on the storage, but {}", + tableNameWithTypeToString(table_name.database, table_name.table, false), + uuid == UUIDHelpers::Nil ? "its database doesn't give tables UUIDs" : "the backup records no UUID for it"); + + if (uuid != uuid_in_backup) + throw Exception( + ErrorCodes::CANNOT_RESTORE_TABLE, + "Cannot restore {}: it is on a storage policy marked backup_metadata_only, so its data is not restored and it " + "must have its UUID from the backup to find the data on the storage, but the table with UUID {} in the backup " + "has UUID {}", + tableNameWithTypeToString(table_name.database, table_name.table, false), + toString(uuid_in_backup), + toString(uuid)); + + LOG_INFO(log, "Not restoring the data of {}: it is on a storage policy marked backup_metadata_only", + tableNameWithTypeToString(table_name.database, table_name.table, false)); +} + void RestorerFromBackup::insertDataToTableImpl(const QualifiedTableName & table_name, StoragePtr storage, const String & data_path_in_backup, const std::optional & partitions) { try diff --git a/src/Backups/RestorerFromBackup.h b/src/Backups/RestorerFromBackup.h index fa754a049a1d7..aad3d2eb4232e 100644 --- a/src/Backups/RestorerFromBackup.h +++ b/src/Backups/RestorerFromBackup.h @@ -107,6 +107,10 @@ class RestorerFromBackup : public BackupMetadataFinder void insertDataToTable(const QualifiedTableName & table_name); void insertDataToTableImpl(const QualifiedTableName & table_name, StoragePtr storage, const String & data_path_in_backup, const std::optional & partitions); + /// The data of a table on a storage policy marked `backup_metadata_only` is not restored, see `BackupUtils`. + void skipTableDataOnBackupMetadataOnlyStoragePolicy( + const QualifiedTableName & table_name, const StoragePtr & storage, const std::optional & partitions, const ASTPtr & create_table_query_from_backup) const; + void runDataRestoreTasks(); void finalizeTables(); diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f47f4f2c2bce5..e0e76478534e0 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -511,6 +511,10 @@ if (USE_ARROW) add_object_library(clickhouse_processors_formats_impl_arrow_ipc Processors/Formats/Impl/ArrowIPC) endif() +if (USE_ORC) + add_object_library(clickhouse_processors_formats_impl_orc Processors/Formats/Impl/ORC) +endif() + if (TARGET ch_contrib::nuraft) add_object_library(clickhouse_coordination Coordination) add_object_library(clickhouse_coordination_storage Coordination/Storage) @@ -824,11 +828,6 @@ endif() target_link_libraries (clickhouse_common_io PUBLIC ch_contrib::fast_float) -if (USE_ORC) - target_link_libraries(dbms PUBLIC ${ORC_LIBRARIES}) - target_include_directories(dbms SYSTEM BEFORE PUBLIC ${ORC_INCLUDE_DIR} "${PROJECT_BINARY_DIR}/contrib/orc/c++/include") -endif () - if (TARGET ch_contrib::rocksdb) target_link_libraries(dbms PUBLIC ch_contrib::rocksdb) endif() @@ -978,6 +977,10 @@ if (ENABLE_TESTS) target_link_libraries(unit_tests_dbms PRIVATE ch_contrib::krb5) endif() + if (TARGET ch_contrib::brotli) + target_link_libraries(unit_tests_dbms PRIVATE ch_contrib::brotli) + endif() + if (TARGET ch_contrib::simdjson) target_link_libraries(unit_tests_dbms PRIVATE ch_contrib::simdjson) endif() diff --git a/src/Client/BuzzHouse/Generator/TableSettings.cpp b/src/Client/BuzzHouse/Generator/TableSettings.cpp index de0bd32094c6e..d1aa71b132b6d 100644 --- a/src/Client/BuzzHouse/Generator/TableSettings.cpp +++ b/src/Client/BuzzHouse/Generator/TableSettings.cpp @@ -295,7 +295,7 @@ static std::unordered_map mergeTreeTableSettings = { {"0", "1", "5", "25"}, false)}, {"max_replicated_fetches_network_bandwidth", bytesRangeSetting}, - {"max_replicated_logs_to_keep", highRangeSetting}, + {"max_replicated_logs_to_keep", highRangeNonZeroSetting}, {"max_replicated_merges_in_queue", CHSetting( [](RandomGenerator & rg, FuzzConfig &) { return std::to_string(rg.thresholdGenerator(0.2, 0.2, 0, 100)); }, diff --git a/src/Client/ClientBase.cpp b/src/Client/ClientBase.cpp index 5ab920a0665ff..925b73cb464a0 100644 --- a/src/Client/ClientBase.cpp +++ b/src/Client/ClientBase.cpp @@ -4763,13 +4763,13 @@ void ClientBase::runInteractive() #if USE_REPLXX - replxx::Replxx::highlighter_callback_with_pos_t highlight_callback{}; + ReplxxLineReader::Highlighter highlight_callback{}; if (ConfigHelper::getBool(getClientConfiguration(), "highlight", true)) { - highlight_callback = [this](const String & query, std::vector & colors, int pos) + highlight_callback = [this](const String & query, std::vector & colors, int pos, bool & last_is_delimiter) { - highlight(query, colors, *client_context, pos, rainbow_parentheses); + highlight(query, colors, *client_context, pos, rainbow_parentheses, &last_is_delimiter); }; } diff --git a/src/Client/ClientBaseHelpers.cpp b/src/Client/ClientBaseHelpers.cpp index 6b485bab3c582..1d9fd59c18080 100644 --- a/src/Client/ClientBaseHelpers.cpp +++ b/src/Client/ClientBaseHelpers.cpp @@ -247,7 +247,7 @@ static void highlightWithLexer(const String & query, std::vector & colors, const Context & context, int cursor_position, bool rainbow_parentheses, bool lexer_fallback) +void highlight(const String & query, std::vector & colors, const Context & context, int cursor_position, bool rainbow_parentheses, bool * last_is_delimiter, bool lexer_fallback) { using namespace replxx; @@ -602,14 +602,16 @@ void highlight(const String & query, std::vector & colors } /// This is a callback for the client/local app to better find query end. Note: this is a kludge, remove it. + if (!last_is_delimiter) + return; if (last_token.type == TokenType::Semicolon || last_token.type == TokenType::VerticalDelimiter || query.ends_with(';') || query.ends_with("\\G")) /// This is for raw data in INSERT queries, which is not necessarily tokenized. { - ReplxxLineReader::setLastIsDelimiter(true); + *last_is_delimiter = true; } else if (last_token.type != TokenType::Whitespace) { - ReplxxLineReader::setLastIsDelimiter(false); + *last_is_delimiter = false; } } @@ -618,7 +620,7 @@ String highlighted(const String & query, const Context & context, bool rainbow_p const size_t num_code_points = countCodePointsWithSeqLength(query, query.data() + query.size()); std::vector colors(num_code_points, replxx::Replxx::Color::DEFAULT); - highlight(query, colors, context, 0, rainbow_parentheses, lexer_fallback); + highlight(query, colors, context, 0, rainbow_parentheses, nullptr, lexer_fallback); String res; size_t query_size = query.size(); diff --git a/src/Client/ClientBaseHelpers.h b/src/Client/ClientBaseHelpers.h index d106413dee0de..97191753a4cfe 100644 --- a/src/Client/ClientBaseHelpers.h +++ b/src/Client/ClientBaseHelpers.h @@ -27,7 +27,9 @@ bool isCloudEndpoint(const std::string & host); /// When `lexer_fallback` is set, input that cannot be parsed as complete queries (e.g. the query /// fragments found in the documentation) is highlighted token-by-token using the lexer instead of being /// left unhighlighted. The line editor keeps the default (parser-only) behaviour. -void highlight(const String & query, std::vector & colors, const Context & context, int cursor_position, bool rainbow_parentheses, bool lexer_fallback = false); +/// If `last_is_delimiter` is not null, it is set to whether the query ends with a delimiter (and left +/// intact if the query ends with whitespace). +void highlight(const String & query, std::vector & colors, const Context & context, int cursor_position, bool rainbow_parentheses, bool * last_is_delimiter, bool lexer_fallback = false); String highlighted(const String & query, const Context & context, bool rainbow_parentheses, bool lexer_fallback = false); #endif diff --git a/src/Client/ReplxxLineReader.cpp b/src/Client/ReplxxLineReader.cpp index 9ef83c3974eac..6dd54968f79da 100644 --- a/src/Client/ReplxxLineReader.cpp +++ b/src/Client/ReplxxLineReader.cpp @@ -328,12 +328,6 @@ void convertHistoryFile(const std::string & path, replxx::Replxx & rx) namespace DB { -static bool replxx_last_is_delimiter = false; -void ReplxxLineReader::setLastIsDelimiter(bool flag) -{ - replxx_last_is_delimiter = flag; -} - ReplxxLineReader::ReplxxLineReader(ReplxxLineReader::Options && options) : LineReader ( @@ -385,8 +379,8 @@ ReplxxLineReader::ReplxxLineReader(ReplxxLineReader::Options && options) } } - /// The handler of `SIGWINCH` is process-wide. In the embedded mode the process is the server, - /// and the pseudoterminal is not its controlling terminal, so the signal is never delivered. + /// Signals are process-wide: an embedded client must not take over `SIGWINCH` of the server. + /// Its terminal size is updated by the server, and replxx queries it on every refresh. if (!options.embedded_mode) rx.install_window_change_handler(); @@ -417,7 +411,12 @@ ReplxxLineReader::ReplxxLineReader(ReplxxLineReader::Options && options) rx.set_indent_multiline(false); if (highlighter) - rx.set_highlighter_callback(highlighter); + { + rx.set_highlighter_callback([this](const String & query, std::vector & colors, int cursor_position) + { + highlighter(query, colors, cursor_position, last_is_delimiter); + }); + } /// As-you-type autocompletion: show the matching suggestions as inline "ghost" hints, with /// the same priority ordering as Tab completion. replxx renders a single hint inline after @@ -564,9 +563,9 @@ ReplxxLineReader::ReplxxLineReader(ReplxxLineReader::Options && options) /// a newline into the edit buffer and does not use the lexer, so the paste stays under a /// single prompt even with `--highlight 0`. The explicit `--multiline` continuation still /// requires the highlighter, preserving the previous behavior. - if (!replxx_last_is_delimiter && ((highlighter && multiline) || hasInputData())) + if (!last_is_delimiter && ((highlighter && multiline) || hasInputData())) return rx.invoke(Replxx::ACTION::NEW_LINE, code); - replxx_last_is_delimiter = false; + last_is_delimiter = false; return rx.invoke(Replxx::ACTION::COMMIT_LINE, code); }; /// bind C-j to ENTER action. diff --git a/src/Client/ReplxxLineReader.h b/src/Client/ReplxxLineReader.h index f7813514374da..1603acf773e6d 100644 --- a/src/Client/ReplxxLineReader.h +++ b/src/Client/ReplxxLineReader.h @@ -12,6 +12,11 @@ namespace DB class ReplxxLineReader : public LineReader { public: + /// The highlighter callback of replxx, which also reports whether the last token of the + /// query is a delimiter: it sets `last_is_delimiter` to true or false, or leaves it intact + /// when this cannot be determined (e.g. the query ends with whitespace). + using Highlighter = std::function & colors, int cursor_position, bool & last_is_delimiter)>; + struct Options { Suggest & suggest; @@ -26,7 +31,7 @@ class ReplxxLineReader : public LineReader Patterns extenders; Patterns delimiters; std::span word_break_characters; - replxx::Replxx::highlighter_callback_with_pos_t highlighter; + Highlighter highlighter; std::istream & input_stream = std::cin; std::ostream & output_stream = std::cout; int in_fd = STDIN_FILENO; @@ -40,10 +45,6 @@ class ReplxxLineReader : public LineReader void enableBracketedPaste() override; void disableBracketedPaste() override; - /// If highlight is on, we will set a flag to denote whether the last token is a delimiter. - /// This is useful to determine the behavior of key when multiline is enabled. - static void setLastIsDelimiter(bool flag); - /// Set text to be prepopulated in the next readLine call void setInitialText(const String & text) override; private: @@ -76,7 +77,13 @@ class ReplxxLineReader : public LineReader bool hintChosen(); replxx::Replxx rx; - replxx::Replxx::highlighter_callback_with_pos_t highlighter; + Highlighter highlighter; + + /// If highlight is on, the highlighter reports whether the last token is a delimiter. + /// This is used to determine the behavior of key when multiline is enabled. + /// It is a member rather than a global, because several readers can work concurrently + /// in one process (client sessions embedded into the server). + bool last_is_delimiter = false; /// Suggestion dictionary; used both for Tab completion and as-you-type hints, and updated /// with identifiers from committed queries to prioritize recently-used words. diff --git a/src/Columns/ColumnArray.cpp b/src/Columns/ColumnArray.cpp index 983e8647d420e..17bf48ecdbd20 100644 --- a/src/Columns/ColumnArray.cpp +++ b/src/Columns/ColumnArray.cpp @@ -1015,17 +1015,17 @@ ColumnPtr ColumnArray::filterGeneric(const Filter & filt, ssize_t result_size_hi memset(&nested_filt[offsetAt(i)], 0, sizeAt(i)); } - auto res = ColumnArray::create(data->cloneEmpty()); - ssize_t nested_result_size_hint = 0; if (result_size_hint < 0) nested_result_size_hint = result_size_hint; else if (result_size_hint && result_size_hint < 1000000000 && data->size() < 1000000000) /// Avoid overflow. nested_result_size_hint = result_size_hint * data->size() / size; - res->data = data->filter(nested_filt, nested_result_size_hint); + /// See the comment in `indexImpl` about building the nested result first. + ColumnPtr res_data = data->filter(nested_filt, nested_result_size_hint); - Offsets & res_offsets = res->getOffsets(); + auto res_offsets_column = ColumnOffsets::create(); + Offsets & res_offsets = res_offsets_column->getData(); if (result_size_hint) res_offsets.reserve_exact(result_size_hint > 0 ? result_size_hint : size); @@ -1039,7 +1039,7 @@ ColumnPtr ColumnArray::filterGeneric(const Filter & filt, ssize_t result_size_hi } } - return res; + return ColumnArray::create(res_data, std::move(res_offsets_column)); } ColumnPtr ColumnArray::filterNullable(const Filter & filt, ssize_t result_size_hint) const @@ -1273,10 +1273,8 @@ ColumnPtr ColumnArray::indexImpl(const PaddedPODArray & indexes, size_t limit PaddedPODArray & nested_indexes = nested_indexes_column->getData(); nested_indexes.reserve_exact(getOffsets().back()); - auto res = ColumnArray::create(data->cloneEmpty()); - - Offsets & res_offsets = res->getOffsets(); - res_offsets.resize_exact(limit); + auto res_offsets_column = ColumnOffsets::create(limit); + Offsets & res_offsets = res_offsets_column->getData(); size_t current_offset = 0; for (size_t i = 0; i < limit; ++i) @@ -1287,10 +1285,10 @@ ColumnPtr ColumnArray::indexImpl(const PaddedPODArray & indexes, size_t limit res_offsets[i] = current_offset; } - if (current_offset != 0) - res->data = data->index(*nested_indexes_column, current_offset); - - return res; + /// Build the nested result first: an empty clone of `data` copies its whole nested structure, which for + /// `Dynamic` is as deep as the data, and every level of this recursion would hold one. + ColumnPtr res_data = current_offset != 0 ? data->index(*nested_indexes_column, current_offset) : data->cloneEmpty(); + return ColumnArray::create(res_data, std::move(res_offsets_column)); } INSTANTIATE_INDEX_IMPL(ColumnArray) diff --git a/src/Columns/ColumnDynamic.cpp b/src/Columns/ColumnDynamic.cpp index 073730faf6037..6937da418c568 100644 --- a/src/Columns/ColumnDynamic.cpp +++ b/src/Columns/ColumnDynamic.cpp @@ -338,6 +338,8 @@ void ColumnDynamic::get(size_t n, Field & res) const void ColumnDynamic::getValueNameImpl(WriteBufferFromOwnString & name_buf, size_t n, const Options & options) const { + checkStackSize(); + const auto & variant_col = getVariantColumn(); const auto discr = variant_col.globalDiscriminatorAt(n); if (discr == ColumnVariant::NULL_DISCRIMINATOR) @@ -374,6 +376,8 @@ void ColumnDynamic::insertFrom(const IColumn & src_, size_t n) void ColumnDynamic::doInsertFrom(const IColumn & src_, size_t n) #endif { + checkStackSize(); + const auto & dynamic_src = assert_cast(src_); /// Same variants: copy the Variant column directly. Skip this fast path when the source has values in @@ -453,6 +457,8 @@ void ColumnDynamic::insertRangeFrom(const IColumn & src_, size_t start, size_t l void ColumnDynamic::doInsertRangeFrom(const IColumn & src_, size_t start, size_t length) #endif { + checkStackSize(); + if (start + length > src_.size()) throw Exception(ErrorCodes::PARAMETER_OUT_OF_BOUND, "Parameter out of bound in ColumnDynamic::insertRangeFrom method. " "[start({}) + length({}) > src.size()({})]", start, length, src_.size()); @@ -685,6 +691,8 @@ void ColumnDynamic::insertManyFrom(const IColumn & src_, size_t position, size_t void ColumnDynamic::doInsertManyFrom(const IColumn & src_, size_t position, size_t length) #endif { + checkStackSize(); + const auto & dynamic_src = assert_cast(src_); auto & variant_col = getVariantColumn(); @@ -900,6 +908,8 @@ void ColumnDynamic::skipSerializedInArena(ReadBuffer & in) const void ColumnDynamic::updateHashWithValue(size_t n, SipHash & hash) const { + checkStackSize(); + const auto & variant_col = getVariantColumn(); auto discr = variant_col.globalDiscriminatorAt(n); if (discr == ColumnVariant::NULL_DISCRIMINATOR) @@ -930,6 +940,7 @@ void ColumnDynamic::updateHashWithValue(size_t n, SipHash & hash) const void ColumnDynamic::updateHashWithValueRange(size_t begin, size_t end, SipHash & hash) const { + checkStackSize(); variant_column_ptr->updateHashWithValueRange(begin, end, hash); } @@ -980,6 +991,8 @@ int ColumnDynamic::compareAt(size_t n, size_t m, const IColumn & rhs, int nan_di int ColumnDynamic::doCompareAt(size_t n, size_t m, const IColumn & rhs, int nan_direction_hint) const #endif { + checkStackSize(); + const auto & left_variant = getVariantColumn(); const auto & right_dynamic = assert_cast(rhs); const auto & right_variant = right_dynamic.getVariantColumn(); @@ -1106,6 +1119,7 @@ void ColumnDynamic::updatePermutation(IColumn::PermutationSortDirection directio ColumnPtr ColumnDynamic::compress(bool force_compression) const { + checkStackSize(); ColumnPtr variant_compressed = variant_column_ptr->compress(force_compression); size_t byte_size = variant_compressed->byteSize(); return ColumnCompressed::create(size(), byte_size, @@ -1117,6 +1131,7 @@ ColumnPtr ColumnDynamic::compress(bool force_compression) const ColumnCheckpointPtr ColumnDynamic::getCheckpoint() const { + checkStackSize(); UnorderedMapWithMemoryTracking variants_checkpoints; for (const auto & [name, discr] : variant_info.variant_name_to_discriminator) variants_checkpoints[name] = variant_column_ptr->getVariantByGlobalDiscriminator(discr).getCheckpoint(); @@ -1125,6 +1140,7 @@ ColumnCheckpointPtr ColumnDynamic::getCheckpoint() const void ColumnDynamic::updateCheckpoint(ColumnCheckpoint & checkpoint) const { + checkStackSize(); auto & variants_checkpoints = assert_cast(checkpoint).variants_checkpoints; for (const auto & [name, discr] : variant_info.variant_name_to_discriminator) @@ -1142,6 +1158,7 @@ void ColumnDynamic::updateCheckpoint(ColumnCheckpoint & checkpoint) const void ColumnDynamic::rollback(const ColumnCheckpoint & checkpoint) { + checkStackSize(); const auto & variants_checkpoints = assert_cast(checkpoint).variants_checkpoints; /// Manually rollback internals of Variant column @@ -1214,6 +1231,7 @@ void ColumnDynamic::getAllTypeNamesInto(UnorderedSetWithMemoryTracking & void ColumnDynamic::prepareForSquashing(const VectorWithMemoryTracking & source_columns, size_t factor) { + checkStackSize(); if (source_columns.empty()) return; @@ -1237,6 +1255,7 @@ void ColumnDynamic::prepareForSquashing(const VectorWithMemoryTracking & source_columns, size_t factor) { + checkStackSize(); /// Internal variants of source dynamic columns may differ. /// We want to preallocate memory for all variants we will have after squashing. /// It may happen that the total number of variants in source columns will @@ -1341,6 +1360,8 @@ void ColumnDynamic::prepareVariantsForSquashing(const VectorWithMemoryTracking(&rhs)) return max_dynamic_types == rhs_concrete->max_dynamic_types && global_max_dynamic_types == rhs_concrete->global_max_dynamic_types && variant_info.variant_name == rhs_concrete->variant_info.variant_name @@ -1350,6 +1371,8 @@ bool ColumnDynamic::dynamicStructureEquals(const IColumn & rhs) const void ColumnDynamic::chooseDynamicStructureForMerge(const VectorWithMemoryTracking & source_columns, std::optional max_dynamic_subcolumns) { + checkStackSize(); + if (!empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "chooseDynamicStructureForMerge should be called only on empty Dynamic column"); @@ -1480,6 +1503,8 @@ void ColumnDynamic::chooseDynamicStructureForMerge(const VectorWithMemoryTrackin void ColumnDynamic::takeExactDynamicStructureFrom(const IColumn & source) { + checkStackSize(); + if (!empty()) throw Exception(ErrorCodes::LOGICAL_ERROR, "takeExactDynamicStructureFrom should be called only on empty Dynamic column"); @@ -1502,6 +1527,8 @@ void ColumnDynamic::takeExactDynamicStructureFrom(const IColumn & source) void ColumnDynamic::fixDynamicStructure() { + checkStackSize(); + /// Reduce max_dynamic_types to the number of selected variants, so there will be no possibility /// to extend selected variants on inserts into this column. /// -1 because we don't count shared variant in the limit. @@ -1536,6 +1563,8 @@ ColumnDynamic::StatisticsPtr ColumnDynamic::getOrCalculateStatistics() const void ColumnDynamic::takeOrCalculateStatisticsFrom(const VectorWithMemoryTracking & source_columns) { + checkStackSize(); + /// Assumes dynamic structure has already been set by `takeExactDynamicStructureFrom` or `chooseDynamicStructureForMerge`. Statistics new_statistics; /// Collect total sizes for variants that are not in our structure (candidates for shared variant statistics). diff --git a/src/Columns/ColumnDynamic.h b/src/Columns/ColumnDynamic.h index 66979244f0e63..118561cf921d0 100644 --- a/src/Columns/ColumnDynamic.h +++ b/src/Columns/ColumnDynamic.h @@ -7,6 +7,7 @@ #include #include #include +#include namespace DB @@ -24,6 +25,10 @@ namespace DB * When the limit on number of dynamic types is exceeded, all values * with new types are inserted into special shared variant with type String * that contains values and their types in binary format. + * + * Values nest into each other through variants like Array(Dynamic) to a depth that comes + * from the data, not from the declared type, so methods that recurse into the Variant + * column call checkStackSize. */ class ColumnDynamic final : public COWHelper, ColumnDynamic> { @@ -116,12 +121,14 @@ class ColumnDynamic final : public COWHelper, Colum MutableColumnPtr cloneEmpty() const override { + checkStackSize(); /// Keep current dynamic structure return Base::create(variant_column->cloneEmpty(), variant_info, max_dynamic_types, global_max_dynamic_types, statistics); } MutableColumnPtr cloneResized(size_t size) const override { + checkStackSize(); return Base::create(variant_column->cloneResized(size), variant_info, max_dynamic_types, global_max_dynamic_types, statistics); } @@ -181,6 +188,7 @@ class ColumnDynamic final : public COWHelper, Colum void popBack(size_t n) override { + checkStackSize(); variant_column_ptr->popBack(n); } @@ -207,42 +215,50 @@ class ColumnDynamic final : public COWHelper, Colum void updateHashFast(SipHash & hash) const override { + checkStackSize(); variant_column_ptr->updateHashFast(hash); } ColumnPtr filter(const Filter & filt, ssize_t result_size_hint) const override { + checkStackSize(); return create(variant_column_ptr->filter(filt, result_size_hint), variant_info, max_dynamic_types, global_max_dynamic_types, statistics); } void filter(const Filter & filt) override { + checkStackSize(); variant_column_ptr->filter(filt); statistics.reset(); } void expand(const Filter & mask, bool inverted) override { + checkStackSize(); variant_column_ptr->expand(mask, inverted); } ColumnPtr permute(const Permutation & perm, size_t limit) const override { + checkStackSize(); return create(variant_column_ptr->permute(perm, limit), variant_info, max_dynamic_types, global_max_dynamic_types, statistics); } ColumnPtr index(const IColumn & indexes, size_t limit) const override { + checkStackSize(); return create(variant_column_ptr->index(indexes, limit), variant_info, max_dynamic_types, global_max_dynamic_types, statistics); } ColumnPtr replicate(const Offsets & replicate_offsets) const override { + checkStackSize(); return create(variant_column_ptr->replicate(replicate_offsets), variant_info, max_dynamic_types, global_max_dynamic_types, statistics); } VectorWithMemoryTracking scatter(size_t num_columns, const Selector & selector) const override { + checkStackSize(); VectorWithMemoryTracking scattered_variant_columns = variant_column_ptr->scatter(num_columns, selector); VectorWithMemoryTracking scattered_columns; scattered_columns.reserve(num_columns); @@ -266,6 +282,7 @@ class ColumnDynamic final : public COWHelper, Colum bool hasEqualValues() const override { + checkStackSize(); return variant_column_ptr->hasEqualValues(); } @@ -296,31 +313,37 @@ class ColumnDynamic final : public COWHelper, Colum void shrinkToFit() override { + checkStackSize(); variant_column_ptr->shrinkToFit(); } void ensureOwnership() override { + checkStackSize(); variant_column_ptr->ensureOwnership(); } size_t byteSize() const override { + checkStackSize(); return variant_column_ptr->byteSize(); } size_t byteSizeAt(size_t n) const override { + checkStackSize(); return variant_column_ptr->byteSizeAt(n); } size_t allocatedBytes() const override { + checkStackSize(); return variant_column_ptr->allocatedBytes(); } void protect() override { + checkStackSize(); variant_column_ptr->protect(); } @@ -332,14 +355,21 @@ class ColumnDynamic final : public COWHelper, Colum void forEachMutableSubcolumn(MutableColumnCallback callback) override { + /// Callers like IColumn::mutate recurse into the subcolumns through this method. + checkStackSize(); callback(variant_column); variant_column_ptr = assert_cast(variant_column.get()); } - void forEachSubcolumn(ColumnCallback callback) const override { callback(variant_column); } + void forEachSubcolumn(ColumnCallback callback) const override + { + checkStackSize(); + callback(variant_column); + } void forEachMutableSubcolumnRecursively(RecursiveMutableColumnCallback callback) override { + checkStackSize(); callback(*variant_column); variant_column_ptr = assert_cast(variant_column.get()); variant_column->forEachMutableSubcolumnRecursively(callback); @@ -347,6 +377,7 @@ class ColumnDynamic final : public COWHelper, Colum void forEachSubcolumnRecursively(RecursiveColumnCallback callback) const override { + checkStackSize(); callback(*variant_column); variant_column->forEachSubcolumnRecursively(callback); } @@ -377,11 +408,13 @@ class ColumnDynamic final : public COWHelper, Colum void finalize() override { + checkStackSize(); variant_column_ptr->finalize(); } bool isFinalized() const override { + checkStackSize(); return variant_column_ptr->isFinalized(); } diff --git a/src/Columns/ColumnLowCardinality.cpp b/src/Columns/ColumnLowCardinality.cpp index b40c289a53c69..febbfee4ac5fd 100644 --- a/src/Columns/ColumnLowCardinality.cpp +++ b/src/Columns/ColumnLowCardinality.cpp @@ -254,7 +254,7 @@ void ColumnLowCardinality::insertRangeFromFullColumn(const IColumn & src, size_t idx.insertIndexesRange(*inserted_indexes, 0, length); } -static void checkIndexesAreLimited(const IColumn & indexes, UInt64 limit) +void ColumnLowCardinality::checkIndexesAreLimited(const IColumn & indexes, UInt64 limit) { auto check_for_type = [&](auto type) { @@ -268,14 +268,14 @@ static void checkIndexesAreLimited(const IColumn & indexes, UInt64 limit) size_t num_rows = data.size(); if (num_rows == 0) return true; - UInt64 max_position = 0; + ColumnType max_position = 0; for (size_t i = 0; i < num_rows; ++i) - max_position = std::max(max_position, data[i]); + max_position = std::max(max_position, data[i]); - if (max_position >= limit) + if (static_cast(max_position) >= limit) throw Exception(ErrorCodes::INCORRECT_DATA, "Index for LowCardinality is out of range. Dictionary size is {}, " - "but found index with value {}", limit, max_position); + "but found index with value {}", limit, static_cast(max_position)); return true; }; diff --git a/src/Columns/ColumnLowCardinality.h b/src/Columns/ColumnLowCardinality.h index 701c9ce702968..f8f60b05e76f3 100644 --- a/src/Columns/ColumnLowCardinality.h +++ b/src/Columns/ColumnLowCardinality.h @@ -95,6 +95,7 @@ class ColumnLowCardinality final : public COWHelper +#include #include #include #include @@ -16,6 +17,12 @@ namespace DB namespace ErrorCodes { extern const int LOGICAL_ERROR; + extern const int MEMORY_LIMIT_EXCEEDED; +} + +namespace FailPoints +{ + extern const char reverse_index_build_index_throw[]; } template @@ -441,17 +448,25 @@ void ReverseIndex::buildIndex() if (!column) throw Exception(ErrorCodes::LOGICAL_ERROR, "ReverseIndex can't build index because index column wasn't set."); + /// Build everything into local variables and publish them only at the end, so that an exception + /// thrown midway (for example, `MEMORY_LIMIT_EXCEEDED` from an allocation in `calcHashes`) does not + /// leave a half-built index behind: a non-null `index` with a null `saved_hash` would make the next + /// `insert` dereference a null pointer. auto size = column->size(); - index = std::make_unique(size); + auto new_index = std::make_unique(size); + fiu_do_on(FailPoints::reverse_index_build_index_throw, + throw Exception(ErrorCodes::MEMORY_LIMIT_EXCEEDED, "Injected failure while building ReverseIndex");); + + ColumnUInt64::MutablePtr new_saved_hash; if constexpr (use_saved_hash) - saved_hash = calcHashes(); + new_saved_hash = calcHashes(); - auto & state = index->getState(); + auto & state = new_index->getState(); state.index_column = column; state.base_index = base_index; if constexpr (use_saved_hash) - state.saved_hash_column = &saved_hash->getData(); + state.saved_hash_column = &new_saved_hash->getData(); using IteratorType = typename IndexMapType::iterator; IteratorType iterator; @@ -461,15 +476,18 @@ void ReverseIndex::buildIndex() { UInt64 hash = 0; if constexpr (use_saved_hash) - hash = saved_hash->getElement(row); + hash = new_saved_hash->getElement(row); else hash = getHash(column->getDataAt(row)); - index->reverseIndexEmplace(row + base_index, iterator, inserted, hash, column->getDataAt(row)); + new_index->reverseIndexEmplace(row + base_index, iterator, inserted, hash, column->getDataAt(row)); if (!inserted) throw Exception(ErrorCodes::LOGICAL_ERROR, "Duplicating keys found in ReverseIndex."); } + + index = std::move(new_index); + saved_hash = std::move(new_saved_hash); } template @@ -493,37 +511,58 @@ UInt64 ReverseIndex::insert(std::string_view data) if (!index) buildIndex(); - using IteratorType = typename IndexMapType::iterator; - IteratorType iterator; - bool inserted = false; - - auto hash = getHash(data); - UInt64 num_rows = size(); - - if constexpr (use_saved_hash) + try { - auto & column_data = saved_hash->getData(); - if (column_data.size() <= num_rows) - column_data.resize(num_rows + 1); - column_data[num_rows] = hash; - } - else - column->insertData(data.data(), data.size()); + using IteratorType = typename IndexMapType::iterator; + IteratorType iterator; + bool inserted = false; - index->reverseIndexEmplace(num_rows + base_index, iterator, inserted, hash, data); + auto hash = getHash(data); + UInt64 num_rows = size(); - if constexpr (use_saved_hash) - { - if (inserted) + if constexpr (use_saved_hash) + { + auto & column_data = saved_hash->getData(); + if (column_data.size() <= num_rows) + column_data.resize(num_rows + 1); + column_data[num_rows] = hash; + } + else column->insertData(data.data(), data.size()); + + index->reverseIndexEmplace(num_rows + base_index, iterator, inserted, hash, data); + + if constexpr (use_saved_hash) + { + if (inserted) + column->insertData(data.data(), data.size()); + } + else + { + if (!inserted) + column->popBack(1); + } + + return iterator->getValue(); } - else + catch (...) { - if (!inserted) - column->popBack(1); - } + /// The hash table may now disagree with the column: for example, the key was already added to the + /// hash table, but `insertData` failed, so the key refers to a row that does not exist. Drop the index, + /// it is rebuilt from the column on the next use. + index = nullptr; + saved_hash = nullptr; - return iterator->getValue(); + if constexpr (std::is_same_v) + { + /// `ColumnString::insertData` appends to `chars` before it grows `offsets`. If the latter failed, the bytes + /// of the row that was never added are still there and would become a prefix of the next inserted value. + const auto & offsets = column->getOffsets(); + column->getChars().resize_assume_reserved(offsets.empty() ? 0 : offsets.back()); + } + + throw; + } } template diff --git a/src/Columns/tests/gtest_reverse_index_exception_safety.cpp b/src/Columns/tests/gtest_reverse_index_exception_safety.cpp new file mode 100644 index 0000000000000..59de5cc52968d --- /dev/null +++ b/src/Columns/tests/gtest_reverse_index_exception_safety.cpp @@ -0,0 +1,126 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace DB::ErrorCodes +{ + extern const int MEMORY_LIMIT_EXCEEDED; +} + +namespace DB::FailPoints +{ + extern const char reverse_index_build_index_throw[]; +} + +using namespace DB; + +/// Mimics what happens during an asynchronous insert flush under memory pressure: the first insert into a +/// `LowCardinality` dictionary throws `MEMORY_LIMIT_EXCEEDED` while the reverse index is being built, the error +/// is handled by the caller, and the same dictionary is reused for the next entry. The index must either be +/// fully built or absent, never a non-null hash table paired with a null saved-hash array. +TEST(ReverseIndex, ExceptionWhileBuildingIndexLeavesNoHalfBuiltIndex) +{ + auto column = ColumnUnique::create(DataTypeString()); + + FailPointInjection::enableFailPoint(FailPoints::reverse_index_build_index_throw); + try + { + column->uniqueInsertData("a", 1); + FAIL() << "Expected an exception from the failpoint"; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::MEMORY_LIMIT_EXCEEDED); + } + FailPointInjection::disableFailPoint(FailPoints::reverse_index_build_index_throw); + + /// Index 0 is the default value (empty string). + EXPECT_EQ(column->size(), 1u); + EXPECT_EQ(column->uniqueInsertData("a", 1), 1u); + EXPECT_EQ(column->uniqueInsertData("b", 1), 2u); + EXPECT_EQ(column->uniqueInsertData("a", 1), 1u); + EXPECT_EQ(column->uniqueInsertData("", 0), 0u); + EXPECT_EQ(column->size(), 3u); + EXPECT_EQ(column->getDataAt(1), "a"); + EXPECT_EQ(column->getDataAt(2), "b"); +} + +/// The same reuse after a failure inside `insert` itself: the key is already in the hash table and the string bytes are +/// already appended to `chars` when growing `offsets` fails. The dictionary must come back to the state before the +/// insert, so that the next inserted value does not get the failed one as a prefix and lookups stay correct. +TEST(ReverseIndex, ExceptionWhileInsertingDataKeepsDictionaryConsistent) +{ + MainThreadStatus::getInstance(); + auto & thread_tracker = CurrentThread::get().memory_tracker; + + auto column = ColumnUnique::create(DataTypeString()); + ASSERT_EQ(column->uniqueInsertData("a", 1), 1u); + + /// Leave `offsets` without spare capacity and `chars` with plenty, so that growing `offsets` is the only + /// allocation of the next insert. + auto & nested = assert_cast(column->getNestedColumn()->assumeMutableRef()); + nested.getOffsets().shrink_to_fit(); + nested.getChars().reserve(nested.getChars().size() + 1024); + + const Int64 saved_untracked_limit = CurrentThread::get().untracked_memory_limit; + const Int64 saved_thread_hard_limit = thread_tracker.getHardLimit(); + const Int64 saved_total_hard_limit = total_memory_tracker.getHardLimit(); + const UInt64 saved_min_alloc_to_throw = CurrentMemoryTracker::getMinAllocationSizeBytesToThrow(); + + auto lift_limits = [&] + { + total_memory_tracker.setHardLimit(saved_total_hard_limit); + thread_tracker.setHardLimit(saved_thread_hard_limit); + CurrentThread::get().untracked_memory_limit = saved_untracked_limit; + CurrentMemoryTracker::setMinAllocationSizeBytesToThrow(saved_min_alloc_to_throw); + }; + SCOPE_EXIT_SAFE(lift_limits()); + + /// Make even a tiny allocation reach the trackers and fail: no per-thread untracked buffer, limits of one byte. + CurrentMemoryTracker::setMinAllocationSizeBytesToThrow(1); + CurrentThread::get().untracked_memory_limit = 0; + CurrentThread::flushUntrackedMemory(); + total_memory_tracker.resetCounters(); + thread_tracker.resetCounters(); + total_memory_tracker.setHardLimit(1); + thread_tracker.setHardLimit(1); + + bool thrown = false; + int code = 0; + try + { + column->uniqueInsertData("bb", 2); + } + catch (const Exception & e) + { + thrown = true; + code = e.code(); + } + + /// Lift the limits before anything else, the assertions allocate. + lift_limits(); + + ASSERT_TRUE(thrown) << "the insert was expected to fail under the memory limit"; + EXPECT_EQ(code, ErrorCodes::MEMORY_LIMIT_EXCEEDED); + + EXPECT_EQ(column->size(), 2u); + EXPECT_EQ(nested.getChars().size(), nested.getOffsets().back()); + + EXPECT_EQ(column->uniqueInsertData("c", 1), 2u); + EXPECT_EQ(column->getDataAt(2), "c"); + EXPECT_EQ(column->uniqueInsertData("bb", 2), 3u); + EXPECT_EQ(column->getDataAt(3), "bb"); + EXPECT_EQ(column->uniqueInsertData("a", 1), 1u); + EXPECT_EQ(column->uniqueInsertData("c", 1), 2u); + EXPECT_EQ(column->size(), 4u); +} diff --git a/src/Common/ColumnsHashing.h b/src/Common/ColumnsHashing.h index a12369c22bfa8..f5f3a87317e76 100644 --- a/src/Common/ColumnsHashing.h +++ b/src/Common/ColumnsHashing.h @@ -528,7 +528,7 @@ struct HashMethodSerialized precomputed_hashes_initialized = true; calibration_row = first_row + PrefetchingHelper::iterationsToMeasure(); - if (min_bytes_for_prefetch != 0 && data.getBufferSizeInBytes() <= min_bytes_for_prefetch) + if (min_bytes_for_prefetch != 0 && bytesForPrefetch(data) <= min_bytes_for_prefetch) { can_precompute_hashes = false; return; diff --git a/src/Common/CurrentMetrics.cpp b/src/Common/CurrentMetrics.cpp index 8be37f19d9536..b3e428d1171fb 100644 --- a/src/Common/CurrentMetrics.cpp +++ b/src/Common/CurrentMetrics.cpp @@ -218,6 +218,9 @@ M(DistributedIndexAnalysisThreads, "Number of threads in the thread pool for distributed index analysis.") \ M(DistributedIndexAnalysisThreadsActive, "Number of threads in the thread pool for distributed index analysis running a task.") \ M(DistributedIndexAnalysisThreadsScheduled, "Number of queued or active jobs in the distributed index analysis thread pool.") \ + M(AIRequestThreads, "Number of threads in the AI request thread pool.") \ + M(AIRequestThreadsActive, "Number of threads in the AI request thread pool running a task.") \ + M(AIRequestThreadsScheduled, "Number of queued or active jobs in the AI request thread pool.") \ M(ObjectStorageQueueRegisteredServers, "Number of registered servers in StorageS3(Azure)Queue")\ M(IcebergCatalogThreads, "Number of threads in the IcebergCatalog thread pool.") \ M(IcebergCatalogThreadsActive, "Number of threads in the IcebergCatalog thread pool running a task.") \ diff --git a/src/Common/DNSResolver.cpp b/src/Common/DNSResolver.cpp index 8774eec4c105f..d589842e624bd 100644 --- a/src/Common/DNSResolver.cpp +++ b/src/Common/DNSResolver.cpp @@ -161,6 +161,12 @@ std::unordered_set reverseResolveImpl(const Poco::Net::IPAddress & addre { return ptr_resolver->resolve(address.toString()); } + if (address.isIPv4Mapped()) + { + /// An IPv4-mapped address is an IPv4 peer of a dual-stack socket, so its PTR record is the IPv4 one. + Poco::Net::IPAddress ipv4(reinterpret_cast(address.addr()) + 12, 4); + return ptr_resolver->resolve(ipv4.toString()); + } return ptr_resolver->resolve_v6(address.toString()); } diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index abeebe2da0a6f..91b870818faac 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -34,6 +34,7 @@ static struct InitFiu #define APPLY_FOR_FAILPOINTS(ONCE, REGULAR, PAUSEABLE_ONCE, PAUSEABLE) \ ONCE(replicated_merge_tree_commit_zk_fail_after_op) \ ONCE(replicated_queue_fail_next_entry) \ + ONCE(reverse_index_build_index_throw) \ REGULAR(replicated_queue_unfail_entries) \ ONCE(replicated_merge_tree_insert_quorum_fail_0) \ REGULAR(replicated_merge_tree_commit_zk_fail_when_recovering_from_hw_fault) \ @@ -159,6 +160,7 @@ static struct InitFiu PAUSEABLE_ONCE(query_metric_log_pause_before_finish) \ PAUSEABLE_ONCE(replicated_table_remove_zk_before_get_children) \ PAUSEABLE_ONCE(replicated_table_remove_zk_before_final_multi) \ + PAUSEABLE_ONCE(rmt_restore_replica_pause_before_metadata_version_raise) \ PAUSEABLE_ONCE(kafka2_remove_zk_before_get_children) \ PAUSEABLE_ONCE(kafka2_remove_zk_before_final_multi) \ PAUSEABLE_ONCE(keeper_map_delete_pause_before_multi) \ @@ -228,6 +230,7 @@ static struct InitFiu REGULAR(database_replicated_delay_recovery) \ REGULAR(database_replicated_delay_entry_execution) \ PAUSEABLE(database_replicated_stop_entry_execution) \ + PAUSEABLE_ONCE(database_replicated_pause_before_initial_query_execution) \ PAUSEABLE_ONCE(database_replicated_pause_after_reading_log_pointer) \ PAUSEABLE_ONCE(database_replicated_pause_after_snapshot_identity_check) \ REGULAR(remove_merge_tree_part_delay) \ @@ -242,6 +245,7 @@ static struct InitFiu REGULAR(slowdown_skip_index_read_result_build) \ ONCE(iceberg_writes_cleanup) \ REGULAR(iceberg_slow_manifest_read) \ + PAUSEABLE_ONCE(iceberg_metadata_prefetch_pause) \ REGULAR(storage_cluster_read_sleep) \ ONCE(backup_add_empty_memory_table) \ ONCE(backup_from_snapshot_fail_after_batch) \ diff --git a/src/Common/HashTable/Prefetching.h b/src/Common/HashTable/Prefetching.h index 6a609e1444464..e887d8a4d57eb 100644 --- a/src/Common/HashTable/Prefetching.h +++ b/src/Common/HashTable/Prefetching.h @@ -52,4 +52,18 @@ class PrefetchingHelper Stopwatch watch; }; +/// The size compared with `min_bytes_for_prefetch`. A two-level table allocates all its buckets up front, so until one +/// grows its buffer is a constant; count its keys instead, as the smallest table that holds them at load factor 1/2. +template +size_t bytesForPrefetch(const Table & table) +{ + const size_t buffer_bytes = table.getBufferSizeInBytes(); + if constexpr (requires { Table::NUM_BUCKETS; Table::Impl::initial_buffer_bytes; }) + { + if (buffer_bytes == Table::NUM_BUCKETS * Table::Impl::initial_buffer_bytes) + return std::min(buffer_bytes, 2 * table.size() * sizeof(typename Table::cell_type)); + } + return buffer_bytes; +} + } diff --git a/src/Common/HashTable/StringHashTable.h b/src/Common/HashTable/StringHashTable.h index ec1506898e940..05a933c2f05ce 100644 --- a/src/Common/HashTable/StringHashTable.h +++ b/src/Common/HashTable/StringHashTable.h @@ -200,6 +200,7 @@ struct StringHashTableEmpty // NOLINT(cppcoreguidelines-pro-type-member-init,hic void readText(DB::ReadBuffer & rb) { zeroValue()->readText(rb); } size_t size() const { return hasZero() ? 1 : 0; } bool empty() const { return !hasZero(); } + static constexpr size_t initial_buffer_bytes = sizeof(Cell); size_t getBufferSizeInBytes() const { return sizeof(Cell); } size_t getCollisions() const { return 0; } }; @@ -296,6 +297,10 @@ class StringHashTable : private boost::noncopyable using value_type = typename Ts::value_type; using cell_type = typename Ts::cell_type; + /// What `getBufferSizeInBytes` returns before any submap grows. + static constexpr size_t initial_buffer_bytes = T0::initial_buffer_bytes + T1::initial_buffer_bytes + + T2::initial_buffer_bytes + T3::initial_buffer_bytes + Ts::initial_buffer_bytes; + using LookupResult = StringHashTableLookupResult; using ConstLookupResult = StringHashTableLookupResult; diff --git a/src/Common/ProfileEvents.cpp b/src/Common/ProfileEvents.cpp index 3a17b07f8f6a7..76ff1b08cd0a7 100644 --- a/src/Common/ProfileEvents.cpp +++ b/src/Common/ProfileEvents.cpp @@ -1663,6 +1663,9 @@ The server successfully detected this situation and will download merged part fr M(AIInputTokens, "Total prompt tokens consumed across all AI function calls in the query.", ValueType::Number) \ M(AIOutputTokens, "Total completion tokens consumed across all AI function calls in the query.", ValueType::Number) \ M(AIAPICalls, "Number of HTTP requests dispatched to AI providers.", ValueType::Number) \ + M(AIAPICallsRetried, "Number of AI provider requests retried after a transient failure.", ValueType::Number) \ + M(AIAPICallsFailed, "Number of AI provider requests that failed after all retries.", ValueType::Number) \ + M(AIAPICallsThrottled, "Number of AI provider requests rejected with HTTP 429 (Too Many Requests).", ValueType::Number) \ M(AIRowsProcessed, "Number of rows that received an AI result.", ValueType::Number) \ M(AIRowsSkipped, "Number of rows that received a default value due to quota or error.", ValueType::Number) \ \ diff --git a/src/Common/ProgressIndication.cpp b/src/Common/ProgressIndication.cpp index 172f2b735e52a..def1f44b11f8d 100644 --- a/src/Common/ProgressIndication.cpp +++ b/src/Common/ProgressIndication.cpp @@ -129,7 +129,6 @@ void ProgressIndication::writeProgress(WriteBufferFromFileDescriptor & message, { std::lock_guard lock(progress_mutex); - static size_t increment = 0; static const char * indicators[8] = { "\033[1;30m→\033[0m", "\033[1;31m↘\033[0m", @@ -141,7 +140,7 @@ void ProgressIndication::writeProgress(WriteBufferFromFileDescriptor & message, "\033[1m↗\033[0m", }; - const char * indicator = indicators[increment % 8]; + const char * indicator = indicators[indicator_frame % 8]; auto [terminal_width, terminal_height] = getTerminalSize(in_fd, err_fd); @@ -287,7 +286,7 @@ void ProgressIndication::writeProgress(WriteBufferFromFileDescriptor & message, } message << CLEAR_TO_END_OF_LINE; - ++increment; + ++indicator_frame; message.next(); } diff --git a/src/Common/ProgressIndication.h b/src/Common/ProgressIndication.h index 517a23f0c4a87..68df926b9b81b 100644 --- a/src/Common/ProgressIndication.h +++ b/src/Common/ProgressIndication.h @@ -106,6 +106,10 @@ class ProgressIndication /// to check whether progress output needs to be cleared. size_t written_progress_chars = 0; + /// Selects the frame of the spinning indicator. It is per instance, because several clients + /// (e.g. the sessions of the web terminal) can show progress concurrently in one process. + size_t indicator_frame = 0; + /// The server periodically sends information about how much data was read since last time. /// This information is stored here. Progress progress; diff --git a/src/Common/RemoteHostFilter.cpp b/src/Common/RemoteHostFilter.cpp index 6f953f48d48b0..5b5da5ec7c6b9 100644 --- a/src/Common/RemoteHostFilter.cpp +++ b/src/Common/RemoteHostFilter.cpp @@ -110,6 +110,10 @@ bool RemoteHostFilter::checkForDirectEntry(const std::string & str) const /// Allow everything by default. return true; + /// Connectors take the host as a C string and stop at the first NUL, so the checked string must not contain one. + if (str.contains('\0')) + return false; + std::lock_guard guard(hosts_mutex); if (primary_hosts.contains(str)) diff --git a/src/Common/Scheduler/CPUSlotsAllocation.cpp b/src/Common/Scheduler/CPUSlotsAllocation.cpp index a8965c3990c94..729030c4004f7 100644 --- a/src/Common/Scheduler/CPUSlotsAllocation.cpp +++ b/src/Common/Scheduler/CPUSlotsAllocation.cpp @@ -7,6 +7,7 @@ #include #include #include +#include namespace ProfileEvents { @@ -72,6 +73,10 @@ CPUSlotsAllocation::CPUSlotsAllocation(SlotCount master_slots_, SlotCount worker for (CPUSlotRequest & request : requests) request.allocation = this; + wait_thread_group = CurrentThread::getGroup(); + if (wait_thread_group) + wait_counters = &wait_thread_group->performance_counters; + std::unique_lock lock{schedule_mutex}; while (allocated < total_slots) { @@ -79,7 +84,7 @@ CPUSlotsAllocation::CPUSlotsAllocation(SlotCount master_slots_, SlotCount worker { queue->enqueueRequest(current_request); scheduled_slot_increment.emplace(CurrentMetrics::ConcurrencyControlScheduled); - wait_timer.emplace(CurrentThread::getProfileEvents().timer(ProfileEvents::ConcurrencyControlWaitMicroseconds)); + wait_timer.emplace(wait_counters->timer(ProfileEvents::ConcurrencyControlWaitMicroseconds)); break; } else // noncompeting slot - provide for free diff --git a/src/Common/Scheduler/CPUSlotsAllocation.h b/src/Common/Scheduler/CPUSlotsAllocation.h index c7f94f2557d9d..03e18a4c8a116 100644 --- a/src/Common/Scheduler/CPUSlotsAllocation.h +++ b/src/Common/Scheduler/CPUSlotsAllocation.h @@ -18,6 +18,7 @@ namespace DB { class CPUSlotsAllocation; +class ThreadGroup; // Represents a resource request for a cpu slot for a single thread class CPUSlotRequest final : public ResourceRequest @@ -114,6 +115,9 @@ class CPUSlotsAllocation final : public ISlotAllocation std::vector requests; // Requests per every slot CPUSlotRequest * current_request; std::optional scheduled_slot_increment; + /// `wait_timer` may end on another thread after the creating one has exited, so it counts into the query's group, kept alive here. + std::shared_ptr wait_thread_group; + ProfileEvents::Counters * wait_counters = &ProfileEvents::global_counters; std::optional wait_timer; }; diff --git a/src/Common/Scheduler/IWorkloadNode.h b/src/Common/Scheduler/IWorkloadNode.h index 26662046efb40..44e2cdd86980a 100644 --- a/src/Common/Scheduler/IWorkloadNode.h +++ b/src/Common/Scheduler/IWorkloadNode.h @@ -102,6 +102,10 @@ class IWorkloadNode /// Returns true if this workload node has workload children virtual bool hasWorkloadChildren() const = 0; + /// Fails the requests and allocations still in the queue of this leaf workload. Called on the scheduler thread when + /// the workload is dropped, before the node is destroyed: query threads may still reach the queue through older versions. + virtual void purgeQueue() = 0; + /// Introspection. Calls a visitor for self and every internal node. Do not recurse into workload children. virtual void forEachSchedulerNode(std::function visitor) = 0; }; diff --git a/src/Common/Scheduler/Nodes/SpaceShared/AllocationQueue.cpp b/src/Common/Scheduler/Nodes/SpaceShared/AllocationQueue.cpp index 8e680060124ce..715f1b80411b7 100644 --- a/src/Common/Scheduler/Nodes/SpaceShared/AllocationQueue.cpp +++ b/src/Common/Scheduler/Nodes/SpaceShared/AllocationQueue.cpp @@ -181,6 +181,9 @@ void AllocationQueue::purgeQueue() allocated = 0; allocations = 0; is_not_usable = true; + + // Older hierarchy versions can retain this retired queue after its workload is destroyed. + workload = nullptr; } void AllocationQueue::propagateUpdate(ISpaceSharedNode &, Update &&) diff --git a/src/Common/Scheduler/Nodes/TimeShared/FifoQueue.h b/src/Common/Scheduler/Nodes/TimeShared/FifoQueue.h index 52909645aca91..9079a5b60436e 100644 --- a/src/Common/Scheduler/Nodes/TimeShared/FifoQueue.h +++ b/src/Common/Scheduler/Nodes/TimeShared/FifoQueue.h @@ -127,6 +127,9 @@ class FifoQueue : public ISchedulerQueue requests_to_fail.push_back(request); } cancelActivation(); + + // Older hierarchy versions can retain this retired queue after its workload is destroyed. + workload = nullptr; } // Now notify all collected requests about the failure without holding the mutex auto exception = std::make_exception_ptr( diff --git a/src/Common/Scheduler/Nodes/WorkloadNode.h b/src/Common/Scheduler/Nodes/WorkloadNode.h index ea055c8842743..6c644005ca317 100644 --- a/src/Common/Scheduler/Nodes/WorkloadNode.h +++ b/src/Common/Scheduler/Nodes/WorkloadNode.h @@ -442,6 +442,7 @@ class WorkloadNodeCommon : public BaseNode, public IWorkloadNode struct QueueOrChildrenBranch { NodePtr queue; /// FifoQueue or AllocationQueue node is used if there are no children + bool queue_dropped = false; /// `queue` has been purged by `purgeDroppedQueue` ChildrenBranch branch; /// Used if there is at least one child WorkloadSettings settings; CostUnit unit = CostUnit::IOByte; @@ -495,6 +496,17 @@ class WorkloadNodeCommon : public BaseNode, public IWorkloadNode Traits::updateQueue(queue, settings, unit); } + /// Fails the requests and allocations of a dropped workload, the same way `removeQueue` does + /// when the workload gets its first child. The queue itself stays alive while older hierarchy + /// versions reference it. + void purgeDroppedQueue() + { + if (!queue || std::exchange(queue_dropped, true)) + return; + detach(queue); + Traits::purgeQueue(queue); + } + private: void removeQueue() { @@ -639,6 +651,18 @@ class WorkloadNodeCommon : public BaseNode, public IWorkloadNode : BaseNode(event_queue_, SchedulerNodeInfo(settings)) {} + ~WorkloadNodeCommon() override + { + /// Queues and constraints outlive this node while older versions are still referenced by classifiers. + /// The queue clears its own pointer in `purgeQueue` under its mutex, because query threads read it there. + purgeQueue(); // No-op if the workload was dropped through `purgeQueue` already + forEachSchedulerNode([this](ISchedulerNode * node) + { + if (node != impl.branch.queue.get()) + node->workload = nullptr; + }); + } + void attachWorkloadChild(const WorkloadNodePtr & child) final { SCHED_DBG("{} -- attachWorkloadChild(child={})", this->getPath(), child->getWorkload()); @@ -716,6 +740,11 @@ class WorkloadNodeCommon : public BaseNode, public IWorkloadNode } } + void purgeQueue() final + { + impl.branch.purgeDroppedQueue(); + } + bool hasWorkloadChildren() const final { return impl.branch.queue == nullptr; diff --git a/src/Common/Scheduler/Nodes/tests/gtest_workload_resource_manager.cpp b/src/Common/Scheduler/Nodes/tests/gtest_workload_resource_manager.cpp index bd1d35a18af27..8c00ef44a95b7 100644 --- a/src/Common/Scheduler/Nodes/tests/gtest_workload_resource_manager.cpp +++ b/src/Common/Scheduler/Nodes/tests/gtest_workload_resource_manager.cpp @@ -13,6 +13,8 @@ #include #include #include +#include +#include #include #include #include @@ -65,6 +67,7 @@ namespace ProfileEvents { extern const Event ConcurrencyControlUpscales; extern const Event ConcurrencyControlDownscales; + extern const Event ConcurrencyControlWaitMicroseconds; } namespace CurrentMetrics @@ -475,6 +478,45 @@ TEST(SchedulerWorkloadResourceManager, DropNotEmptyQueue) // failed the request), the thread-local `Request` must be reset back to `Finished`. Otherwise // the next `ResourceGuard` on the same thread chasserts in `enqueue` because the request is // still in `Dequeued` state with `exception` set. +TEST(SchedulerWorkloadResourceManager, DropWorkloadFailsPendingRequests) +{ + ResourceTest t; + + t.query("CREATE RESOURCE res1 (WRITE DISK disk, READ DISK disk)"); + t.query("CREATE WORKLOAD all SETTINGS max_io_requests = 1"); + t.query("CREATE WORKLOAD intermediate IN all"); + + std::barrier sync_before_enqueue(2); + std::barrier sync_before_drop(3); + std::barrier sync_after_drop(2); + t.async("intermediate", "res1", [&] (ResourceLink link) + { + TestGuard g(t, link, 1); + sync_before_enqueue.arrive_and_wait(); + sync_before_drop.arrive_and_wait(); // 1st resource request is consuming + sync_after_drop.arrive_and_wait(); // 1st resource request is still consuming + }); + + sync_before_enqueue.arrive_and_wait(); // to maintain correct order of resource requests + + t.async("intermediate", "res1", [&] (ResourceLink link) + { + { + TestGuard g(t, link, 1, EnqueueOnly); + sync_before_drop.arrive_and_wait(); // 2nd resource request is enqueued + g.waitFailed("is about to be destructed"); + } + // The dropped workload's queue is kept alive by this classifier, but it must not accept new requests. + EXPECT_THROW(ResourceGuard(ResourceGuard::Metrics::getIOWrite(), link, 1), DB::Exception); + }); + + sync_before_drop.arrive_and_wait(); + t.query("DROP WORKLOAD intermediate"); + sync_after_drop.arrive_and_wait(); + + t.wait(); // Wait for threads to finish before destructing locals +} + TEST(SchedulerWorkloadResourceManager, ResourceGuardDefaultLockResetsOnFailure) { ResourceTest t; @@ -1292,6 +1334,41 @@ TEST(SchedulerWorkloadResourceManager, CPUSchedulingRoundRobin) t.wait(); } +/// The wait for a CPU slot is accounted to the query even when the allocation outlives the thread that created it +/// (here: destroyed on another thread while its request is still queued, like a `KILL QUERY` of a waiting query). +TEST(SchedulerWorkloadResourceManager, CPUSlotsWaitTimerOutlivesCreatingThread) +{ + ResourceTest t; + t.query("CREATE RESOURCE cpu (MASTER THREAD, WORKER THREAD)"); + t.query("CREATE WORKLOAD all SETTINGS max_concurrent_threads = 1"); + ClassifierPtr c = t.manager->acquire("all"); + ResourceLink link = c->get("cpu"); + + /// Takes the only slot, so the next allocation keeps its request queued. + auto blocker = std::make_shared(1, 0, link, link); + auto blocker_slot = blocker->acquire(); + + std::weak_ptr weak_group; + SlotAllocationPtr allocation; + std::thread([&] + { + DB::ThreadStatus thread_status; + auto group = std::make_shared(Context::getGlobalContextInstance(), 0); + weak_group = group; + CurrentThread::attachToGroupIfDetached(group); + allocation = std::make_shared(1, 0, link, link); + CurrentThread::detachFromGroupIfNotDetached(); + }).join(); + + ASSERT_TRUE(allocation->isRequesting()); + /// The creating thread is gone, so only the allocation can still own the group. + ThreadGroupPtr group = weak_group.lock(); + ASSERT_NE(group, nullptr); + std::this_thread::sleep_for(std::chrono::milliseconds(10)); + allocation.reset(); + EXPECT_GE(group->performance_counters[ProfileEvents::ConcurrencyControlWaitMicroseconds], 10'000); +} + TEST(SchedulerWorkloadResourceManager, CPUSchedulingFairness) { ResourceTest t; @@ -2697,6 +2774,102 @@ TEST(SchedulerWorkloadResourceManager, MemoryReservationDropQueueWhilePending) EXPECT_EQ(threw.load(), waiter_count); } +TEST(SchedulerWorkloadResourceManager, RetiredAllocationQueueOutlivesWorkload) +{ + ResourceTest t; + + t.query("CREATE RESOURCE memory (MEMORY RESERVATION)"); + // This branch allows a single root workload, so `A` and `B` share a parent. + t.query("CREATE WORKLOAD all"); + t.query("CREATE WORKLOAD A IN all SETTINGS max_memory = 100"); + t.query("CREATE WORKLOAD B IN all SETTINGS max_memory = 100"); + + // Only `B`'s classifier retains the old hierarchy version after the temporary classifier for `A` is released. + ClassifierPtr classifier_b = t.manager->acquire("B"); + auto * retired_queue = t.manager->acquire("A")->get("memory").allocation_queue; + ASSERT_NE(retired_queue, nullptr); + t.executeFromScheduler("memory", [&] + { + EXPECT_EQ(retired_queue->getWorkloadName(), "A"); + }); + + t.query("CREATE WORKLOAD child IN A"); + t.executeFromScheduler("memory", [&] + { + // Check retirement while `A` is still alive: this fails deterministically without relying on + // ASan or on the contents of freed memory when the retired queue is eventually destroyed. + EXPECT_TRUE(retired_queue->getWorkloadName().empty()); + }); + + t.query("DROP WORKLOAD child"); + EXPECT_NE(retired_queue, t.manager->acquire("A")->get("memory").allocation_queue); + t.query("DROP WORKLOAD A"); + + // Destroy the original queue after its workload, through the classifier for a different workload. + classifier_b.reset(); +} + +TEST(SchedulerWorkloadResourceManager, MemoryReservationDropWorkloadFailsRunningAllocation) +{ + ResourceTest t; + + t.query("CREATE RESOURCE memory (MEMORY RESERVATION)"); + t.query("CREATE WORKLOAD all SETTINGS max_memory = 100"); + t.query("CREATE WORKLOAD A IN all"); + t.query("CREATE WORKLOAD B IN all"); + + ClassifierPtr c_a = t.manager->acquire("A"); + TestAllocation a(c_a->get("memory"), "a", 60); + a.waitSync(); + + // The classifier keeps `A`'s queue alive, but the allocation must not keep running outside the limits of `all`. + t.query("DROP WORKLOAD A"); + EXPECT_THROW(a.waitSync(), Exception); + a.setSize(80); + EXPECT_THROW(a.waitSync(), Exception); + + // The memory of the failed allocation is available to the rest of the hierarchy. + ClassifierPtr c_b = t.manager->acquire("B"); + TestAllocation b(c_b->get("memory"), "b", 100); + b.waitSync(); +} + +TEST(SchedulerWorkloadResourceManager, MemoryReservationDropWorkloadWhileAllocating) +{ + ResourceTest t; + + t.query("CREATE RESOURCE memory (MEMORY RESERVATION)"); + t.query("CREATE WORKLOAD A SETTINGS max_memory = 100, max_waiting_queries = 0"); + + ClassifierPtr c_a = t.manager->acquire("A"); + ResourceLink link = c_a->get("memory"); + + // Keep inserting allocations into `A`'s queue while `A` is dropped. Every one is rejected and the rejection reads + // the workload's name: by `max_waiting_queries` before the purge, as an unusable queue after it. This lets TSan + // check those reads against the drop and the destruction of the workload node. + std::atomic dropped{false}; + std::atomic rejected{0}; + std::thread allocator([&] + { + while (!dropped.load() || rejected.load() < 100) + { + try + { + TestAllocation allocation(link, "racer", 10); + } + catch (const Exception &) + { + rejected.fetch_add(1); + } + } + }); + + t.query("DROP WORKLOAD A"); + dropped.store(true); + allocator.join(); + EXPECT_GE(rejected.load(), 100); +} + // Regression: simulate the race where the owner's destructor reaches `queue.removeAllocation` // AFTER `updateMinMaxAllocated`/`updateQueueLimit` has already rejected the allocation. The // guard in `removeAllocation` must keep an already-failed allocation out of diff --git a/src/Common/Scheduler/WorkloadResourceManager.cpp b/src/Common/Scheduler/WorkloadResourceManager.cpp index 0348710be62b8..9255595a94e8e 100644 --- a/src/Common/Scheduler/WorkloadResourceManager.cpp +++ b/src/Common/Scheduler/WorkloadResourceManager.cpp @@ -148,6 +148,10 @@ void WorkloadResourceManager::Resource::deleteNode(const NodeInfo & info) root_node.reset(); } + // Fail the queue's requests while the node is fully alive: query threads may still reach the queue through + // older versions, and the queue's error messages read the workload's name. + n->purgeQueue(); + node_for_workload.erase(info.name); updateCurrentVersion(); diff --git a/src/Common/ThreadStatus.cpp b/src/Common/ThreadStatus.cpp index eb4970f9b0b5e..419a52981100d 100644 --- a/src/Common/ThreadStatus.cpp +++ b/src/Common/ThreadStatus.cpp @@ -145,7 +145,10 @@ ThreadStatus::ThreadStatus() /// memory_tracker is already parented to total_memory_tracker, so a thread that never attaches /// to a group still honors total_memory_tracker_sample_probability. resolveMemorySampleConfig(); - log = getLogger("ThreadStatus"); + /// A `ThreadStatus` is created for every job of a thread pool, and `getLogger` locks the global mutex of the + /// registry of loggers, so all threads of the pools would contend on it. + static const LoggerPtr thread_status_log = getLogger("ThreadStatus"); + log = thread_status_log; current_thread = this; diff --git a/src/Common/ZooKeeper/ZooKeeperConstants.h b/src/Common/ZooKeeper/ZooKeeperConstants.h index 68f9459ddd671..36de2427dbbef 100644 --- a/src/Common/ZooKeeper/ZooKeeperConstants.h +++ b/src/Common/ZooKeeper/ZooKeeperConstants.h @@ -95,6 +95,8 @@ static constexpr int32_t PASSWORD_LENGTH = 16; static constexpr int32_t MAX_STRING_OR_ARRAY_SIZE = 1 << 28; /// 256 MiB /// Upper bound on elements reserved up front when reading an array; the declared size is attacker-controlled static constexpr int32_t MAX_ARRAY_PREALLOCATION = 4096; +/// Bytes read per step when reading a string; the declared size is attacker-controlled, so memory grows with the bytes received +static constexpr int32_t MAX_STRING_PREALLOCATION = 1 << 16; /// 64 KiB /// Request length prefix is a signed int32, so this is a hard limit static constexpr uint64_t MAX_REQUEST_SIZE_HARD_LIMIT = std::numeric_limits::max(); /// Smallest server-advertised max_request_size we accept; anything below is treated as bogus and ignored diff --git a/src/Common/ZooKeeper/ZooKeeperIO.cpp b/src/Common/ZooKeeper/ZooKeeperIO.cpp index 1a2956e7fbe26..10c6d05591fc1 100644 --- a/src/Common/ZooKeeper/ZooKeeperIO.cpp +++ b/src/Common/ZooKeeper/ZooKeeperIO.cpp @@ -93,11 +93,24 @@ void read(std::string & s, ReadBuffer & in) if (size > MAX_STRING_OR_ARRAY_SIZE) throw Exception(Error::ZMARSHALLINGERROR,"Too large string size while reading from ZooKeeper {}", size); - s.resize(size); - size_t read_bytes = in.read(s.data(), size); - if (read_bytes != static_cast(size)) - throw Exception( - Error::ZMARSHALLINGERROR, "Buffer size read from Zookeeper is not big enough. Expected {}. Got {}", size, read_bytes); + /// Memory must track bytes actually read, not the declared size: + /// do not pre-allocate for an attacker-controlled length. + s.clear(); + size_t remaining = static_cast(size); + while (remaining > 0) + { + const size_t chunk = std::min(remaining, MAX_STRING_PREALLOCATION); + const size_t old_size = s.size(); + s.resize(old_size + chunk); + const size_t read_bytes = in.read(s.data() + old_size, chunk); + if (read_bytes != chunk) + throw Exception( + Error::ZMARSHALLINGERROR, + "Buffer size read from Zookeeper is not big enough. Expected {}. Got {}", + size, + old_size + read_bytes); + remaining -= chunk; + } } void read(ACL & acl, ReadBuffer & in) diff --git a/src/Common/maskURIPassword.h b/src/Common/maskURIPassword.h index b49fd22240075..81559987db593 100644 --- a/src/Common/maskURIPassword.h +++ b/src/Common/maskURIPassword.h @@ -1,33 +1,66 @@ #pragma once +#include #include #include #include +#include namespace DB { -/** Mask the value of `=` in a connection string, where the value runs up to the next - * ';' or to the end of the string. Only the first occurrence is masked, and nothing is masked if - * the key is absent. - * - * This replaces the regular expressions `AccountKey=.*?(;|$)` and `SharedAccessSignature=.*?(;|$)` - * that used to be applied here, so that masking a secret does not require a regex engine. +/** Hides the secrets of an Azure connection string as the Azure SDK reads it (`Azure::Storage::_internal::ParseConnectionString`, + * `contrib/azure/sdk/storage/azure-storage-common/src/storage_credential.cpp`): a key runs to '=' and its value to ';', keys + * match exactly, and the last copy of a repeated key wins. Splitting at every ';' first can only hide more than the SDK reads. + * - secret keys: `AccountName=a;AccountKey=K;SharedAccessSignature=S` -> `AccountName=a;AccountKey=[HIDDEN];SharedAccessSignature=[HIDDEN]` + * - repeated keys: `AccountKey=K1;AccountKey=K2` -> `AccountKey=[HIDDEN];AccountKey=[HIDDEN]` + * - an endpoint is a url, so a query in it is a SAS: `BlobEndpoint=https://h/?sig=S` or `BlobEndpoint=https://u@h/` -> `BlobEndpoint=[HIDDEN]` + * - the whole string is hidden when a '?', '#' or '@' is still shown, or when it has one and holds a url outside the pair + * values (':' or '/' in a key or in a part without '='; no key has them): `https://h/?sig=S` or `https://u:p=s@h/` -> `[HIDDEN]` + * Returns whether anything was hidden. */ -inline bool maskConnectionStringKey(std::string & str, std::string_view key_with_eq) +inline bool maskAzureConnectionStringSecrets(std::string & str) { - size_t key_position = str.find(key_with_eq); - if (key_position == std::string::npos) - return false; + static constexpr std::array secret_keys = {"AccountKey", "SharedAccessSignature"}; + auto is_secret_key = [](std::string_view key) { return std::ranges::find(secret_keys, key) != secret_keys.end(); }; + + const bool has_url_delimiters = str.find_first_of("?#@") != std::string::npos; + bool url_outside_values = false; + std::string result; + size_t copied = 0; + bool masked = false; + for (size_t part_begin = 0; part_begin <= str.length();) + { + const size_t part_end = std::min(str.find(';', part_begin), str.length()); + const std::string_view part = std::string_view(str).substr(part_begin, part_end - part_begin); + const size_t equals = part.find('='); + if (part.substr(0, equals).find_first_of(":/") != std::string_view::npos) + url_outside_values = true; + if (equals != std::string_view::npos + && (is_secret_key(part.substr(0, equals)) || part.substr(equals + 1).find_first_of("?@") != std::string_view::npos)) + { + const size_t value_begin = part_begin + equals + 1; + result.append(str, copied, value_begin - copied); + result.append("[HIDDEN]"); + copied = part_end; + masked = true; + } + part_begin = part_end + 1; + } - size_t value_begin = key_position + key_with_eq.length(); - size_t value_end = str.find(';', value_begin); - if (value_end == std::string::npos) - value_end = str.length(); + if (masked) + { + result.append(str, copied, std::string::npos); + str = std::move(result); + } - str.replace(value_begin, value_end - value_begin, "[HIDDEN]"); - return true; + if (str.find_first_of("?#@") != std::string::npos || (url_outside_values && has_url_delimiters)) + { + str = "[HIDDEN]"; + return true; + } + return masked; } /** Replace the password in a URI of the form `scheme://user:password@host` with `[HIDDEN]`. diff --git a/src/Common/mysqlxx/Connection.cpp b/src/Common/mysqlxx/Connection.cpp index 3f9accfb97eb0..281f8f33c9be2 100644 --- a/src/Common/mysqlxx/Connection.cpp +++ b/src/Common/mysqlxx/Connection.cpp @@ -120,6 +120,14 @@ void Connection::connect(const char* db, if (mysql_ssl_set(driver.get(), ifNotEmpty(ssl_key), ifNotEmpty(ssl_cert), ifNotEmpty(ssl_ca), nullptr, nullptr)) throw ConnectionFailed(errorMessage(driver.get()), mysql_errno(driver.get())); + /// Without an explicit socket, use TCP to the given port: otherwise `localhost` goes to the library's default socket. + if (!ifNotEmpty(socket)) + { + unsigned protocol = MYSQL_PROTOCOL_TCP; + if (mysql_options(driver.get(), MYSQL_OPT_PROTOCOL, &protocol)) + throw ConnectionFailed(errorMessage(driver.get()), mysql_errno(driver.get())); + } + if (!mysql_real_connect(driver.get(), server, user, password, db, port, ifNotEmpty(socket), driver->client_flag)) throw ConnectionFailed(errorMessage(driver.get()), mysql_errno(driver.get())); diff --git a/src/Common/mysqlxx/Pool.cpp b/src/Common/mysqlxx/Pool.cpp index 4f7c4ac57d171..112f0c370be8e 100644 --- a/src/Common/mysqlxx/Pool.cpp +++ b/src/Common/mysqlxx/Pool.cpp @@ -9,11 +9,61 @@ #include #include #include +#include #include #include #include +namespace DB::ErrorCodes +{ + extern const int BAD_ARGUMENTS; +} + +namespace +{ + +/// `mysql_real_connect` reinterprets these hosts and port 0; reject them so it dials exactly what `remote_url_allow_hosts` checked. +void checkConnectionTarget(const std::string & server, unsigned port, const std::string & socket) +{ + std::string_view meaning; + if (server.contains('\0')) + meaning = "the end of the host"; + else if (server.find_first_of(";=") != std::string::npos) + meaning = "a connection string"; + else if (server.contains("://")) + meaning = "a connection plugin URL"; + else if (server.contains(',')) + meaning = "a list of hosts"; + + if (!meaning.empty()) + { + std::string printable; + for (const char c : server) + printable += c == '\0' ? std::string_view("\\0") : std::string_view(&c, 1); + + throw DB::Exception( + DB::ErrorCodes::BAD_ARGUMENTS, + "MySQL host '{}' contains syntax which the MySQL client library would read as {}. " + "Specify a plain host and port, and use the `socket` and `ssl_*` settings instead", + printable, meaning); + } + + if (socket.contains('\0')) + throw DB::Exception( + DB::ErrorCodes::BAD_ARGUMENTS, "MySQL socket for host '{}' contains a NUL byte, where the MySQL client library would end it", server); + + /// The client library uses the socket only for an empty host or `localhost`, otherwise it connects over TCP. + const bool uses_socket = !socket.empty() && (server.empty() || server == "localhost"); + if (port == 0 && !uses_socket) + throw DB::Exception( + DB::ErrorCodes::BAD_ARGUMENTS, + "MySQL port 0 for host '{}' would be replaced by the default port of the MySQL client library. Specify the port explicitly", + server); +} + +} + namespace mysqlxx { @@ -380,6 +430,9 @@ void Pool::initialize() Pool::Connection * Pool::allocConnection(bool dont_throw_if_failed_first_time) { + /// All connections of the pool are created here. + checkConnectionTarget(server, port, socket); + auto conn_ptr = std::make_unique(); try diff --git a/src/Common/setThreadName.h b/src/Common/setThreadName.h index 3c062991b6ddf..cf5f56738fcd9 100644 --- a/src/Common/setThreadName.h +++ b/src/Common/setThreadName.h @@ -9,6 +9,7 @@ namespace DB M(AGGREGATED_ZOOKEEPER_LOG, "AggrZooLog") \ M(AGGREGATOR_DESTRUCTION, "AggregDestruct") \ M(AGGREGATOR_POOL, "AggregatorPool") \ + M(AI_REQUEST, "AIRequest") \ M(ARROW_FILE, "ArrowFile") \ M(ARROW_FLIGHT, "ArrowFlight") \ M(ARROW_FLIGHT_EXPR, "ArrowFlightExpr") \ @@ -181,6 +182,7 @@ namespace DB M(BLOB_KILLER_TASK, "BlobKillerTask") \ M(BLOB_COPIER_TASK, "BlobCopierTask") \ M(DISK_OBJECT_STORAGE_COPY, "DiskObjStCopy") \ + M(DISK_OBJECT_STORAGE_WRITE_BLOB, "DiskObjStWrBlob") \ M(DISTRIBUTED_CACHE, "DistribCache") \ M(DISTRIBUTED_CACHE_DROP, "DropDistCache") \ diff --git a/src/Common/tests/gtest_distrusted_certificates.cpp b/src/Common/tests/gtest_distrusted_certificates.cpp new file mode 100644 index 0000000000000..5931feea9f6b4 --- /dev/null +++ b/src/Common/tests/gtest_distrusted_certificates.cpp @@ -0,0 +1,137 @@ +#include "config.h" + +#if USE_SSL + +#include + +#include +#include + +#include +#include +#include + +#include +#include + + +namespace +{ + +/// `emSign ECC Root CA - G3`, distrusted. +constexpr std::string_view emsign_ecc_root_g3 = R"( +-----BEGIN CERTIFICATE----- +MIICTjCCAdOgAwIBAgIKPPYHqWhwDtqLhDAKBggqhkjOPQQDAzBrMQswCQYDVQQG +EwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNo +bm9sb2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0g +RzMwHhcNMTgwMjE4MTgzMDAwWhcNNDMwMjE4MTgzMDAwWjBrMQswCQYDVQQGEwJJ +TjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBUZWNobm9s +b2dpZXMgTGltaXRlZDEgMB4GA1UEAxMXZW1TaWduIEVDQyBSb290IENBIC0gRzMw +djAQBgcqhkjOPQIBBgUrgQQAIgNiAAQjpQy4LRL1KPOxst3iAhKAnjlfSU2fySU0 +WXTsuwYc58Byr+iuL+FBVIcUqEqy6HyC5ltqtdyzdc6LBtCGI79G1Y4PPwT01xyS +fvalY8L1X44uT6EYGQIrMgqCZH0Wk9GjQjBAMB0GA1UdDgQWBBR8XQKEE9TMipuB +zhccLikenEhjQjAOBgNVHQ8BAf8EBAMCAQYwDwYDVR0TAQH/BAUwAwEB/zAKBggq +hkjOPQQDAwNpADBmAjEAvvNhzwIQHWSVB7gYboiFBS+DCBeQyh+KTOgNG3qxrdWB +CUfvO6wIBHxcmbHtRwfSAjEAnbpV/KlK6O3t5nYBQnvI+GDZjVGLVTv7jHvrZQnD ++JbNR6iC8hZVdyR+EhCVBCyj +-----END CERTIFICATE----- + +)"; + +/// `EM DV TLS CA - G3B` cross-signed by `AAA Certificate Services`, distrusted by the key of the certificate issued by `emSign Root TLS CA - G3`. +constexpr std::string_view em_dv_tls_ca_g3b_cross_signed = R"( +-----BEGIN CERTIFICATE----- +MIIDzjCCAragAwIBAgIRAKAk/fS6Zlx4XOSgIEg4WLcwDQYJKoZIhvcNAQELBQAw +ezELMAkGA1UEBhMCR0IxGzAZBgNVBAgMEkdyZWF0ZXIgTWFuY2hlc3RlcjEQMA4G +A1UEBwwHU2FsZm9yZDEaMBgGA1UECgwRQ29tb2RvIENBIExpbWl0ZWQxITAfBgNV +BAMMGEFBQSBDZXJ0aWZpY2F0ZSBTZXJ2aWNlczAeFw0yNTA1MTQwMDAwMDBaFw0y +ODEyMzEyMzU5NTlaMD8xCzAJBgNVBAYTAk5MMRMwEQYDVQQKDAplTXVkaHJhIEJW +MRswGQYDVQQDDBJFTSBEViBUTFMgQ0EgLSBHM0IwdjAQBgcqhkjOPQIBBgUrgQQA +IgNiAATnntnASpdzxHNeUREdqFrCvV5kqzdLgOEsvCA47B5PDjuBL8mFnh5nr7BO +L4vJf3SFumWqkR7opORbYtXMA9Vr5nizw4y03JN8okYOI4RTyJKTXSnUT5HhVga2 +PM7atGijggE1MIIBMTAfBgNVHSMEGDAWgBSgEQojPpbxB+zirynvgqV/0DCktDAd +BgNVHQ4EFgQU4h4v5xwZOQ5VS2LBICRXNSSKJRQwDgYDVR0PAQH/BAQDAgEGMBIG +A1UdEwEB/wQIMAYBAf8CAQAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMC +MDEGA1UdIAQqMCgwCAYGZ4EMAQIBMA0GCysGAQQBg44hAQABMA0GCysGAQQBg44h +AQJkMEMGA1UdHwQ8MDowOKA2oDSGMmh0dHA6Ly9jcmwuY29tb2RvY2EuY29tL0FB +QUNlcnRpZmljYXRlU2VydmljZXMuY3JsMDQGCCsGAQUFBwEBBCgwJjAkBggrBgEF +BQcwAYYYaHR0cDovL29jc3AuY29tb2RvY2EuY29tMA0GCSqGSIb3DQEBCwUAA4IB +AQBzQ1IoqpdotM6gFrR4Q3YrKifOLeaj5XXunldpVE/vZn2TxC2y0s8oYiLR5ICE +wqVXSB7N6BM1Mp+Nif7Jbhb2IswAfSR4Iik8+B0iRmi7lYoNlycCgHjmsjOmZaxM +QkPmjXAJbG5f4V41OsvJZmS7lI3r0IuvEDVX3GAZ13on6IImSs9ybNpHQ7NCqkMQ +7wH4uAO1dtpFycgJSdHaCOhHjrox2BTYGLdpUrYwY3EOAjlZ8UGgsxs7mG+xquMl +bN1LFURwfnkVMP3T/DBIZ/ys2+lNFyxK3FlArrV4/000uTnVB1lO8oGORYdXoT5V +GUyRCEIDhF5TBUuKF4E3kf1h +-----END CERTIFICATE-----)"; + +/// `emSign Root CA - G1`, not affected. +constexpr std::string_view emsign_root_g1 = R"( +-----BEGIN CERTIFICATE----- +MIIDlDCCAnygAwIBAgIKMfXkYgxsWO3W2DANBgkqhkiG9w0BAQsFADBnMQswCQYD +VQQGEwJJTjETMBEGA1UECxMKZW1TaWduIFBLSTElMCMGA1UEChMcZU11ZGhyYSBU +ZWNobm9sb2dpZXMgTGltaXRlZDEcMBoGA1UEAxMTZW1TaWduIFJvb3QgQ0EgLSBH +MTAeFw0xODAyMTgxODMwMDBaFw00MzAyMTgxODMwMDBaMGcxCzAJBgNVBAYTAklO +MRMwEQYDVQQLEwplbVNpZ24gUEtJMSUwIwYDVQQKExxlTXVkaHJhIFRlY2hub2xv +Z2llcyBMaW1pdGVkMRwwGgYDVQQDExNlbVNpZ24gUm9vdCBDQSAtIEcxMIIBIjAN +BgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAk0u76WaK7p1b1TST0Bsew+eeuGQz +f2N4aLTNLnF115sgxk0pvLZoYIr3IZpWNVrzdr3YzZr/k1ZLpVkGoZM0Kd0WNHVO +8oG0x5ZOrRkVUkr+PHB1cM2vK6sVmjM8qrOLqs1D/fXqcP/tzxE7lM5OMhbTI0Aq +d7OvPAEsbO2ZLIvZTmmYsvePQbAyeGHWDV/D+qJAkh1cF+ZwPjXnorfCYuKrpDhM +tTk1b+oDafo6VGiFbdbyL0NVHpENDtjVaqSW0RM8LHhQ6DqS0hdW5TUaQBw+jSzt +Od9C4INBdN+jzcKGYEho42kLVACL5HZpIQ15TjQIXhTCzLG3rdd8cIrHhQIDAQAB +o0IwQDAdBgNVHQ4EFgQU++8Nhp6w492pufEhF38+/PB3KxowDgYDVR0PAQH/BAQD +AgEGMA8GA1UdEwEB/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAFn/8oz1h31x +PaOfG1vR2vjTnGs2vZupYeveFix0PZ7mddrXuqe8QhfnPZHr5X3dPpzxz5KsbEjM +wiI/aTvFthUvozXGaCocV685743QNcMYDHsAVhzNixl03r4PEuDQqqE/AjSxcM6d +GNYIAwlG7mDgfrbESQRRfXBgvKqy/3lyeqYdPV8q+Mri/Tm3R7nrft8EI6/6nAYH +6ftjk4BAtcZsCjEozgyfz7MjNYBBjWzEN3uBL4ChQEKF6dk4jeihU80Bv2noWgby +RQuQ+q7hv53yrlc8pa6yVvSLZUDp/TGBLPQ5Cdjua6e0ph0VpZj3AYHYhX3zUVxx +iN66zB+Afko= +-----END CERTIFICATE----- + +)"; + +using X509Ptr = std::unique_ptr; + +X509Ptr parseCertificate(std::string_view pem) +{ + std::unique_ptr bio(BIO_new_mem_buf(pem.data(), static_cast(pem.size())), BIO_free); + X509Ptr certificate(PEM_read_bio_X509(bio.get(), nullptr, nullptr, nullptr), X509_free); + EXPECT_NE(certificate, nullptr); + return certificate; +} + +/// Verifies a trusted self-signed certificate the same way a TLS connection of a Poco context does it. +int verifyTrustedRoot(std::string_view pem) +{ + Poco::Net::Context context(Poco::Net::Context::CLIENT_USE, Poco::Net::Context::Params{}); + X509Ptr certificate = parseCertificate(pem); + X509_STORE * store = SSL_CTX_get_cert_store(context.sslContext()); + EXPECT_EQ(X509_STORE_add_cert(store, certificate.get()), 1); + + std::unique_ptr store_ctx(X509_STORE_CTX_new(), X509_STORE_CTX_free); + EXPECT_EQ(X509_STORE_CTX_init(store_ctx.get(), store, certificate.get(), nullptr), 1); + X509_STORE_CTX_set_verify_cb(store_ctx.get(), SSL_CTX_get_verify_callback(context.sslContext())); + + if (X509_verify_cert(store_ctx.get()) == 1) + return X509_V_OK; + return X509_STORE_CTX_get_error(store_ctx.get()); +} + +} + + +TEST(DistrustedCertificates, Check) +{ + EXPECT_EQ(Poco::Net::checkDistrustedCertificate(parseCertificate(emsign_ecc_root_g3).get()), X509_V_ERR_CERT_REVOKED); + EXPECT_EQ(Poco::Net::checkDistrustedCertificate(parseCertificate(em_dv_tls_ca_g3b_cross_signed).get()), X509_V_ERR_CERT_REVOKED); + EXPECT_EQ(Poco::Net::checkDistrustedCertificate(parseCertificate(emsign_root_g1).get()), X509_V_OK); +} + +TEST(DistrustedCertificates, RejectedEvenIfTrusted) +{ + EXPECT_EQ(verifyTrustedRoot(emsign_ecc_root_g3), X509_V_ERR_CERT_REVOKED); + EXPECT_EQ(verifyTrustedRoot(emsign_root_g1), X509_V_OK); +} + +#endif diff --git a/src/Common/tests/gtest_file_channel_rotation_failure.cpp b/src/Common/tests/gtest_file_channel_rotation_failure.cpp new file mode 100644 index 0000000000000..d4ca3bee239bc --- /dev/null +++ b/src/Common/tests/gtest_file_channel_rotation_failure.cpp @@ -0,0 +1,133 @@ +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/// Log rotation must not leave `FileChannel` with a deleted `LogFile` when the process has no free file descriptors. + +namespace +{ + +/// glibc defines `RLIMIT_NOFILE` as a self-referential macro, which trips -Wdisabled-macro-expansion inside gtest macros. +const auto nofile_resource = RLIMIT_NOFILE; + +struct DescriptorTableGuard +{ + struct rlimit original_limit{}; + std::vector descriptors; + + void release() + { + for (int fd : descriptors) + ::close(fd); + descriptors.clear(); + EXPECT_EQ(0, setrlimit(nofile_resource, &original_limit)); + } + + ~DescriptorTableGuard() { release(); } +}; + +int setSoftLimit(rlim_t value, const struct rlimit & original_limit) +{ + struct rlimit limit = original_limit; + limit.rlim_cur = value; + return setrlimit(nofile_resource, &limit); +} + +int fillDescriptorTable(DescriptorTableGuard & guard) +{ + while (true) + { + int fd = ::open("/dev/null", O_RDONLY | O_CLOEXEC); + if (fd < 0) + return errno; + guard.descriptors.push_back(fd); + } +} + +Poco::Message makeMessage(const std::string & text) +{ + return Poco::Message("gtest_file_channel_rotation_failure", text, Poco::Message::PRIO_INFORMATION); +} + +void expectNoFreeDescriptors(Poco::FileChannel & channel, const std::string & text) +{ + try + { + channel.log(makeMessage(text)); + ADD_FAILURE() << "Expected EMFILE while logging: " << text; + } + catch (const Poco::FileException & e) + { + EXPECT_EQ(EMFILE, e.code()) << e.displayText(); + } +} + +std::string readFile(const std::string & path) +{ + std::ifstream in(path); + return std::string((std::istreambuf_iterator(in)), std::istreambuf_iterator()); +} + +} + +TEST(FileChannelRotation, NoFreeDescriptorsDuringRotation) +{ + const auto dir = std::filesystem::temp_directory_path() / ("gtest_file_channel_rotation_" + std::to_string(::getpid())); + std::filesystem::remove_all(dir); + std::filesystem::create_directories(dir); + struct DirectoryGuard + { + std::filesystem::path path; + ~DirectoryGuard() + { + std::error_code ec; + std::filesystem::remove_all(path, ec); + } + } directory_guard{dir}; + const std::string path = (dir / "server.log").string(); + + Poco::AutoPtr channel = new Poco::FileChannel(path); + channel->setProperty(Poco::FileChannel::PROP_ROTATION, "1K"); + channel->setProperty(Poco::FileChannel::PROP_ARCHIVE, "number"); + channel->setProperty(Poco::FileChannel::PROP_COMPRESS, "false"); + + DescriptorTableGuard guard; + ASSERT_EQ(0, getrlimit(nofile_resource, &guard.original_limit)); + const rlim_t limit = std::min({guard.original_limit.rlim_cur, guard.original_limit.rlim_max - 1, 1024}); + + /// The log file gets descriptor `limit`, so closing it in `archive` frees no descriptor below the limit. + ASSERT_EQ(0, setSoftLimit(limit, guard.original_limit)); + ASSERT_EQ(EMFILE, fillDescriptorTable(guard)); + ASSERT_EQ(0, setSoftLimit(limit + 1, guard.original_limit)); + ASSERT_NO_THROW(channel->open()); + ASSERT_EQ(0, setSoftLimit(limit, guard.original_limit)); + + ASSERT_NO_THROW(channel->log(makeMessage(std::string(2048, 'x')))); + + expectNoFreeDescriptors(*channel, "rotation without free descriptors"); + EXPECT_TRUE(std::filesystem::exists(path + ".0")); + EXPECT_EQ(0u, channel->size()); + expectNoFreeDescriptors(*channel, "still no free descriptors"); + + guard.release(); + + const std::string marker = "logged after descriptors were released"; + ASSERT_NO_THROW(channel->log(makeMessage(marker))); + channel->close(); + EXPECT_NE(std::string::npos, readFile(path).find(marker)); +} diff --git a/src/Coordination/Changelog.cpp b/src/Coordination/Changelog.cpp index a5d204ea5bce5..88e2e7ebc3b89 100644 --- a/src/Coordination/Changelog.cpp +++ b/src/Coordination/Changelog.cpp @@ -4580,6 +4580,8 @@ void Changelog::applyEntriesFromBuffer(uint64_t index, nuraft::buffer & buffer) { uint64_t cur_index = index + i; int buf_size = buffer.get_int(); + if (buf_size < 0 || static_cast(buf_size) > buffer.size() - buffer.pos()) + throw Exception(ErrorCodes::CORRUPTED_DATA, "Log entry {} in a log pack has invalid size {}", cur_index, buf_size); nuraft::ptr buf_local = nuraft::buffer::alloc(buf_size); buffer.get(buf_local); diff --git a/src/Coordination/KeeperStorageImpl.cpp b/src/Coordination/KeeperStorageImpl.cpp index 92b5405c561c2..0ffb06436dab7 100644 --- a/src/Coordination/KeeperStorageImpl.cpp +++ b/src/Coordination/KeeperStorageImpl.cpp @@ -960,7 +960,9 @@ processLocal(const Coordination::ZooKeeperCheckWatchRequest & zk_request, Keeper ProfileEvents::increment(ProfileEvents::KeeperCheckWatchRequest); auto response = zk_request.makeResponse(); - if (!storage.containsWatch(zk_request.getPath(), zk_request.type)) + if (!magic_enum::enum_contains(zk_request.type)) + response->error = Coordination::Error::ZBADARGUMENTS; + else if (!storage.containsWatch(zk_request.getPath(), zk_request.type)) response->error = Coordination::Error::ZNOWATCHER; return response; } @@ -979,7 +981,9 @@ processLocal(const Coordination::ZooKeeperRemoveWatchRequest & zk_request, Keepe ProfileEvents::increment(ProfileEvents::KeeperRemoveWatchRequest); auto response = zk_request.makeResponse(); - if (!storage.removePersistentWatch(zk_request.path, zk_request.type, session_id)) + if (!magic_enum::enum_contains(zk_request.type)) + response->error = Coordination::Error::ZBADARGUMENTS; + else if (!storage.removePersistentWatch(zk_request.path, zk_request.type, session_id)) response->error = Coordination::Error::ZNOWATCHER; return response; @@ -999,7 +1003,10 @@ processLocal(const Coordination::ZooKeeperAddWatchRequest & zk_request, KeeperSt ProfileEvents::increment(ProfileEvents::KeeperAddWatchRequest); auto response = zk_request.makeResponse(); - storage.addPersistentWatch(zk_request.path, zk_request.mode, session_id); + if (!magic_enum::enum_contains(zk_request.mode)) + response->error = Coordination::Error::ZBADARGUMENTS; + else + storage.addPersistentWatch(zk_request.path, zk_request.mode, session_id); return response; } diff --git a/src/Coordination/tests/gtest_coordination.cpp b/src/Coordination/tests/gtest_coordination.cpp index daf984bb6ac67..39d64147a3656 100644 --- a/src/Coordination/tests/gtest_coordination.cpp +++ b/src/Coordination/tests/gtest_coordination.cpp @@ -427,6 +427,69 @@ TEST_P(CoordinationTest, CreateRequestDoesNotPreallocateOversizedACLs) } } +TEST(CoordinationSerialization, StringReadDoesNotPreallocateDeclaredSize) +{ + /// Invariant: memory must track bytes actually read, not the declared string length. + const size_t chunk = Coordination::MAX_STRING_PREALLOCATION; + + auto expect_marshalling_error = [](DB::ReadBuffer & rbuf, std::string & s) + { + try + { + Coordination::read(s, rbuf); + FAIL() << "Expected ZMARSHALLINGERROR"; + } + catch (const Coordination::Exception & e) + { + EXPECT_EQ(e.code, Coordination::Error::ZMARSHALLINGERROR); + } + }; + + { + /// The largest allowed length is declared, but no payload bytes follow. + DB::WriteBufferFromNuraftBuffer wbuf; + Coordination::write(Coordination::MAX_STRING_OR_ARRAY_SIZE, wbuf); + + DB::ReadBufferFromNuraftBuffer rbuf(wbuf.getBuffer()); + std::string s; + expect_marshalling_error(rbuf, s); + EXPECT_LE(s.capacity(), 2 * chunk); + } + + { + /// The input ends after one complete chunk. + DB::WriteBufferFromNuraftBuffer wbuf; + Coordination::write(Coordination::MAX_STRING_OR_ARRAY_SIZE, wbuf); + const std::string first_chunk(chunk, 'x'); + wbuf.write(first_chunk.data(), first_chunk.size()); + + DB::ReadBufferFromNuraftBuffer rbuf(wbuf.getBuffer()); + std::string s; + expect_marshalling_error(rbuf, s); + EXPECT_LE(s.capacity(), 4 * chunk); + } + + { + /// A string that needs more than one chunk is read completely, and the next field is aligned. + std::string expected(chunk + 1, '\0'); + for (size_t i = 0; i < expected.size(); ++i) + expected[i] = static_cast(i % 251); + + DB::WriteBufferFromNuraftBuffer wbuf; + Coordination::write(expected, wbuf); + Coordination::write(static_cast(42), wbuf); + + DB::ReadBufferFromNuraftBuffer rbuf(wbuf.getBuffer()); + std::string s; + int32_t next = 0; + Coordination::read(s, rbuf); + Coordination::read(next, rbuf); + EXPECT_EQ(s, expected); + EXPECT_EQ(next, 42); + EXPECT_TRUE(rbuf.eof()); + } +} + template struct SimpliestRaftServer { diff --git a/src/Coordination/tests/gtest_coordination_changelog.cpp b/src/Coordination/tests/gtest_coordination_changelog.cpp index ef2a02920b17f..cf601aebd88df 100644 --- a/src/Coordination/tests/gtest_coordination_changelog.cpp +++ b/src/Coordination/tests/gtest_coordination_changelog.cpp @@ -772,6 +772,46 @@ TEST_P(CoordinationTestWithCompression, ChangelogTestBatchOperationsEmpty) changelog_reader.init(5, 0); } +TEST_P(CoordinationTestWithCompression, ChangelogTestBatchOperationsInvalidSize) +{ + ChangelogDirTest test("./logs"); + this->setLogDirectory("./logs"); + + DB::KeeperLogStore changelog( + DB::LogFileSettings{.force_sync = true, .compress_logs = this->enable_compression, .rotate_interval = 100}, + DB::FlushSettings(), + DB::ReadAheadSettings{}, + this->keeper_context); + changelog.init(0, 0); + auto entry = getLogEntry("hello_world", 10); + changelog.append(entry); + changelog.end_of_append_batch(0, 0); + waitDurableLogs(changelog); + + /// A pack of one entry holds the entry count, the entry size and the entry itself. A peer controls the size, + /// so only the size is rewritten here. + auto entries = changelog.pack(1, 1); + const auto entry_size = static_cast(entries->size() - 2 * sizeof(int32_t)); + auto apply_with_size = [&](int32_t size) + { + entries->pos(sizeof(int32_t)); + entries->put(size); + changelog.apply_pack(2, *entries); + }; + + /// On master a size past the entry over-reads the pack in `nuraft::buffer::get` and a negative size + /// wraps the allocation size in `nuraft::buffer::alloc`. + EXPECT_THROW(apply_with_size(entry_size + 1), DB::Exception); + EXPECT_THROW(apply_with_size(-1), DB::Exception); + EXPECT_EQ(changelog.next_slot(), 2); + + apply_with_size(entry_size); + changelog.apply_pack(3, *changelog.pack(1, 0)); + changelog.end_of_append_batch(0, 0); + EXPECT_EQ(changelog.next_slot(), 3); + EXPECT_EQ(changelog.entry_at(2)->get_term(), 10); +} + TEST_P(CoordinationTestWithCompression, ChangelogTestWriteAtPreviousFile) { diff --git a/src/Core/BackgroundSchedulePool.cpp b/src/Core/BackgroundSchedulePool.cpp index 7fb3095594de8..fbcfcc137c5df 100644 --- a/src/Core/BackgroundSchedulePool.cpp +++ b/src/Core/BackgroundSchedulePool.cpp @@ -49,6 +49,12 @@ Poco::Timestamp::TimeDiff delayToMicroseconds(size_t milliseconds) return static_cast(milliseconds) * 1000; } +size_t maxParallelTasksPerType(size_t pool_size, double ratio) +{ + size_t limit = static_cast(static_cast(pool_size) * ratio); + return limit ? limit : pool_size; +} + } /// @@ -297,12 +303,12 @@ Coordination::WatchCallbackPtr BackgroundSchedulePoolTaskInfo::getWatchCallback( /// BackgroundSchedulePool /// -BackgroundSchedulePoolPtr BackgroundSchedulePool::create(size_t size, size_t initial_size, size_t max_parallel_tasks_per_type, CurrentMetrics::Metric tasks_metric, CurrentMetrics::Metric size_metric, ThreadName thread_name) +BackgroundSchedulePoolPtr BackgroundSchedulePool::create(size_t size, size_t initial_size, double max_parallel_tasks_per_type_ratio, CurrentMetrics::Metric tasks_metric, CurrentMetrics::Metric size_metric, ThreadName thread_name) { - return std::shared_ptr(new BackgroundSchedulePool(size, initial_size, max_parallel_tasks_per_type, tasks_metric, size_metric, thread_name)); + return std::shared_ptr(new BackgroundSchedulePool(size, initial_size, max_parallel_tasks_per_type_ratio, tasks_metric, size_metric, thread_name)); } -BackgroundSchedulePool::BackgroundSchedulePool(size_t size_, size_t initial_size_, size_t max_parallel_tasks_per_type_, CurrentMetrics::Metric tasks_metric_, CurrentMetrics::Metric size_metric_, ThreadName thread_name_) +BackgroundSchedulePool::BackgroundSchedulePool(size_t size_, size_t initial_size_, double max_parallel_tasks_per_type_ratio_, CurrentMetrics::Metric tasks_metric_, CurrentMetrics::Metric size_metric_, ThreadName thread_name_) : logger(getLogger(fmt::format("BackgroundSchedulePool/{}", toString(thread_name_)))) , tasks_metric(tasks_metric_) /// The metric reports the configured cap (so `system.server_settings` displays @@ -310,7 +316,7 @@ BackgroundSchedulePool::BackgroundSchedulePool(size_t size_, size_t initial_size /// not the number of currently spawned workers — that is tracked separately by `threads.size()`. , size_metric(size_metric_, size_) , thread_name(thread_name_) - , max_parallel_tasks_per_type(max_parallel_tasks_per_type_ ? max_parallel_tasks_per_type_ : size_) + , max_parallel_tasks_per_type_ratio(max_parallel_tasks_per_type_ratio_) { const size_t initial_size = std::min(size_, initial_size_); LOG_INFO(logger, "Create BackgroundSchedulePool with {} initial threads (grows lazily up to {})", initial_size, size_); @@ -320,6 +326,7 @@ BackgroundSchedulePool::BackgroundSchedulePool(size_t size_, size_t initial_size { std::lock_guard tasks_lock(tasks_mutex); max_size = size_; + max_parallel_tasks_per_type = maxParallelTasksPerType(size_, max_parallel_tasks_per_type_ratio); threads.reserve(size_); for (size_t i = 0; i < initial_size; ++i) spawnThreadLocked(); @@ -393,9 +400,32 @@ void BackgroundSchedulePool::increaseThreadsCount(size_t new_threads_count) return; } + /// Never lowered: a group may already run more tasks than a lower limit allows. + const size_t new_max_parallel_tasks_per_type = maxParallelTasksPerType(new_threads_count, max_parallel_tasks_per_type_ratio); + const bool raise_limit = new_max_parallel_tasks_per_type > max_parallel_tasks_per_type; + if (raise_limit) + runnable_task_types.reserve(task_groups.size()); + max_size = new_threads_count; size_metric.changeTo(new_threads_count); + if (raise_limit) + { + max_parallel_tasks_per_type = new_max_parallel_tasks_per_type; + + /// Groups paused at the old limit are runnable under the new one. + for (auto & [task_type, group] : task_groups) + { + if (!group.runnable_list_pos && !group.tasks.empty()) + { + group.runnable_list_pos = runnable_task_types.size(); + runnable_task_types.push_back(task_type); + } + } + /// Wake parked workers: the spawn below counts them as available. + tasks_cond_var.notify_all(); + } + /// Consume already-queued demand with the freshly-raised cap. A saturated pool has no parked /// workers to wake and may receive no further scheduleTask call for a while, so without spawning /// here it would keep executing at the old concurrency until some unrelated future task happens diff --git a/src/Core/BackgroundSchedulePool.h b/src/Core/BackgroundSchedulePool.h index bd418101030df..a618c8ae98eab 100644 --- a/src/Core/BackgroundSchedulePool.h +++ b/src/Core/BackgroundSchedulePool.h @@ -57,9 +57,10 @@ class BackgroundSchedulePool : public std::enable_shared_from_this; /// STYLE_CHECK_ALLOW_STD_CONTAINERS /// @param thread_name_ cannot be longer then 13 bytes (2 bytes is reserved for "/D" suffix for delayExecutionThreadFunction()) - BackgroundSchedulePool(size_t size_, size_t initial_size_, size_t max_parallel_tasks_per_type_, CurrentMetrics::Metric tasks_metric_, CurrentMetrics::Metric size_metric_, ThreadName thread_name_); + BackgroundSchedulePool(size_t size_, size_t initial_size_, double max_parallel_tasks_per_type_ratio_, CurrentMetrics::Metric tasks_metric_, CurrentMetrics::Metric size_metric_, ThreadName thread_name_); void threadFunction(); void delayExecutionThreadFunction(); @@ -154,7 +155,8 @@ class BackgroundSchedulePool : public std::enable_shared_from_this::changes() const template void BaseSettings::checkShorthandChange(const SettingChange & change) const { - /// `SET name` without a value means `SET name = true`, which only makes sense for a Bool - /// setting. This is where the settings schema is known, so this is where it is checked. + /// `SET name` without a value means `SET name = true`, which only makes sense for a Bool* + /// settings. This is where the settings schema is known, so this is where it is checked. if (!change.shorthand) return; - if (std::string_view type = getTypeName(change.name); type != "Bool") + if (std::string_view type = getTypeName(change.name); !type.starts_with("Bool")) BaseSettingsHelpers::throwValuelessSettingIsNotBool(change.name, type); - /// The type alone is not enough. The parser writes Bool `true` for the valueless form, but a + /// The type alone is not enough. The parser writes `Bool` `true` for the valueless form, but a /// `SettingChange` can also arrive from the AST JSON dialect, which is free to pair the flag - /// with any other value - and for a `Bool` setting the check above lets that through. Such a + /// with any other value - and for a `Bool*` setting the check above lets that through. Such a /// change would execute with the carried value while `ASTSetQuery::formatImpl` renders the bare /// name, so `system.query_log` and every other formatter would under-report what ran. This is /// the first point after deserialization where an exception is logged with the AST masked diff --git a/src/Core/BaseSettingsProgramOptions.h b/src/Core/BaseSettingsProgramOptions.h index e1bf34f060bf1..f7b2cd213cf36 100644 --- a/src/Core/BaseSettingsProgramOptions.h +++ b/src/Core/BaseSettingsProgramOptions.h @@ -11,7 +11,7 @@ template void addProgramOptionAsMultitoken(T &cmd_settings, boost::program_options::options_description & options, std::string_view name, const typename T::SettingFieldRef & field) { auto on_program_option = boost::function1([&cmd_settings, name](const Strings & values) { cmd_settings.set(name, values.back()); }); - if (field.getTypeName() == "Bool") + if (std::string_view type = field.getTypeName(); type.starts_with("Bool")) { options.add( boost::shared_ptr(new boost::program_options::option_description( @@ -54,7 +54,7 @@ template void addProgramOption(T & cmd_settings, boost::program_options::options_description & options, std::string_view name, const typename T::SettingFieldRef & field) { auto on_program_option = boost::function1([&cmd_settings, name](const std::string & value) { cmd_settings.set(name, value); }); - if (field.getTypeName() == "Bool") + if (std::string_view type = field.getTypeName(); type.starts_with("Bool")) { options.add(boost::shared_ptr(new boost::program_options::option_description( name.data(), boost::program_options::value()->composing()->implicit_value("1")->notifier(on_program_option), field.getDescription().data()))); // NOLINT diff --git a/src/Core/Field.cpp b/src/Core/Field.cpp index 5e9998838ce8a..5d27e08fa202d 100644 --- a/src/Core/Field.cpp +++ b/src/Core/Field.cpp @@ -1,5 +1,6 @@ #include #include +#include #include #include #include @@ -370,6 +371,9 @@ bool Field::operator== (const Field & rhs) const static Field getBinaryValue(UInt8 type, ReadBuffer & buf) { + /// Containers are read recursively, and the nesting depth comes from the data. + checkStackSize(); + switch (static_cast(type)) { case Field::Types::Null: diff --git a/src/Core/FormatFactorySettings.h b/src/Core/FormatFactorySettings.h index f4bb179a550f3..9f610dedf5878 100644 --- a/src/Core/FormatFactorySettings.h +++ b/src/Core/FormatFactorySettings.h @@ -1519,6 +1519,13 @@ Note that `LowCardinality` columns can produce Arrow batches much larger or smal Buffering blocks can increase memory use and delay the first record batch until the query finishes. Leave both targets at `0` to write record batches as blocks arrive. `0` (the default) disables the byte target. Try `1048576` (1 MiB) as a starting value. +)", 0) \ + DECLARE(UInt64, output_format_arrow_max_record_batch_size_bytes, 2147483646, R"( +Upper bound on the `Utf8`, `Binary` and `List` buffers of a single record batch in the `Arrow` and `ArrowStream` output formats. A block whose strings, binary values or list elements exceed this is written as several record batches. + +Unlike [output_format_arrow_record_batch_size_bytes](#output_format_arrow_record_batch_size_bytes), which is a target that combines small blocks, this is a limit that splits large ones. The default is the largest value Arrow can address, because it offsets those buffers with 32-bit integers; a larger value has no effect. + +Only the buffers named above are bounded: a column of a fixed-width type has no offsets to overflow and stays in one batch. A single row never splits either, so a row above the limit still produces a batch that exceeds it. )", 0) \ \ DECLARE(Bool, output_format_orc_string_as_string, true, R"( diff --git a/src/Core/PostgreSQLProtocol.h b/src/Core/PostgreSQLProtocol.h index 0adea1b15bb52..72cfde989a1bc 100644 --- a/src/Core/PostgreSQLProtocol.h +++ b/src/Core/PostgreSQLProtocol.h @@ -55,6 +55,14 @@ namespace PostgreSQLProtocol namespace Messaging { +/** Upper bound for the declared length of a message that is parsed before the client is authenticated. + * The payload of such a message is bounded by its declared length, but that length is chosen by an + * unauthenticated client, so the length itself must be bounded as well. Otherwise a client declares + * the Int32 maximum and makes the server buffer up to 2 GiB of payload before it gives up. + * PostgreSQL uses the same limit for its startup packet. + */ +constexpr Int32 max_unauthenticated_message_size = 10000; + enum class FrontMessageType : Int32 { // first message types @@ -307,14 +315,26 @@ class FrontMessage : public IMessage virtual void deserialize(ReadBuffer & in) = 0; protected: + /** Parses the payload of a message inside a buffer that ends at the declared length. + * `max_size` bounds the declared length itself. Pass it for every message that is parsed before the + * client is authenticated, because that client chooses the length. A frame rejected by `max_size` is + * left unread in the stream, so only a caller whose error closes the connection may pass a limit. + */ template - static void deserializePayload(ReadBuffer & in, std::string_view message_name, F && deserialize_payload) + static void deserializePayload( + ReadBuffer & in, + std::string_view message_name, + F && deserialize_payload, + Int32 max_size = std::numeric_limits::max()) { Int32 size = 0; readBinaryBigEndian(size, in); if (size < 4) throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT, "Wrong message length {} in {}, it must be at least 4", size, message_name); + if (size > max_size) + throw Exception(ErrorCodes::UNKNOWN_PACKET_FROM_CLIENT, + "Wrong message length {} in {}, it must not exceed {}", size, message_name, max_size); const size_t payload_size = static_cast(size - 4); LimitReadBuffer payload_in(in, {.read_no_less = payload_size, .read_no_more = payload_size}); @@ -598,7 +618,7 @@ class SASLInitialResponse : public Messaging::FrontMessage "Wrong SASL mechanism length {} in SASLInitialResponse, it must not be less than -1", size_sasl_mechanism); if (size_sasl_mechanism > 0) readStringOfDeclaredSize(sasl_mechanism, size_sasl_mechanism, payload_in); - }); + }, max_unauthenticated_message_size); } MessageType getMessageType() const override @@ -646,7 +666,7 @@ class SASLResponse : public Messaging::FrontMessage deserializePayload(in, "SASLResponse message", [this](ReadBuffer & payload_in) { readStringUntilEOF(sasl_mechanism, payload_in); - }); + }, max_unauthenticated_message_size); } MessageType getMessageType() const override @@ -687,7 +707,7 @@ class PasswordMessage : FrontMessage deserializePayload(in, "PasswordMessage", [this](ReadBuffer & payload_in) { readNullTerminated(password, payload_in); - }); + }, max_unauthenticated_message_size); } MessageType getMessageType() const override diff --git a/src/Core/ServerSettings.cpp b/src/Core/ServerSettings.cpp index 5d47a1b810c50..8aeef728f2cb1 100644 --- a/src/Core/ServerSettings.cpp +++ b/src/Core/ServerSettings.cpp @@ -193,6 +193,15 @@ Maximum number of idle standby threads to keep in the thread pool for decoding I )", 0) \ DECLARE(UInt64, iceberg_manifest_decode_thread_pool_queue_size, 10000, R"( The maximum number of jobs that can be scheduled on the thread pool for decoding Iceberg data manifest files. +)", 0) \ + DECLARE(NonZeroUInt64, ai_function_max_thread_pool_size, 1000, R"( +Maximum total number of threads to use for requests to AI providers. One thread holds one request, so this caps how many AI provider requests the whole server has in flight at once, across all queries. [`ai_function_max_concurrent_requests_per_thread`](/reference/settings/session-settings/ai-function#ai_function_max_concurrent_requests_per_thread) bounds each query thread's share of it. +)", 0) \ + DECLARE(UInt64, ai_function_max_thread_pool_free_size, 0, R"( +Maximum number of idle standby threads to keep in the thread pool for requests to AI providers. +)", 0) \ + DECLARE(UInt64, ai_function_thread_pool_queue_size, 10'000, R"( +The maximum number of jobs that can be scheduled on the thread pool for requests to AI providers. )", 0) \ DECLARE(UInt64, max_format_parsing_thread_pool_free_size, 0, R"( Maximum number of idle standby threads to keep in the thread pool for parsing input. @@ -885,6 +894,11 @@ A value of `0` means no limitation. ```xml 400 ``` +)", 0) \ + DECLARE(Bool, ai_function_allow_sql_named_collections, true, R"( +Allows AI functions (such as `aiGenerate`, `aiEmbed`, `aiSimilarity`) to use named collections created with `CREATE NAMED COLLECTION`. +When disabled, AI functions accept only named collections defined in the server configuration file, and using any other collection throws an exception. +This applies both to the `credentials` parameter and to the `ai_function_text_default_credentials` and `ai_function_embedding_default_credentials` settings. )", 0) \ DECLARE(UInt64, max_table_num_to_throw, 0lu, R"( If number of tables is greater than this value, server will throw an exception. @@ -3665,6 +3679,12 @@ ChangeableSettingsMap collectChangeableServerSettings(ContextPtr context) {getIcebergManifestDecodeThreadPool().isInitialized() ? std::to_string(getIcebergManifestDecodeThreadPool().get().getMaxFreeThreads()) : "0", ChangeableWithoutRestart::Yes}}, {"iceberg_manifest_decode_thread_pool_queue_size", {getIcebergManifestDecodeThreadPool().isInitialized() ? std::to_string(getIcebergManifestDecodeThreadPool().get().getQueueSize()) : "0", ChangeableWithoutRestart::Yes}}, + {"ai_function_max_thread_pool_size", + {getAIRequestThreadPool().isInitialized() ? std::to_string(getAIRequestThreadPool().get().getMaxThreads()) : "0", ChangeableWithoutRestart::Yes}}, + {"ai_function_max_thread_pool_free_size", + {getAIRequestThreadPool().isInitialized() ? std::to_string(getAIRequestThreadPool().get().getMaxFreeThreads()) : "0", ChangeableWithoutRestart::Yes}}, + {"ai_function_thread_pool_queue_size", + {getAIRequestThreadPool().isInitialized() ? std::to_string(getAIRequestThreadPool().get().getQueueSize()) : "0", ChangeableWithoutRestart::Yes}}, {"abort_on_logical_error", {std::to_string(DB::abort_on_logical_error), ChangeableWithoutRestart::Yes}}, diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 3beb8564864b8..21c96f7a0d4cd 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -1462,6 +1462,10 @@ Possible values: Since v25.4, `INSERT ... SELECT` from a `ReplicatedMergeTree` or `SharedMergeTree` source can also be parallelized across replicas. To enable it: - `parallel_distributed_insert_select = 2` - `enable_parallel_replicas = 1` + +:::note +The rewritten query, including every expression of the `SELECT` (subqueries, UDFs, and so on), is executed on the shards under the credentials of the cluster configuration — the same trust model as other shard-side processing, such as the subqueries of a non-`GLOBAL` `IN`. Configure an interserver secret (the `secret` element of the cluster configuration) if shard-side reads must be authorized as the calling user. +::: )", 0) \ DECLARE(UInt64, distributed_group_by_no_merge, 0, R"( Do not merge aggregation states from different servers for distributed query processing, you can use this in case it is for certain that there are different keys on different shards @@ -4551,6 +4555,13 @@ Reject patterns which will likely be expensive to evaluate with hyperscan (due t DECLARE(Bool, allow_simdjson, true, R"( Allow using simdjson library in 'JSON*' functions if AVX2 instructions are available. If disabled rapidjson will be used. )", 0) \ + DECLARE(Bool, json_extract_use_cast_string_to_date_time_mode, true, R"( +Controls which setting the `JSONExtract` family of functions follows when parsing a JSON string value into `DateTime` or `DateTime64`. When enabled, the parsing mode is taken from [cast_string_to_date_time_mode](#cast_string_to_date_time_mode), consistently with `CAST`. When disabled, it is taken from [date_time_input_format](/reference/settings/formats/input-format#date_time_input_format), as before version 26.8. + +When the chosen parser cannot read the value, the function returns the default value of the type (`1970-01-01 00:00:00`) without an error, so a mismatch between the two settings silently changes the extracted values. +)", 0, \ + {"26.11", true, true, "New setting to choose whether the `JSONExtract` family parses `DateTime` values according to `cast_string_to_date_time_mode` (since 26.8) or `date_time_input_format` (before 26.8)."}, \ + {"26.8", false, true, "`JSONExtract` parses `DateTime` values according to `cast_string_to_date_time_mode` instead of `date_time_input_format`. `compatibility` with versions before 26.8 restores the previous behavior."}) \ DECLARE(Bool, allow_introspection_functions, false, R"( Enables or disables [introspection functions](/reference/functions/regular-functions/introspection) for query profiling. @@ -4705,7 +4716,7 @@ Allow using `from_zk` substitutions in the dynamic disk configuration (i.e. in t Disabled by default. )", 0) \ DECLARE(Bool, s3_allow_server_credentials_in_user_queries, false, R"( -Allow S3 access that originates from user SQL to use server-managed credentials. +Allow S3 and MongoDB access that originates from user SQL to use server-managed credentials. When disabled (the default), the `s3`/`s3Cluster` table functions, the `S3`/`S3Queue` engines, S3 named collections, dynamic `disk(type=s3, ...)` definitions, `BACKUP`/`RESTORE TO S3`, DataLake table-data reads, and `DataLakeCatalog` databases (Glue, BigLake) may not resolve credentials from the environment, instance metadata (IMDS), IRSA, ECS, instance profile, SSO, AWS config/credentials files, or the GCP OAuth metadata service. A request that asks for one of those server-managed sources (for example `use_environment_credentials = 1` or `http_client = gcp_oauth`) without supplying usable explicit credentials is rejected with `ACCESS_DENIED`. A request that asks for none of them is sent unsigned (anonymous), the same as if `NOSIGN` had been given. @@ -4725,6 +4736,8 @@ For `BACKUP`/`RESTORE ... ON CLUSTER` the initiator's value of this setting is p Durability for persistent `S3` and `S3Queue` tables: enabling this only per session or profile is not durable across a restart. When the server reloads such a table from its stored definition (startup or `RESTORE`) it rebuilds the S3 client and re-applies the restriction with the startup context, so a table that relied on server-managed credentials and was created only under a session/profile `s3_allow_server_credentials_in_user_queries = 1` is created successfully but becomes inaccessible after a restart (the table is left in place; queries against it fail until its credentials resolve to a permitted source again). The server itself still starts. Give such tables explicit credentials for durable access; alternatively, enabling the setting server-wide keeps them loading across restarts, at the cost of relaxing the restriction for all reloads. +MongoDB is covered by this setting as well. When it is disabled, a query that reads with the `mongodb` table function or from a `MongoDB` table is rejected with `ACCESS_DENIED` if the connection string makes the server authenticate with its own credentials: with the authentication mechanism `MONGODB-OIDC` (the identity the server runs under: a Kubernetes service account token, or a token from the Azure or GCP instance metadata service), `MONGODB-AWS`, `GSSAPI` or `MONGODB-X509`, or with the connection option `tlsCertificateKeyFile` (or its deprecated alias `sslClientCertificateKeyFile`), which makes the server present a client certificate from its own filesystem. Only `SCRAM-SHA-1`, `SCRAM-SHA-256` and `PLAIN` are accepted, or no `authMechanism` at all. The check uses the settings of the session that reads, also for a table created by another user, so a session where this setting is enabled should not read MongoDB tables, or views of them, defined by untrusted users. Creating a `MongoDB` table is not checked. A MongoDB dictionary created with a DDL query may not use these mechanisms or `tlsCertificateKeyFile` whatever the value of this setting (except in `clickhouse-local`), because the server loads it with its own context, after the named collection it refers to may have changed; define such a dictionary in the server configuration files. Dictionaries defined in the server configuration files are not restricted, also when they refer to a named collection created with a DDL query, so whoever may alter that collection controls their connection string. + To keep it disabled for untrusted users, pin it in their profile by both setting the value explicitly to `0` and marking it `readonly`: ```xml @@ -6840,7 +6853,6 @@ Possible values: - 0 - Disable - 1 - Enable )", 0) \ - DECLARE(Bool, query_plan_read_in_order_through_join, true, "Keep reading in order from the left table in JOIN operations, which can be utilized by subsequent steps.", 0) \ DECLARE(Bool, query_plan_aggregation_in_order, true, R"( Toggles the aggregation in-order query-plan-level optimization. Only takes effect if setting [`query_plan_enable_optimizations`](#query_plan_enable_optimizations) is 1. @@ -6853,6 +6865,11 @@ Possible values: - 0 - Disable - 1 - Enable +)", 0) \ + DECLARE(BoolForce, query_plan_read_in_order_through_join, true, R"( +Keep reading in order from the left table in JOIN operations, which can be utilized by subsequent steps. + +For a join other than `LEFT`, the read stays in order only when every part of the left table announces a virtual row (see [`read_in_order_use_virtual_row`](#read_in_order_use_virtual_row)); without the virtual rows the merge after the join may read far more than the limit needs. The value `force` keeps the read in order regardless. )", 0) \ DECLARE(Bool, query_plan_remove_redundant_sorting, true, R"( Toggles a query-plan-level optimization which removes redundant sorting steps, e.g. in subqueries. @@ -8782,11 +8799,11 @@ Control the minimum number of payload columns from the left side required for en DECLARE(UInt64, ai_function_request_timeout_sec, 60, R"( Timeout in seconds for individual HTTP requests made by AI functions (AI chat completions and embedding API calls). If a request does not complete within this time, it is considered failed and may be retried according to `ai_function_max_retries`. )", BETA) \ - DECLARE(UInt64, ai_function_max_retries, 1, R"( -Maximum number of retry attempts for transient errors per individual API request. Each retry uses exponential backoff starting from `ai_function_retry_initial_delay_ms`. + DECLARE(UInt64, ai_function_max_retries, 3, R"( +Maximum number of retry attempts for transient errors per individual API request. Each retry uses exponential backoff starting from `ai_function_retry_initial_delay_ms`. At most 5. )", BETA) \ DECLARE(UInt64, ai_function_retry_initial_delay_ms, 1000, R"( -Initial delay in milliseconds before the first retry of a failed AI function API request. The delay doubles on each subsequent attempt (exponential backoff). For example, with default settings: 1000ms, 2000ms, 4000ms. +Initial delay in milliseconds before the first retry of a failed AI function API request. The delay doubles on each subsequent attempt (exponential backoff). For example, with default settings: 1000ms, 2000ms, 4000ms. At most 10000. )", BETA) \ DECLARE(Bool, ai_function_throw_on_error, true, R"( If true (default), an AI function call that fails permanently after exhausting all retries aborts the query with an exception. If false, the failed row receives the default value for the column type (empty string for String) and processing continues. @@ -8809,6 +8826,18 @@ If true (default), exceeding an AI function quota limit (`ai_function_max_input_ )", BETA) \ DECLARE(NonZeroUInt64, ai_function_embedding_max_batch_size, 100, R"( Maximum number of texts to include in a single HTTP request made by the embedding functions (`aiEmbed`, `aiSimilarity`). Texts are grouped into batches of this size to reduce API call overhead. For example, 500 unique texts with a batch size of 100 result in 5 HTTP requests. +)", BETA) \ + DECLARE(NonZeroUInt64, ai_function_max_concurrent_requests_per_thread, 8, R"( +Maximum number of provider requests one query thread has in flight at the same time. `1` issues requests one at a time. + +A query running on several threads (at most `max_threads`) can have up to this many requests in flight per thread. Use a [settings profile constraint](/concepts/features/configuration/settings/constraints-on-settings) (`...`) to put a ceiling on it that a query cannot raise. The ceiling is still per thread. + +A value above `1` loosens two guarantees: + +- A request's token usage is only known once its response arrives, so the token quotas (`ai_function_max_input_tokens_per_query`, `ai_function_max_output_tokens_per_query`) may overshoot by one request's worth per request in flight - that is, by up to this many requests' worth rather than by one. +- A whole wave of requests is dispatched before any of their errors is seen, so a query that fails on its first row may still have issued, and been billed for, up to this many requests. + +`ai_function_max_api_calls_per_query` is unaffected and stays an exact cap, because a slot is reserved before each request is dispatched. )", BETA) \ DECLARE(String, ai_function_text_default_credentials, "", R"( Name of the named collection used by the text AI functions (`aiGenerate`, `aiClassify`, `aiFilter`, `aiExtract`, `aiTranslate`, `aiRedact`) when the call does not pass `credentials` in its parameter map. Empty means no default: such calls must pass `credentials` explicitly. A chat-completions endpoint differs from an embeddings one, so this is separate from `ai_function_embedding_default_credentials`. diff --git a/src/Core/Settings.h b/src/Core/Settings.h index 449589b2431fd..18f53168d5418 100644 --- a/src/Core/Settings.h +++ b/src/Core/Settings.h @@ -52,6 +52,7 @@ class WriteBuffer; M(CLASS_NAME, ArrowFlightDescriptorType) \ M(CLASS_NAME, Bool) \ M(CLASS_NAME, BoolAuto) \ + M(CLASS_NAME, BoolForce) \ M(CLASS_NAME, CapnProtoEnumComparingMode) \ M(CLASS_NAME, Char) \ M(CLASS_NAME, DateTimeInputFormat) \ diff --git a/src/Core/SettingsChangesHistory.cpp b/src/Core/SettingsChangesHistory.cpp index faefbc68af3a8..8c5e3a229cb0c 100644 --- a/src/Core/SettingsChangesHistory.cpp +++ b/src/Core/SettingsChangesHistory.cpp @@ -47,6 +47,7 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() }); addSettingsChanges(settings_changes_history, "26.8", { + {"json_extract_use_cast_string_to_date_time_mode", false, true, "`JSONExtract` parses `DateTime` values according to `cast_string_to_date_time_mode` instead of `date_time_input_format`. `compatibility` with versions before 26.8 restores the previous behavior."}, {"backup_s3_expired_token_retry_timeout_ms", 0, 120000, "Retry S3 backup and restore requests after an expired token; 0 restores the previous behavior."}, {"allow_named_collection_override_by_default", true, true, "Obsolete. Overriding named collection keys requires `SHOW NAMED COLLECTIONS SECRETS`."}, {"use_aggregation_memory_tracker", false, false, "New setting. In this release it is disabled by default, which restores the two-level conversion of `GROUP BY` from before 26.7 (https://github.com/ClickHouse/ClickHouse/pull/107490)."}, @@ -55,7 +56,8 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"validate_group_by_all_key_types", true, true, "The validation of the key types that `GROUP BY ALL` expands the `SELECT` expressions into is kept under `compatibility` with 26.7: the previous value is deliberately equal to the new one, because 26.7 already rejected such a key and only a version before 26.7 restores the earlier acceptance."}, {"allow_executable_tables", true, true, "New setting to disable reading through the `executable` table function and from `Executable` and `ExecutablePool` tables."}, {"allow_experimental_ai_functions", false, false, "The setting is obsolete, AI functions are beta now and enabled by default."}, - {"ai_function_max_retries", 0, 1, "Retry a transient API error once by default, so a single 429 or 5xx from the provider does not fail the query."}, + {"ai_function_max_retries", 0, 3, "Retry a transient API error up to three times by default, so a 429 or 5xx from the provider does not fail the query and an overloaded rate-limited provider is retried under concurrent requests."}, + {"ai_function_max_concurrent_requests_per_thread", 1, 8, "New setting"}, {"adaptive_aggregator_freeze_threshold_bytes", 4194304, 4194304, "New setting bounding the adaptive aggregator's frozen local tables in bytes, whichever of it and the key-count threshold is reached first; 0 disables the byte bound."}, {"query_plan_aggregation_bucket_top_k", false, true, "New setting to toggle the plan optimization that materializes only each two-level bucket's best n groups when a final aggregation feeds ORDER BY over its outputs with LIMIT n and the per-bucket selection is provably exact."}, {"enable_group_by_top_k_optimization", false, true, "New setting to control the TopK filtering optimization during aggregation in `GROUP BY key ORDER BY key LIMIT N` queries."}, @@ -170,6 +172,8 @@ const VersionToSettingsChangesMap & getSettingsChangesHistory() {"output_format_arrow_record_batch_size", 0, 0, "New setting to combine small blocks in `Arrow` and `ArrowStream` output using a target row count. The default `0` preserves one record batch per block."}, {"output_format_arrow_record_batch_size_bytes", 0, 0, "New setting to combine small blocks in `Arrow` and `ArrowStream` output using a target size in bytes of accumulated data. The default `0` preserves one record batch per block."}, {"query_plan_filter_push_down_below_limit_by", false, true, "New setting to control pushing a filter on the `LIMIT BY` key columns below the `LIMIT BY` step. Set it to false to keep the filter above the `LIMIT BY`."}, + {"output_format_arrow_max_record_batch_size_bytes", 2147483646, 2147483646, "New setting bounding the size of one record batch in `Arrow` and `ArrowStream` output. The default is Arrow's own addressing limit, so it does not change behavior."}, + {"query_plan_read_in_order_through_join", true, true, "Added the value `force`: keep reading in order through a join even when the parts announce no virtual rows."}, }); addSettingsChanges(settings_changes_history, "26.7", { diff --git a/src/Core/SettingsFields.cpp b/src/Core/SettingsFields.cpp index 001edbfeb6290..faeaadc691e0a 100644 --- a/src/Core/SettingsFields.cpp +++ b/src/Core/SettingsFields.cpp @@ -562,6 +562,83 @@ void SettingFieldURI::readBinary(ReadBuffer & in) } +static bool isTriBoolKeyword(const Field & f, std::string_view keyword) +{ + return f.getType() == Field::Types::String && f.safeGet() == keyword; +} + +template +SettingFieldTriBool::SettingFieldTriBool(const Field & f) : is_keyword(isTriBoolKeyword(f, keyword)) +{ + if (!is_keyword) + base = SettingFieldBool(f); +} + +template +SettingFieldTriBool & SettingFieldTriBool::operator=(const Field & f) +{ + changed = true; + is_keyword = isTriBoolKeyword(f, keyword); + if (!is_keyword) + base = f; + return *this; +} + +template +SettingFieldTriBool::operator Field() const +{ + return is_keyword ? Field(String(keyword)) : Field(base); +} + +template +String SettingFieldTriBool::toString() const +{ + return is_keyword ? String(keyword) : base.toString(); +} + +template +void SettingFieldTriBool::parseFromString(const String & str) +{ + changed = true; + is_keyword = (str == keyword); + if (!is_keyword) + base.parseFromString(str); +} + +template +void SettingFieldTriBool::writeBinary(WriteBuffer & out) const +{ + if (is_keyword) + writeStringBinary(keyword, out); + else + base.writeBinary(out); +} + +template +void SettingFieldTriBool::readBinary(ReadBuffer & in) +{ + changed = true; + + /// A Bool is a varint 0/1 (see `SettingFieldNumber`); the keyword string starts with its length, which is longer. + UInt64 x = 0; + readVarUInt(x, in); + is_keyword = x > 1; + if (!is_keyword) + { + base = SettingFieldBool(x != 0); + return; + } + + if (x != keyword.size()) + throw Exception(ErrorCodes::CANNOT_PARSE_BOOL, "Cannot parse a bool or '{}' from the binary value of a setting", keyword); + String str(x, '\0'); + in.readStrict(str.data(), x); + if (str != keyword) + throw Exception(ErrorCodes::CANNOT_PARSE_BOOL, "Cannot parse a bool or '{}' from the binary value '{}' of a setting", keyword, str); +} + +template struct SettingFieldTriBool; + void SettingFieldEnumHelpers::writeBinary(std::string_view str, WriteBuffer & out) { writeStringBinary(str, out); diff --git a/src/Core/SettingsFields.h b/src/Core/SettingsFields.h index 0c06a6e8e3efd..c217c594b85da 100644 --- a/src/Core/SettingsFields.h +++ b/src/Core/SettingsFields.h @@ -200,6 +200,44 @@ using SettingFieldUInt32Auto = SettingAutoWrapper; using SettingFieldInt32Auto = SettingAutoWrapper; using SettingFieldDoubleAuto = SettingAutoWrapper; +/// A Bool setting with a third, named value: accepts everything a Bool accepts plus `Keyword::value`. +/// Mirrors `SettingAutoWrapper`: `base` holds the Bool, `is_keyword` tells that the third value is set. +/// The members are defined in SettingsFields.cpp, which instantiates the struct for every keyword. +template +struct SettingFieldTriBool +{ + constexpr static std::string_view keyword = Keyword::value; + /// In the binary format a Bool is a varint 0/1 and a string starts with its length (see `readBinary`). + static_assert(keyword.size() > 1); + + SettingFieldBool base; + bool is_keyword = false; + bool changed = false; + + explicit SettingFieldTriBool(bool x = false) : base(x) {} + explicit SettingFieldTriBool(const Field & f); + SettingFieldTriBool(const SettingFieldTriBool &) = default; + SettingFieldTriBool & operator=(const SettingFieldTriBool &) = default; + SettingFieldTriBool & operator=(const Field & f); + + bool isChanged() const { return changed; } + void setChanged(bool changed_) { changed = changed_; } + + explicit operator Field() const; + String toString() const; + void parseFromString(const String & str); + + void writeBinary(WriteBuffer & out) const; + void readBinary(ReadBuffer & in); +}; + +struct SettingFieldKeywordForce +{ + constexpr static std::string_view value = "force"; +}; + +using SettingFieldBoolForce = SettingFieldTriBool; + /* Similar to SettingFieldUInt64Auto with small differences to behave like regular UInt64, supported to compatibility. * When setting to 'auto' it becomes equal to the number of processor cores without taking into account SMT. * A value of 0 is also treated as 'auto', so 'auto' is parsed and serialized in the same way as 0. diff --git a/src/Core/Streaming/CursorTree.cpp b/src/Core/Streaming/CursorTree.cpp index 7f5933bd00bf3..eccbe6c4fdb2b 100644 --- a/src/Core/Streaming/CursorTree.cpp +++ b/src/Core/Streaming/CursorTree.cpp @@ -17,8 +17,14 @@ namespace DB namespace ErrorCodes { extern const int INVALID_CURSOR_LOOKUP; + extern const int BAD_ARGUMENTS; } +/// The tree is traversed and destroyed recursively, so its depth must be bounded. +/// Real cursors are shallow (e.g. `partition_id.block_number`), but the dotted paths may come +/// from a query (`STREAM CURSOR`) or from external storage (the refresh cursor in data lake metadata). +static constexpr size_t MAX_CURSOR_TREE_DEPTH = 256; + namespace { @@ -182,6 +188,9 @@ CursorTreeNodePtr buildCursorTree(const Map & collapsed_tree) VectorWithMemoryTracking path; boost::split(path, dotted_path, boost::is_any_of(".")); + if (path.size() > MAX_CURSOR_TREE_DEPTH) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cursor path is too deep: {} components, the maximum is {}", path.size(), MAX_CURSOR_TREE_DEPTH); + CursorTreeNode * node = root.get(); for (size_t i = 0; i + 1 < path.size(); ++i) node = node->getSubtreeOrCreate(path[i]).get(); diff --git a/src/Core/tests/gtest_BackgroundSchedulePool.cpp b/src/Core/tests/gtest_BackgroundSchedulePool.cpp index 3720301acbfc5..1b66e25a45079 100644 --- a/src/Core/tests/gtest_BackgroundSchedulePool.cpp +++ b/src/Core/tests/gtest_BackgroundSchedulePool.cpp @@ -403,3 +403,225 @@ TEST(BackgroundSchedulePool, ScheduleAfterHugeDelayDoesNotFire) ASSERT_EQ(task->deactivate(), true); pool->join(); } + +/// Raising the pool size at runtime (config reload) must also raise the number of tasks of one +/// type that may run at once, including for a group already paused at the old limit. +TEST(BackgroundSchedulePool, IncreaseThreadsCountRaisesSameTypeLimit) +{ + constexpr size_t initial_threads = 2; + constexpr size_t increased_threads = 4; + constexpr size_t num_tasks = increased_threads; + + auto pool = BackgroundSchedulePool::create(initial_threads, initial_threads, 0, CurrentMetrics::end(), CurrentMetrics::end(), ThreadName::TEST_SCHEDULER); + + std::mutex mutex; + std::condition_variable running_changed; + std::condition_variable release_tasks; + size_t running = 0; + bool release = false; + + /// A single lambda expression ⇒ one closure type ⇒ all tasks share one task group. + auto task_body = [&] + { + std::unique_lock lock(mutex); + ++running; + running_changed.notify_all(); + release_tasks.wait(lock, [&] { return release; }); + }; + + std::vector tasks; + tasks.reserve(num_tasks); + for (size_t i = 0; i < num_tasks; ++i) + tasks.emplace_back(pool->createTask(StorageID::createEmpty(), "increase_same_type", task_body)); + + for (auto & task : tasks) + EXPECT_TRUE(task->activateAndSchedule()); + + bool initial_ran = false; + bool all_ran = false; + size_t running_before_increase = 0; + size_t running_after_increase = 0; + { + std::unique_lock lock(mutex); + initial_ran = running_changed.wait_for(lock, std::chrono::seconds(60), [&] { return running >= initial_threads; }); + running_before_increase = running; + if (initial_ran) + { + lock.unlock(); + pool->increaseThreadsCount(increased_threads); + lock.lock(); + all_ran = running_changed.wait_for(lock, std::chrono::seconds(60), [&] { return running >= num_tasks; }); + } + running_after_increase = running; + release = true; + release_tasks.notify_all(); + } + + /// The pool's destructor requires join(), so a fatal assertion must not precede it. + pool->join(); + + EXPECT_TRUE(initial_ran); + EXPECT_EQ(running_before_increase, initial_threads); + EXPECT_TRUE(all_ran) << "only " << running_after_increase << " of " << num_tasks << " tasks of one type run after raising the pool size"; + EXPECT_EQ(running_after_increase, num_tasks); +} + +/// With a ratio, the raised limit is the new size times the ratio, exactly as at startup. +TEST(BackgroundSchedulePool, IncreaseThreadsCountKeepsPerTypeRatio) +{ + constexpr size_t initial_threads = 4; + constexpr size_t increased_threads = 8; + constexpr double ratio = 0.5; + constexpr size_t initial_limit = 2; + constexpr size_t increased_limit = 4; + constexpr size_t num_tasks = 6; + + auto pool = BackgroundSchedulePool::create(initial_threads, initial_threads, ratio, CurrentMetrics::end(), CurrentMetrics::end(), ThreadName::TEST_SCHEDULER); + + std::mutex mutex; + std::condition_variable running_changed; + std::condition_variable release_tasks; + size_t running = 0; + bool release = false; + + /// A single lambda expression ⇒ one closure type ⇒ all tasks share one task group. + auto task_body = [&] + { + std::unique_lock lock(mutex); + ++running; + running_changed.notify_all(); + release_tasks.wait(lock, [&] { return release; }); + }; + + std::vector tasks; + tasks.reserve(num_tasks); + for (size_t i = 0; i < num_tasks; ++i) + tasks.emplace_back(pool->createTask(StorageID::createEmpty(), "increase_keeps_ratio", task_body)); + + for (auto & task : tasks) + EXPECT_TRUE(task->activateAndSchedule()); + + bool initial_ran = false; + bool increased_ran = false; + size_t running_before_increase = 0; + size_t running_after_increase = 0; + size_t running_later = 0; + { + std::unique_lock lock(mutex); + initial_ran = running_changed.wait_for(lock, std::chrono::seconds(60), [&] { return running >= initial_limit; }); + running_before_increase = running; + if (initial_ran) + { + lock.unlock(); + pool->increaseThreadsCount(increased_threads); + lock.lock(); + /// The pool still has idle workers here, so they must be woken to take the extra tasks. + increased_ran = running_changed.wait_for(lock, std::chrono::seconds(60), [&] { return running >= increased_limit; }); + running_after_increase = running; + if (increased_ran) + { + lock.unlock(); + std::this_thread::sleep_for(std::chrono::seconds(1)); + lock.lock(); + running_later = running; + } + } + release = true; + release_tasks.notify_all(); + } + + /// The pool's destructor requires join(), so a fatal assertion must not precede it. + pool->join(); + + EXPECT_TRUE(initial_ran); + EXPECT_EQ(running_before_increase, initial_limit); + EXPECT_TRUE(increased_ran) << "only " << running_after_increase << " of " << increased_limit << " tasks of one type run after raising the pool size"; + EXPECT_EQ(running_after_increase, increased_limit); + EXPECT_EQ(running_later, increased_limit) << "the per-type ratio is not kept after raising the pool size"; +} + +/// The recomputed limit can be lower than the current one (0.4 * 2 truncates to 0, which means the +/// pool size 2, while 0.4 * 3 gives 1). It must be ignored: a second wave of tasks of the type, +/// scheduled after the resize, must still run two at once. +TEST(BackgroundSchedulePool, IncreaseThreadsCountNeverLowersPerTypeLimit) +{ + constexpr size_t initial_threads = 2; + constexpr size_t increased_threads = 3; + constexpr double ratio = 0.4; + constexpr size_t limit = 2; + constexpr size_t num_tasks = 2 * limit; + + auto pool = BackgroundSchedulePool::create(initial_threads, initial_threads, ratio, CurrentMetrics::end(), CurrentMetrics::end(), ThreadName::TEST_SCHEDULER); + + std::mutex mutex; + std::condition_variable state_changed; + size_t started = 0; + size_t running = 0; + size_t released = 0; + size_t finished = 0; + + /// A single lambda expression ⇒ one closure type ⇒ all tasks share one task group. + auto task_body = [&] + { + std::unique_lock lock(mutex); + const size_t ordinal = started++; + ++running; + state_changed.notify_all(); + state_changed.wait(lock, [&] { return ordinal < released; }); + --running; + ++finished; + state_changed.notify_all(); + }; + + std::vector tasks; + tasks.reserve(num_tasks); + for (size_t i = 0; i < num_tasks; ++i) + tasks.emplace_back(pool->createTask(StorageID::createEmpty(), "increase_never_lowers", task_body)); + + for (size_t i = 0; i < limit; ++i) + EXPECT_TRUE(tasks[i]->activateAndSchedule()); + + bool first_wave_ran = false; + bool first_wave_finished = false; + bool second_wave_ran = false; + size_t running_before_increase = 0; + size_t running_in_second_wave = 0; + { + std::unique_lock lock(mutex); + first_wave_ran = state_changed.wait_for(lock, std::chrono::seconds(60), [&] { return running >= limit; }); + running_before_increase = running; + if (first_wave_ran) + { + lock.unlock(); + pool->increaseThreadsCount(increased_threads); + lock.lock(); + released = limit; + state_changed.notify_all(); + first_wave_finished = state_changed.wait_for(lock, std::chrono::seconds(60), [&] { return finished >= limit; }); + } + if (first_wave_finished) + { + lock.unlock(); + /// The pool decrements its running count after the task function returns. + const auto deadline = std::chrono::steady_clock::now() + std::chrono::seconds(60); + while (!pool->getTasks().empty() && std::chrono::steady_clock::now() < deadline) + std::this_thread::yield(); + for (size_t i = limit; i < num_tasks; ++i) + EXPECT_TRUE(tasks[i]->activateAndSchedule()); + lock.lock(); + second_wave_ran = state_changed.wait_for(lock, std::chrono::seconds(60), [&] { return running >= limit; }); + running_in_second_wave = running; + } + released = num_tasks; + state_changed.notify_all(); + } + + /// The pool's destructor requires join(), so a fatal assertion must not precede it. + pool->join(); + + EXPECT_TRUE(first_wave_ran); + EXPECT_EQ(running_before_increase, limit); + EXPECT_TRUE(first_wave_finished); + EXPECT_TRUE(second_wave_ran) << "only " << running_in_second_wave << " of " << limit << " tasks of one type run after raising the pool size"; + EXPECT_EQ(running_in_second_wave, limit); +} diff --git a/src/DataTypes/DataTypeMap.cpp b/src/DataTypes/DataTypeMap.cpp index be88ea3a19b6f..3e8346cae6449 100644 --- a/src/DataTypes/DataTypeMap.cpp +++ b/src/DataTypes/DataTypeMap.cpp @@ -388,6 +388,8 @@ When a data part is written with `with_buckets` serialization: When a query reads a specific key (`m['key']`), the optimizer rewrites the expression to a key subcolumn (`m.key_`). The serialization layer computes which bucket the requested key belongs to and reads only that single bucket from disk. +This rewrite happens only for tables that use `with_buckets` serialization: with `basic` serialization reading a key subcolumn +reads the whole `Map` anyway. When the full map is read (e.g., `SELECT m`), all buckets are read and reassembled into the original map. This is slower than `basic` serialization due to the overhead of reading and merging multiple substreams. diff --git a/src/DataTypes/IDataType.cpp b/src/DataTypes/IDataType.cpp index ccc8e24295bef..c573881c50837 100644 --- a/src/DataTypes/IDataType.cpp +++ b/src/DataTypes/IDataType.cpp @@ -213,6 +213,7 @@ std::unique_ptr IDataType::getSubcolumnData( settings.enumerate_dynamic_streams = false; settings.enumerate_virtual_streams = true; settings.array_level = initial_array_level; + settings.subcolumn_name = subcolumn_name; data.serialization->enumerateStreams(settings, callback_with_data, data); if (!res && data.type->hasDynamicSubcolumnsData()) diff --git a/src/DataTypes/Serializations/ISerialization.h b/src/DataTypes/Serializations/ISerialization.h index b94270507e95b..b6e854b799128 100644 --- a/src/DataTypes/Serializations/ISerialization.h +++ b/src/DataTypes/Serializations/ISerialization.h @@ -19,6 +19,7 @@ #include #include #include +#include #include namespace DB @@ -362,6 +363,8 @@ class ISerialization : private boost::noncopyable, public std::enable_shared_fro struct EnumerateStreamsSettings { SubstreamPath path; + /// When resolving a subcolumn, serializers may omit streams that cannot contribute to its lookup. + std::optional subcolumn_name; bool position_independent_encoding = true; /// If set to false, don't enumerate dynamic subcolumns /// (such as dynamic types in Dynamic column or dynamic paths in JSON column). diff --git a/src/DataTypes/Serializations/SerializationLowCardinality.cpp b/src/DataTypes/Serializations/SerializationLowCardinality.cpp index 99271b598501a..5f6406bee6a73 100644 --- a/src/DataTypes/Serializations/SerializationLowCardinality.cpp +++ b/src/DataTypes/Serializations/SerializationLowCardinality.cpp @@ -702,6 +702,13 @@ void SerializationLowCardinality::deserializeBinaryBulkWithMultipleStreams( bool has_additional_keys = low_cardinality_state->index_type.has_additional_keys; bool column_is_empty = low_cardinality_column.empty(); + /// Indexes come from the stream. + if (low_cardinality_state->index_type.need_global_dictionary) + { + const size_t num_additional_keys = additional_keys ? additional_keys->size() : 0; + ColumnLowCardinality::checkIndexesAreLimited(*indexes_column, global_dictionary->size() + num_additional_keys); + } + if (!low_cardinality_state->index_type.need_global_dictionary) { if (additional_keys == nullptr) diff --git a/src/DataTypes/Serializations/SerializationObject.cpp b/src/DataTypes/Serializations/SerializationObject.cpp index 35ad4488a2880..934a510c66221 100644 --- a/src/DataTypes/Serializations/SerializationObject.cpp +++ b/src/DataTypes/Serializations/SerializationObject.cpp @@ -206,8 +206,7 @@ void SerializationObject::enumerateStreams(EnumerateStreamsSettings & settings, settings.path.push_back(Substream::ObjectData); - /// First, iterate over typed paths in sorted order, we will always serialize them. - for (const auto & path : sorted_typed_paths) + auto enumerate_typed_path = [&](const String & path) { settings.path.back().creator = std::make_shared(path); settings.path.push_back(Substream::ObjectTypedPath); @@ -222,6 +221,37 @@ void SerializationObject::enumerateStreams(EnumerateStreamsSettings & settings, serialization->enumerateStreams(settings, callback, path_data); settings.path.pop_back(); settings.path.back().creator.reset(); + }; + + if (settings.subcolumn_name) + { + /// Include the separator using the canonical stream naming, including empty path elements. + settings.path.push_back(Substream::ObjectTypedPath); + const auto prefix = getSubcolumnNameForStream(settings.path); + settings.path.pop_back(); + auto name = *settings.subcolumn_name; + if (name.starts_with(prefix)) + { + name.remove_prefix(prefix.size()); + /// Only an exact typed path or its descendants can match the requested name. + /// Prefixes are already in the same order as `sorted_typed_paths`, preserving collisions. + size_t end = 0; + while (true) + { + end = name.find('.', end); + if (auto it = typed_paths_serializations.find(String(name.substr(0, end))); it != typed_paths_serializations.end()) + enumerate_typed_path(it->first); + if (end == std::string_view::npos) + break; + ++end; + } + } + } + else + { + /// Full enumeration retains every typed path in serialization order. + for (const auto & path : sorted_typed_paths) + enumerate_typed_path(path); } /// If column or deserialization state was provided, iterate over dynamic paths, diff --git a/src/DataTypes/Serializations/tests/gtest_low_cardinality_serialization.cpp b/src/DataTypes/Serializations/tests/gtest_low_cardinality_serialization.cpp new file mode 100644 index 0000000000000..9aa26ad5cd997 --- /dev/null +++ b/src/DataTypes/Serializations/tests/gtest_low_cardinality_serialization.cpp @@ -0,0 +1,179 @@ +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace DB::ErrorCodes +{ + extern const int INCORRECT_DATA; +} + +using namespace DB; + +namespace +{ + +/// The MergeTree layout: the dictionary in one stream, the indexes and the additional keys in another. +template +auto makeGetter(Buffer & keys, Buffer & indexes) +{ + return [&keys, &indexes](const ISerialization::SubstreamPath & path) -> BufferPtr + { + if (!path.empty() && path.back().type == ISerialization::Substream::DictionaryKeys) + return &keys; + return &indexes; + }; +} + +struct Streams +{ + std::string keys; + std::string indexes; +}; + +Streams serialize(const DataTypePtr & type, const IColumn & column, size_t max_dictionary_size) +{ + WriteBufferFromOwnString keys; + WriteBufferFromOwnString indexes; + auto serialization = type->getDefaultSerialization(); + ISerialization::SerializeBinaryBulkSettings settings; + settings.getter = makeGetter(keys, indexes); + settings.low_cardinality_max_dictionary_size = max_dictionary_size; + settings.low_cardinality_use_single_dictionary_for_part = false; + ISerialization::SerializeBinaryBulkStatePtr state; + serialization->serializeBinaryBulkStatePrefix(column, settings, state); + serialization->serializeBinaryBulkWithMultipleStreams(column, 0, column.size(), settings, state); + serialization->serializeBinaryBulkStateSuffix(settings, state); + keys.finalize(); + indexes.finalize(); + return {keys.str(), indexes.str()}; +} + +ColumnPtr deserialize(const DataTypePtr & type, const Streams & streams, size_t rows) +{ + ReadBufferFromString keys(streams.keys); + ReadBufferFromString indexes(streams.indexes); + auto serialization = type->getDefaultSerialization(); + ISerialization::DeserializeBinaryBulkSettings settings; + settings.getter = makeGetter(keys, indexes); + settings.native_format = false; + ISerialization::DeserializeBinaryBulkStatePtr state; + serialization->deserializeBinaryBulkStatePrefix(settings, state, nullptr); + auto column = type->createColumn(); + serialization->deserializeBinaryBulkWithMultipleStreams(*column, rows, settings, state, nullptr); + return column; +} + +void expectIncorrectData(const DataTypePtr & type, const Streams & streams, size_t rows) +{ + try + { + deserialize(type, streams, rows); + FAIL() << "accepted an index past the keys"; + } + catch (const Exception & e) + { + ASSERT_EQ(e.code(), ErrorCodes::INCORRECT_DATA) << e.message(); + } +} + +/// Calls `check` for `LowCardinality(String)` and `LowCardinality(Nullable(UInt64))`, each serialized with a dictionary +/// of 2 keys (so most values become additional keys) and with one large enough for all values (no additional keys). +template +void forEachCase(Check && check) +{ + for (bool nullable : {false, true}) + { + DataTypePtr inner = nullable ? makeNullable(std::make_shared()) : std::make_shared(); + auto type = std::make_shared(inner); + auto column = type->createColumn(); + for (size_t i = 0; i < 20; ++i) + { + if (nullable) + column->insert(i % 6 == 0 ? Field() : Field(UInt64(i % 6))); + else + column->insert(String(1, static_cast('a' + i % 5))); + } + + for (size_t max_dictionary_size : {2, 8192}) + { + SCOPED_TRACE(type->getName() + " with max dictionary size " + std::to_string(max_dictionary_size)); + auto streams = serialize(type, *column, max_dictionary_size); + + constexpr UInt64 need_global_dictionary_bit = 1ULL << 8; + constexpr UInt64 has_additional_keys_bit = 1ULL << 9; + const UInt64 index_type = unalignedLoadLittleEndian(streams.indexes.data()); + const bool has_additional_keys = (index_type & has_additional_keys_bit) != 0; + ASSERT_NE(index_type & need_global_dictionary_bit, 0u); + ASSERT_EQ(has_additional_keys, max_dictionary_size == 2); + ASSERT_LE(index_type & 0xff, 3u); + + /// The keys stream starts with the version, then the number of keys; the indexes stream starts with the index type, + /// then the number of additional keys if there are any. + const UInt64 num_keys = unalignedLoadLittleEndian(streams.keys.data() + 8) + + (has_additional_keys ? unalignedLoadLittleEndian(streams.indexes.data() + 8) : 0); + const size_t index_width = 1ULL << (index_type & 0xff); + + check(type, *column, std::move(streams), num_keys, index_width); + } + } +} + +} + +TEST(SerializationLowCardinality, ValidStreamsRoundTrip) +{ + MainThreadStatus::getInstance(); + forEachCase([](const DataTypePtr & type, const IColumn & column, Streams streams, UInt64, size_t) + { + auto result = deserialize(type, streams, column.size()); + ASSERT_EQ(result->size(), column.size()); + for (size_t i = 0; i < column.size(); ++i) + ASSERT_EQ((*result)[i], column[i]) << "row " << i; + }); +} + +/// The indexes come from the data, so they have to be checked against the keys of the dictionary and the additional keys. +TEST(SerializationLowCardinality, IndexPastTheKeysIsRejected) +{ + MainThreadStatus::getInstance(); + forEachCase([](const DataTypePtr & type, const IColumn & column, Streams streams, UInt64 num_keys, size_t index_width) + { + const size_t rows = column.size(); + const size_t indexes_begin = streams.indexes.size() - rows * index_width; + + /// The same indexes as `UInt64` (index type 3, the low byte of the index type) read back the same values... + Streams wide{streams.keys, streams.indexes.substr(0, indexes_begin)}; + wide.indexes[0] = 3; + for (size_t row = 0; row < rows; ++row) + { + UInt64 index = 0; + for (size_t i = 0; i < index_width; ++i) + index |= UInt64(static_cast(streams.indexes[indexes_begin + row * index_width + i])) << (8 * i); + for (size_t i = 0; i < 8; ++i) + wide.indexes.push_back(static_cast(index >> (8 * i))); + } + auto result = deserialize(type, wide, rows); + ASSERT_EQ(result->size(), rows); + for (size_t row = 0; row < rows; ++row) + ASSERT_EQ((*result)[row], column[row]) << "row " << row; + + /// ...and the largest possible last index is rejected before `mapIndexWithAdditionalKeys` sizes its overflow map by it. + std::fill(wide.indexes.end() - 8, wide.indexes.end(), '\xff'); + expectIncorrectData(type, wide, rows); + + /// The last index of the original stream becomes the first value past the keys. + ASSERT_LE(num_keys, std::numeric_limits::max() >> (64 - 8 * index_width)); + for (size_t i = 0; i < index_width; ++i) + streams.indexes[streams.indexes.size() - index_width + i] = static_cast(num_keys >> (8 * i)); + expectIncorrectData(type, streams, rows); + }); +} diff --git a/src/DataTypes/tests/gtest_DataTypeObject_create.cpp b/src/DataTypes/tests/gtest_DataTypeObject_create.cpp index 8563b3d8343d0..eb8548a3c9322 100644 --- a/src/DataTypes/tests/gtest_DataTypeObject_create.cpp +++ b/src/DataTypes/tests/gtest_DataTypeObject_create.cpp @@ -1,4 +1,7 @@ +#include +#include #include +#include #include #include #include @@ -10,6 +13,29 @@ using namespace DB; +namespace +{ + +void expectSubcolumnType(const IDataType & type, std::string_view subcolumn, std::string_view expected_name) +{ + SCOPED_TRACE(String(subcolumn)); + auto result = type.tryGetSubcolumnType(subcolumn); + EXPECT_EQ(type.hasSubcolumn(subcolumn), !expected_name.empty()); + + if (expected_name.empty()) + { + EXPECT_EQ(result, nullptr); + EXPECT_THROW(type.getSubcolumnType(subcolumn), Exception); + return; + } + + ASSERT_NE(result, nullptr); + EXPECT_EQ(result->getName(), expected_name); + EXPECT_TRUE(result->equals(*type.getSubcolumnType(subcolumn))); +} + +} + /// Regression tests for STID 1477-2777 (BuzzHouse serverfuzz, amd_msan, 2026-04-25): /// `DataTypeObject::createObject` accessed `object_type_argument->parameter`, /// `path_with_type`, and `function->arguments->children[0]` / `[1]` without @@ -195,3 +221,106 @@ TEST(DataTypeObject, CreateJSONWithValidAST) auto type_default = factory.get("JSON"); ASSERT_NE(type_default, nullptr); } + +TEST(DataTypeObject, SubcolumnLookup) +{ + auto type = DataTypeFactory::instance().get( + "JSON(" + "max_dynamic_types=5, max_dynamic_paths=3, " + "typed_string String, typed_array Array(UInt64), " + "nested Array(JSON(inner UInt32)), nullable Nullable(String), " + "a Array(JSON), a.b Int64, `escaped.dot` UInt8, `spaced path` UInt8, obj.x UInt16)"); + const auto & object = assert_cast(*type); + + const std::vector> subcolumns = { + {"typed_string", "String"}, + {"typed_array", "Array(UInt64)"}, + {"typed_array.size0", "UInt64"}, + {"arbitrary", "Dynamic(max_types=5)"}, + {"arbitrary.:`Int64`", "Nullable(Int64)"}, + {"arbitrary.:`Array(String)`.size0", "UInt64"}, + {"nested.inner", "Array(UInt32)"}, + {"nested.inner.:`UInt32`", "Array(UInt32)"}, + {"a.x", "Array(Dynamic)"}, + {"a.b", "Int64"}, + {"a.b.c", "Array(Dynamic)"}, + {"a.:`Array(JSON)`.x", "Array(Dynamic)"}, + {"a.:`String`", "Array(Dynamic)"}, + {"nullable", "Nullable(String)"}, + {"escaped.dot", "UInt8"}, + {"spaced path", "UInt8"}, + {"^`obj`", "JSON(max_dynamic_types=5, max_dynamic_paths=3, x UInt16)"}, + {"@`obj`.x", "UInt16"}, + {"@`missing`", "Dynamic(max_types=5)"}, + {DataTypeObject::SPECIAL_SUBCOLUMN_NAME_FOR_DISTINCT_PATHS_CALCULATION, "Array(String)"}, + {":`Int64`", ""}, + }; + + for (const auto & [subcolumn, expected_name] : subcolumns) + expectSubcolumnType(object, subcolumn, expected_name); +} + +TEST(DataTypeObject, SubcolumnLookupPreservesSerializationPrecedence) +{ + auto & factory = DataTypeFactory::instance(); + + auto array_collision = factory.get("JSON(a Array(UInt64), a.size0 String)"); + expectSubcolumnType(*array_collision, "a.size0", "UInt64"); + + auto tuple_collision = factory.get("JSON(a Tuple(size0 String), a.size0 UInt64)"); + expectSubcolumnType(*tuple_collision, "a.size0", "String"); + + auto multiple_prefixes = factory.get("JSON(a Array(JSON), a.b Int64)"); + expectSubcolumnType(*multiple_prefixes, "a.b", "Int64"); + expectSubcolumnType(*multiple_prefixes, "a.b.c", "Array(Dynamic)"); + + auto special_name_collision = factory.get(fmt::format( + "JSON({} UInt8)", DataTypeObject::SPECIAL_SUBCOLUMN_NAME_FOR_DISTINCT_PATHS_CALCULATION)); + expectSubcolumnType( + *special_name_collision, DataTypeObject::SPECIAL_SUBCOLUMN_NAME_FOR_DISTINCT_PATHS_CALCULATION, "UInt8"); + + auto prefixed_name_collision = std::make_shared( + DataTypeObject::SchemaFormat::JSON, + std::unordered_map{{"^`obj`", factory.get("JSON(x UInt8)")}}); + expectSubcolumnType(*prefixed_name_collision, "^`obj`.x", "UInt8"); +} + +TEST(DataTypeObject, SubcolumnLookupUsesCanonicalPathNames) +{ + auto nested_type = DataTypeFactory::instance().get("JSON(x UInt8)"); + auto object = std::make_shared( + DataTypeObject::SchemaFormat::JSON, + std::unordered_map{ + {"escaped.dot", nested_type}, + {R"(back\slash)", nested_type}, + {"back`tick", nested_type}, + {"spaced path", nested_type}, + {"", nested_type}, + {".x", DataTypeFactory::instance().get("UInt16")}, + }); + + expectSubcolumnType(*object, "escaped.dot.x", "UInt8"); + expectSubcolumnType(*object, R"(back\slash.x)", "UInt8"); + expectSubcolumnType(*object, "back`tick.x", "UInt8"); + expectSubcolumnType(*object, "spaced path.x", "UInt8"); + expectSubcolumnType(*object, ".x", "UInt8"); +} + +TEST(DataTypeObject, SubcolumnLookupDefersToCustomSerialization) +{ + auto object = std::make_shared( + DataTypeObject::SchemaFormat::JSON, + std::unordered_map{{"a", DataTypeFactory::instance().get("UInt8")}}); + auto serialization = object->getDefaultSerialization(); + object->setCustomization(std::make_unique(DataTypeCustomNamePtr{}, std::move(serialization))); + + expectSubcolumnType(*object, "a", "UInt8"); +} + +TEST(DataTypeObject, SubcolumnsThroughContainers) +{ + auto & factory = DataTypeFactory::instance(); + expectSubcolumnType(*factory.get("Array(JSON(x UInt8))"), "x", "Array(UInt8)"); + expectSubcolumnType(*factory.get("Tuple(j JSON(x UInt8))"), "j.x", "UInt8"); + expectSubcolumnType(*factory.get("Dynamic"), "JSON.x", "Dynamic"); +} diff --git a/src/Databases/DataLake/HiveCatalog.cpp b/src/Databases/DataLake/HiveCatalog.cpp index 086f389e81a1f..421fe2b73d9d8 100644 --- a/src/Databases/DataLake/HiveCatalog.cpp +++ b/src/Databases/DataLake/HiveCatalog.cpp @@ -10,8 +10,10 @@ #include #include #include +#include #include #include +#include #include #include @@ -74,8 +76,9 @@ std::pair parseHostPort(const String & url) } -HiveCatalog::HiveCatalog(const std::string & warehouse_, const std::string & base_url_, DB::ContextPtr) +HiveCatalog::HiveCatalog(const std::string & warehouse_, const std::string & base_url_, DB::ContextPtr context_) : ICatalog(warehouse_) + , DB::WithContext(context_) , base_url(base_url_) { std::lock_guard lock(client_mutex); @@ -98,6 +101,10 @@ void HiveCatalog::reconnectUnlocked() const TSA_REQUIRES(client_mutex) auto [host, port] = parseHostPort(base_url); + /// Also rejects control characters such as NUL, which `getaddrinfo` would truncate the host at. + getContext()->getRemoteHostFilter().checkAndGetCanonicalHostAndPort( + host + ":" + std::to_string(port), static_cast(port), "Hive metastore address"); + socket = std::make_shared(host, port); transport = std::make_shared(socket); protocol = std::make_shared(transport); @@ -129,6 +136,11 @@ void HiveCatalog::executeWithRetry(Func && func) const { reconnectUnlocked(); } + catch (const DB::Exception &) + { + /// Non-transient (e.g. `UNACCEPTABLE_URL` from `remote_url_allow_hosts`), retrying cannot help. + throw; + } catch (const std::exception & ex) { LOG_TRACE( diff --git a/src/Databases/DatabaseReplicated.cpp b/src/Databases/DatabaseReplicated.cpp index d83f38975ccab..e432535a3e36f 100644 --- a/src/Databases/DatabaseReplicated.cpp +++ b/src/Databases/DatabaseReplicated.cpp @@ -64,6 +64,7 @@ #include #include #include +#include #include #include @@ -860,14 +861,14 @@ bool DatabaseReplicated::waitForReplicaToProcessAllEntries(UInt64 timeout_ms, Sy { auto component_guard = Coordination::setCurrentComponent("DatabaseReplicated::waitForReplicaToProcessAllEntries"); chassert(mode == SyncReplicaMode::DEFAULT || mode == SyncReplicaMode::STRICT); - { - std::lock_guard lock{ddl_worker_mutex}; - if (!ddl_worker || is_probably_dropped) - return false; - } + /// The worker can be stopped and replaced concurrently (`SYSTEM RESTORE DATABASE REPLICA`, `DROP DATABASE`): + /// the reference keeps it alive, and a stopped worker returns from the wait. + auto worker = tryGetDDLWorker(); + if (!worker) + return false; if (mode == SyncReplicaMode::DEFAULT) - return ddl_worker->waitForReplicaToProcessAllEntries(timeout_ms); + return worker->waitForReplicaToProcessAllEntries(timeout_ms); /// In STRICT mode the sync is satisfied as soon as our replication lag is within the /// max_replication_lag_to_enqueue threshold - we do not require every entry to be fully @@ -879,7 +880,7 @@ bool DatabaseReplicated::waitForReplicaToProcessAllEntries(UInt64 timeout_ms, Sy Stopwatch elapsed; while (true) { - UInt32 our_log_ptr = ddl_worker->getLogPointer(); + UInt32 our_log_ptr = worker->getLogPointer(); UInt32 max_log_ptr = parse(getZooKeeper()->get(fs::path(zookeeper_path) / "max_log_ptr")); bool became_synced = our_log_ptr + db_settings[DatabaseReplicatedSetting::max_replication_lag_to_enqueue] >= max_log_ptr; if (became_synced) @@ -893,8 +894,12 @@ bool DatabaseReplicated::waitForReplicaToProcessAllEntries(UInt64 timeout_ms, Sy /// processing (our_log_ptr == max_log_ptr) is strictly stronger than the STRICT /// threshold, so return immediately in that case; otherwise loop and re-check the /// threshold above, re-reading max_log_ptr, which might have advanced while we waited. - if (ddl_worker->waitForReplicaToProcessAllEntries(std::min(wait_step_ms, timeout_ms - elapsed_ms))) + if (worker->waitForReplicaToProcessAllEntries(std::min(wait_step_ms, timeout_ms - elapsed_ms))) return true; + + /// The worker was stopped (e.g. by `SYSTEM RESTORE DATABASE REPLICA` or `DROP DATABASE`), it will not make progress. + if (worker->isStopped()) + return false; } } @@ -1066,11 +1071,36 @@ void DatabaseReplicated::initDDLWorkerUnlocked() chassert(!is_probably_dropped.load()); chassert(!ddl_worker_initialized.load()); - ddl_worker = std::make_unique(this, getContext()); + replaceDDLWorkerUnlocked(std::make_shared(this, getContext())); ddl_worker->startup(); ddl_worker_initialized = true; } +std::shared_ptr DatabaseReplicated::tryGetDDLWorker() const +{ + std::lock_guard lock{ddl_worker_mutex}; + if (!ddl_worker || is_probably_dropped) + return nullptr; + return ddl_worker; +} + +std::shared_ptr DatabaseReplicated::replaceDDLWorkerUnlocked(std::shared_ptr new_worker) +{ + std::lock_guard lock{ddl_worker_ptr_mutex}; + std::swap(ddl_worker, new_worker); + return new_worker; +} + +bool DatabaseReplicated::isDDLWorkerCurrentlyActive() const +{ + std::shared_ptr worker; + { + std::lock_guard lock{ddl_worker_ptr_mutex}; + worker = ddl_worker; + } + return worker && worker->isCurrentlyActive(); +} + void DatabaseReplicated::restoreDatabaseNodesInKeeper(const ZooKeeperPtr & zookeeper) { auto local_context = getContext(); @@ -1129,7 +1159,7 @@ void DatabaseReplicated::reinitializeDDLWorker() ddl_worker->shutdown(); { ddl_worker_initialized = false; - ddl_worker = nullptr; + replaceDDLWorkerUnlocked(nullptr); } } @@ -1359,7 +1389,7 @@ bool DatabaseReplicated::checkDigestValid(const ContextPtr & local_context) cons LOG_TEST(log, "Current in-memory metadata digest: {}", tables_metadata_digest); /// Database is probably being dropped - if (!local_context->getZooKeeperMetadataTransaction() && (!ddl_worker || !ddl_worker->isCurrentlyActive())) + if (!local_context->getZooKeeperMetadataTransaction() && !isDDLWorkerCurrentlyActive()) return true; /// SYSTEM RESTART REPLICA temporarily removes a table from the in-memory tables map @@ -1465,13 +1495,12 @@ BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, Contex if (is_readonly) throw Exception(ErrorCodes::NO_ZOOKEEPER, "Database is in readonly mode, because it cannot connect to ZooKeeper"); - String host_fqdn_id; - { - std::lock_guard lock{ddl_worker_mutex}; - if (!ddl_worker || is_probably_dropped) - throw Exception(ErrorCodes::DATABASE_REPLICATION_FAILED, "Database is not initialized or is being dropped"); - host_fqdn_id = ddl_worker->getCommonHostID(); - } + /// Hold our own reference: `SYSTEM RESTORE DATABASE REPLICA` or `DROP DATABASE` can stop and replace the worker + /// while this query waits in `tryEnqueueAndExecuteEntry`. A stopped worker makes it throw instead of executing. + auto worker = tryGetDDLWorker(); + if (!worker) + throw Exception(ErrorCodes::DATABASE_REPLICATION_FAILED, "Database is not initialized or is being dropped"); + String host_fqdn_id = worker->getCommonHostID(); if (!flags.internal && query_context->isDDLOrOnClusterInternal()) throw Exception(ErrorCodes::INCORRECT_QUERY, "It's not initial query. ON CLUSTER is not allowed for Replicated database."); @@ -1486,7 +1515,7 @@ BlockIO DatabaseReplicated::tryEnqueueReplicatedDDL(const ASTPtr & query, Contex entry.tracing_context = OpenTelemetry::CurrentContext(); entry.initial_query_id = query_context->getClientInfo().initial_query_id; entry.is_backup_restore = flags.distributed_backup_restore; - String node_path = ddl_worker->tryEnqueueAndExecuteEntry(entry, query_context, flags.internal); + String node_path = worker->tryEnqueueAndExecuteEntry(entry, query_context, flags.internal); Strings hosts_to_wait; Strings unfiltered_hosts = getZooKeeper()->getChildren(zookeeper_path + "/replicas"); @@ -2375,10 +2404,63 @@ void DatabaseReplicated::restoreDatabaseInKeeper(ContextPtr) auto component_guard = Coordination::setCurrentComponent("DatabaseReplicated::restoreDatabaseMetadataInKeeper"); waitDatabaseStarted(); + /// Concurrent restores of the same database would stop and re-create the DDL worker under each other + /// and race on the replication state that `initDatabaseReplica` rewrites (e.g. `max_log_ptr_at_creation`). + std::lock_guard restore_lock{restore_in_keeper_mutex}; + auto zookeeper = getZooKeeper(); if (!zookeeper) throw Exception(ErrorCodes::NO_ZOOKEEPER, "No ZooKeeper"); + /// Stop this replica's DDL worker before the calls below re-initialize replication state: + /// its recovery reads `max_log_ptr_at_creation` and compares the Keeper metadata with the + /// local table set, which those calls rewrite, and a recovery running against a half-updated + /// view detaches local tables into `_broken_replicated_tables`. + std::shared_ptr old_worker; + { + std::lock_guard lock{ddl_worker_mutex}; + old_worker = ddl_worker; + } + + /// Initial queries admitted by the worker execute and commit on their own threads, and `shutdown` does not wait + /// for them. Wait for them before rewriting the replication state: otherwise a query could commit its metadata + /// while the recovery of the new worker compares the Keeper snapshot with the local tables, and the recovery + /// would move the just created table to `_broken_replicated_tables` (or drop it). + /// Stop admitting new initial queries, but keep the worker running while waiting: an admitted query may depend + /// on it, e.g. an `ALTER` of a `ReplicatedMergeTree` table with `alter_sync` waits for its `ALTER_METADATA` entry, + /// which the table executes only while `canExecuteReplicatedMetadataAlter` is true. + /// Not under `ddl_worker_mutex`, because these queries may need it. + if (old_worker) + { + LOG_TRACE(log, "Waiting for initial queries before restoring database metadata in Keeper."); + old_worker->stopInitialQueries(); + old_worker->waitForForegroundTasks(); + } + + { + std::lock_guard lock{ddl_worker_mutex}; + /// Only `restoreDatabaseInKeeper` (serialized by `restore_in_keeper_mutex`) creates a new worker after startup, + /// so the worker is either still the same or was removed by a concurrent `DROP DATABASE`. + chassert(!ddl_worker || ddl_worker == old_worker); + if (ddl_worker) + { + LOG_TRACE(log, "Stopping DDL worker before restoring database metadata in Keeper."); + ddl_worker->shutdown(); + ddl_worker_initialized = false; + replaceDDLWorkerUnlocked(nullptr); + } + } + old_worker.reset(); + + /// If the restore fails, reinitialize the DDL worker so the database remains functional. + /// SAFE: the body runs while an exception propagates and can itself throw (thread + /// creation), which must not replace the restore's own error. + bool need_reinitialize_ddl_worker = true; + SCOPE_EXIT_SAFE({ + if (need_reinitialize_ddl_worker) + reinitializeDDLWorker(); + }); + try { restoreDatabaseNodesInKeeper(zookeeper); @@ -2396,6 +2478,8 @@ void DatabaseReplicated::restoreDatabaseInKeeper(ContextPtr) /// Force the database to recover to update the restored metadata auto current_zookeeper = getZooKeeper(); current_zookeeper->set(replica_path + "/digest", DatabaseReplicatedDDLWorker::FORCE_AUTO_RECOVERY_DIGEST); + + need_reinitialize_ddl_worker = false; reinitializeDDLWorker(); } @@ -2490,7 +2574,7 @@ void DatabaseReplicated::shutdown() { std::lock_guard lock{ddl_worker_mutex}; ddl_worker_initialized = false; - ddl_worker = nullptr; + replaceDDLWorkerUnlocked(nullptr); } DatabaseAtomic::shutdown(); @@ -2505,7 +2589,7 @@ void DatabaseReplicated::dropTable(ContextPtr local_context, const String & tabl waitDatabaseStarted(); auto txn = local_context->getZooKeeperMetadataTransaction(); - chassert(!ddl_worker || !ddl_worker->isCurrentlyActive() || txn || startsWith(table_name, ".inner_id.") || startsWith(table_name, ".tmp.inner_id.")); + chassert(txn || !isDDLWorkerCurrentlyActive() || startsWith(table_name, ".inner_id.") || startsWith(table_name, ".tmp.inner_id.")); if (txn && txn->isInitialQuery() && !txn->isCreateOrReplaceQuery()) { String metadata_zk_path = zookeeper_path + "/metadata/" + escapeForFileName(table_name); @@ -2617,7 +2701,7 @@ void DatabaseReplicated::commitCreateTable(const ASTCreateQuery & query, const S ContextPtr query_context) { auto txn = query_context->getZooKeeperMetadataTransaction(); - chassert(!ddl_worker->isCurrentlyActive() || txn); + chassert(txn || !isDDLWorkerCurrentlyActive()); String statement = getObjectDefinitionFromCreateQuery(query.clone()); @@ -2654,7 +2738,7 @@ void DatabaseReplicated::commitAlterTable(const StorageID & table_id, const String & statement, ContextPtr query_context) { auto txn = query_context->getZooKeeperMetadataTransaction(); - chassert(!ddl_worker || !ddl_worker->isCurrentlyActive() || txn); + chassert(txn || !isDDLWorkerCurrentlyActive()); if (txn && txn->isInitialQuery()) { String metadata_zk_path = zookeeper_path + "/metadata/" + escapeForFileName(table_id.table_name); @@ -2694,7 +2778,7 @@ void DatabaseReplicated::detachTablePermanently(ContextPtr local_context, const throw Exception(ErrorCodes::SUPPORT_IS_DISABLED, "Support for DETACH TABLE PERMANENTLY is disabled"); auto txn = local_context->getZooKeeperMetadataTransaction(); - chassert(!ddl_worker->isCurrentlyActive() || txn); + chassert(txn || !isDDLWorkerCurrentlyActive()); if (txn && txn->isInitialQuery()) { /// We have to remove metadata from zookeeper, because we do not distinguish permanently detached tables @@ -2719,7 +2803,7 @@ void DatabaseReplicated::removeDetachedPermanentlyFlag(ContextPtr local_context, waitDatabaseStarted(); auto txn = local_context->getZooKeeperMetadataTransaction(); - chassert(!ddl_worker->isCurrentlyActive() || txn); + chassert(txn || !isDDLWorkerCurrentlyActive()); if (txn && txn->isInitialQuery() && attach) { String metadata_zk_path = zookeeper_path + "/metadata/" + escapeForFileName(table_name); diff --git a/src/Databases/DatabaseReplicated.h b/src/Databases/DatabaseReplicated.h index 68fd64c8e93b2..adb9063fe751c 100644 --- a/src/Databases/DatabaseReplicated.h +++ b/src/Databases/DatabaseReplicated.h @@ -249,6 +249,15 @@ class DatabaseReplicated : public DatabaseAtomic void stopLoading() override; void initDDLWorkerUnlocked() TSA_REQUIRES(ddl_worker_mutex); + /// Returns a reference to the DDL worker taken under `ddl_worker_mutex`, or nullptr if the database + /// is not initialized yet or is being dropped. + std::shared_ptr tryGetDDLWorker() const; + /// Assigns `ddl_worker` and returns the previous worker. Requires both mutexes, see `ddl_worker_ptr_mutex`. + std::shared_ptr replaceDDLWorkerUnlocked(std::shared_ptr new_worker) + TSA_REQUIRES(ddl_worker_mutex); + /// Whether the current DDL worker exists and is active. Does not take `ddl_worker_mutex`, + /// so it is safe to call from the callbacks of DDL queries executed by the worker. + bool isDDLWorkerCurrentlyActive() const; void restoreDatabaseNodesInKeeper(const ZooKeeperPtr & zookeeper); void reinitializeDDLWorker(); @@ -276,9 +285,20 @@ class DatabaseReplicated : public DatabaseAtomic std::atomic_bool is_probably_dropped = false; std::atomic_bool is_recovering = false; std::atomic_bool ddl_worker_initialized = false; - std::unique_ptr ddl_worker; + /// Shared, so that a query thread that uses the worker outside of `ddl_worker_mutex` (e.g. waiting in + /// `tryEnqueueAndExecuteEntry`) keeps it alive while `shutdown`, `restoreDatabaseInKeeper` or + /// `reinitializeDDLWorker` concurrently stop and replace it. Such a thread must take its own reference + /// under the mutex with `tryGetDDLWorker` and never dereference this member without the lock. + std::shared_ptr ddl_worker; mutable std::mutex ddl_worker_mutex; + /// Protects only the `ddl_worker` pointer itself: it is assigned under both mutexes and can be read under either. + /// Unlike `ddl_worker_mutex`, it is never held while the worker is shut down (and its threads are joined), + /// so the callbacks of DDL queries (`commitCreateTable`, `dropTable`, ...) can take it with `isDDLWorkerCurrentlyActive` + /// even when they are executed by the worker's threads or concurrently with `SYSTEM RESTORE DATABASE REPLICA`. + mutable std::mutex ddl_worker_ptr_mutex; UInt32 max_log_ptr_at_creation = 0; + /// Serializes `restoreDatabaseInKeeper`. Taken before `ddl_worker_mutex`. + std::mutex restore_in_keeper_mutex; /// Usually operation with metadata are single-threaded because of the way replication works, /// but StorageReplicatedMergeTree may call alterTable outside from DatabaseReplicatedDDLWorker causing race conditions. diff --git a/src/Databases/DatabaseReplicatedWorker.cpp b/src/Databases/DatabaseReplicatedWorker.cpp index 363a2452e8204..bcc650f292e92 100644 --- a/src/Databases/DatabaseReplicatedWorker.cpp +++ b/src/Databases/DatabaseReplicatedWorker.cpp @@ -1,4 +1,5 @@ #include +#include #include #include @@ -52,6 +53,7 @@ namespace FailPoints extern const char database_replicated_delay_recovery[]; extern const char database_replicated_delay_entry_execution[]; extern const char database_replicated_stop_entry_execution[]; + extern const char database_replicated_pause_before_initial_query_execution[]; } @@ -163,9 +165,35 @@ void DatabaseReplicatedDDLWorker::shutdown() active_node_holder.reset(); active_node_holder_zookeeper.reset(); + /// Waiters in `tryEnqueueAndExecuteEntry` and `waitForReplicaToProcessAllEntries` (which may outlive the + /// shutdown, holding a reference to this worker) check `stop_flag` under `mutex`. Taking it before notifying + /// guarantees that each of them either sees the flag or receives the notification. + { + std::lock_guard lock{mutex}; + for (const auto & event : log_ptr_waiters) + event->set(); + } + wait_current_task_change.notify_all(); +} + +void DatabaseReplicatedDDLWorker::stopInitialQueries() +{ + { + std::lock_guard lock{mutex}; + initial_queries_stopped = true; + } wait_current_task_change.notify_all(); } +void DatabaseReplicatedDDLWorker::waitForForegroundTasks() +{ + std::unique_lock lock{mutex}; + chassert(stop_flag || initial_queries_stopped); + if (foreground_tasks) + LOG_TRACE(log, "Waiting for {} admitted initial queries", foreground_tasks); + foreground_tasks_finished.wait(lock, [&] { return foreground_tasks == 0; }); +} + void DatabaseReplicatedDDLWorker::initializeReplication() { /// Check if we need to recover replica. @@ -367,19 +395,31 @@ bool DatabaseReplicatedDDLWorker::waitForReplicaToProcessAllEntries(UInt64 timeo return zookeeper->expired() || current_task >= max_log || stop_flag; }); - if (!processed) + if (!processed || stop_flag) return false; } - /// Lets now wait for max_log_ptr to be processed - Coordination::Stat stat; + /// Lets now wait for max_log_ptr to be processed. + /// The event is set either by the watch or by `shutdown`, so a stopped worker does not wait for the whole timeout. auto event_ptr = std::make_shared(); + { + std::lock_guard lock{mutex}; + if (stop_flag) + return false; + log_ptr_waiters.insert(event_ptr); + } + SCOPE_EXIT({ + std::lock_guard lock{mutex}; + log_ptr_waiters.erase(event_ptr); + }); + + Coordination::Stat stat; auto new_log = zookeeper->get(our_log_ptr_path, &stat, event_ptr); if (new_log == toString(max_log_ptr)) return true; - return event_ptr->tryWait(timeout_ms); + return event_ptr->tryWait(timeout_ms) && !stop_flag; } @@ -483,9 +523,9 @@ String DatabaseReplicatedDDLWorker::tryEnqueueAndExecuteEntry(DDLLogEntry & entr { chassert(zookeeper->expired() || current_task <= entry_name); - if (zookeeper->expired() || stop_flag) + if (zookeeper->expired() || stop_flag || initial_queries_stopped) { - LOG_TRACE(log, "Not enqueueing query: {}", stop_flag ? "replication stopped" : "ZooKeeper session expired"); + LOG_TRACE(log, "Not enqueueing query: {}", zookeeper->expired() ? "ZooKeeper session expired" : "replication stopped"); throw Exception(ErrorCodes::DATABASE_REPLICATION_FAILED, "ZooKeeper session expired or replication stopped, try again"); } @@ -501,8 +541,21 @@ String DatabaseReplicatedDDLWorker::tryEnqueueAndExecuteEntry(DDLLogEntry & entr if (!processed) throw Exception(ErrorCodes::UNFINISHED, "Timeout: Cannot enqueue query on this replica, " "most likely because replica is busy with previous queue entries"); + + /// The predicate above has checked `stop_flag` and `initial_queries_stopped` under `mutex`, + /// so `waitForForegroundTasks` will wait for this query. + ++foreground_tasks; } + SCOPE_EXIT({ + std::lock_guard lock{mutex}; + --foreground_tasks; + if (foreground_tasks == 0) + foreground_tasks_finished.notify_all(); + }); + + FailPointInjection::pauseFailPoint(FailPoints::database_replicated_pause_before_initial_query_execution); + if (entry.parent_table_uuid.has_value() && !checkParentTableExists(entry.parent_table_uuid.value())) throw Exception(ErrorCodes::TABLE_IS_DROPPED, "Parent table doesn't exist"); diff --git a/src/Databases/DatabaseReplicatedWorker.h b/src/Databases/DatabaseReplicatedWorker.h index 5583f112662ff..1f03d07229552 100644 --- a/src/Databases/DatabaseReplicatedWorker.h +++ b/src/Databases/DatabaseReplicatedWorker.h @@ -2,6 +2,9 @@ #include #include #include +#include + +#include namespace DB { @@ -31,8 +34,21 @@ class DatabaseReplicatedDDLWorker : public DDLWorker void shutdown() override; + /// Returns false on timeout or if the worker is stopped before or during the wait. bool waitForReplicaToProcessAllEntries(UInt64 timeout_ms); + /// The worker was shut down. A stopped worker is never restarted, the database replaces it with a new one. + bool isStopped() const { return stop_flag; } + + /// Makes `tryEnqueueAndExecuteEntry` reject new initial queries (as if the worker was stopped), + /// while the worker itself keeps running. Irreversible: the database replaces such a worker afterwards. + void stopInitialQueries(); + + /// Waits until the initial queries that `tryEnqueueAndExecuteEntry` admitted for execution finish. + /// They execute and commit on the submitting threads, so `shutdown` (which joins only the worker's + /// own threads) does not wait for them. Requires `stopInitialQueries` or `shutdown` to be called before. + void waitForForegroundTasks(); + static String enqueueQueryImpl(const ZooKeeperPtr & zookeeper, DDLLogEntry & entry, DatabaseReplicated * const database, bool committed = false, Coordination::Requests additional_checks = {}); /// NOLINT @@ -68,6 +84,15 @@ class DatabaseReplicatedDDLWorker : public DDLWorker std::condition_variable wait_current_task_change; String current_task; + /// The number of initial queries being executed by `tryEnqueueAndExecuteEntry` on the submitting threads. + /// Protected by `mutex`, see `waitForForegroundTasks`. + size_t foreground_tasks = 0; + std::condition_variable foreground_tasks_finished; + /// Set by `stopInitialQueries`. Protected by `mutex`. + bool initial_queries_stopped = false; + /// Events of `waitForReplicaToProcessAllEntries` waiting for our `log_ptr` in Keeper to change. + /// `shutdown` sets them, so that the waiters return without waiting for the timeout. + std::unordered_set> log_ptr_waiters; std::atomic logs_to_keep = std::numeric_limits::max(); std::atomic_bool unsynced_after_recovery = false; diff --git a/src/Databases/MySQL/DatabaseMySQL.cpp b/src/Databases/MySQL/DatabaseMySQL.cpp index 8c93a25e5800c..5302b8e74c0fb 100644 --- a/src/Databases/MySQL/DatabaseMySQL.cpp +++ b/src/Databases/MySQL/DatabaseMySQL.cpp @@ -39,6 +39,7 @@ # include # include # include +# include # include #if CLICKHOUSE_CLOUD @@ -682,6 +683,14 @@ void registerDatabaseMySQL(DatabaseFactory & factory) configuration.username = safeGetLiteralValue(arguments[2], engine_name); configuration.password = safeGetLiteralValue(arguments[3], engine_name); } + + /// Same policy as the `PostgreSQL` database engine: skip only the server's own metadata replay. + if (!(args.is_metadata_replay && args.mode >= LoadingStrictnessLevel::ATTACH)) + { + for (const auto & address : configuration.addresses) + args.context->getRemoteHostFilter().checkHostAndPort(address.first, toString(address.second)); + } + mysql_settings->loadFromQueryContext(args.context, *engine_define); if (engine_define->settings) mysql_settings->loadFromQuery(*engine_define); @@ -740,7 +749,7 @@ ENGINE = MySQL('host:port', ['database' | database], 'user', 'password') **Engine Parameters** -- `host:port` — MySQL server address. +- `host:port` — MySQL server address. The host must be a plain host name or IP address and is reached over TCP at the given port, also for `localhost`: Unix sockets are not supported. Other host syntax of the MySQL client library, such as connection strings or comma-separated host lists, is rejected. - `database` — Remote database name. - `user` — MySQL user. - `password` — User password. diff --git a/src/Dictionaries/ClickHouseDictionarySource.cpp b/src/Dictionaries/ClickHouseDictionarySource.cpp index 250a3a504f3de..12a897ba958da 100644 --- a/src/Dictionaries/ClickHouseDictionarySource.cpp +++ b/src/Dictionaries/ClickHouseDictionarySource.cpp @@ -241,11 +241,10 @@ std::string ClickHouseDictionarySource::doInvalidateQuery(const std::string & re return result; } - /// We pass empty block to RemoteQueryExecutor, because we don't know the structure of the result. - auto invalidate_sample_block = std::make_shared(Block{}); - QueryPipeline pipeline(std::make_shared( - std::make_shared(pool, request, invalidate_sample_block, context_copy), false, false, false)); - return readInvalidateQuery(pipeline); + /// The structure of the result is not known in advance, so the executor gets an empty header + /// and is read directly, not through a `RemoteSource`. + RemoteQueryExecutor executor(pool, request, std::make_shared(Block{}), context_copy); + return readInvalidateQuery(executor); } void registerDictionarySourceClickHouse(DictionarySourceFactory & factory); diff --git a/src/Dictionaries/DirectDictionary.cpp b/src/Dictionaries/DirectDictionary.cpp index 05d98fc6c70a9..c51159bfebda8 100644 --- a/src/Dictionaries/DirectDictionary.cpp +++ b/src/Dictionaries/DirectDictionary.cpp @@ -347,7 +347,7 @@ class SourceFromQueryPipeline : public ISource public: explicit SourceFromQueryPipeline(QueryPipeline & pipeline_) : ISource(std::make_shared(pipeline_.getSharedHeader()->cloneEmpty())) - , executor(pipeline_) + , executor(std::in_place, pipeline_) { pipeline_.setConcurrencyControl(false); } @@ -355,7 +355,7 @@ class SourceFromQueryPipeline : public ISource explicit SourceFromQueryPipeline(BlockIO io) : ISource(std::make_shared(io.pipeline.getSharedHeader()->cloneEmpty())) , io_holder(std::move(io)) - , executor(io_holder->pipeline) + , executor(std::in_place, io_holder->pipeline) { io_holder->pipeline.setConcurrencyControl(false); } @@ -370,7 +370,7 @@ class SourceFromQueryPipeline : public ISource try { Chunk chunk; - while (executor.pull(chunk)) + while (executor->pull(chunk)) { if (chunk) return chunk; @@ -380,6 +380,7 @@ class SourceFromQueryPipeline : public ISource { if (io_holder) { + executor.reset(); io_holder->onException(); } throw; @@ -387,6 +388,7 @@ class SourceFromQueryPipeline : public ISource if (io_holder) { + executor.reset(); io_holder->onFinish(); } return {}; @@ -400,7 +402,9 @@ class SourceFromQueryPipeline : public ISource /// * We need to run BlockIO::onFinish or BlockIO::onException. std::optional io_holder; - TExecutor executor; + /// Reset before `io_holder` tears its pipeline down: the executor shares the pipeline's processors, + /// and they must not outlive the table locks that `onFinish` / `onException` release. + std::optional executor; }; Pipe getSourcePipe(QueryPipeline & pipeline, const bool use_async_executor) diff --git a/src/Dictionaries/MongoDBDictionarySource.cpp b/src/Dictionaries/MongoDBDictionarySource.cpp index 3ea596d5a5e25..6ff705c680ebf 100644 --- a/src/Dictionaries/MongoDBDictionarySource.cpp +++ b/src/Dictionaries/MongoDBDictionarySource.cpp @@ -9,6 +9,7 @@ #include #include +#include #include #include @@ -46,7 +47,7 @@ void registerDictionarySourceMongoDB(DictionarySourceFactory & factory) Block & sample_block, ContextPtr context, const std::string & /* default_database */, - bool /* created_from_ddl */) + bool created_from_ddl) { const auto config_prefix = root_config_prefix + ".mongodb"; auto configuration = std::make_shared(); @@ -103,6 +104,10 @@ void registerDictionarySourceMongoDB(DictionarySourceFactory & factory) configuration->checkHosts(context); configuration->checkCollection(); + /// A DDL dictionary is loaded by the server with the global context, so no session setting can permit server credentials here. + if (created_from_ddl && context->getApplicationType() != Context::ApplicationType::LOCAL) + configuration->checkServerCredentialsForDictionary(); + return std::make_unique(dict_struct, std::move(configuration), std::make_shared(sample_block)); }; #else diff --git a/src/Dictionaries/MySQLDictionarySource.cpp b/src/Dictionaries/MySQLDictionarySource.cpp index e18fdb935deef..391d4de233007 100644 --- a/src/Dictionaries/MySQLDictionarySource.cpp +++ b/src/Dictionaries/MySQLDictionarySource.cpp @@ -344,7 +344,11 @@ The `table` or `where` fields cannot be used together with the `query` field. An There is no explicit parameter `secure`. When establishing an SSL-connection security is mandatory. -MySQL can be connected to on a local host via sockets. To do this, set `host` and `socket`. +MySQL can be connected to on a local host via sockets. To do this, set `host` to `localhost` and `socket`. For any other `host`, `socket` is ignored and the connection goes over TCP. Without `socket`, the connection goes over TCP to `port`, also for `localhost`. + + +`remote_url_allow_hosts` checks only `host` and `port`: the path in `socket` is not checked. + Example of settings: diff --git a/src/Dictionaries/SSDCacheDictionaryStorage.h b/src/Dictionaries/SSDCacheDictionaryStorage.h index 3edd9815022c8..873f9d533e16c 100644 --- a/src/Dictionaries/SSDCacheDictionaryStorage.h +++ b/src/Dictionaries/SSDCacheDictionaryStorage.h @@ -12,6 +12,7 @@ #include #include #include +#include # include # include @@ -480,24 +481,53 @@ class SSDCacheFileBuffer : private boost::noncopyable ProfileEvents::increment(ProfileEvents::FileOpen); + /// Another storage with the same path may still be reading the old file, so it is replaced, not truncated. + /// rename(2) overwrites a symlink, not the file it points to, so the link is resolved first. + std::filesystem::path target_path = file_path; + for (size_t hops = 0; target_path.has_filename() && FS::isSymlinkNoThrow(target_path); ++hops) + { + if (hops == 40) + ErrnoException::throwFromPathWithErrno(ErrorCodes::CANNOT_OPEN_FILE, file_path, ELOOP, "Cannot open file {}", file_path); + target_path = target_path.parent_path() / FS::readSymlink(target_path); + } + const std::string target_file_path = target_path.string(); + + /// rename(2) ignores the replaced file's permissions: a file the server cannot read and write must not be replaced. + if (::access(target_file_path.c_str(), R_OK | W_OK) != 0 && errno != ENOENT) + ErrnoException::throwFromPath(ErrorCodes::CANNOT_OPEN_FILE, file_path, "Cannot open file {}", target_file_path); + + const std::string new_file_path = (target_path.parent_path() / fmt::format("ssd_cache_{}.tmp", randomSeed())).string(); + #if defined(OS_DARWIN) /// macOS has no O_DIRECT; F_NOCACHE (set below) is the closest equivalent. - file.fd = ::open(file_path.c_str(), O_RDWR | O_CREAT | O_TRUNC, 0666); + file.fd = ::open(new_file_path.c_str(), O_RDWR | O_CREAT | O_EXCL, 0666); #else - file.fd = ::open(file_path.c_str(), O_RDWR | O_CREAT | O_TRUNC | O_DIRECT, 0666); + file.fd = ::open(new_file_path.c_str(), O_RDWR | O_CREAT | O_EXCL | O_DIRECT, 0666); #endif if (file.fd == -1) { auto error_code = (errno == ENOENT) ? ErrorCodes::FILE_DOESNT_EXIST : ErrorCodes::CANNOT_OPEN_FILE; - ErrnoException::throwFromPath(error_code, file_path, "Cannot open file {}", file_path); + ErrnoException::throwFromPath(error_code, file_path, "Cannot create file {} to replace {}", new_file_path, target_file_path); } - #if defined(OS_DARWIN) - if (::fcntl(file.fd, F_NOCACHE, 1) == -1) - ErrnoException::throwFromPath(ErrorCodes::CANNOT_OPEN_FILE, file_path, "Cannot set F_NOCACHE on file {}", file_path); - #endif + try + { + #if defined(OS_DARWIN) + if (::fcntl(file.fd, F_NOCACHE, 1) == -1) + ErrnoException::throwFromPath(ErrorCodes::CANNOT_OPEN_FILE, file_path, "Cannot set F_NOCACHE on file {}", file_path); + #endif + + allocateSizeForNextPartition(); - allocateSizeForNextPartition(); + if (::rename(new_file_path.c_str(), target_file_path.c_str()) != 0) + ErrnoException::throwFromPath( + ErrorCodes::CANNOT_OPEN_FILE, file_path, "Cannot rename {} to {}", new_file_path, target_file_path); + } + catch (...) + { + ::unlink(new_file_path.c_str()); + throw; + } } void allocateSizeForNextPartition() @@ -1396,9 +1426,10 @@ class SSDCacheDictionaryStorage final : public ICacheDictionaryStorage if (memory_buffer_partitions.size() < configuration.max_partitions_count) { /// Try tro create next partition without reusing old partitions - ++current_partition_index; file_buffer.allocateSizeForNextPartition(); memory_buffer_partitions.emplace_back(configuration.block_size, configuration.write_buffer_blocks_size); + /// Advance only after both calls succeed, so the index stays valid if one of them throws. + ++current_partition_index; } else { diff --git a/src/Dictionaries/YAMLRegExpTreeDictionarySource.cpp b/src/Dictionaries/YAMLRegExpTreeDictionarySource.cpp index 20c2c6bc0d152..28f1da3ca9bbe 100644 --- a/src/Dictionaries/YAMLRegExpTreeDictionarySource.cpp +++ b/src/Dictionaries/YAMLRegExpTreeDictionarySource.cpp @@ -334,7 +334,7 @@ void parseMatchList(UInt64 parent_id, UInt64 & id, const YAML::Node & node, Resu { if (!node.IsSequence()) { - throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Configuration {} must be a yaml list of match rules", node.as()); + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, "Configuration must be a yaml list of match rules"); } for (const auto & child_node : node) diff --git a/src/Dictionaries/readInvalidateQuery.cpp b/src/Dictionaries/readInvalidateQuery.cpp index a4aaa63cc84fd..00a4107733719 100644 --- a/src/Dictionaries/readInvalidateQuery.cpp +++ b/src/Dictionaries/readInvalidateQuery.cpp @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -19,12 +20,15 @@ namespace ErrorCodes extern const int RECEIVED_EMPTY_DATA; } -std::string readInvalidateQuery(QueryPipeline & pipeline) +namespace { - PullingPipelineExecutor executor(pipeline); +/// `pull` fills the next block of the result and returns false when there are no more blocks. +template +std::string readInvalidateQueryImpl(Pull && pull) +{ Block block; - while (executor.pull(block)) + while (pull(block)) if (!block.empty()) break; @@ -45,7 +49,7 @@ std::string readInvalidateQuery(QueryPipeline & pipeline) auto & column_type = block.getByPosition(0); column_type.type->getDefaultSerialization()->serializeTextQuoted(*column_type.column->convertToFullColumnIfConst(), 0, out, FormatSettings()); - while (executor.pull(block)) + while (pull(block)) if (block.rows() > 0) throw Exception(ErrorCodes::TOO_MANY_ROWS, "Expected single row in resultset, got at least {}", rows + 1); @@ -53,3 +57,24 @@ std::string readInvalidateQuery(QueryPipeline & pipeline) } } + +std::string readInvalidateQuery(QueryPipeline & pipeline) +{ + PullingPipelineExecutor executor(pipeline); + return readInvalidateQueryImpl([&](Block & block) { return executor.pull(block); }); +} + +std::string readInvalidateQuery(RemoteQueryExecutor & executor) +{ + executor.sendQuery(); + auto result = readInvalidateQueryImpl([&](Block & block) + { + /// An empty block marks the end of the stream. + block = executor.readBlock(); + return !block.empty(); + }); + executor.finish(); + return result; +} + +} diff --git a/src/Dictionaries/readInvalidateQuery.h b/src/Dictionaries/readInvalidateQuery.h index 99e20ad54e4b6..8ad34982f28e5 100644 --- a/src/Dictionaries/readInvalidateQuery.h +++ b/src/Dictionaries/readInvalidateQuery.h @@ -6,8 +6,13 @@ namespace DB { class QueryPipeline; +class RemoteQueryExecutor; /// Using in MySQLDictionarySource and XDBCDictionarySource after processing invalidate_query. std::string readInvalidateQuery(QueryPipeline & pipeline); +/// For a remote query whose result structure is not known in advance, so it cannot be a pipeline source: +/// a port with an empty header does not accept the received columns. +std::string readInvalidateQuery(RemoteQueryExecutor & executor); + } diff --git a/src/Disks/DiskFromAST.cpp b/src/Disks/DiskFromAST.cpp index 4053efc211b5b..4057e1092eeda 100644 --- a/src/Disks/DiskFromAST.cpp +++ b/src/Disks/DiskFromAST.cpp @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -18,48 +19,76 @@ #include #include +#include namespace DB { namespace ErrorCodes { + extern const int ACCESS_DENIED; extern const int BAD_ARGUMENTS; } -static std::string getOrCreateCustomDisk( +/// Resolve `include`, `from_env` and `from_zk` in a disk definition. +static Poco::AutoPtr resolveDiskConfiguration( + Poco::AutoPtr xml_document, ContextPtr context, bool attach) +{ + Poco::AutoPtr name_pool(new Poco::XML::NamePool()); + Poco::XML::DOMParser dom_parser(name_pool); + + std::vector> substitutions; + zkutil::ZooKeeperNodeCache zk_node_cache([&]() { return context->getZooKeeper(); }); + + ConfigProcessor::processIncludes( + xml_document, + substitutions, + context->getConfigRef().getString("include_from", ""), + /* throw_on_bad_incl= */!attach, + dom_parser, + getLogger("getOrCreateCustomDisk"), + /*contributing_zk_paths=*/ {}, + /*contributing_files=*/ {}, + &zk_node_cache); + + Poco::AutoPtr config(new Poco::Util::XMLConfiguration()); + config->load(xml_document); + return config; +} + +static std::string getCredentialsMarker(const ASTPtr & arg) +{ + const auto * eq = arg->as(); + if (!eq || eq->name != "equals" || !eq->arguments || eq->arguments->children.size() < 2) + return {}; + const auto * key = eq->arguments->children[0]->as(); + if (key && key->name() == S3_SERVER_CREDENTIALS_MARKER) + return key->name(); + return {}; +} + +static bool hasCredentialsMarker(const ASTs & args, const std::string & marker) +{ + return std::ranges::any_of(args, [&](const ASTPtr & arg) { return getCredentialsMarker(arg) == marker; }); +} + +/// `serializations` holds the definition, then other forms of it (without the credentials marker, other disks below it). +/// Returns the indexes and names of the registered disks the forms match, the one used first. The others are returned +/// only if this session may use them. If none matches, the first form is created. +static std::vector> getOrCreateCustomDisk( const ASTs & disk_args, - const std::string & serialization, + const std::vector & serializations, ContextPtr context, bool attach, bool for_system_database) { - const auto & server_config = context->getConfigRef(); - std::string include_from_path = server_config.getString("include_from", ""); - - Poco::AutoPtr config(new Poco::Util::XMLConfiguration()); + Poco::AutoPtr config; + bool uses_server_credentials = false; + bool restriction_exempt = false; { DynamicS3DiskCredentialInfo s3_disk_info; auto xml_document = getDiskConfigurationFromASTImpl(disk_args, context, attach, &s3_disk_info, for_system_database); - - Poco::AutoPtr name_pool(new Poco::XML::NamePool()); - Poco::XML::DOMParser dom_parser(name_pool); - - std::vector> substitutions; - zkutil::ZooKeeperNodeCache zk_node_cache([&]() { return context->getZooKeeper(); }); - - ConfigProcessor::processIncludes( - xml_document, - substitutions, - include_from_path, - /* throw_on_bad_incl= */!attach, - dom_parser, - getLogger("getOrCreateCustomDisk"), - /*contributing_zk_paths=*/ {}, - /*contributing_files=*/ {}, - &zk_node_cache); - - config->load(xml_document); + config = resolveDiskConfiguration(xml_document, context, attach); /// Applied after `processIncludes`. If the pre-resolution check already decided the disk must load /// anonymously, enforce it now (an `include` cannot re-introduce server credentials). Otherwise @@ -69,8 +98,13 @@ static std::string getOrCreateCustomDisk( forceAnonymousS3DiskConfig(*config); else validateResolvedS3DiskCredentials(*config, context, attach, s3_disk_info); + + uses_server_credentials = diskUsesServerCredentials(*config, s3_disk_info); + restriction_exempt = s3_disk_info.restriction_exempt; } + const bool restricted = !attach && !restriction_exempt && context->shouldRestrictUserQueryS3Credentials(); + Poco::Util::AbstractConfiguration::Keys disk_settings_keys; config->keys(disk_settings_keys); /// Check that no settings are defined when disk from the config is referred. @@ -83,28 +117,64 @@ static std::string getOrCreateCustomDisk( throw Exception( ErrorCodes::BAD_ARGUMENTS, "Disk function `{}` must have other arguments apart from `name`, which describe disk configuration. Invalid disk description.", - serialization); - - auto disk_settings_hash = sipHash128(serialization.data(), serialization.size()); + serializations[0]); - std::string disk_name; - if (config->has("name")) - { - disk_name = config->getString("name"); - } - else + auto get_disk_name = [&](UInt128 hash) -> std::string { + if (config->has("name")) + return config->getString("name"); /// We need a unique name for a created custom disk, but it needs to be the same /// after table is reattached or server is restarted, so take a hash of the disk /// configuration serialized ast as a disk name suffix. - disk_name = DiskSelector::TMP_INTERNAL_DISK_PREFIX + toString(disk_settings_hash); + return DiskSelector::TMP_INTERNAL_DISK_PREFIX + toString(hash); + }; + + std::vector hashes; + for (const auto & serialization : serializations) + hashes.push_back(sipHash128(serialization.data(), serialization.size())); + + /// A disk declared by an older version is registered under the form of the definition that version stored. Reusing it + /// is safe: this session passed the credential checks for its definition, the forms differ only in markers and in the + /// equivalent disks below, and the caller stores the form of the reused disk. + std::vector> matches; + if (!attach) + { + const auto disks = context->getDisksMap(); + for (size_t i = 0; i < serializations.size(); ++i) + { + auto name = get_disk_name(hashes[i]); + const auto it = disks.find(name); + if (it == disks.end() || !it->second->isCustomDisk() || it->second->getCustomDiskSettings() != hashes[i]) + continue; + if (!matches.empty() && restricted && it->second->customDiskUsesServerCredentials()) + continue; + matches.emplace_back(i, std::move(name)); + } } + if (matches.empty()) + matches.emplace_back(0, get_disk_name(hashes[0])); + + const auto disk_settings_hash = hashes[matches.front().first]; + const auto & disk_name = matches.front().second; auto disk = context->getOrCreateDisk(disk_name, [&](const DisksMap & disks_map) -> DiskPtr { + /// A wrapper (`encrypted`, `cache`) names a disk another session may have created with the opt-in. It is looked up + /// in the map the wrapper is created from, so that it cannot be registered in between. + const auto wrapped = disks_map.find(config->getString(".disk", "")); + const bool wraps_server_credentials = wrapped != disks_map.end() && wrapped->second->customDiskUsesServerCredentials(); + if (wraps_server_credentials && restricted) + throw Exception( + ErrorCodes::ACCESS_DENIED, + "A dynamic disk created from user SQL may not wrap the disk `{}`, which uses the server's own credentials." +#if !CLICKHOUSE_CLOUD + " Enable the setting `s3_allow_server_credentials_in_user_queries` to allow it." +#endif + , wrapped->first); + auto result = DiskFactory::instance().create( disk_name, *config, /* config_path */"", context, disks_map, /* attach */attach, /* custom_disk */true); /// Mark that disk can be used without storage policy. - result->markDiskAsCustom(disk_settings_hash); + result->markDiskAsCustom(disk_settings_hash, uses_server_credentials || wraps_server_credentials); return result; }); @@ -138,7 +208,18 @@ static std::string getOrCreateCustomDisk( disk_path_expected_prefix); } - return disk_name; + /// The disk may have been created by a session with the opt-in, from a configuration that resolved differently then + /// (`include` or `from_zk` changed since), so the check of this session's definition does not cover it. + if (restricted && disk->customDiskUsesServerCredentials()) + throw Exception( + ErrorCodes::ACCESS_DENIED, + "The disk `{}` uses the server's own credentials, which a dynamic disk created from user SQL may not use." +#if !CLICKHOUSE_CLOUD + " Enable the setting `s3_allow_server_credentials_in_user_queries` to allow it." +#endif + , disk_name); + + return matches; } template @@ -150,29 +231,129 @@ static void visitBottomUp(ASTPtr & ast, typename Matcher::Data & data) Matcher::visit(ast, data); } -class DiskConfigurationFlattener + +static ASTs & getDiskFunctionArguments(const ASTPtr & ast) { -public: - struct Data + return assert_cast(*ast->as().arguments).children; +} + +static ASTPtr makeCredentialsMarker(const std::string & marker) +{ + return makeASTFunction("equals", make_intrusive(marker), make_intrusive(static_cast(1))); +} + + +/// How a form of a disk definition stored by an older version differs from the current one. +struct CredentialsMarkersChange +{ + std::set drop; +}; + +static void applyCredentialsMarkersChange(ASTs & args, const CredentialsMarkersChange & change) +{ + args.erase(std::remove_if(args.begin(), args.end(), [&](const ASTPtr & arg) { return change.drop.contains(getCredentialsMarker(arg)); }), args.end()); +} + +struct DiskConfigurationFlattenerData +{ + ContextPtr context; + bool attach; + bool for_system_database; + bool stored_definition; +}; + +/// A registered disk a disk function may resolve to, and the changes of credentials markers in the stored definition +/// (of the function and of the disk functions below it) that give it. +struct DiskAlternative +{ + std::string name; + std::vector> edits; +}; + +/// A disk function already replaced by the name of its disk, and the disks it may resolve to, the used one first. +struct FlattenedDisk +{ + ASTPtr * literal; + std::vector alternatives; +}; + +/// Replace every disk function in `flat` (bottom-up) with the name of its disk. `original` is the AST `flat` was cloned +/// from, which is stored in metadata; the caller applies the edits of the alternative it uses to it. Returns the disk +/// functions directly in `flat`, or `flat` itself if it is one. +static std::vector flattenDiskConfiguration(ASTPtr & flat, const ASTPtr & original, const DiskConfigurationFlattenerData & data) +{ + checkStackSize(); + std::vector below; + for (size_t i = 0; i < flat->children.size(); ++i) + std::ranges::move(flattenDiskConfiguration(flat->children[i], original->children[i], data), std::back_inserter(below)); + + if (!isDiskFunction(flat)) + return below; + + const auto & args = getDiskFunctionArguments(flat); + std::vector changes{{}}; + if (!data.attach && !data.stored_definition) { - ContextPtr context; - bool attach; - bool for_system_database; + /// The form stored by older versions: versions before 26.7 had no marker. + if (hasCredentialsMarker(args, S3_SERVER_CREDENTIALS_MARKER)) + changes.push_back({.drop = {S3_SERVER_CREDENTIALS_MARKER}}); + } + + /// Every combination of the disks below (an older wrapper names the disk of the older form), the used ones first. + std::vector> combinations{{}}; + for (const auto & disk : below) + { + std::vector> extended; + for (size_t alternative = 0; alternative < disk.alternatives.size(); ++alternative) + { + for (const auto & combination : combinations) + { + extended.push_back(combination); + extended.back().push_back(alternative); + } + } + combinations = std::move(extended); + } + + auto set_disks_below = [&](const std::vector & combination) + { + for (size_t i = 0; i < below.size(); ++i) + *below[i].literal = make_intrusive(below[i].alternatives[combination[i]].name); }; - static void visit(ASTPtr & ast, Data & data) + std::vector serializations; + std::vector forms; + for (const auto & combination : combinations) { - if (isDiskFunction(ast)) + set_disks_below(combination); + for (const auto & change : changes) { - const auto * function = ast->as(); - const auto * function_args_expr = assert_cast(function->arguments.get()); - const auto & function_args = function_args_expr->children; - auto disk_setting_string = function->formatWithSecretsOneLine(); - auto disk_name = getOrCreateCustomDisk(function_args, disk_setting_string, data.context, data.attach, data.for_system_database); - ast = make_intrusive(disk_name); + auto candidate = flat->clone(); + applyCredentialsMarkersChange(getDiskFunctionArguments(candidate), change); + serializations.push_back(candidate->formatWithSecretsOneLine()); + + DiskAlternative form{.name = {}, .edits = {}}; + if (!change.drop.empty()) + form.edits.emplace_back(original, change); + for (size_t i = 0; i < below.size(); ++i) + std::ranges::copy(below[i].alternatives[combination[i]].edits, std::back_inserter(form.edits)); + forms.push_back(std::move(form)); } } -}; + /// The disk is created, if it is, from the disks used below. + set_disks_below(combinations.front()); + + const auto matches = getOrCreateCustomDisk(args, serializations, data.context, data.attach, data.for_system_database); + + std::vector alternatives; + for (const auto & [index, name] : matches) + { + alternatives.push_back(std::move(forms[index])); + alternatives.back().name = name; + } + flat = make_intrusive(alternatives.front().name); + return {FlattenedDisk{.literal = &flat, .alternatives = std::move(alternatives)}}; +} /// Persist the credential opt-in into the stored disk definition. For each leaf dynamic S3 disk that resolves /// server-managed credentials and is currently allowed (the session opted in), add a `_server_credentials_allowed` @@ -198,20 +379,11 @@ class ServerCredentialMarkerInjector return; auto & args = args_expr->children; - auto is_marker = [](const ASTPtr & arg) - { - const auto * eq = arg->as(); - if (!eq || eq->name != "equals" || !eq->arguments || eq->arguments->children.size() != 2) - return false; - const auto * key = eq->arguments->children[0]->as(); - return key && key->name() == "_server_credentials_allowed"; - }; - /// Strip any marker the user put in the definition. This runs only on a fresh create (not on metadata /// load), so a server-written marker in stored metadata is never stripped. As a result a persisted /// marker is only ever one this injector wrote below for a disk that actually needed server credentials /// while the session had the opt-in -- a user cannot pre-seed it to bypass the restriction on reload. - args.erase(std::remove_if(args.begin(), args.end(), is_marker), args.end()); + applyCredentialsMarkersChange(args, {.drop = {S3_SERVER_CREDENTIALS_MARKER}}); /// A wrapper disk (a nested `disk = disk(...)`) is not itself an S3 disk; the marker belongs on the /// inner leaf S3 disk, which the visitor reaches separately. (Markers were already stripped above.) @@ -226,15 +398,13 @@ class ServerCredentialMarkerInjector DynamicS3DiskCredentialInfo info; getDiskConfigurationFromASTImpl(args, data.context, /* is_loading_from_existing_metadata */ false, &info, data.for_system_database); if (info.persist_server_credentials_allowance) - args.push_back(makeASTFunction( - "equals", - make_intrusive("_server_credentials_allowed"), - make_intrusive(static_cast(1)))); + args.push_back(makeCredentialsMarker(S3_SERVER_CREDENTIALS_MARKER)); } }; -std::string DiskFromAST::createCustomDisk(const ASTPtr & disk_function_ast, ContextPtr context, bool attach, bool for_system_database) +std::string DiskFromAST::createCustomDisk( + const ASTPtr & disk_function_ast, ContextPtr context, bool attach, bool for_system_database, bool stored_definition) { if (!isDiskFunction(disk_function_ast)) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Expected a disk function"); @@ -244,7 +414,7 @@ std::string DiskFromAST::createCustomDisk(const ASTPtr & disk_function_ast, Cont /// disk's name is a hash of its definition: the marker has to be part of that definition so the disk /// resolves to the same name (and storage path) when reloaded from metadata. The marker itself is an unknown /// key to the disk factory and is ignored when building the client. - if (!attach) + if (!attach && !stored_definition) { ASTPtr to_mark = disk_function_ast; ServerCredentialMarkerInjector::Data inject_data{context, for_system_database}; @@ -252,13 +422,36 @@ std::string DiskFromAST::createCustomDisk(const ASTPtr & disk_function_ast, Cont } auto ast = disk_function_ast->clone(); - - DiskConfigurationFlattener::Data data{context, attach, for_system_database}; - visitBottomUp(ast, data); + const auto disks = flattenDiskConfiguration(ast, disk_function_ast, {context, attach, for_system_database, stored_definition}); + for (const auto & [disk_function, change] : disks.front().alternatives.front().edits) + applyCredentialsMarkersChange(getDiskFunctionArguments(disk_function), change); return assert_cast(*ast).value.safeGet(); } +void DiskFromAST::ensureDiskIsAllowed(const DiskPtr & disk, ContextPtr context) +{ + /// A custom disk can be named by another session than the one that created it. + if (disk->customDiskUsesServerCredentials() && context->shouldRestrictUserQueryS3Credentials()) + throw Exception( + ErrorCodes::ACCESS_DENIED, + "The disk `{}` uses the server's own credentials, which may not be used from user SQL." +#if !CLICKHOUSE_CLOUD + " Enable the setting `s3_allow_server_credentials_in_user_queries` to allow it." +#endif + , disk->getName()); +} + +void DiskFromAST::ensureStoragePolicyIsAllowed(const std::string & storage_policy_name, ContextPtr context) +{ + /// `__` is the policy a table on a disk gets, so it names a custom disk as well. + if (const auto storage_policy = context->tryGetStoragePolicy(storage_policy_name)) + { + for (const auto & disk : storage_policy->getDisks()) + ensureDiskIsAllowed(disk, context); + } +} + void DiskFromAST::ensureDiskIsNotCustom(const std::string & disk_name, ContextPtr context) { auto disk = context->getDisk(disk_name); diff --git a/src/Disks/DiskFromAST.h b/src/Disks/DiskFromAST.h index cd0f52b731cdb..22a9e7220b35c 100644 --- a/src/Disks/DiskFromAST.h +++ b/src/Disks/DiskFromAST.h @@ -1,4 +1,5 @@ #pragma once +#include #include #include #include @@ -6,11 +7,21 @@ namespace DB { +class IDisk; +using DiskPtr = std::shared_ptr; + namespace DiskFromAST { void ensureDiskIsNotCustom(const std::string & name, ContextPtr context); - std::string createCustomDisk(const ASTPtr & disk_function, ContextPtr context, bool attach, bool for_system_database = false); + /// Refuse, for a session without `s3_allow_server_credentials_in_user_queries`, a custom disk that uses the server's own + /// credentials, or a storage policy with one. For a table definition selecting a disk or a policy by name. + void ensureDiskIsAllowed(const DiskPtr & disk, ContextPtr context); + void ensureStoragePolicyIsAllowed(const std::string & storage_policy_name, ContextPtr context); + /// `stored_definition`: the definition was read from the metadata the server stored for the table (short `ATTACH`), + /// so its credentials markers are kept as they are: they keep the identity of the disk, and are not honored unless `attach`. + std::string createCustomDisk( + const ASTPtr & disk_function, ContextPtr context, bool attach, bool for_system_database = false, bool stored_definition = false); } } diff --git a/src/Disks/DiskObjectStorage/DiskObjectStorageTransaction.cpp b/src/Disks/DiskObjectStorage/DiskObjectStorageTransaction.cpp index ff752d7c0272a..3ce962c4d0e7f 100644 --- a/src/Disks/DiskObjectStorage/DiskObjectStorageTransaction.cpp +++ b/src/Disks/DiskObjectStorage/DiskObjectStorageTransaction.cpp @@ -5,7 +5,9 @@ #include #include #include +#include #include +#include #if ENABLE_DISTRIBUTED_CACHE #include #endif @@ -50,7 +52,6 @@ namespace ErrorCodes { extern const int NOT_IMPLEMENTED; extern const int FAULT_INJECTED; - extern const int LOGICAL_ERROR; } void DiskObjectStorageTransaction::waitBlobRemoval(const StoredObjects & blobs) const @@ -437,38 +438,60 @@ void DiskObjectStorageTransaction::submitBlobForRemoval(const std::string & remo } /// This function is a simplified and adapted version of DiskObjectStorageTransaction::writeFile(). -/// TODO: This function is incorrect for multiple object storages, we need to remove it and use everywhere regular writeFile +/// TODO: Remove it and use regular writeFile everywhere. void DiskObjectStorageTransaction::writeFileUsingBlobWritingFunction( const String & path, WriteMode mode, WriteBlobFunction && write_blob_function) { - if (cluster->getConfiguration().size() > 1) - throw Exception(ErrorCodes::LOGICAL_ERROR, "writeFileUsingBlobWritingFunction not supported for replicated setup"); + const Location local_location = cluster->getLocalLocation(); + const auto local_object_storage = object_storages->takePointingTo(local_location); StoredObject object(metadata_transaction->generateObjectKeyForPath(path).serialize(), path); - written_blobs[cluster->getLocalLocation()].push_back(object); + written_blobs[local_location].push_back(object); /// See DiskObjectStorage::getBlobPath(). Strings blob_path; blob_path.reserve(2); blob_path.emplace_back(object.remote_path); - String objects_namespace = object_storages->takePointingTo(cluster->getLocalLocation())->getObjectsNamespace(); + String objects_namespace = local_object_storage->getObjectsNamespace(); if (!objects_namespace.empty()) blob_path.emplace_back(objects_namespace); /// We always use mode Rewrite because we simulate append using metadata and different files object.bytes_size = std::move(write_blob_function)(blob_path, WriteMode::Rewrite, /*object_attributes=*/std::nullopt); - addOperation([object, mode](MetadataTransactionPtr tx) + LocationSet written_locations{local_location}; + for (const auto & location : cluster->getEnabledLocations()) + if (written_locations.insert(location).second) + written_blobs[location].push_back(object); + + { + const ReadSettings read_settings = getReadSettings(); + const WriteSettings write_settings = getWriteSettings(); + ThreadPoolCallbackRunnerLocal runner(*copy_object_pool, ThreadName::DISK_OBJECT_STORAGE_WRITE_BLOB); + for (const auto & location : written_locations) + { + if (location == local_location) + continue; + + runner.enqueueAndKeepTrack([&, location] + { + local_object_storage->copyObjectToAnotherObjectStorage(object, object, read_settings, write_settings, *object_storages->takePointingTo(location)); + }); + } + runner.waitForAllToFinishAndRethrowFirstError(); + } + + addOperation([object, mode, missing_locations = cluster->findComplement(std::move(written_locations))](MetadataTransactionPtr tx) { if (mode == WriteMode::Rewrite) { LOG_TEST(getLogger("DiskObjectStorageTransaction"), "Writing blob for path {}, key {}, size {}", object.local_path, object.remote_path, object.bytes_size); - tx->recordBlobsReplication(object, /*missing_locations=*/{}); + tx->recordBlobsReplication(object, missing_locations); tx->createMetadataFile(object.local_path, {object}); } else { - tx->recordBlobsReplication(object, /*missing_locations=*/{}); + tx->recordBlobsReplication(object, missing_locations); tx->addBlobToMetadata(object.local_path, object); } }); diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp index 02a5af057f7a7..bf73ed4f56bfa 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.cpp @@ -88,6 +88,7 @@ namespace ServerSetting namespace ErrorCodes { + extern const int AZURE_BLOB_STORAGE_ERROR; extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; } @@ -227,6 +228,28 @@ String ConnectionParams::getConnectionURL() const return endpoint.storage_account_url; } +String ConnectionParams::getConnectionURLWithoutQuery() const +{ + if (std::holds_alternative(auth_method)) + { + try + { + /// `ParseConnectionString` appends the SAS of the connection string to `BlobServiceUrl` as its query. + auto url = Azure::Storage::_internal::ParseConnectionString(endpoint.storage_account_url).BlobServiceUrl; + const auto query = url.GetQueryParameters(); + for (const auto & [key, value] : query) + url.RemoveQueryParameter(key); + return url.GetAbsoluteUrl(); + } + catch (const std::logic_error & e) + { + translateAzureSdkParseError(e); + } + } + + return endpoint.storage_account_url.substr(0, endpoint.storage_account_url.find_first_of("?#")); +} + std::unique_ptr ConnectionParams::createForService() const { try @@ -376,6 +399,13 @@ std::unique_ptr getContainerClient(const ConnectionParams & par } } +String getETag(const Azure::ETag & etag, const String & blob_path) +{ + if (!etag.HasValue()) + throw Exception(ErrorCodes::AZURE_BLOB_STORAGE_ERROR, "Azure Blob Storage endpoint did not return an ETag for blob {}", blob_path); + return etag.ToString(); +} + AuthMethod getAuthMethod(const Poco::Util::AbstractConfiguration & config, const String & config_prefix) { if (config.has(config_prefix + ".account_key") && config.has(config_prefix + ".account_name")) diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h b/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h index 79ff8caeaedb8..399183f094531 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h +++ b/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureBlobStorageCommon.h @@ -149,6 +149,8 @@ struct ConnectionParams String getContainer() const { return endpoint.container_name; } String getConnectionURL() const; + /// Without the query, which can be a SAS, for where the url only names the endpoint (logs, errors, cache keys). + String getConnectionURLWithoutQuery() const; std::unique_ptr createForService() const; std::unique_ptr createForContainer() const; @@ -166,6 +168,10 @@ BlobClientOptions getClientOptions( AuthMethod getAuthMethod(const Poco::Util::AbstractConfiguration & config, const String & config_prefix); +/// `ETag` is an optional response header, and `Azure::ETag::ToString` aborts the process when the tag is absent +/// (also in release builds). Use this function instead: it throws an exception when the endpoint did not send the tag. +String getETag(const Azure::ETag & etag, const String & blob_path); + #endif Endpoint processEndpoint(const Poco::Util::AbstractConfiguration & config, const String & config_prefix); diff --git a/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp b/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp index 4e37f48d50392..ecc285029a5d4 100644 --- a/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp +++ b/src/Disks/DiskObjectStorage/ObjectStorages/AzureBlobStorage/AzureObjectStorage.cpp @@ -106,7 +106,7 @@ class AzureIteratorAsync final : public IObjectStorageIteratorAsync .last_modified = Poco::Timestamp::fromEpochTime( std::chrono::duration_cast( static_cast(blob.Details.LastModified).time_since_epoch()).count()), - .etag = blob.Details.ETag.ToString(), + .etag = AzureBlobStorage::getETag(blob.Details.ETag, blob.Name), .tags = {}, .attributes = {}, })); @@ -217,7 +217,7 @@ void AzureObjectStorage::listObjects(const std::string & path, RelativePathsWith .last_modified = Poco::Timestamp::fromEpochTime( std::chrono::duration_cast( static_cast(blob.Details.LastModified).time_since_epoch()).count()), - .etag = blob.Details.ETag.ToString(), + .etag = AzureBlobStorage::getETag(blob.Details.ETag, blob.Name), .tags = {}, .attributes = {}, })); @@ -651,7 +651,7 @@ ObjectMetadata AzureObjectStorage::getObjectMetadata(const std::string & path, b ObjectMetadata result; result.size_bytes = properties.BlobSize; - result.etag = properties.ETag.ToString(); + result.etag = AzureBlobStorage::getETag(properties.ETag, path); if (!properties.Metadata.empty()) { result.attributes.emplace(); diff --git a/src/Disks/IDisk.h b/src/Disks/IDisk.h index 478795f523f1f..b7ce685fdc3dd 100644 --- a/src/Disks/IDisk.h +++ b/src/Disks/IDisk.h @@ -570,7 +570,13 @@ class IDisk : public Space /// Was disk created to be used without storage configuration? bool isCustomDisk() const { return custom_disk_settings_hash != 0; } UInt128 getCustomDiskSettings() const { return custom_disk_settings_hash; } - void markDiskAsCustom(UInt128 settings_hash) { custom_disk_settings_hash = settings_hash; } + void markDiskAsCustom(UInt128 settings_hash, bool uses_server_credentials) + { + custom_disk_settings_hash = settings_hash; + custom_disk_uses_server_credentials = uses_server_credentials; + } + /// A custom disk that uses the server's own credentials, directly or through the disk it wraps. + bool customDiskUsesServerCredentials() const { return custom_disk_uses_server_credentials; } virtual DiskPtr getDelegateDiskIfExists() const { return nullptr; } @@ -608,6 +614,7 @@ class IDisk : public Space std::unique_ptr copying_thread_pool; // 0 means the disk is not custom, the disk is predefined in the config UInt128 custom_disk_settings_hash = 0; + bool custom_disk_uses_server_credentials = false; /// True if underlying filesystem is case-insensitive, /// e.g. file_name and FILE_NAME are the same files. diff --git a/src/Disks/IO/AsynchronousBoundedReadBuffer.cpp b/src/Disks/IO/AsynchronousBoundedReadBuffer.cpp index 0596c8a5fd6a3..593fc9c278578 100644 --- a/src/Disks/IO/AsynchronousBoundedReadBuffer.cpp +++ b/src/Disks/IO/AsynchronousBoundedReadBuffer.cpp @@ -86,6 +86,7 @@ AsynchronousBoundedReadBuffer::AsynchronousBoundedReadBuffer( { use_page_cache = true; buffer_size = page_cache_block_size; + padded = false; } LOG_TEST(log, "Using buffer size {} while reading {}", buffer_size, file_name); diff --git a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp index 5724a16deef8b..68a794c9a4a47 100644 --- a/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp +++ b/src/Disks/IO/ReadBufferFromAzureBlobStorage.cpp @@ -569,7 +569,7 @@ void ReadBufferFromAzureBlobStorage::setMetadataFromResponse(const Azure::Storag { ObjectMetadata new_metadata; new_metadata.size_bytes = blob_size; - new_metadata.etag = details.ETag.ToString(); + new_metadata.etag = AzureBlobStorage::getETag(details.ETag, path); new_metadata.last_modified = static_cast(details.LastModified).time_since_epoch().count(); if (!details.Metadata.empty()) { diff --git a/src/Disks/IStoragePolicy.h b/src/Disks/IStoragePolicy.h index d80d0b89a2ffb..66226f4cc425c 100644 --- a/src/Disks/IStoragePolicy.h +++ b/src/Disks/IStoragePolicy.h @@ -60,6 +60,9 @@ class IStoragePolicy DiskPtr getDiskByName(const String & disk_name) const; /// Get free space from most free disk virtual UInt64 getMaxUnreservedFreeSpace() const = 0; + /// The storage provides the data of tables on this policy (``), so `BACKUP` stores only their + /// metadata and `RESTORE` doesn't restore their data. + virtual bool isBackupMetadataOnly() const = 0; /// Reserves space on any volume with index > min_volume_index or returns error virtual ReservationPtrOrError reserve(UInt64 bytes, size_t min_volume_index) const = 0; /// Returns valid reservation or error diff --git a/src/Disks/StoragePolicy.cpp b/src/Disks/StoragePolicy.cpp index eab83e5abd0f3..7311e1e69aebc 100644 --- a/src/Disks/StoragePolicy.cpp +++ b/src/Disks/StoragePolicy.cpp @@ -117,11 +117,40 @@ StoragePolicy::StoragePolicy( "Disk move factor have to be in [0., 1.] interval, but set to {} in storage policy {}", move_factor, backQuote(name)); + backup_metadata_only = config.getBool(config_prefix + ".backup_metadata_only", false); + if (backup_metadata_only) + checkVolumesAreReadOnly(); + buildVolumeIndices(); LOG_TRACE(log, "Storage policy {} created, total volumes {}", name, volumes.size()); } +bool StoragePolicy::isBackupMetadataOnly() const +{ + /// Rechecked on every call, because a disk can become writable while the policy is live: `DiskLocal` clears its + /// read-only flag itself once its periodic check writes to the root directory again, and a reload of the + /// configuration that makes a disk writable keeps this policy, since the policy built from the new configuration + /// fails the check in the constructor and is discarded. So the check cannot be left to where a policy is built. + if (backup_metadata_only) + checkVolumesAreReadOnly(); + return backup_metadata_only; +} + +void StoragePolicy::checkVolumesAreReadOnly() const +{ + /// The data of a table on such a policy is not backed up, so it must come from the storage, not from writes to it. + for (const auto & volume : volumes) + { + if (!volume->isReadOnly()) + throw Exception(ErrorCodes::INVALID_CONFIG_PARAMETER, + "Storage policy {} is marked backup_metadata_only, but its volume {} is not read-only, " + "so BACKUP would leave out the data written to it", + backQuote(name), backQuote(volume->getName())); + } +} + + StoragePolicy::StoragePolicy(String name_, Volumes volumes_, double move_factor_) : volumes(std::move(volumes_)) , name(std::move(name_)) diff --git a/src/Disks/StoragePolicy.h b/src/Disks/StoragePolicy.h index eef7dc1aef764..fc65c63af5960 100644 --- a/src/Disks/StoragePolicy.h +++ b/src/Disks/StoragePolicy.h @@ -50,6 +50,9 @@ class StoragePolicy : public IStoragePolicy /// Get free space from most free disk UInt64 getMaxUnreservedFreeSpace() const override; + /// Throws if the policy is marked `backup_metadata_only` but one of its volumes is not read-only. + bool isBackupMetadataOnly() const override; + const String & getName() const override{ return name; } /// Returns valid reservation or nullptr @@ -101,7 +104,10 @@ class StoragePolicy : public IStoragePolicy /// filled more than total_size * move_factor double move_factor = 0.1; /// by default move factor is 10% + bool backup_metadata_only = false; + void buildVolumeIndices(); + void checkVolumesAreReadOnly() const; LoggerPtr log; }; diff --git a/src/Disks/getDiskConfigurationFromAST.cpp b/src/Disks/getDiskConfigurationFromAST.cpp index 1f3bc0d903b51..e9ab532c30afc 100644 --- a/src/Disks/getDiskConfigurationFromAST.cpp +++ b/src/Disks/getDiskConfigurationFromAST.cpp @@ -175,7 +175,7 @@ Poco::AutoPtr getDiskConfigurationFromASTImpl(const ASTs & use_environment_credentials_off = !indirect && (value_str == "0" || boost::iequals(value_str, "false")); else if (key == "role_arn") has_role_arn = !value_str.empty() && !indirect; - else if (key == "_server_credentials_allowed") + else if (key == S3_SERVER_CREDENTIALS_MARKER) has_server_credentials_marker = !indirect && value_str != "0" && !boost::iequals(value_str, "false"); if (key == "include") @@ -260,6 +260,7 @@ Poco::AutoPtr getDiskConfigurationFromASTImpl(const ASTs & /// (the session opted in), so the disk is not re-restricted on reload. info->persist_server_credentials_allowance = !is_loading_from_existing_metadata && relies_on_server_credentials && !has_server_credentials_marker && !context->shouldRestrictUserQueryS3Credentials(); + info->uses_server_credentials = relies_on_server_credentials; } if (relies_on_server_credentials && context->shouldRestrictUserQueryS3Credentials() && !restriction_exempt) @@ -277,7 +278,10 @@ Poco::AutoPtr getDiskConfigurationFromASTImpl(const ASTs & /// Force the disk anonymous once `include` is resolved (see forceAnonymousS3DiskConfig); we /// cannot rewrite the DOM here because DiskFromAST processes `include` after this returns. if (info) + { info->load_anonymously = true; + info->uses_server_credentials = false; + } } else throw Exception( @@ -349,6 +353,20 @@ void forceAnonymousS3DiskConfigAtPrefix(Poco::Util::AbstractConfiguration & conf namespace { +/// The disk root and every `locations.` child of a resolved dynamic disk configuration. +std::vector getS3BackendPrefixes(const Poco::Util::AbstractConfiguration & config) +{ + std::vector prefixes{""}; + if (config.has("locations")) + { + Poco::Util::AbstractConfiguration::Keys locations; + config.keys("locations", locations); + for (const auto & location : locations) + prefixes.push_back("locations." + location + "."); + } + return prefixes; +} + /// Whether one resolved object-storage backend (the disk root, or a single `locations.` child selected by /// `prefix`) is an S3 backend that resolved to credentials the AST did not prove explicit, and so must be /// downgraded to anonymous or rejected. Returns false for a non-S3 backend or one whose resolved auth is safe. @@ -419,17 +437,8 @@ void validateResolvedS3DiskCredentials( /// Check the disk root and every `locations.` child: a multi-location `DiskObjectStorage` builds one /// object storage per child, each with its own `object_storage_type`, so an `include` can hide an S3 child /// (built with `for_disk_s3 = true`) behind a root that proves non-S3. - std::vector prefixes{""}; - if (config.has("locations")) - { - Poco::Util::AbstractConfiguration::Keys locations; - config.keys("locations", locations); - for (const auto & location : locations) - prefixes.push_back("locations." + location + "."); - } - std::vector restricted_prefixes; - for (const auto & prefix : prefixes) + for (const auto & prefix : getS3BackendPrefixes(config)) if (resolvedS3BackendIsRestricted(config, prefix, /* is_root */ prefix.empty(), info)) restricted_prefixes.push_back(prefix); @@ -462,6 +471,15 @@ void validateResolvedS3DiskCredentials( ); } +bool diskUsesServerCredentials(const Poco::Util::AbstractConfiguration & config, const DynamicS3DiskCredentialInfo & info) +{ + if (info.uses_server_credentials) + return true; + /// Every backend counts, because a multi-location disk writes to all of them. + return std::ranges::any_of(getS3BackendPrefixes(config), + [&](const String & prefix) { return resolvedS3BackendIsRestricted(config, prefix, /* is_root */ prefix.empty(), info); }); +} + DiskConfigurationPtr getDiskConfigurationFromAST(const ASTs & disk_args, ContextPtr context) { DynamicS3DiskCredentialInfo info; diff --git a/src/Disks/getDiskConfigurationFromAST.h b/src/Disks/getDiskConfigurationFromAST.h index e7c428d965ff9..0c1572a763a8a 100644 --- a/src/Disks/getDiskConfigurationFromAST.h +++ b/src/Disks/getDiskConfigurationFromAST.h @@ -40,6 +40,8 @@ struct DynamicS3DiskCredentialInfo /// should persist a `_server_credentials_allowed` marker in the stored definition. The marker is honored /// (only) on later metadata loads, so the disk keeps working across restart without re-opting in. bool persist_server_credentials_allowance = false; + /// The disk is built with server-managed S3 credentials. + bool uses_server_credentials = false; /// The safe credential forms the AST itself supplied with literal values; the resolved auth mode is /// validated against these. bool ast_has_explicit_key_pair = false; /// literal `access_key_id` + `secret_access_key` @@ -71,6 +73,13 @@ void forceAnonymousS3DiskConfigAtPrefix(Poco::Util::AbstractConfiguration & conf /// anonymous when loading from existing metadata (see `s3_load_table_anonymously_if_credentials_restricted`). void validateResolvedS3DiskCredentials(Poco::Util::AbstractConfiguration & config, ContextPtr context, bool is_loading_from_existing_metadata, const DynamicS3DiskCredentialInfo & info); +/// Persisted in the stored definition of a dynamic S3 disk created with `s3_allow_server_credentials_in_user_queries`. +inline constexpr auto S3_SERVER_CREDENTIALS_MARKER = "_server_credentials_allowed"; + +/// Whether a resolved dynamic disk may use the server's own credentials on any of its backends (the root or any +/// `locations.` child). The disk a wrapper names is not checked here. +bool diskUsesServerCredentials(const Poco::Util::AbstractConfiguration & config, const DynamicS3DiskCredentialInfo & info); + /* * A reverse function. */ diff --git a/src/Disks/tests/gtest_azure_read_buffer_missing_etag.cpp b/src/Disks/tests/gtest_azure_read_buffer_missing_etag.cpp new file mode 100644 index 0000000000000..161256caba5ba --- /dev/null +++ b/src/Disks/tests/gtest_azure_read_buffer_missing_etag.cpp @@ -0,0 +1,106 @@ +#include "config.h" + +#if USE_AZURE_BLOB_STORAGE + +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +namespace DB::ErrorCodes +{ + extern const int AZURE_BLOB_STORAGE_ERROR; +} + +namespace +{ + +class OwningBodyStream : public Azure::Core::IO::BodyStream +{ +public: + explicit OwningBodyStream(std::vector data_) : data(std::move(data_)) { } + + int64_t Length() const override { return static_cast(data.size()); } + + void Rewind() override { position = 0; } + +private: + size_t OnRead(uint8_t * buffer, size_t count, const Azure::Core::Context &) override + { + const size_t to_read = std::min(count, data.size() - position); + if (to_read != 0) + memcpy(buffer, data.data() + position, to_read); + position += to_read; + return to_read; + } + + std::vector data; + size_t position = 0; +}; + +/// A fake Azure endpoint that serves a 10-byte blob in a response without the optional `ETag` header. +class EndpointWithoutETag : public Azure::Core::Http::HttpTransport +{ +public: + std::unique_ptr Send(Azure::Core::Http::Request &, const Azure::Core::Context &) override + { + constexpr size_t blob_size = 10; + auto response = std::make_unique( + 1, 1, Azure::Core::Http::HttpStatusCode::PartialContent, "Partial Content"); + response->SetHeader("Content-Length", std::to_string(blob_size)); + response->SetHeader("Content-Range", "bytes 0-" + std::to_string(blob_size - 1) + "/" + std::to_string(blob_size)); + response->SetHeader("Last-Modified", "Wed, 21 Oct 2015 07:28:00 GMT"); + response->SetHeader("x-ms-blob-type", "BlockBlob"); + response->SetBodyStream(std::make_unique(std::vector(blob_size, 'a'))); + return response; + } +}; + +} + +/// `Azure::ETag::ToString` aborts the process on an absent tag, also in release builds. +/// A download response without `ETag` must fail the read with an exception instead. +TEST(AzureReadBuffer, MissingETagIsAnException) +{ + Azure::Storage::Blobs::BlobClientOptions client_options; + client_options.Retry.MaxRetries = 0; + client_options.Transport.Transport = std::make_shared(); + + auto container_client = std::make_shared( + Azure::Storage::Blobs::BlobContainerClient("http://azure.invalid/container", client_options), /* blob_prefix */ ""); + + DB::ReadBufferFromAzureBlobStorage buffer( + container_client, + "blob", + DB::ReadSettings{}, + /* max_single_read_retries */ 1, + /* max_single_download_retries */ 1); + + std::optional error_code; + try + { + std::string data; + DB::readStringUntilEOF(data, buffer); + } + catch (const DB::Exception & e) + { + error_code = e.code(); + } + ASSERT_EQ(error_code, std::optional(DB::ErrorCodes::AZURE_BLOB_STORAGE_ERROR)); +} + +#endif diff --git a/src/Formats/FormatFactory.cpp b/src/Formats/FormatFactory.cpp index e3ed598c84a9a..4db92386e44d7 100644 --- a/src/Formats/FormatFactory.cpp +++ b/src/Formats/FormatFactory.cpp @@ -350,6 +350,7 @@ FormatSettings getFormatSettings(const ContextPtr & context, const Settings & se format_settings.arrow.output_unsupported_types_as_binary = settings[Setting::output_format_arrow_unsupported_types_as_binary]; format_settings.arrow.output_record_batch_rows = settings[Setting::output_format_arrow_record_batch_size]; format_settings.arrow.output_record_batch_bytes = settings[Setting::output_format_arrow_record_batch_size_bytes]; + format_settings.arrow.output_max_record_batch_bytes = settings[Setting::output_format_arrow_max_record_batch_size_bytes]; format_settings.orc.allow_missing_columns = settings[Setting::input_format_orc_allow_missing_columns]; format_settings.orc.row_batch_size = settings[Setting::input_format_orc_row_batch_size]; format_settings.orc.skip_columns_with_unsupported_types_in_schema_inference = settings[Setting::input_format_orc_skip_columns_with_unsupported_types_in_schema_inference]; diff --git a/src/Formats/FormatSchemaInfo.cpp b/src/Formats/FormatSchemaInfo.cpp index 9dc88c55f0e2a..a59ea6fe5d122 100644 --- a/src/Formats/FormatSchemaInfo.cpp +++ b/src/Formats/FormatSchemaInfo.cpp @@ -25,6 +25,7 @@ #include #include #include +#include #include namespace DB { @@ -288,7 +289,9 @@ void FormatSchemaInfo::storeSchemaOnDisk(const fs::path & file_path, const Strin fs::create_directory(dir_path); } - auto temp_path = fs::path(file_path.string() + ".tmp"); + /// The final file name is a hash of the schema source, so every writer of one schema targets + /// the same path; a shared temporary name would let them overwrite and delete each other's files. + auto temp_path = fs::path(file_path.string() + "." + getRandomASCIIString(8) + ".tmp"); try { diff --git a/src/Formats/FormatSettings.h b/src/Formats/FormatSettings.h index 99385f454d624..a4d4115b77129 100644 --- a/src/Formats/FormatSettings.h +++ b/src/Formats/FormatSettings.h @@ -5,9 +5,15 @@ #include #include +#include + namespace DB { +/// Arrow offsets its `Utf8`, `Binary` and `List` buffers with 32-bit integers, and the Apache Arrow +/// builders additionally reject the last representable value. +constexpr UInt64 MAX_ARROW_BUFFER_SIZE = std::numeric_limits::max() - 1; + /** * Various tweaks for input/output formats. Text serialization/deserialization * of data types also depend on some of these settings. It is different from @@ -203,6 +209,8 @@ struct FormatSettings bool output_unsupported_types_as_binary = true; UInt64 output_record_batch_rows = 0; UInt64 output_record_batch_bytes = 0; + /// Upper bound on one record batch, as opposed to the targets above, which combine small blocks. + UInt64 output_max_record_batch_bytes = MAX_ARROW_BUFFER_SIZE; } arrow{}; struct AvroSchemaRegistryTimeouts diff --git a/src/Functions/AI/AIQuotaTracker.cpp b/src/Functions/AI/AIQuotaTracker.cpp index 374e22d370371..0d10a70c0db42 100644 --- a/src/Functions/AI/AIQuotaTracker.cpp +++ b/src/Functions/AI/AIQuotaTracker.cpp @@ -1,4 +1,5 @@ #include +#include #include namespace DB @@ -7,6 +8,8 @@ namespace DB namespace ErrorCodes { extern const int LIMIT_EXCEEDED; + extern const int QUERY_WAS_CANCELLED; + extern const int TIMEOUT_EXCEEDED; } bool AIQuotaTracker::quotasExceededLocked() @@ -39,14 +42,44 @@ bool AIQuotaTracker::quotasExceededLocked() return false; } +void AIQuotaTracker::throwIfCancelled() const +{ + if (!isCancelled()) + return; + + /// `checkTimeLimit` throws on `KILL QUERY` and when `max_execution_time` is exceeded. + /// Under `timeout_overflow_mode = 'break'`, it returns false instead of throwing. + /// A function cannot return fewer rows than it received, so throw in that case too. + if (auto status = query_status.lock(); status && !status->checkTimeLimit()) + throw Exception(ErrorCodes::TIMEOUT_EXCEEDED, "Timeout exceeded: elapsed time limit reached in an AI function"); + + /// The query has finished, and a request it submitted is still queued. + throw Exception(ErrorCodes::QUERY_WAS_CANCELLED, "Query was cancelled"); +} + +bool AIQuotaTracker::isCancelled() const +{ + if (cancelled) + return true; + + auto status = query_status.lock(); + if (status && !status->checkTimeLimitSoft()) + cancelled = true; + return cancelled; +} + bool AIQuotaTracker::checkQuotas() { + throwIfCancelled(); + std::lock_guard lock(mutex); return quotasExceededLocked(); } bool AIQuotaTracker::recordApiCall() { + throwIfCancelled(); + std::lock_guard lock(mutex); /// Don't start a new request once any quota is known-exhausted (e.g. another thread's response diff --git a/src/Functions/AI/AIQuotaTracker.h b/src/Functions/AI/AIQuotaTracker.h index 515821b0f9f64..b2d018863b857 100644 --- a/src/Functions/AI/AIQuotaTracker.h +++ b/src/Functions/AI/AIQuotaTracker.h @@ -3,11 +3,15 @@ #include #include +#include +#include #include namespace DB { +class QueryStatus; + /// Tracks AI-function quota usage for one query. A single instance is shared by every AI function /// call in the query context (owned by the query `Context`) and updated concurrently from the /// pipeline threads, so the counters are guarded by `mutex`. It is per query-execution context: a @@ -16,21 +20,26 @@ namespace DB /// /// The API-call limit is a hard cap within a context, while token limits are best effort (we only /// know the usage after the call returns). +/// +/// `checkQuotas` and `recordApiCall` also throw once the query is killed or exceeds +/// `max_execution_time`, so no new request is dispatched for it. class AIQuotaTracker { public: AIQuotaTracker( UInt64 max_input_tokens_, UInt64 max_output_tokens_, - UInt64 max_api_calls_, bool throw_on_quota_exceeded_) + UInt64 max_api_calls_, bool throw_on_quota_exceeded_, + std::weak_ptr query_status_) : max_input_tokens(max_input_tokens_) , max_output_tokens(max_output_tokens_) , max_api_calls(max_api_calls_) , throw_on_quota_exceeded(throw_on_quota_exceeded_) + , query_status(std::move(query_status_)) {} /// Check the token quotas (and the sticky exceeded flag). Returns true if a limit is met or - /// exceeded, false otherwise. Should be called before issuing an API call. The API-call limit is - /// enforced separately by `recordApiCall`. + /// exceeded, false otherwise. Lets a caller skip building and submitting a request that + /// `recordApiCall` would refuse anyway. The API-call limit is enforced separately by `recordApiCall`. bool checkQuotas(); /// Count one outbound API call against the request quota, only while under the limit. Should be @@ -45,12 +54,21 @@ class AIQuotaTracker /// so this is kept separate and called only after the response is parsed. void recordTokens(UInt64 in_tokens, UInt64 out_tokens); + /// Throws if the query was killed or exceeded its time limit. + void throwIfCancelled() const; + + /// Whether the query was killed or exceeded its time limit. Does not throw. + bool isCancelled() const; private: const UInt64 max_input_tokens; const UInt64 max_output_tokens; const UInt64 max_api_calls; const bool throw_on_quota_exceeded; + const std::weak_ptr query_status; + + /// Sticky: used to check for requests submitted by a cancelled query (may outlive its `QueryStatus`). + mutable std::atomic cancelled = false; std::mutex mutex; bool quota_exceeded TSA_GUARDED_BY(mutex) = false; diff --git a/src/Functions/AI/AIRequestExecutor.cpp b/src/Functions/AI/AIRequestExecutor.cpp new file mode 100644 index 0000000000000..9dad0e2aad2b0 --- /dev/null +++ b/src/Functions/AI/AIRequestExecutor.cpp @@ -0,0 +1,269 @@ +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +namespace ProfileEvents +{ + extern const Event AIInputTokens; + extern const Event AIOutputTokens; + extern const Event AIAPICalls; + extern const Event AIAPICallsRetried; + extern const Event AIAPICallsFailed; + extern const Event AIAPICallsThrottled; +} + +namespace DB +{ + +namespace ErrorCodes +{ + extern const int AI_PROVIDER_RESPONSE_TRUNCATED; + extern const int AI_PROVIDER_RESPONSE_INCOMPLETE; + extern const int CANNOT_SCHEDULE_TASK; +} + +namespace +{ + +/// Whether a failed provider request should be retried: transient network failures and +/// transient/server-side HTTP responses are retriable, deterministic argument/usage errors are not. +bool isRetriableProviderError(std::exception_ptr exception) +{ + try + { + std::rethrow_exception(exception); + } + catch (const AIProviderHTTPException & http_exception) + { + return isRetriableHTTPError(http_exception.getHTTPStatus()); + } + catch (const NetException &) + { + /// ClickHouse-level network error (e.g. a DNS failure raised by the HTTP connection pool). + return true; + } + catch (const Poco::Net::NetException &) + { + /// Connection refused/reset, TLS connect failure, or an unreachable advertised address. + return true; + } + catch (const Poco::TimeoutException &) + { + /// Connect or receive timeout. + return true; + } + catch (const Poco::IOException & io_exception) + { + /// Write-side transient I/O failure, e.g. a broken pipe (`EPIPE`) when the peer resets the + /// connection mid-request. Out-of-file-descriptors (`EMFILE`) is not retriable. + return io_exception.code() != POCO_EMFILE; + } + catch (...) + { + /// Ok: any other exception is a deterministic and non-retrieable error, e.g. a malformed + /// provider response, bad configuration, JSON parse failure, etc. + return false; + } +} + +bool isThrottledProviderError(std::exception_ptr exception) +{ + try + { + std::rethrow_exception(exception); + } + catch (const AIProviderHTTPException & http_exception) + { + return http_exception.getHTTPStatus() == Poco::Net::HTTPResponse::HTTP_TOO_MANY_REQUESTS; + } + catch (...) + { + /// Ok: any other exception is not a rate limit. + return false; + } +} + +/// Reject a response the model did not finish. Throws a plain `Exception`, which +/// `isRetriableProviderError` classifies as non-retriable. +void checkResponseIsComplete(const AIResponse & response) +{ + /// `raw_finish_reason` is provider-controlled text; sanitize control characters before + /// interpolating it into an exception message that reaches the logs and `system.query_log`. + const String safe_finish_reason = sanitize(response.raw_finish_reason); + + switch (response.finish_reason) + { + case FinishReason::Complete: + case FinishReason::Unknown: /// Don't throw on Unknown, could be new valid reason in new API version + return; + case FinishReason::Truncated: + /// Differentiate between model hitting our output cap and exhausting its context window + throw Exception( + ErrorCodes::AI_PROVIDER_RESPONSE_TRUNCATED, + "AI provider returned a truncated response (finish_reason='{}'): {}", + safe_finish_reason, + response.raw_finish_reason == "model_context_window_exceeded" + ? "the model ran out of context window before completing its answer. " + "Reduce the input or use a model with a larger context window." + : "the model hit the output token limit before completing its answer. " + "Increase max_tokens or reduce the input."); + case FinishReason::ContentFilter: + throw Exception( + ErrorCodes::AI_PROVIDER_RESPONSE_INCOMPLETE, + "AI provider withheld or filtered the response (finish_reason='{}'): the returned answer " + "is incomplete.", + safe_finish_reason); + case FinishReason::RequiresAction: + throw Exception( + ErrorCodes::AI_PROVIDER_RESPONSE_INCOMPLETE, + "AI provider stopped expecting further caller action (finish_reason='{}') instead of " + "returning a completed answer.", + safe_finish_reason); + } +} + +/// The attempt loop every request kind shares: reserve an API-call slot, run one attempt, retry a +/// transient failure after a backoff. Returns nothing when the API-call quota is exhausted, or when +/// the request failed (after all retries) and `throw_on_error` is disabled. +template +std::optional runRequest(const AIRequestPolicy & policy, AIQuotaTracker & quota, Attempt && attempt_fn) +{ + for (UInt64 attempt = 0; attempt <= policy.max_retries; ++attempt) + { + /// Reserve an API-call slot before each request; this also performs a quota check. + if (!quota.recordApiCall()) + return {}; + + try + { + /// Count the call before issuing it, so a failed request is still counted. + ProfileEvents::increment(ProfileEvents::AIAPICalls); + return attempt_fn(); + } + catch (...) + { + if (isThrottledProviderError(std::current_exception())) + ProfileEvents::increment(ProfileEvents::AIAPICallsThrottled); + + if (attempt < policy.max_retries && isRetriableProviderError(std::current_exception())) + { + ProfileEvents::increment(ProfileEvents::AIAPICallsRetried); + /// Exponential backoff. `makeRequestPolicy` bounds both settings, so the longest sleep is 160 seconds. + /// Sleep in short steps, so the retry of a cancelled query does not hold the worker. + using Clock = std::chrono::steady_clock; + const Clock::time_point deadline = Clock::now() + std::chrono::milliseconds(policy.retry_initial_delay_ms << attempt); + while (!quota.isCancelled()) + { + const Clock::time_point now = Clock::now(); + if (now >= deadline) + break; + std::this_thread::sleep_for(std::min(deadline - now, std::chrono::milliseconds(100))); + } + continue; + } + + ProfileEvents::increment(ProfileEvents::AIAPICallsFailed); + + if (!policy.throw_on_error) + return {}; + + throw; + } + } + + return {}; +} + +void runAttempt(IAIProvider & provider, const AIChatRequest & request, const ConnectionTimeouts & timeouts, AIResponse & response) +{ + response = provider.call(request, timeouts); + checkResponseIsComplete(response); +} + +void runAttempt(IAIProvider & provider, const AIEmbeddingRequest & request, const ConnectionTimeouts & timeouts, AIEmbeddingResponse & response) +{ + response = provider.embed(request, timeouts); +} + +template +std::future> submit( + std::shared_ptr provider, Request request, AIRequestPolicy policy, AIQuotaTrackerPtr quota) +{ + auto task = [my_provider = std::move(provider), + my_request = std::move(request), + my_policy = std::move(policy), + my_quota = quota] + { + return runRequest(my_policy, *my_quota, [&] + { + Response response; + + /// A response rejected by `checkResponseIsComplete` still reports the usage the provider billed for. + SCOPE_EXIT({ + my_quota->recordTokens(response.input_tokens, response.output_tokens); + ProfileEvents::increment(ProfileEvents::AIInputTokens, response.input_tokens); + ProfileEvents::increment(ProfileEvents::AIOutputTokens, response.output_tokens); + }); + + runAttempt(*my_provider, my_request, my_policy.timeouts, response); + return response; + }); + }; + + using Task = detail::CallbackRunnerTask, decltype(task)>; + auto runner_task = std::make_shared(getCurrentThreadGroup(), ThreadName::AI_REQUEST, std::move(task)); + auto future = runner_task->promise.get_future(); + + /// Wait for a free slot in the queue in short steps, so a killed or timed-out query stops waiting. + constexpr Int64 wait_microseconds = 100'000; + auto & pool = getAIRequestThreadPool().get(); + while (true) + { + Stopwatch watch; + try + { + pool.scheduleOrThrow([runner_task] { runner_task->run(); }, {}, wait_microseconds); + break; + } + catch (const Exception & e) + { + /// Only throw if we hit an unrelated error or it happened before the timeout + if (e.code() != ErrorCodes::CANNOT_SCHEDULE_TASK || watch.elapsedMicroseconds() < static_cast(wait_microseconds)) + throw; + } + quota->throwIfCancelled(); + } + + return future; +} + +} + +std::future> submitAIRequest( + std::shared_ptr provider, AIChatRequest request, AIRequestPolicy policy, AIQuotaTrackerPtr quota) +{ + return submit(std::move(provider), std::move(request), std::move(policy), std::move(quota)); +} + +std::future> submitAIRequest( + std::shared_ptr provider, AIEmbeddingRequest request, AIRequestPolicy policy, AIQuotaTrackerPtr quota) +{ + return submit(std::move(provider), std::move(request), std::move(policy), std::move(quota)); +} + +} diff --git a/src/Functions/AI/AIRequestExecutor.h b/src/Functions/AI/AIRequestExecutor.h new file mode 100644 index 0000000000000..438ab43bef879 --- /dev/null +++ b/src/Functions/AI/AIRequestExecutor.h @@ -0,0 +1,41 @@ +#pragma once + +#include +#include + +#include +#include +#include + +namespace DB +{ + +class AIQuotaTracker; +using AIQuotaTrackerPtr = std::shared_ptr; + +/// How one AI provider request is executed. Resolved once per query from the `ai_function_*` +/// settings and shared by every request that query issues. +struct AIRequestPolicy +{ + ConnectionTimeouts timeouts; + + /// Number of extra attempts after a retriable failure, and the delay before the first of them. + UInt64 max_retries = 0; + UInt64 retry_initial_delay_ms = 0; + + /// `ai_function_throw_on_error`. When disabled, a request that failed (after retries) does not + /// throw, and the caller outputs a default value. + bool throw_on_error = true; +}; + +/// Issue one chat-completion request on the AI request thread pool (`getAIRequestThreadPool`) and +/// check that the model produced a complete answer. Also handles retries with backoff, API-call +/// reservation, and token accounting. +std::future> submitAIRequest( + std::shared_ptr provider, AIChatRequest request, AIRequestPolicy policy, AIQuotaTrackerPtr quota); + +/// The same, for one batch of texts sent to an embeddings endpoint. +std::future> submitAIRequest( + std::shared_ptr provider, AIEmbeddingRequest request, AIRequestPolicy policy, AIQuotaTrackerPtr quota); + +} diff --git a/src/Functions/AI/AnthropicProvider.cpp b/src/Functions/AI/AnthropicProvider.cpp index fb8ae2afd4e87..720f7af8fc8d4 100644 --- a/src/Functions/AI/AnthropicProvider.cpp +++ b/src/Functions/AI/AnthropicProvider.cpp @@ -40,7 +40,7 @@ HTTPHeaderEntries AnthropicProvider::getHeaders() const return headers; } -AIResponse AnthropicProvider::call(const AIRequest & ai_request, const ConnectionTimeouts & timeouts) +AIResponse AnthropicProvider::call(const AIChatRequest & ai_request, const ConnectionTimeouts & timeouts) { Poco::JSON::Object::Ptr root = new Poco::JSON::Object; root->set("model", ai_request.model); diff --git a/src/Functions/AI/AnthropicProvider.h b/src/Functions/AI/AnthropicProvider.h index cd91752b31cc9..122e1c1b383ae 100644 --- a/src/Functions/AI/AnthropicProvider.h +++ b/src/Functions/AI/AnthropicProvider.h @@ -22,7 +22,7 @@ class AnthropicProvider : public IAIProvider public: AnthropicProvider(const String & endpoint_, const String & api_key_, const String & api_version_); - AIResponse call(const AIRequest & request, const ConnectionTimeouts & timeouts) override; + AIResponse call(const AIChatRequest & request, const ConnectionTimeouts & timeouts) override; HTTPHeaderEntries getHeaders() const override; private: diff --git a/src/Functions/AI/IAIProvider.cpp b/src/Functions/AI/IAIProvider.cpp index d88ec81183c2d..072231adbf8cd 100644 --- a/src/Functions/AI/IAIProvider.cpp +++ b/src/Functions/AI/IAIProvider.cpp @@ -18,7 +18,7 @@ namespace ErrorCodes extern const int RECEIVED_ERROR_FROM_REMOTE_IO_SERVER; } -String sanitizeForLog(std::string_view input) +String sanitize(std::string_view input) { String output; output.reserve(input.size()); @@ -45,7 +45,7 @@ String formatProviderError(int status_code, const String & response_body) String msg = err->optValue("message", ""); String type = err->optValue("type", ""); if (!msg.empty()) - return fmt::format("HTTP {} [{}]: {}", status_code, sanitizeForLog(type), sanitizeForLog(msg)); + return fmt::format("HTTP {} [{}]: {}", status_code, sanitize(type), sanitize(msg)); } } } @@ -55,7 +55,7 @@ String formatProviderError(int status_code, const String & response_body) constexpr size_t max_len = 256; const std::string_view snippet = std::string_view(response_body).substr(0, max_len); - return fmt::format("HTTP {}: {}", status_code, sanitizeForLog(snippet)); + return fmt::format("HTTP {}: {}", status_code, sanitize(snippet)); } AIProviderHTTPException::AIProviderHTTPException(Poco::Net::HTTPResponse::HTTPStatus http_status_, PreformattedMessage msg) diff --git a/src/Functions/AI/IAIProvider.h b/src/Functions/AI/IAIProvider.h index 850218e093347..bae33011fc9d4 100644 --- a/src/Functions/AI/IAIProvider.h +++ b/src/Functions/AI/IAIProvider.h @@ -31,12 +31,25 @@ class AIProviderHTTPException : public Exception const char * className() const noexcept override { return "DB::AIProviderHTTPException"; } }; +/// Extra info sent with each request. Useful for debugging. +struct AIRequest +{ + /// Name of the AI SQL function that produced this request, e.g. "aiGenerate". + String function_name; + + /// The current query id. Different on different shards of a distributed query. + String query_id; + + /// The initial query id. Same on all shards of a distributed query, empty for a background mutation. + String initial_query_id; +}; + /** Parameters for a single AI chat completion request. * - * Each row processed by an AI function produces one AIRequest. + * Each row processed by an AI chat function produces one AIChatRequest. * The provider serializes it into the HTTP body format expected by the API. */ -struct AIRequest +struct AIChatRequest : AIRequest { /// Constant instruction that guides the model's behavior (persona, format, constraints). /// Sent as a system message (OpenAI) or top-level field (Anthropic). @@ -63,10 +76,6 @@ struct AIRequest /// Maximum number of tokens the model may generate in its response. This is a per-request limit, not a per-query limit. UInt64 max_tokens = 0; - - /// SQL name of the AI function that produced this request (e.g. "aiGenerate"). - /// Emitted by OpenAIProvider as the `X-ClickHouse-AI-Function` header; ignored by other providers. - String function_name; }; /// Canonical, provider-independent reason the model stopped generating. Each provider maps its @@ -104,7 +113,7 @@ struct AIResponse * Embedding APIs typically accept multiple inputs per call, so inputs is a vector. * The provider serializes it into the HTTP body format expected by the API. */ -struct AIEmbeddingRequest +struct AIEmbeddingRequest : AIRequest { /// Texts to embed. Providers send these in a single batched HTTP request. VectorWithMemoryTracking inputs; @@ -115,10 +124,6 @@ struct AIEmbeddingRequest /// Optional target dimensionality for the output vectors. 0 means use the model's native size. /// Supported by OpenAI's `text-embedding-3-*` models; providers that ignore it return the native size. UInt64 dimensions = 0; - - /// SQL name of the AI function that produced this request (e.g. "aiEmbed"). - /// Emitted by OpenAIProvider as the `X-ClickHouse-AI-Function` header; ignored by other providers. - String function_name; }; /// Response from a single embedding request. `embeddings` is aligned 1:1 with `AIEmbeddingRequest::inputs`. @@ -129,6 +134,9 @@ struct AIEmbeddingResponse /// Number of tokens in the input, as reported by the provider. Used for quota tracking. UInt64 input_tokens = 0; + + /// Always 0: embedding endpoints produce no output tokens. + static constexpr UInt64 output_tokens = 0; }; /** Abstract interface for AI provider HTTP clients. @@ -143,7 +151,7 @@ class IAIProvider virtual ~IAIProvider() = default; /// Send a chat completion request and return the parsed response. - virtual AIResponse call(const AIRequest & ai_request, const ConnectionTimeouts & timeouts) = 0; + virtual AIResponse call(const AIChatRequest & ai_request, const ConnectionTimeouts & timeouts) = 0; /// Provider-specific headers (credentials, API version) sent with every request. virtual HTTPHeaderEntries getHeaders() const = 0; @@ -165,8 +173,8 @@ AIProviderPtr createAIProvider( /// Build an error message from a provider's non-200 HTTP response, for use in an exception that is logged. String formatProviderError(int status_code, const String & response_body); -/// Replace control characters (including `\t \n \r`) with spaces so provider-controlled text cannot -/// forge log lines or corrupt a terminal when embedded in a logged exception. -String sanitizeForLog(std::string_view input); +/// Replace control characters (including `\t \n \r`) with spaces. +/// Needed to block provider-controlled text from forging log lines or corrupt a terminal when embedded in a logged exception. +String sanitize(std::string_view input); } diff --git a/src/Functions/AI/OpenAIProvider.cpp b/src/Functions/AI/OpenAIProvider.cpp index c307b2cfa6a16..44ab7006c1483 100644 --- a/src/Functions/AI/OpenAIProvider.cpp +++ b/src/Functions/AI/OpenAIProvider.cpp @@ -19,6 +19,21 @@ namespace ErrorCodes extern const int MALFORMED_AI_PROVIDER_RESPONSE; } +namespace +{ + +void setCommonHeaders(Poco::Net::HTTPRequest & http_request, const AIRequest & ai_request) +{ + chassert(!ai_request.function_name.empty()); + http_request.set("X-ClickHouse-AI-Function", ai_request.function_name); + if (!ai_request.query_id.empty()) + http_request.set("X-ClickHouse-Query-Id", sanitize(ai_request.query_id)); + if (!ai_request.initial_query_id.empty()) + http_request.set("X-ClickHouse-Initial-Query-Id", sanitize(ai_request.initial_query_id)); +} + +} + OpenAIProvider::OpenAIProvider(const String & endpoint_, const String & api_key_) : endpoint(endpoint_) , api_key(api_key_) @@ -33,7 +48,7 @@ HTTPHeaderEntries OpenAIProvider::getHeaders() const return {{"Authorization", "Bearer " + api_key}}; } -AIResponse OpenAIProvider::call(const AIRequest & ai_request, const ConnectionTimeouts & timeouts) +AIResponse OpenAIProvider::call(const AIChatRequest & ai_request, const ConnectionTimeouts & timeouts) { Poco::JSON::Object::Ptr root = new Poco::JSON::Object; root->set("model", ai_request.model); @@ -70,8 +85,7 @@ AIResponse OpenAIProvider::call(const AIRequest & ai_request, const ConnectionTi http_request.setContentType("application/json"); for (const auto & header : getHeaders()) http_request.set(header.name, header.value); - chassert(!ai_request.function_name.empty()); - http_request.set("X-ClickHouse-AI-Function", ai_request.function_name); + setCommonHeaders(http_request, ai_request); http_request.setContentLength(body.size()); auto & out_stream = session->sendRequest(http_request); @@ -181,8 +195,7 @@ AIEmbeddingResponse OpenAIProvider::embed(const AIEmbeddingRequest & ai_embeddin http_request.setContentType("application/json"); for (const auto & header : getHeaders()) http_request.set(header.name, header.value); - chassert(!ai_embedding_request.function_name.empty()); - http_request.set("X-ClickHouse-AI-Function", ai_embedding_request.function_name); + setCommonHeaders(http_request, ai_embedding_request); http_request.setContentLength(body.size()); auto & out_stream = session->sendRequest(http_request); diff --git a/src/Functions/AI/OpenAIProvider.h b/src/Functions/AI/OpenAIProvider.h index 38d65fc6b8138..bcfe89c6d411b 100644 --- a/src/Functions/AI/OpenAIProvider.h +++ b/src/Functions/AI/OpenAIProvider.h @@ -33,7 +33,7 @@ class OpenAIProvider : public IAIProvider public: OpenAIProvider(const String & endpoint_, const String & api_key_); - AIResponse call(const AIRequest & ai_request, const ConnectionTimeouts & timeouts) override; + AIResponse call(const AIChatRequest & ai_request, const ConnectionTimeouts & timeouts) override; HTTPHeaderEntries getHeaders() const override; bool supportsEmbeddings() const override { return true; } diff --git a/src/Functions/FunctionBaseAI.cpp b/src/Functions/FunctionBaseAI.cpp index 7aea1b1826401..b657fffb7731a 100644 --- a/src/Functions/FunctionBaseAI.cpp +++ b/src/Functions/FunctionBaseAI.cpp @@ -1,37 +1,36 @@ #include + #include #include -#include +#include +#include #include -#include -#include -#include -#include -#include -#include -#include -#include #include +#include #include -#include -#include -#include -#include -#include +#include +#include +#include +#include #include +#include #include -#include -#include #include -#include -#include +#include +#include + +#include +#include + +#include +#include +#include #include +#include +#include namespace ProfileEvents { - extern const Event AIInputTokens; - extern const Event AIOutputTokens; - extern const Event AIAPICalls; extern const Event AIRowsProcessed; extern const Event AIRowsSkipped; } @@ -43,17 +42,23 @@ namespace Setting { extern const SettingsUInt64 ai_function_request_timeout_sec; extern const SettingsUInt64 ai_function_max_retries; + extern const SettingsNonZeroUInt64 ai_function_max_concurrent_requests_per_thread; extern const SettingsUInt64 ai_function_retry_initial_delay_ms; extern const SettingsBool ai_function_throw_on_error; extern const SettingsString ai_function_text_default_credentials; extern const SettingsBool ai_function_allow_insecure_endpoint; } +namespace ServerSetting +{ + extern const ServerSettingsBool ai_function_allow_sql_named_collections; +} + namespace ErrorCodes { + extern const int ACCESS_DENIED; extern const int BAD_ARGUMENTS; - extern const int AI_PROVIDER_RESPONSE_TRUNCATED; - extern const int AI_PROVIDER_RESPONSE_INCOMPLETE; + extern const int NOT_IMPLEMENTED; } namespace @@ -175,6 +180,33 @@ AIParamSpecs FunctionBaseAI::allParams() const namespace { +constexpr auto cancellation_check_interval = std::chrono::milliseconds(100); + +/// Waits for a submitted request. Throws once the query is killed or exceeds its time limit, +/// so a request queued behind other queries' requests does not hold the query. +template +std::optional waitForRequest(std::future> & request, const AIQuotaTracker & quota) +{ + while (request.wait_for(cancellation_check_interval) != std::future_status::ready) + quota.throwIfCancelled(); + return request.get(); +} + +/// Waits for the submitted requests to finish, until the query is killed or exceeds its time limit. +/// The tasks own everything they use, so they may outlive the query. +template +void drainRequests(VectorWithMemoryTracking>> & requests, const AIQuotaTracker & quota) +{ + for (auto & request : requests) + { + if (!request.valid()) + continue; + while (request.wait_for(cancellation_check_interval) != std::future_status::ready) + if (quota.isCancelled()) + return; + } +} + const Field & getResolvedAIParam(const AIParamValues & values, std::string_view key) { auto it = values.find(key); @@ -250,6 +282,11 @@ FunctionBaseAI::AIParams FunctionBaseAI::resolveAIParams( context->checkAccess(AccessType::NAMED_COLLECTION, credentials); const auto & collection = NamedCollectionFactory::instance().get(credentials); + if (!context->getServerSettings()[ServerSetting::ai_function_allow_sql_named_collections] + && collection->getSourceId() != NamedCollection::SourceId::CONFIG) + throw Exception(ErrorCodes::ACCESS_DENIED, + "AI named collection '{}' is not defined in the server configuration file, and the server setting " + "'ai_function_allow_sql_named_collections' is disabled", credentials); AIParams params; params.collection.collection_name = credentials; @@ -305,52 +342,28 @@ FunctionBaseAI::AIParams FunctionBaseAI::resolveAIParams( return params; } -UInt64 FunctionBaseAI::computeRetryBackoffMs(UInt64 initial_delay_ms, UInt64 attempt) +AIRequestPolicy FunctionBaseAI::makeRequestPolicy(const ContextPtr & context) { - constexpr UInt64 max_retry_delay_ms = 60'000; - UInt64 delay_ms = std::min(initial_delay_ms, max_retry_delay_ms); - for (UInt64 i = 0; i < attempt && delay_ms < max_retry_delay_ms; ++i) - delay_ms = std::min(delay_ms * 2, max_retry_delay_ms); - return delay_ms; -} + const auto & settings = context->getSettingsRef(); + + AIRequestPolicy policy; + policy.timeouts = ConnectionTimeouts::getHTTPTimeouts(settings, context->getServerSettings()); + policy.timeouts.receive_timeout + = Poco::Timespan(static_cast(settings[Setting::ai_function_request_timeout_sec].value) /*s*/, 0 /*us*/); + policy.max_retries = settings[Setting::ai_function_max_retries].value; + policy.retry_initial_delay_ms = settings[Setting::ai_function_retry_initial_delay_ms].value; + policy.throw_on_error = settings[Setting::ai_function_throw_on_error].value; + + constexpr UInt64 max_retries_limit = 5; + constexpr UInt64 retry_initial_delay_ms_limit = 10'000; + if (policy.max_retries > max_retries_limit) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Setting 'ai_function_max_retries' is {}, the maximum is {}", policy.max_retries, max_retries_limit); + if (policy.retry_initial_delay_ms > retry_initial_delay_ms_limit) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "Setting 'ai_function_retry_initial_delay_ms' is {}, the maximum is {}", policy.retry_initial_delay_ms, retry_initial_delay_ms_limit); -bool FunctionBaseAI::isRetriableProviderError(std::exception_ptr exception) -{ - try - { - std::rethrow_exception(exception); - } - catch (const AIProviderHTTPException & exception) - { - return isRetriableHTTPError(exception.getHTTPStatus()); - } - catch (const NetException &) - { - /// ClickHouse-level network error (e.g. a DNS failure raised by the HTTP connection pool). - return true; - } - catch (const Poco::Net::NetException &) - { - /// Connection refused/reset, TLS connect failure, or an unreachable advertised address. - return true; - } - catch (const Poco::TimeoutException &) - { - /// Connect or receive timeout. - return true; - } - catch (const Poco::IOException & exception) - { - /// Write-side transient I/O failure, e.g. a broken pipe (`EPIPE`) when the peer resets the - /// connection mid-request. Out-of-file-descriptors (`EMFILE`) is not retriable. - return exception.code() != POCO_EMFILE; - } - catch (...) - { - /// Ok: any other exception is a deterministic and non-retrieable error, e.g. a malformed - /// provider response, bad configuration, JSON parse failure, etc. - return false; - } + return policy; } void FunctionBaseAI::insertProcessedResult(IColumn & column, const String & processed) const @@ -358,6 +371,32 @@ void FunctionBaseAI::insertProcessedResult(IColumn & column, const String & proc column.insertData(processed.data(), processed.size()); } +void FunctionBaseAI::checkHasUser(const ContextPtr & context) +{ + if (context->getUserID()) + return; + + /// A background operation is retried, and its failure is visible only in the logs and system tables. + context->addOrUpdateWarningMessage( + Context::WarningType::AI_FUNCTION_IN_BACKGROUND_OPERATION, + PreformattedMessage::create( + "A background operation (merge, mutation or `Buffer` flush) failed because it evaluates an AI function, " + "which is executed only in queries run by a user. {}Remove the AI function from the table's keys, TTL, " + "skip indexes, projections, `DEFAULT` and `MATERIALIZED` columns and constraints.", + context->getCurrentQueryId().empty() ? "" : fmt::format("Last failed task: {}. ", context->getCurrentQueryId()))); + + throw Exception(ErrorCodes::ACCESS_DENIED, + "AI functions cannot be executed without a user, e.g. in background merges, mutations or `Buffer` flushes"); +} + +void FunctionBaseAI::throwIfBackgroundExpression(const ContextPtr & context) +{ + if (context->isAnalyzingBackgroundExpressions()) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, + "AI functions cannot be used in expressions executed in the background: mutations, keys, TTL, skip indexes " + "and projections. Use a lightweight `UPDATE` or `INSERT ... SELECT` instead"); +} + AIParamSpecs FunctionBaseAI::embeddingParams() { return { @@ -366,94 +405,102 @@ AIParamSpecs FunctionBaseAI::embeddingParams() }; } -FunctionBaseAI::EmbeddingResult FunctionBaseAI::embedTexts( - IAIProvider & provider, +void FunctionBaseAI::embedTexts( + const std::shared_ptr & provider, const String & model, UInt64 dimensions, const String & function_name, + const String & query_id, + const String & initial_query_id, const VectorWithMemoryTracking & inputs, size_t max_batch_size, - UInt64 max_retries, - UInt64 retry_delay_ms, - bool throw_on_error, - AIQuotaTracker & quota, - const ConnectionTimeouts & timeouts) + size_t max_concurrent_requests, + const AIRequestPolicy & policy, + const AIQuotaTrackerPtr & quota, + EmbeddingResult & result) { - EmbeddingResult result; result.embeddings.resize(inputs.size()); - for (size_t batch_start = 0; batch_start < inputs.size(); batch_start += max_batch_size) - { - if (quota.checkQuotas()) - { - result.texts_skipped += inputs.size() - batch_start; - break; - } + /// Equivalent to (n + max_batch_size - 1) / max_batch_size but can't overflow + const size_t num_batches = inputs.empty() ? 0 : 1 + (inputs.size() - 1) / max_batch_size; + const size_t concurrency = std::min(max_concurrent_requests, num_batches); - size_t batch_end = std::min(batch_start + max_batch_size, inputs.size()); + /// Batches go out in waves of size `concurrency` and each completed wave is applied before the next + /// one starts, so at most that many of this call's requests are in flight at a time. + VectorWithMemoryTracking>> wave; + wave.reserve(concurrency); - AIEmbeddingRequest ai_embedding_request; - ai_embedding_request.model = model; - ai_embedding_request.dimensions = dimensions; - ai_embedding_request.function_name = function_name; - ai_embedding_request.inputs.reserve(batch_end - batch_start); - for (size_t k = batch_start; k < batch_end; ++k) - ai_embedding_request.inputs.emplace_back(inputs[k]); + /// Wait for all waves to complete on error, so that API-call and request counts are recorded + /// They were already dispatched and billed either way. A cancelled query does not wait. + SCOPE_EXIT_SAFE({ drainRequests(wave, *quota); }); - AIEmbeddingResponse ai_embedding_response; - bool batch_ok = false; - for (UInt64 attempt = 0; attempt <= max_retries; ++attempt) - { - /// Reserve an API-call slot before each request; this also performs a quota check. - /// Kept outside the `try` so a `throw_on_quota_exceeded` exception isn't caught by the retry handler. - if (!quota.recordApiCall()) - break; + for (size_t wave_begin = 0; wave_begin < num_batches; wave_begin += concurrency) + { + const size_t wave_end = std::min(wave_begin + concurrency, num_batches); - try + wave.clear(); + /// fire off `concurrency` batches at a time + for (size_t batch = wave_begin; batch < wave_end; ++batch) + { + /// Once the quota is exhausted nothing more is issued, so the batch's slot stays an + /// empty future and its inputs stay empty. + if (quota->checkQuotas()) { - ++result.api_calls; - ai_embedding_response = provider.embed(ai_embedding_request, timeouts); - result.input_tokens += ai_embedding_response.input_tokens; - quota.recordTokens(ai_embedding_response.input_tokens, 0); - batch_ok = true; - break; + wave.emplace_back(); + continue; } - catch (...) - { - if (attempt < max_retries && isRetriableProviderError(std::current_exception())) - { - std::this_thread::sleep_for(std::chrono::milliseconds(computeRetryBackoffMs(retry_delay_ms, attempt))); - continue; - } - if (!throw_on_error) /// Skip to next batch, this batch's inputs stay empty. - break; + const size_t begin = batch * max_batch_size; + const size_t end = std::min(begin + max_batch_size, inputs.size()); - throw; - } + AIEmbeddingRequest ai_embedding_request; + ai_embedding_request.model = model; + ai_embedding_request.dimensions = dimensions; + ai_embedding_request.function_name = function_name; + ai_embedding_request.query_id = query_id; + ai_embedding_request.initial_query_id = initial_query_id; + ai_embedding_request.inputs.reserve(end - begin); + for (size_t k = begin; k < end; ++k) + ai_embedding_request.inputs.emplace_back(inputs[k]); + + wave.push_back(submitAIRequest(provider, std::move(ai_embedding_request), policy, quota)); } - if (!batch_ok) + /// get the results for our `concurrency` batches + for (size_t k = 0; k < wave.size(); ++k) { - result.texts_skipped += batch_end - batch_start; - continue; - } + const size_t begin = (wave_begin + k) * max_batch_size; + const size_t end = std::min(begin + max_batch_size, inputs.size()); - chassert(ai_embedding_response.embeddings.size() == ai_embedding_request.inputs.size(), - "Number of inputs does not match number of output embeddings"); + /// Nothing when no request was issued for this batch, or when it failed and + /// `ai_function_throw_on_error` is disabled; either way its inputs stay empty. + std::optional ai_embedding_response; + if (wave[k].valid()) + ai_embedding_response = waitForRequest(wave[k], *quota); - for (size_t k = 0; k < ai_embedding_response.embeddings.size(); ++k) - { - result.embeddings[batch_start + k] = std::move(ai_embedding_response.embeddings[k]); - ++result.texts_embedded; + if (!ai_embedding_response) + { + result.texts_skipped += end - begin; + continue; + } + + chassert(ai_embedding_response->embeddings.size() == end - begin, + "Number of inputs does not match number of output embeddings"); + + for (size_t j = 0; j < ai_embedding_response->embeddings.size(); ++j) + { + result.embeddings[begin + j] = std::move(ai_embedding_response->embeddings[j]); + ++result.texts_embedded; + } } } - - return result; } ColumnPtr FunctionBaseAI::executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const { + /// Before resolving the credentials: their validation errors must not reveal a named collection. + checkHasUser(getContext()); + const auto & settings = getContext()->getSettingsRef(); auto params = resolveAIParams(getContext(), arguments, allParams(), settings[Setting::ai_function_text_default_credentials]); @@ -466,7 +513,10 @@ ColumnPtr FunctionBaseAI::executeImpl(const ColumnsWithTypeAndName & arguments, checkSanityBeforeExecuteImpl(arguments, result_type, input_rows_count); String system_prompt = sanitizeForModel(buildSystemPrompt(arguments, params)); auto response_format = buildResponseFormat(arguments); - auto provider = createAIProvider( + + /// Shared with the submitted requests so each one is self-contained and nothing dangles even + /// if this call stops waiting for a future it handed out. + std::shared_ptr provider = createAIProvider( getContext(), params.collection.provider, params.collection.endpoint, params.collection.api_key, params.collection.api_version); if (input_rows_count == 0) @@ -482,146 +532,99 @@ ColumnPtr FunctionBaseAI::executeImpl(const ColumnsWithTypeAndName & arguments, prompt_nullable = typeid_cast(prompt_column.get()); } - UInt64 timeout_sec = settings[Setting::ai_function_request_timeout_sec].value; - UInt64 max_retries = settings[Setting::ai_function_max_retries].value; - UInt64 retry_delay_ms = settings[Setting::ai_function_retry_initial_delay_ms].value; - - bool throw_on_error = settings[Setting::ai_function_throw_on_error].value; - /// Shared across every AI function call in the query auto quota_tracker = getContext()->getAIQuotaTracker(); - auto timeouts = ConnectionTimeouts::getHTTPTimeouts(settings, getContext()->getServerSettings()); - timeouts.receive_timeout = Poco::Timespan(static_cast(timeout_sec) /*s*/, 0 /*us*/); + const auto policy = makeRequestPolicy(getContext()); + + const String query_id = getContext()->getCurrentQueryId(); + const String & initial_query_id = getContext()->getClientInfo().initial_query_id; auto result_col = removeNullable(result_type)->createColumn(); auto null_map_col = prompt_nullable ? ColumnUInt8::create(input_rows_count, static_cast(0)) : nullptr; - UInt64 total_api_calls = 0; - UInt64 total_input_tokens = 0; - UInt64 total_output_tokens = 0; UInt64 rows_processed = 0; UInt64 rows_skipped = 0; - /// Increment ProfileEvents counters upon destruction, to avoid underreporting on error + /// Increment ProfileEvents counters upon destruction, to avoid underreporting on error. + /// The per-request counters (`AIAPICalls`, `AIInputTokens`, `AIOutputTokens`) are incremented by + /// the submitted requests as they complete. SCOPE_EXIT({ - ProfileEvents::increment(ProfileEvents::AIAPICalls, total_api_calls); - ProfileEvents::increment(ProfileEvents::AIInputTokens, total_input_tokens); - ProfileEvents::increment(ProfileEvents::AIOutputTokens, total_output_tokens); ProfileEvents::increment(ProfileEvents::AIRowsProcessed, rows_processed); ProfileEvents::increment(ProfileEvents::AIRowsSkipped, rows_skipped); }); - for (size_t i = 0; i < input_rows_count; ++i) + const size_t concurrency = std::min(settings[Setting::ai_function_max_concurrent_requests_per_thread].value, input_rows_count); + + /// Requests go out in waves of `concurrency` rows, and each completed wave is applied to the + /// result column in row order. A wave waits for its slowest request before the next one starts, + /// which gives up a little throughput next to a sliding window, but keeps row ordering, quota + /// accounting and error propagation identical to issuing the requests one at a time. + VectorWithMemoryTracking>> wave; + wave.reserve(concurrency); + + /// Wait for all waves to complete on error, so that API-call and request counts are recorded + /// They were already dispatched and billed either way. A cancelled query does not wait. + SCOPE_EXIT_SAFE({ drainRequests(wave, *quota_tracker); }); + + for (size_t wave_begin = 0; wave_begin < input_rows_count; wave_begin += concurrency) { - if (prompt_nullable && prompt_nullable->getNullMapData()[i]) - { - result_col->insertDefault(); - null_map_col->getData()[i] = 1; - continue; - } + const size_t wave_end = std::min(wave_begin + concurrency, input_rows_count); - if (quota_tracker->checkQuotas()) + wave.clear(); + for (size_t row = wave_begin; row < wave_end; ++row) { - result_col->insertDefault(); - ++rows_skipped; - continue; - } + /// A NULL prompt produces NULL, and once the quota is exhausted a row keeps its default + /// value. Neither issues a request, so the row's slot stays an empty future. + if ((prompt_nullable && prompt_nullable->getNullMapData()[row]) || quota_tracker->checkQuotas()) + { + wave.emplace_back(); + continue; + } - String user_message = sanitizeForModel(buildUserMessage(arguments, i)); - String result; - bool success = false; + AIChatRequest ai_request; + ai_request.system_prompt = system_prompt; + ai_request.user_message = sanitizeForModel(buildUserMessage(arguments, row)); + ai_request.response_format = response_format; + ai_request.model = model; + ai_request.temperature = temperature; + ai_request.max_tokens = max_tokens; + ai_request.function_name = getName(); + ai_request.query_id = query_id; + ai_request.initial_query_id = initial_query_id; + + wave.push_back(submitAIRequest(provider, std::move(ai_request), policy, quota_tracker)); + } - for (UInt64 attempt = 0; attempt <= max_retries; ++attempt) + for (size_t k = 0; k < wave.size(); ++k) { - /// Reserve an API-call slot before each request; this also performs a quota check. - /// Kept outside the `try` so a `throw_on_quota_exceeded` exception isn't caught by the retry handler. - if (!quota_tracker->recordApiCall()) - break; + const size_t row = wave_begin + k; - try + /// A NULL prompt produces NULL without a request. + if (prompt_nullable && prompt_nullable->getNullMapData()[row]) { - AIRequest ai_request; - ai_request.system_prompt = system_prompt; - ai_request.user_message = user_message; - ai_request.response_format = response_format; - ai_request.model = model; - ai_request.temperature = temperature; - ai_request.max_tokens = max_tokens; - ai_request.function_name = getName(); - - ++total_api_calls; - - auto ai_response = provider->call(ai_request, timeouts); - - quota_tracker->recordTokens(ai_response.input_tokens, ai_response.output_tokens); - total_input_tokens += ai_response.input_tokens; - total_output_tokens += ai_response.output_tokens; - - /// `raw_finish_reason` is provider-controlled text; sanitize control characters before - /// interpolating it into an exception message that reaches the logs and `system.query_log`. - const String safe_finish_reason = sanitizeForLog(ai_response.raw_finish_reason); - - /// Reject incomplete responses, throw plain DB::Exception so it is classified as non-retriable - switch (ai_response.finish_reason) - { - case FinishReason::Complete: - case FinishReason::Unknown: /// Don't throw on Unknown, could be new valid reason in new API version - break; - case FinishReason::Truncated: - /// Differentiate between model hitting our output cap and exhausting its context window - throw Exception( - ErrorCodes::AI_PROVIDER_RESPONSE_TRUNCATED, - "AI provider returned a truncated response (finish_reason='{}'): {}", - safe_finish_reason, - ai_response.raw_finish_reason == "model_context_window_exceeded" - ? "the model ran out of context window before completing its answer. " - "Reduce the input or use a model with a larger context window." - : "the model hit the output token limit before completing its answer. " - "Increase max_tokens or reduce the input."); - case FinishReason::ContentFilter: - throw Exception( - ErrorCodes::AI_PROVIDER_RESPONSE_INCOMPLETE, - "AI provider withheld or filtered the response (finish_reason='{}'): the returned answer " - "is incomplete.", - safe_finish_reason); - case FinishReason::RequiresAction: - throw Exception( - ErrorCodes::AI_PROVIDER_RESPONSE_INCOMPLETE, - "AI provider stopped expecting further caller action (finish_reason='{}') instead of " - "returning a completed answer.", - safe_finish_reason); - } - - result = postProcessResponse(ai_response.result); - success = true; - break; + result_col->insertDefault(); + null_map_col->getData()[row] = 1; + continue; } - catch (...) - { - if (attempt < max_retries && isRetriableProviderError(std::current_exception())) - { - std::this_thread::sleep_for(std::chrono::milliseconds(computeRetryBackoffMs(retry_delay_ms, attempt))); - continue; - } - if (!throw_on_error) - break; + /// Nothing when no request was issued because the API-call quota was exhausted, or when + /// the request failed and `ai_function_throw_on_error` is disabled; either way the row + /// keeps its default value. + std::optional ai_response; + if (wave[k].valid()) + ai_response = waitForRequest(wave[k], *quota_tracker); - throw; + if (!ai_response) + { + result_col->insertDefault(); + ++rows_skipped; + continue; } - } - if (success) - { - insertProcessedResult(*result_col, result); + insertProcessedResult(*result_col, postProcessResponse(ai_response->result)); ++rows_processed; } - else - { - result_col->insertDefault(); - ++rows_skipped; - } } if (result_type->isNullable()) diff --git a/src/Functions/FunctionBaseAI.h b/src/Functions/FunctionBaseAI.h index 41751061fa203..5932413e733fc 100644 --- a/src/Functions/FunctionBaseAI.h +++ b/src/Functions/FunctionBaseAI.h @@ -4,11 +4,11 @@ #include #include #include +#include #include #include #include -#include #include #include #include @@ -52,13 +52,14 @@ using AIParamValues = std::map>; // STYLE_CHECK_ALLOW class FunctionBaseAI : public IFunction { public: - explicit FunctionBaseAI(ContextPtr context_) : context(context_) {} + explicit FunctionBaseAI(ContextPtr context_) : context(context_) { throwIfBackgroundExpression(context); } bool isStateful() const override { return true; } bool isDeterministic() const override { return false; } bool isDeterministicInScopeOfQuery() const override { return false; } bool isSuitableForConstantFolding() const override { return false; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } + bool isExpensive() const override { return true; } /// Handle Nullable cols explicitly, since setting this to true may call functions with arbitrary input values bool useDefaultImplementationForNulls() const override { return false; } bool useDefaultImplementationForConstants() const override { return false; } @@ -128,14 +129,9 @@ class FunctionBaseAI : public IFunction /// Parameters common to every AI function. Function-specific params are appended by `functionParams`. static AIParamSpecs commonParams(); - /// Exponential backoff delay capped at one minute, so adversarial values of - /// `ai_function_retry_initial_delay_ms` or `ai_function_max_retries` cannot produce a multi-hour - /// sleep or overflow `std::chrono::milliseconds`. - static UInt64 computeRetryBackoffMs(UInt64 initial_delay_ms, UInt64 attempt); - - /// Whether a failed provider request should be retried: transient network failures and - /// transient/server-side HTTP responses are retriable, deterministic argument/usage errors are not. - static bool isRetriableProviderError(std::exception_ptr exception); + /// Read the `ai_function_*` settings that govern how one provider request runs. The same policy + /// applies to every request the query issues. + static AIRequestPolicy makeRequestPolicy(const ContextPtr & context); /// Parameters accepted by the embedding functions (`aiEmbed`, `aiSimilarity`) in their optional trailing /// `Map(String, String)` argument. Unlike the text functions, they take no `max_tokens`, and `model` is a @@ -143,32 +139,41 @@ class FunctionBaseAI : public IFunction /// target vector size when the model supports it. static AIParamSpecs embeddingParams(); + /// Throws if the context has no user. Background work (merges, mutations, `Buffer` flushes) evaluates + /// expressions with such a context, which has full access, so the privilege checks do not apply to it. + static void checkHasUser(const ContextPtr & context); + + /// Throws if the context analyzes expressions that are executed in the background (mutations, expressions stored + /// in a table definition), where `checkHasUser` throws. + static void throwIfBackgroundExpression(const ContextPtr & context); + /// Result of `embedTexts`. `embeddings` is aligned 1:1 with the `inputs` argument of `embedTexts`. An /// entry is empty when that input was not embedded (quota exceeded, or a failed request with - /// `ai_function_throw_on_error` disabled). The counters feed the AI ProfileEvents. + /// `ai_function_throw_on_error` disabled). struct EmbeddingResult { VectorWithMemoryTracking> embeddings; - UInt64 api_calls = 0; - UInt64 input_tokens = 0; UInt64 texts_embedded = 0; UInt64 texts_skipped = 0; }; - /// Embed a flat list of already-filtered (non-null, non-empty) texts, reusing the shared batching, - /// retry/backoff, and quota logic. Inputs are grouped into batches of up to `max_batch_size` per HTTP call. - static EmbeddingResult embedTexts( - IAIProvider & provider, + /// Embed a flat list of already-filtered (non-null, non-empty) texts, reusing the shared batching + /// and quota logic. Inputs are grouped into batches of up to `max_batch_size` per HTTP call, and + /// up to `max_concurrent_requests` of those calls are in flight at once. + /// Accumulates into `result`, so the waves completed before a throw stay visible to the caller. + static void embedTexts( + const std::shared_ptr & provider, const String & model, UInt64 dimensions, const String & function_name, + const String & query_id, + const String & initial_query_id, const VectorWithMemoryTracking & inputs, size_t max_batch_size, - UInt64 max_retries, - UInt64 retry_delay_ms, - bool throw_on_error, - AIQuotaTracker & quota, - const ConnectionTimeouts & timeouts); + size_t max_concurrent_requests, + const AIRequestPolicy & policy, + const AIQuotaTrackerPtr & quota, + EmbeddingResult & result); protected: ContextPtr context; diff --git a/src/Functions/FunctionDynamicAdaptor.h b/src/Functions/FunctionDynamicAdaptor.h index 47a60c762973a..788b6d825b127 100644 --- a/src/Functions/FunctionDynamicAdaptor.h +++ b/src/Functions/FunctionDynamicAdaptor.h @@ -72,6 +72,8 @@ class FunctionBaseDynamicAdaptor final : public IFunctionBase bool isSpatialPredicate() const override { return function_overload_resolver->isSpatialPredicate(); } + bool isExpensive() const override { return function_overload_resolver->isExpensive(); } + private: /// We remember the original IFunctionOverloadResolver to be able to build function for types inside Dynamic column. std::shared_ptr function_overload_resolver; diff --git a/src/Functions/FunctionVariantAdaptor.h b/src/Functions/FunctionVariantAdaptor.h index ccdd426a1c93c..83dbf3cd587d1 100644 --- a/src/Functions/FunctionVariantAdaptor.h +++ b/src/Functions/FunctionVariantAdaptor.h @@ -78,6 +78,8 @@ class FunctionBaseVariantAdaptor final : public IFunctionBase bool isSpatialPredicate() const override { return function_overload_resolver->isSpatialPredicate(); } + bool isExpensive() const override { return function_overload_resolver->isExpensive(); } + private: /// We remember the original IFunctionOverloadResolver to be able to build function for types inside Variant column. std::shared_ptr function_overload_resolver; diff --git a/src/Functions/FunctionsJSON.cpp b/src/Functions/FunctionsJSON.cpp index 891d0c8c846ef..9ab5f242d2c3d 100644 --- a/src/Functions/FunctionsJSON.cpp +++ b/src/Functions/FunctionsJSON.cpp @@ -51,6 +51,7 @@ namespace Setting { extern const SettingsBool allow_simdjson; extern const SettingsDateTimeInputFormat cast_string_to_date_time_mode; + extern const SettingsBool json_extract_use_cast_string_to_date_time_mode; } namespace ErrorCodes @@ -698,7 +699,9 @@ class JSONOverloadResolver final : public IFunctionOverloadResolver { /// Extracting a string JSON value into a DateTime/DateTime64 column is a string-to-type /// cast, so we honour `cast_string_to_date_time_mode` (rather than `date_time_input_format`). - format_settings.date_time_input_format = context->getSettingsRef()[Setting::cast_string_to_date_time_mode]; + /// Before 26.8 it followed `date_time_input_format`, which `compatibility` restores. + if (context->getSettingsRef()[Setting::json_extract_use_cast_string_to_date_time_mode]) + format_settings.date_time_input_format = context->getSettingsRef()[Setting::cast_string_to_date_time_mode]; } bool isVariadic() const override { return true; } diff --git a/src/Functions/FunctionsMiscellaneous.h b/src/Functions/FunctionsMiscellaneous.h index f1183243713ce..99849c951bd1a 100644 --- a/src/Functions/FunctionsMiscellaneous.h +++ b/src/Functions/FunctionsMiscellaneous.h @@ -100,6 +100,16 @@ class ExecutableFunctionExpression final : public IExecutableFunction SignaturePtr signature; }; +/// Whether a function in the body of a lambda reports `isExpensive`. Nested lambdas are covered through +/// their own `FunctionCapture`, which is a `FUNCTION` node of this body. +inline bool isLambdaBodyExpensive(const ExpressionActions & expression_actions) +{ + for (const auto & node : expression_actions.getActionsDAG().getNodes()) + if (node.type == ActionsDAG::ActionType::FUNCTION && node.function_base->isExpensive()) + return true; + return false; +} + /// Executes expression. Uses for lambda functions implementation. Can't be created from factory. class FunctionExpression final : public IFunctionBase { @@ -134,6 +144,8 @@ class FunctionExpression final : public IFunctionBase bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo & /*arguments*/) const override { return false; } + bool isExpensive() const override { return isLambdaBodyExpensive(*expression_actions); } + const DataTypes & getArgumentTypes() const override { return argument_types; } const DataTypePtr & getResultType() const override { return capture->return_type; } @@ -292,6 +304,8 @@ class FunctionCapture final : public IFunctionBase return true; } + bool isExpensive() const override { return isLambdaBodyExpensive(*expression_actions); } + const DataTypes & getArgumentTypes() const override { return capture->captured_types; } const DataTypePtr & getResultType() const override { return return_type; } diff --git a/src/Functions/IFunction.h b/src/Functions/IFunction.h index 4c564f19dfecc..d3c4e66ffb6ff 100644 --- a/src/Functions/IFunction.h +++ b/src/Functions/IFunction.h @@ -220,6 +220,12 @@ class IFunctionBase : public IResolvedFunction */ virtual bool isVolumeReducing() const { return false; } + /** Returns true if the function costs far more than reading its arguments, e.g. one network + * request per row. Can be used to postpone the execution of such functions as much as possible + * (e.g. PREWHERE vs. WHERE) + */ + virtual bool isExpensive() const { return false; } + /** Returns true if this is a spatial predicate for which bbox-disjoint pruning is safe. * Specifically: if the bounding boxes of the geometry arguments are disjoint, * the function is guaranteed to return 0/false for all such rows. @@ -444,6 +450,9 @@ class IFunctionOverloadResolver : public std::enable_shared_from_thisisVolumeReducing(); } + bool isExpensive() const override { return function->isExpensive(); } + bool isInjective(const ColumnsWithTypeAndName & sample_columns) const override { return function->isInjective(sample_columns); } ComparisonOrderDomain getComparisonOrderDomain() const override @@ -138,6 +140,7 @@ class FunctionToOverloadResolverAdaptor final : public IFunctionOverloadResolver bool isDeterministicInScopeOfQuery() const override { return function->isDeterministicInScopeOfQuery(); } bool isInjective(const ColumnsWithTypeAndName & columns) const override { return function->isInjective(columns); } bool isSpatialPredicate() const override { return function->isSpatialPredicate(); } + bool isExpensive() const override { return function->isExpensive(); } String getName() const override { return function->getName(); } bool isStateful() const override { return function->isStateful(); } diff --git a/src/Functions/aiClassify.cpp b/src/Functions/aiClassify.cpp index 1bc6b5f15c39b..5e2cba24bdb3c 100644 --- a/src/Functions/aiClassify.cpp +++ b/src/Functions/aiClassify.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -38,7 +39,11 @@ class FunctionAiClassify final : public FunctionBaseAI explicit FunctionAiClassify(ContextPtr context_) : FunctionBaseAI(context_) {} - static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } + static FunctionPtr create(ContextPtr context_) + { + context_->checkAccess(AccessType::aiClassify); + return std::make_shared(context_); + } String getName() const override { return name; } bool isVariadic() const override { return true; } diff --git a/src/Functions/aiEmbed.cpp b/src/Functions/aiEmbed.cpp index 26cb68c767f6c..2bd46f24195f0 100644 --- a/src/Functions/aiEmbed.cpp +++ b/src/Functions/aiEmbed.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -19,17 +20,13 @@ #include #include -#include #include -#include #include #include namespace ProfileEvents { - extern const Event AIInputTokens; - extern const Event AIAPICalls; extern const Event AIRowsProcessed; extern const Event AIRowsSkipped; } @@ -39,11 +36,8 @@ namespace DB namespace Setting { - extern const SettingsUInt64 ai_function_request_timeout_sec; - extern const SettingsUInt64 ai_function_max_retries; - extern const SettingsUInt64 ai_function_retry_initial_delay_ms; - extern const SettingsBool ai_function_throw_on_error; extern const SettingsNonZeroUInt64 ai_function_embedding_max_batch_size; + extern const SettingsNonZeroUInt64 ai_function_max_concurrent_requests_per_thread; extern const SettingsString ai_function_embedding_default_credentials; } @@ -60,9 +54,13 @@ class FunctionAiEmbed final : public IFunction public: static constexpr auto name = "aiEmbed"; - static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } + static FunctionPtr create(ContextPtr context) + { + context->checkAccess(AccessType::aiEmbed); + return std::make_shared(context); + } - explicit FunctionAiEmbed(ContextPtr context_) : context(context_) {} + explicit FunctionAiEmbed(ContextPtr context_) : context(context_) { FunctionBaseAI::throwIfBackgroundExpression(context); } String getName() const override { return name; } bool isVariadic() const override { return true; } @@ -78,6 +76,7 @@ class FunctionAiEmbed final : public IFunction bool isSuitableForConstantFolding() const override { return false; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } + bool isExpensive() const override { return true; } /// Handle Nullable cols explicitly, since setting this to true may call func with arbitrary input values bool useDefaultImplementationForNulls() const override { return false; } @@ -99,6 +98,9 @@ class FunctionAiEmbed final : public IFunction ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { + /// Before resolving the credentials: their validation errors must not reveal a named collection. + FunctionBaseAI::checkHasUser(getContext()); + const auto & settings = getContext()->getSettingsRef(); auto params = FunctionBaseAI::resolveAIParams( getContext(), arguments, FunctionBaseAI::embeddingParams(), settings[Setting::ai_function_embedding_default_credentials]); @@ -106,12 +108,8 @@ class FunctionAiEmbed final : public IFunction UInt64 dimensions = params.getUInt("dimensions"); String model(arguments[model_arg_index].column->getDataAt(0)); - auto provider = createAIProvider( - getContext(), - params.collection.provider, - params.collection.endpoint, - params.collection.api_key, - params.collection.api_version); + std::shared_ptr provider = createAIProvider( + getContext(), params.collection.provider, params.collection.endpoint, params.collection.api_key, params.collection.api_version); if (!provider->supportsEmbeddings()) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "AI provider '{}' does not support embeddings", params.collection.provider); @@ -119,17 +117,13 @@ class FunctionAiEmbed final : public IFunction if (input_rows_count == 0) return result_type->createColumn(); - UInt64 timeout_sec = settings[Setting::ai_function_request_timeout_sec].value; - UInt64 max_retries = settings[Setting::ai_function_max_retries].value; - UInt64 retry_delay_ms = settings[Setting::ai_function_retry_initial_delay_ms].value; - bool throw_on_error = settings[Setting::ai_function_throw_on_error].value; size_t max_batch_size = static_cast(settings[Setting::ai_function_embedding_max_batch_size].value); + size_t max_concurrent_requests = static_cast(settings[Setting::ai_function_max_concurrent_requests_per_thread].value); /// Shared across every AI function call in the query auto quota_tracker = getContext()->getAIQuotaTracker(); - auto timeouts = ConnectionTimeouts::getHTTPTimeouts(settings, getContext()->getServerSettings()); - timeouts.receive_timeout = Poco::Timespan(static_cast(timeout_sec) /*s*/, 0 /*us*/); + const auto policy = FunctionBaseAI::makeRequestPolicy(getContext()); /// `isNullAt` and `getDataAt` are virtual on `IColumn`, so a single path covers `ColumnString`, /// `ColumnConst(ColumnString)`, `ColumnNullable` and `ColumnConst(ColumnNullable)` (e.g. @@ -160,8 +154,10 @@ class FunctionAiEmbed final : public IFunction inputs.push_back(value); } - auto embedding_result = FunctionBaseAI::embedTexts( - *provider, model, dimensions, getName(), inputs, max_batch_size, max_retries, retry_delay_ms, throw_on_error, *quota_tracker, timeouts); + FunctionBaseAI::EmbeddingResult embedding_result; + FunctionBaseAI::embedTexts( + provider, model, dimensions, getName(), getContext()->getCurrentQueryId(), getContext()->getClientInfo().initial_query_id, + inputs, max_batch_size, max_concurrent_requests, policy, quota_tracker, embedding_result); auto data_col = ColumnVector::create(); /// float32 is standard embedding API output auto offsets_col = ColumnArray::ColumnOffsets::create(); @@ -193,8 +189,6 @@ class FunctionAiEmbed final : public IFunction for (; cursor < input_rows_count; ++cursor) offsets_vec.push_back(current_offset); - ProfileEvents::increment(ProfileEvents::AIAPICalls, embedding_result.api_calls); - ProfileEvents::increment(ProfileEvents::AIInputTokens, embedding_result.input_tokens); ProfileEvents::increment(ProfileEvents::AIRowsProcessed, embedding_result.texts_embedded); ProfileEvents::increment(ProfileEvents::AIRowsSkipped, embedding_result.texts_skipped); diff --git a/src/Functions/aiExtract.cpp b/src/Functions/aiExtract.cpp index 850665d982e27..d8b456b91350f 100644 --- a/src/Functions/aiExtract.cpp +++ b/src/Functions/aiExtract.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -26,7 +27,11 @@ class FunctionAiExtract final : public FunctionBaseAI explicit FunctionAiExtract(ContextPtr context_) : FunctionBaseAI(context_) {} - static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } + static FunctionPtr create(ContextPtr context_) + { + context_->checkAccess(AccessType::aiExtract); + return std::make_shared(context_); + } String getName() const override { return name; } bool isVariadic() const override { return true; } diff --git a/src/Functions/aiFilter.cpp b/src/Functions/aiFilter.cpp index 51ac1dbffb0e9..6f593160117d3 100644 --- a/src/Functions/aiFilter.cpp +++ b/src/Functions/aiFilter.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -25,7 +26,11 @@ class FunctionAiFilter final : public FunctionBaseAI explicit FunctionAiFilter(ContextPtr context_) : FunctionBaseAI(context_) {} - static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } + static FunctionPtr create(ContextPtr context_) + { + context_->checkAccess(AccessType::aiFilter); + return std::make_shared(context_); + } String getName() const override { return name; } bool isVariadic() const override { return true; } diff --git a/src/Functions/aiGenerate.cpp b/src/Functions/aiGenerate.cpp index 9eda2c0afe5b1..102bff50c6f5c 100644 --- a/src/Functions/aiGenerate.cpp +++ b/src/Functions/aiGenerate.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -22,7 +23,11 @@ class FunctionAiGenerate final : public FunctionBaseAI explicit FunctionAiGenerate(ContextPtr context_) : FunctionBaseAI(context_) {} - static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } + static FunctionPtr create(ContextPtr context_) + { + context_->checkAccess(AccessType::aiGenerate); + return std::make_shared(context_); + } String getName() const override { return name; } bool isVariadic() const override { return true; } diff --git a/src/Functions/aiRedact.cpp b/src/Functions/aiRedact.cpp index fdadbe3507600..0276839927c3b 100644 --- a/src/Functions/aiRedact.cpp +++ b/src/Functions/aiRedact.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -36,7 +37,11 @@ class FunctionAiRedact final : public FunctionBaseAI explicit FunctionAiRedact(ContextPtr context_) : FunctionBaseAI(context_) {} - static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } + static FunctionPtr create(ContextPtr context_) + { + context_->checkAccess(AccessType::aiRedact); + return std::make_shared(context_); + } String getName() const override { return name; } bool isVariadic() const override { return true; } diff --git a/src/Functions/aiSimilarity.cpp b/src/Functions/aiSimilarity.cpp index 7c35c9bf94bd6..ace4995fd91a5 100644 --- a/src/Functions/aiSimilarity.cpp +++ b/src/Functions/aiSimilarity.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -17,9 +18,7 @@ #include #include -#include #include -#include #include #include @@ -29,8 +28,6 @@ namespace ProfileEvents { - extern const Event AIInputTokens; - extern const Event AIAPICalls; extern const Event AIRowsProcessed; extern const Event AIRowsSkipped; } @@ -40,11 +37,8 @@ namespace DB namespace Setting { - extern const SettingsUInt64 ai_function_request_timeout_sec; - extern const SettingsUInt64 ai_function_max_retries; - extern const SettingsUInt64 ai_function_retry_initial_delay_ms; - extern const SettingsBool ai_function_throw_on_error; extern const SettingsNonZeroUInt64 ai_function_embedding_max_batch_size; + extern const SettingsNonZeroUInt64 ai_function_max_concurrent_requests_per_thread; extern const SettingsString ai_function_embedding_default_credentials; } @@ -87,9 +81,13 @@ class FunctionAiSimilarity final : public IFunction public: static constexpr auto name = "aiSimilarity"; - static FunctionPtr create(ContextPtr context) { return std::make_shared(context); } + static FunctionPtr create(ContextPtr context) + { + context->checkAccess(AccessType::aiSimilarity); + return std::make_shared(context); + } - explicit FunctionAiSimilarity(ContextPtr context_) : context(context_) {} + explicit FunctionAiSimilarity(ContextPtr context_) : context(context_) { FunctionBaseAI::throwIfBackgroundExpression(context); } String getName() const override { return name; } bool isVariadic() const override { return true; } @@ -104,6 +102,7 @@ class FunctionAiSimilarity final : public IFunction bool isSuitableForConstantFolding() const override { return false; } bool isSuitableForShortCircuitArgumentsExecution(const DataTypesWithConstInfo &) const override { return true; } + bool isExpensive() const override { return true; } /// Handle Nullable cols explicitly, since setting this to true may call func with arbitrary input values bool useDefaultImplementationForNulls() const override { return false; } @@ -127,6 +126,9 @@ class FunctionAiSimilarity final : public IFunction ColumnPtr executeImpl(const ColumnsWithTypeAndName & arguments, const DataTypePtr & result_type, size_t input_rows_count) const override { + /// Before resolving the credentials: their validation errors must not reveal a named collection. + FunctionBaseAI::checkHasUser(getContext()); + const auto & settings = getContext()->getSettingsRef(); auto params = FunctionBaseAI::resolveAIParams( getContext(), arguments, FunctionBaseAI::embeddingParams(), settings[Setting::ai_function_embedding_default_credentials]); @@ -134,12 +136,10 @@ class FunctionAiSimilarity final : public IFunction UInt64 dimensions = params.getUInt("dimensions"); String model(arguments[model_arg_index].column->getDataAt(0)); - auto provider = createAIProvider( - getContext(), - params.collection.provider, - params.collection.endpoint, - params.collection.api_key, - params.collection.api_version); + /// Shared with the submitted requests so each one is self-contained and nothing dangles + /// even if this call stops waiting for a future it handed out. + std::shared_ptr provider = createAIProvider( + getContext(), params.collection.provider, params.collection.endpoint, params.collection.api_key, params.collection.api_version); if (!provider->supportsEmbeddings()) throw Exception(ErrorCodes::NOT_IMPLEMENTED, "AI provider '{}' does not support embeddings", params.collection.provider); @@ -147,16 +147,13 @@ class FunctionAiSimilarity final : public IFunction if (input_rows_count == 0) return result_type->createColumn(); - UInt64 max_retries = settings[Setting::ai_function_max_retries].value; - UInt64 retry_delay_ms = settings[Setting::ai_function_retry_initial_delay_ms].value; - bool throw_on_error = settings[Setting::ai_function_throw_on_error].value; size_t max_batch_size = static_cast(settings[Setting::ai_function_embedding_max_batch_size].value); + size_t max_concurrent_requests = static_cast(settings[Setting::ai_function_max_concurrent_requests_per_thread].value); /// Shared across every AI function call in the query auto quota_tracker = getContext()->getAIQuotaTracker(); - auto timeouts = ConnectionTimeouts::getHTTPTimeouts(settings, getContext()->getServerSettings()); - timeouts.receive_timeout = Poco::Timespan(static_cast(settings[Setting::ai_function_request_timeout_sec].value) /*s*/, 0 /*us*/); + const auto policy = FunctionBaseAI::makeRequestPolicy(getContext()); /// `isNullAt` and `getDataAt` are virtual on `IColumn`, so a single path covers `ColumnString`, /// `ColumnConst(ColumnString)`, `ColumnNullable` and `ColumnConst(ColumnNullable)`. A constant @@ -198,11 +195,10 @@ class FunctionAiSimilarity final : public IFunction } } - auto embedding_result = FunctionBaseAI::embedTexts( - *provider, model, dimensions, getName(), inputs, max_batch_size, max_retries, retry_delay_ms, throw_on_error, *quota_tracker, timeouts); - - ProfileEvents::increment(ProfileEvents::AIAPICalls, embedding_result.api_calls); - ProfileEvents::increment(ProfileEvents::AIInputTokens, embedding_result.input_tokens); + FunctionBaseAI::EmbeddingResult embedding_result; + FunctionBaseAI::embedTexts( + provider, model, dimensions, getName(), getContext()->getCurrentQueryId(), getContext()->getClientInfo().initial_query_id, + inputs, max_batch_size, max_concurrent_requests, policy, quota_tracker, embedding_result); const auto & embeddings = embedding_result.embeddings; diff --git a/src/Functions/aiTranslate.cpp b/src/Functions/aiTranslate.cpp index cbf2afcb243c2..712ce120b7b5e 100644 --- a/src/Functions/aiTranslate.cpp +++ b/src/Functions/aiTranslate.cpp @@ -1,3 +1,4 @@ +#include #include #include #include @@ -19,7 +20,11 @@ class FunctionAiTranslate final : public FunctionBaseAI explicit FunctionAiTranslate(ContextPtr context_) : FunctionBaseAI(context_) {} - static FunctionPtr create(ContextPtr context_) { return std::make_shared(context_); } + static FunctionPtr create(ContextPtr context_) + { + context_->checkAccess(AccessType::aiTranslate); + return std::make_shared(context_); + } String getName() const override { return name; } bool isVariadic() const override { return true; } diff --git a/src/Functions/parseDateTime.cpp b/src/Functions/parseDateTime.cpp index cdf9157e2fca9..08d1fa749849b 100644 --- a/src/Functions/parseDateTime.cpp +++ b/src/Functions/parseDateTime.cpp @@ -562,8 +562,10 @@ namespace [[nodiscard]] Int64OrError buildDateTime(const DateLUTImpl & time_zone) { - if (is_hour_of_half_day && !is_am) - hour += 12; + /// The hour of a half-day clock is stored in [0, 11], and PM adds 12 hours to it. They are added to a copy: for a + /// time zone name in the string (`z` in Joda syntax), this function also runs before the final conversion, so + /// adding them to `hour` itself added them twice. + const Int32 hour_of_day = is_hour_of_half_day && !is_am ? hour + 12 : hour; // Convert the parsed date/time into a timestamp. Int32 days_since_epoch = 0; @@ -574,11 +576,33 @@ namespace else ASSIGN_RESULT_OR_RETURN_ERROR(days_since_epoch, daysSinceEpochFromDate(year, month, day)) - Int64 seconds_since_epoch = days_since_epoch * 86400UZ + hour * 3600UZ + minute * 60UZ + second; + Int64 seconds_since_epoch = days_since_epoch * 86400UZ + hour_of_day * 3600UZ + minute * 60UZ + second; /// Time zone is not specified, use local time zone if (!has_time_zone_offset) - time_zone_offset = time_zone.timezoneOffset(seconds_since_epoch); + { + /// Convert the local time as `toDateTime` does. The offset at `seconds_since_epoch`, which is the local + /// time read as UTC, can be the one from the other side of a daylight saving time change. + auto local_year = static_cast(year); + auto local_month = static_cast(month); + auto local_day = static_cast(day); + /// For a week date or a day of the year, take the calendar date from the day number. + if (week_date_format || day_of_year_format) + { + const auto values = time_zone.getValues(ExtendedDayNum(days_since_epoch)); + local_year = values.year; + local_month = values.month; + local_day = values.day_of_month; + } + time_zone_offset = seconds_since_epoch + - time_zone.makeDateTime( + local_year, + local_month, + local_day, + static_cast(hour_of_day), + static_cast(minute), + static_cast(second)); + } /// Time zone is specified in format string. seconds_since_epoch -= time_zone_offset; @@ -1786,12 +1810,17 @@ namespace { std::string_view read_time_zone{cur, end}; const DateLUTImpl & date_time_zone = DateLUT::instance(read_time_zone); + const bool has_offset_before_time_zone = parsed_value.has_time_zone_offset; const auto result = parsed_value.buildDateTime(date_time_zone); if (result.has_value()) { - const DateLUTImpl::Time timezone_offset = date_time_zone.timezoneOffset(*result); + /// An offset parsed before the time zone name selects the instant, for example one of the two occurrences + /// of a repeated local time, and the time zone gives its offset at that instant. Without such an offset, + /// keep the offset that `buildDateTime` resolved for the local time: the offset at the resulting instant + /// differs from it for a local time that does not exist. + if (has_offset_before_time_zone) + parsed_value.time_zone_offset = date_time_zone.timezoneOffset(*result); parsed_value.has_time_zone_offset = true; - parsed_value.time_zone_offset = timezone_offset; return end; } else diff --git a/src/IO/CachedInMemoryReadBufferFromFile.cpp b/src/IO/CachedInMemoryReadBufferFromFile.cpp index db1f9c071aa3f..69e38bbda98bc 100644 --- a/src/IO/CachedInMemoryReadBufferFromFile.cpp +++ b/src/IO/CachedInMemoryReadBufferFromFile.cpp @@ -30,6 +30,8 @@ CachedInMemoryReadBufferFromFile::CachedInMemoryReadBufferFromFile( , in(std::move(in_)), read_until_position(file_size.value()) , inner_read_until_position(read_until_position) { + /// The working buffer is a page cache cell, an exact-size allocation: nothing past its end is readable. + padded = false; } bool CachedInMemoryReadBufferFromFile::innerSupportsReadAt() const diff --git a/src/IO/PipelineReadBuffer.cpp b/src/IO/PipelineReadBuffer.cpp index ee70b8746d9e4..d3eed4efb7ae7 100644 --- a/src/IO/PipelineReadBuffer.cpp +++ b/src/IO/PipelineReadBuffer.cpp @@ -17,6 +17,7 @@ PipelineReadBuffer::PipelineReadBuffer(std::unique_ptr executor_ , executor(std::move(executor_)) , read_position(executor->getPosition()) { + padded = false; LOG_DEBUG(log, "Created, total_size={}, read_position={}", executor->totalSize(), read_position); } diff --git a/src/IO/ReadPipeline.h b/src/IO/ReadPipeline.h index 58dc0d2210ae5..24696be3774bf 100644 --- a/src/IO/ReadPipeline.h +++ b/src/IO/ReadPipeline.h @@ -121,7 +121,7 @@ class ReadPipeline void needFilesystemCache(FileCachePtr cache, FilesystemCacheSettings cache_settings, std::shared_ptr cache_log = nullptr); /// Overload with a custom cache key and origin, bypassing the default `FileCacheKey::fromPath` derivation. - /// Used by `StorageObjectStorageSource` where the cache key is `SipHash(path + etag)`. + /// Used by `StorageObjectStorageSource` where the cache key is `SipHash( + path + etag)`. void needFilesystemCache( FileCachePtr cache, FileCacheKey cache_key, @@ -135,7 +135,7 @@ class ReadPipeline /// Overload with a fully custom page cache key (path + file_version), bypassing the default /// `cache_path_prefix + object.remote_path` derivation. - /// Used by `StorageObjectStorageSource` where the key is `"s3:" + path` with `"etag:" + etag`. + /// Used by `StorageObjectStorageSource` where the key is `"s3::" + path` with `"etag:" + etag`. void needMemoryCache( String custom_cache_path, String custom_file_version, diff --git a/src/IO/S3/Client.cpp b/src/IO/S3/Client.cpp index dc9acdbdaa3e4..d552ea34d69d1 100644 --- a/src/IO/S3/Client.cpp +++ b/src/IO/S3/Client.cpp @@ -35,6 +35,7 @@ #include #include +#include #include #include #include @@ -82,6 +83,7 @@ namespace Setting namespace ErrorCodes { + extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; extern const int TOO_MANY_REDIRECTS; } @@ -880,6 +882,10 @@ Client::doRequestWithRetryNetworkErrors(RequestType & request, RequestFn request // do not increment S3ReadRequestsErrors/S3WriteRequestsErrors here, it has been accounted in IO/S3/PocoHTTPClient.cpp + /// Unless retries are coordinated across threads, the SDK has already retried this error. + if (!client_configuration.s3_slow_all_threads_after_retryable_error) + break; + /// Retry attempts are managed by the outer loop, so the attemptedRetries argument can be ignored. if (!client_configuration.retryStrategy->ShouldRetry(outcome.GetError(), /*attemptedRetries*/ -1)) break; @@ -1314,6 +1320,21 @@ std::unique_ptr ClientFactory::create( // NOLINT if (!server_side_encryption_customer_key_base64.empty()) { + /// `base64Decode` throws on characters outside the alphabet but skips whitespace and ignores padding bits, + /// so the round trip below accepts only the canonical base64 of an AES-256 key, the only key SSE-C takes. + String customer_key; + try + { + customer_key = base64Decode(server_side_encryption_customer_key_base64); + } + catch (const Poco::DataFormatException &) + { + throw Exception(ErrorCodes::BAD_ARGUMENTS, "`server_side_encryption_customer_key_base64` must be a base64-encoded 256-bit key"); + } + + if (customer_key.size() != AES256_KEY_BYTES || base64Encode(customer_key) != server_side_encryption_customer_key_base64) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "`server_side_encryption_customer_key_base64` must be a base64-encoded 256-bit key"); + /// See Client::GeneratePresignedUrlWithSSEC(). headers.push_back({Aws::S3::SSEHeaders::SERVER_SIDE_ENCRYPTION_CUSTOMER_ALGORITHM, @@ -1322,10 +1343,8 @@ std::unique_ptr ClientFactory::create( // NOLINT headers.push_back({Aws::S3::SSEHeaders::SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY, server_side_encryption_customer_key_base64}); - Aws::Utils::ByteBuffer buffer = Aws::Utils::HashingUtils::Base64Decode(server_side_encryption_customer_key_base64); - String str_buffer(reinterpret_cast(buffer.GetUnderlyingData()), buffer.GetLength()); headers.push_back({Aws::S3::SSEHeaders::SERVER_SIDE_ENCRYPTION_CUSTOMER_KEY_MD5, - Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::HashingUtils::CalculateMD5(str_buffer))}); + Aws::Utils::HashingUtils::Base64Encode(Aws::Utils::HashingUtils::CalculateMD5(customer_key))}); } // These will be added after request signing diff --git a/src/IO/S3/Client.h b/src/IO/S3/Client.h index ae5312812549a..1c728fccddeca 100644 --- a/src/IO/S3/Client.h +++ b/src/IO/S3/Client.h @@ -349,6 +349,8 @@ class Client : private Aws::S3::S3Client class ClientFactory { public: + static constexpr size_t AES256_KEY_BYTES = 32; + ~ClientFactory(); static ClientFactory & instance(); diff --git a/src/IO/S3/PocoHTTPClient.cpp b/src/IO/S3/PocoHTTPClient.cpp index 303afead2c090..d772eeb241471 100644 --- a/src/IO/S3/PocoHTTPClient.cpp +++ b/src/IO/S3/PocoHTTPClient.cpp @@ -22,6 +22,7 @@ #include #include #include +#include #include #include @@ -637,9 +638,25 @@ void PocoHTTPClient::makeRequestInternalImpl( request.GetContentBody()->seekg(0); setTimeouts(*session, getTimeouts(method, first_attempt, /*first_byte*/ false)); - auto size = Poco::StreamCopier::copyStream(*request.GetContentBody(), request_body_stream); - if (enable_s3_requests_logging) - LOG_TEST(log, "Written {} bytes to request body", size); + try + { + auto size = Poco::StreamCopier::copyStream(*request.GetContentBody(), request_body_stream); + request_body_stream.flush(); + if (enable_s3_requests_logging) + LOG_TEST(log, "Written {} bytes to request body", size); + } + catch (const Poco::IOException &) + { + /// A server may answer with an error and close the connection before reading the whole body. + /// TLS allows no I/O after a failed write. + if (session->secure() + || DB::getSocketState(session->socket()) != DB::SocketState::DataPending + || !session->receiveEarlyResponse(poco_response) + || poco_response.getStatus() < Poco::Net::HTTPResponse::HTTP_MULTIPLE_CHOICES) + throw; + LOG_DEBUG(log, "Failed to send the request body to {}, using the response the server has already sent: {}", + uri, getCurrentExceptionMessage(/* with_stacktrace */ false)); + } } setTimeouts(*session, getTimeouts(method, first_attempt, /*first_byte*/ false)); diff --git a/src/IO/SharedThreadPools.h b/src/IO/SharedThreadPools.h index 07e76d8e12f70..f502cdc3aeac0 100644 --- a/src/IO/SharedThreadPools.h +++ b/src/IO/SharedThreadPools.h @@ -71,7 +71,9 @@ class StaticThreadPool M(MergeTreePrefixesDeserialization, "MergeTreePrefixesDeserializationThreadPool", MergeTreeSubcolumnsReader) \ M(DropDistributedCache, "DropDistributedCacheThreadPool", DropDistributedCache) \ M(FormatParsing, "FormatParsingThreadPool", FormatParsing) \ - M(IcebergManifestDecode, "IcebergManifestDecodeThreadPool", IcebergManifestDecode) + M(IcebergManifestDecode, "IcebergManifestDecodeThreadPool", IcebergManifestDecode) \ + M(ParquetEncoder, "ParquetEncoderThreadPool", ParquetEncoder) \ + M(AIRequest, "AIRequestThreadPool", AIRequest) #define DECLARE_STATIC_THREAD_POOL_GETTER(SUFFIX, NAME, METRIC) StaticThreadPool & get##SUFFIX##ThreadPool(); APPLY_FOR_STATIC_THREAD_POOLS(DECLARE_STATIC_THREAD_POOL_GETTER) diff --git a/src/IO/SnappyFramedReadBuffer.cpp b/src/IO/SnappyFramedReadBuffer.cpp index 0ae482223a2ee..238dc9f45a41c 100644 --- a/src/IO/SnappyFramedReadBuffer.cpp +++ b/src/IO/SnappyFramedReadBuffer.cpp @@ -46,6 +46,7 @@ uint32_t maskedCrc32c(const char * data, size_t size) SnappyFramedReadBuffer::SnappyFramedReadBuffer( std::unique_ptr in_, size_t buf_size, char * existing_memory, size_t alignment) : CompressedReadBufferWrapper(std::move(in_), buf_size, existing_memory, alignment) + , decompress_buffer(MAX_UNCOMPRESSED_CHUNK_SIZE) { } @@ -168,7 +169,6 @@ bool SnappyFramedReadBuffer::nextImpl() "Snappy chunk claims uncompressed length {} which exceeds the framing format limit of {}{}", uncompressed_length, MAX_UNCOMPRESSED_CHUNK_SIZE, getExceptionEntryWithFileName(*in)); - decompress_buffer.resize(uncompressed_length); if (!snappy::RawUncompress(compressed, payload_size, decompress_buffer.data())) throw Exception(ErrorCodes::SNAPPY_UNCOMPRESS_FAILED, "Snappy decompression failed{}", @@ -211,7 +211,7 @@ bool SnappyFramedReadBuffer::nextImpl() /// Snappy framing format encodes the masked CRC-32C as little-endian. uint32_t expected_crc = unalignedLoadLittleEndian(chunk_buf.data()); - decompress_buffer.assign(chunk_buf.data() + 4, payload_size); + memcpy(decompress_buffer.data(), chunk_buf.data() + 4, payload_size); uint32_t actual_crc = maskedCrc32c(decompress_buffer.data(), payload_size); if (actual_crc != expected_crc) diff --git a/src/IO/SnappyFramedReadBuffer.h b/src/IO/SnappyFramedReadBuffer.h index 500b6fa54d968..9f45423209ce1 100644 --- a/src/IO/SnappyFramedReadBuffer.h +++ b/src/IO/SnappyFramedReadBuffer.h @@ -33,7 +33,8 @@ class SnappyFramedReadBuffer : public CompressedReadBufferWrapper /// Read and validate the stream identifier chunk. bool readStreamIdentifier(); - String decompress_buffer; + /// Padded: working_buffer points here and the base reports isPadded(). + Memory<> decompress_buffer; bool identifier_read = false; }; diff --git a/src/IO/tests/gtest_brotli_decoder.cpp b/src/IO/tests/gtest_brotli_decoder.cpp new file mode 100644 index 0000000000000..d7b8ade6a3f3a --- /dev/null +++ b/src/IO/tests/gtest_brotli_decoder.cpp @@ -0,0 +1,58 @@ +#include "config.h" + +#if USE_BROTLI + +#include +#include + +#include +#include +#include +#include +#include + +TEST(BrotliDecoder, RejectInvalidBlockSwitch) +{ + // A block switch is invalid when the stream declares only one block type. + // From https://github.com/nodejs/node/blob/main/test/parallel/test-zlib-invalid-input.js. + constexpr std::array input = { + 0x20, 0x00, 0x10, 0x41, 0x41, 0x41, 0xfc, 0xff, 0xff, 0x07, 0x00, + 0x22, 0x28, 0xf0, 0x0b, 0x60, 0xf7, 0xf4, 0x1f, 0x00, 0x00, 0x00}; + + for (size_t chunk_size : {input.size(), size_t{1}}) + { + std::unique_ptr state( + BrotliDecoderCreateInstance(nullptr, nullptr, nullptr), &BrotliDecoderDestroyInstance); + ASSERT_NE(state.get(), nullptr); + + std::array output{}; + size_t total_output = 0; + BrotliDecoderResult result = BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT; + + for (size_t offset = 0; offset < input.size() && result == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT;) + { + size_t available_input = std::min(chunk_size, input.size() - offset); + const size_t provided_input = available_input; + const uint8_t * next_input = input.data() + offset; + do + { + uint8_t * next_output = output.data(); + size_t available_output = output.size(); + result = BrotliDecoderDecompressStream( + state.get(), &available_input, &next_input, &available_output, &next_output, nullptr); + total_output += output.size() - available_output; + ASSERT_LT(total_output, 64 * 1024 * 1024); + } + while (result == BROTLI_DECODER_RESULT_NEEDS_MORE_OUTPUT); + offset += provided_input - available_input; + if (result == BROTLI_DECODER_RESULT_NEEDS_MORE_INPUT) + ASSERT_GT(provided_input - available_input, 0); + } + + EXPECT_EQ(result, BROTLI_DECODER_RESULT_ERROR); + // -17 is the `BROTLI_DECODER_ERROR_FORMAT_BLOCK_SWITCH` code added by the fix. + EXPECT_EQ(BrotliDecoderGetErrorCode(state.get()), static_cast(-17)); + } +} + +#endif diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index c41978e614779..26f52e0ecdf1f 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -1180,7 +1180,7 @@ void Aggregator::executeImpl( /// its key there (e.g. serializing all key columns) would pay its dominant per-row cost /// twice - far more than the cache miss the prefetch hides. See `has_cheap_key_holder`. const bool prefetch = State::has_cheap_key_holder && params.enable_prefetch - && (method.data.getBufferSizeInBytes() > min_bytes_for_prefetch); + && (bytesForPrefetch(method.data) > min_bytes_for_prefetch); #if USE_EMBEDDED_COMPILER if (compiled_aggregate_functions_holder && !hasSparseArguments(aggregate_instructions)) diff --git a/src/Interpreters/BloomFilter.cpp b/src/Interpreters/BloomFilter.cpp index 95a6c163f2762..240c2a2ad7e17 100644 --- a/src/Interpreters/BloomFilter.cpp +++ b/src/Interpreters/BloomFilter.cpp @@ -21,8 +21,6 @@ namespace ErrorCodes static constexpr UInt64 SEED_GEN_A = 845897321; static constexpr UInt64 SEED_GEN_B = 217728422; -static constexpr UInt64 MAX_BLOOM_FILTER_SIZE = 1 << 30; - namespace { using BloomDivider = libdivide::divider; diff --git a/src/Interpreters/BloomFilter.h b/src/Interpreters/BloomFilter.h index 5cf7d318ea5a8..3a70a4999b92b 100644 --- a/src/Interpreters/BloomFilter.h +++ b/src/Interpreters/BloomFilter.h @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -12,6 +13,10 @@ namespace DB { + +/// The maximum size in bytes of a single bloom filter. +static constexpr size_t MAX_BLOOM_FILTER_SIZE = 1 << 30; + struct BloomFilterHashPair { UInt64 hash1; @@ -35,7 +40,7 @@ class BloomFilter public: using UnderType = UInt64; - using Container = std::vector; + using Container = VectorWithMemoryTracking; explicit BloomFilter(const BloomFilterParameters & params); /// size -- size of filter in bytes. diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 01696d5962828..184dbf95d835e 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -66,6 +66,7 @@ #include #include #include +#include #include #include #include @@ -952,6 +953,10 @@ struct ContextSharedPart : boost::noncopyable tryLogCurrentException(__PRETTY_FUNCTION__); } } + + /// A cancelled query does not wait for its AI requests. Their memory is tracked under the user trackers in + /// `process_list`, which is destroyed after this destructor body. + getAIRequestThreadPool().shutdown(); } void setConfig(const ConfigurationPtr & config_value) @@ -1425,6 +1430,7 @@ ContextData::ContextData(const ContextData &o) : is_background_operation(o.is_background_operation), is_ddl_or_on_cluster_internal(o.is_ddl_or_on_cluster_internal), is_recovery_from_stored_metadata(o.is_recovery_from_stored_metadata), + is_analyzing_background_expressions(o.is_analyzing_background_expressions), is_view_inner_query(o.is_view_inner_query), positional_arguments_already_resolved(o.positional_arguments_already_resolved), join_analyze_mode(o.join_analyze_mode), @@ -3615,6 +3621,13 @@ void Context::clampToSettingsConstraintsWithLock(SettingsChanges & changes, Sett doSettingsSanityCheckClamp(*settings, getLogger("SettingsSanity")); } +void Context::clampToSettingsConstraintsRejectingInvalidChangesWithLock(SettingsChanges & changes, SettingSource source) +{ + getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.clampRejectingInvalidChanges(*settings, changes, source); + if (getApplicationType() == ApplicationType::LOCAL || getApplicationType() == ApplicationType::SERVER) + doSettingsSanityCheckClamp(*settings, getLogger("SettingsSanity")); +} + void Context::checkMergeTreeSettingsConstraintsWithLock(const MergeTreeSettings & merge_tree_settings, const SettingsChanges & changes) const { getSettingsConstraintsAndCurrentProfilesWithLock()->constraints.check(merge_tree_settings, changes); @@ -3699,6 +3712,21 @@ void Context::clampToSettingsConstraints(SettingsChanges & changes, SettingSourc clampToSettingsConstraintsWithLock(changes, source); } +void Context::clampToSettingsConstraintsRejectingInvalidChanges(SettingsChanges & changes, SettingSource source) +{ + if (std::ranges::any_of(changes, isProfileChange)) + { + changes = enforceConstraintsAlongProfileChanges(shared_from_this(), changes, [source](Context & context, SettingsChanges & segment) + { + context.clampToSettingsConstraintsRejectingInvalidChanges(segment, source); + }); + return; + } + + SharedLockGuard lock(mutex); + clampToSettingsConstraintsRejectingInvalidChangesWithLock(changes, source); +} + void Context::checkMergeTreeSettingsConstraints(const MergeTreeSettings & merge_tree_settings, const SettingsChanges & changes) const { SharedLockGuard lock(mutex); @@ -5596,7 +5624,7 @@ BackgroundSchedulePoolPtr Context::getBufferFlushSchedulePool() const auto created_pool = BackgroundSchedulePool::create( shared->server_settings[ServerSetting::background_buffer_flush_schedule_pool_size], shared->server_settings[ServerSetting::background_schedule_pool_initial_size], - /*max_parallel_tasks_per_type*/ 0, + /*max_parallel_tasks_per_type_ratio*/ 0, CurrentMetrics::BackgroundBufferFlushSchedulePoolTask, CurrentMetrics::BackgroundBufferFlushSchedulePoolSize, ThreadName::BACKGROUND_BUFFER_FLUSH_SCHEDULE_POOL); @@ -5657,9 +5685,6 @@ BackgroundTaskSchedulingSettings Context::getBackgroundStreamingTaskSchedulingSe BackgroundSchedulePoolPtr Context::getSchedulePool() const { - size_t max_parallel_tasks_per_type = static_cast( - static_cast(shared->server_settings[ServerSetting::background_schedule_pool_size]) - * static_cast(shared->server_settings[ServerSetting::background_schedule_pool_max_parallel_tasks_per_type_ratio])); callOnce( shared->schedule_pool_initialized, [&] @@ -5667,7 +5692,7 @@ BackgroundSchedulePoolPtr Context::getSchedulePool() const auto created_pool = BackgroundSchedulePool::create( shared->server_settings[ServerSetting::background_schedule_pool_size], shared->server_settings[ServerSetting::background_schedule_pool_initial_size], - max_parallel_tasks_per_type, + static_cast(shared->server_settings[ServerSetting::background_schedule_pool_max_parallel_tasks_per_type_ratio]), CurrentMetrics::BackgroundSchedulePoolTask, CurrentMetrics::BackgroundSchedulePoolSize, DB::ThreadName::BACKGROUND_SCHEDULE_POOL); @@ -5685,7 +5710,7 @@ BackgroundSchedulePoolPtr Context::getDistributedSchedulePool() const auto created_pool = BackgroundSchedulePool::create( shared->server_settings[ServerSetting::background_distributed_schedule_pool_size], shared->server_settings[ServerSetting::background_schedule_pool_initial_size], - /*max_parallel_tasks_per_type*/ 0, + /*max_parallel_tasks_per_type_ratio*/ 0, CurrentMetrics::BackgroundDistributedSchedulePoolTask, CurrentMetrics::BackgroundDistributedSchedulePoolSize, DB::ThreadName::DISTRIBUTED_SCHEDULE_POOL); @@ -5703,7 +5728,7 @@ BackgroundSchedulePoolPtr Context::getMessageBrokerSchedulePool() const auto created_pool = BackgroundSchedulePool::create( shared->server_settings[ServerSetting::background_message_broker_schedule_pool_size], shared->server_settings[ServerSetting::background_schedule_pool_initial_size], - /*max_parallel_tasks_per_type*/ 0, + /*max_parallel_tasks_per_type_ratio*/ 0, CurrentMetrics::BackgroundMessageBrokerSchedulePoolTask, CurrentMetrics::BackgroundMessageBrokerSchedulePoolSize, DB::ThreadName::MSG_BROKER_SCHEDULE_POOL); @@ -5721,7 +5746,7 @@ BackgroundSchedulePoolPtr Context::getIcebergSchedulePool() const auto created_pool = BackgroundSchedulePool::create( shared->server_settings[ServerSetting::iceberg_background_schedule_pool_size], shared->server_settings[ServerSetting::background_schedule_pool_initial_size], - /*max_parallel_tasks_per_type*/ 0, + /*max_parallel_tasks_per_type_ratio*/ 0, CurrentMetrics::IcebergSchedulePoolTask, CurrentMetrics::IcebergSchedulePoolSize, DB::ThreadName::ICEBERG_SCHEDULE_POOL); @@ -5739,7 +5764,7 @@ BackgroundSchedulePoolPtr Context::getStreamingSchedulePool() const auto created_pool = BackgroundSchedulePool::create( shared->server_settings[ServerSetting::background_streaming_schedule_pool_size], shared->server_settings[ServerSetting::background_schedule_pool_initial_size], - /*max_parallel_tasks_per_type*/ 0, + /*max_parallel_tasks_per_type_ratio*/ 0, CurrentMetrics::BackgroundStreamingSchedulePoolTask, CurrentMetrics::BackgroundStreamingSchedulePoolSize, DB::ThreadName::BACKGROUND_STREAMING_SCHEDULE_POOL); @@ -7377,6 +7402,15 @@ StoragePolicyPtr Context::getStoragePolicy(const String & name) const return policy_selector->get(name); } +StoragePolicyPtr Context::tryGetStoragePolicy(const String & name) const +{ + std::lock_guard lock(shared->storage_policies_mutex); + + auto policy_selector = getStoragePolicySelector(lock); + + return policy_selector->tryGet(name); +} + StoragePolicyPtr Context::getStoragePolicyFromDisk(const String & disk_name) const { std::lock_guard lock(shared->storage_policies_mutex); @@ -9004,7 +9038,8 @@ AIQuotaTrackerPtr Context::getAIQuotaTracker() const settings_ref[Setting::ai_function_max_input_tokens_per_query], settings_ref[Setting::ai_function_max_output_tokens_per_query], settings_ref[Setting::ai_function_max_api_calls_per_query], - settings_ref[Setting::ai_function_throw_on_quota_exceeded]); + settings_ref[Setting::ai_function_throw_on_quota_exceeded], + query_context->process_list_elem); } return query_context->ai_quota_tracker; } diff --git a/src/Interpreters/Context.h b/src/Interpreters/Context.h index 916929569a4b1..bfbf999195581 100644 --- a/src/Interpreters/Context.h +++ b/src/Interpreters/Context.h @@ -610,6 +610,9 @@ class ContextData /// Set for CREATE queries a Replicated database replays from a definition it already stored. /// Such a definition describes existing state, so validation that may reject a new one must not run. bool is_recovery_from_stored_metadata = false; + /// Set while analyzing expressions that are executed later in the background, without a user: + /// mutations and the expressions stored in a table definition. + bool is_analyzing_background_expressions = false; /// True when this context belongs to the inner query of an expanded view. /// Positional arguments inside views must be resolved even on remote/secondary nodes where /// enable_positional_arguments would otherwise be skipped (views are expanded on remote nodes, @@ -814,6 +817,7 @@ class Context: public ContextData, public std::enable_shared_from_this /// present to be able to add, remove or update warnings from the table enum class WarningType { + AI_FUNCTION_IN_BACKGROUND_OPERATION, AST_FUZZER_IS_ENABLED, AVAILABLE_DISK_SPACE_TOO_LOW_FOR_DATA, AVAILABLE_DISK_SPACE_TOO_LOW_FOR_LOGS, @@ -1252,6 +1256,7 @@ class Context: public ContextData, public std::enable_shared_from_this /// For the resets of a statement that also changes `profile`: `changes_applied_first` decides their constraints. void checkSettingsConstraintsForSettingsReset(const std::vector & names, const SettingsChanges & changes_applied_first, SettingSource source); void clampToSettingsConstraints(SettingsChanges & changes, SettingSource source); + void clampToSettingsConstraintsRejectingInvalidChanges(SettingsChanges & changes, SettingSource source); void checkMergeTreeSettingsConstraints(const MergeTreeSettings & merge_tree_settings, const SettingsChanges & changes) const; /// Reset settings to default value @@ -1835,6 +1840,8 @@ class Context: public ContextData, public std::enable_shared_from_this /// Provides storage politics schemes StoragePolicyPtr getStoragePolicy(const String & name) const; + /// Returns `nullptr` instead of throwing when the policy is not configured. + StoragePolicyPtr tryGetStoragePolicy(const String & name) const; StoragePolicyPtr getStoragePolicyFromDisk(const String & disk_name) const; @@ -1869,6 +1876,9 @@ class Context: public ContextData, public std::enable_shared_from_this bool isRecoveryFromStoredMetadata() const { return is_recovery_from_stored_metadata; } void setRecoveryFromStoredMetadata(bool value) { is_recovery_from_stored_metadata = value; } + bool isAnalyzingBackgroundExpressions() const { return is_analyzing_background_expressions; } + void setAnalyzingBackgroundExpressions(bool value) { is_analyzing_background_expressions = value; } + bool isViewInnerQuery() const { return is_view_inner_query; } void setIsViewInnerQuery(bool value) { is_view_inner_query = value; } @@ -2098,6 +2108,7 @@ class Context: public ContextData, public std::enable_shared_from_this void checkSettingsConstraintsWithLock(SettingsChanges & changes, SettingSource source); void clampToSettingsConstraintsWithLock(SettingsChanges & changes, SettingSource source); + void clampToSettingsConstraintsRejectingInvalidChangesWithLock(SettingsChanges & changes, SettingSource source); void checkSettingsConstraintsWithLock(const AlterSettingsProfileElements & profile_elements, SettingSource source) const; void clampToSettingsConstraintsWithLock(SettingsChanges & changes, SettingSource source) const; diff --git a/src/Interpreters/DatabaseCatalog.cpp b/src/Interpreters/DatabaseCatalog.cpp index a5289588bb1c9..dcb7839e0b61b 100644 --- a/src/Interpreters/DatabaseCatalog.cpp +++ b/src/Interpreters/DatabaseCatalog.cpp @@ -1297,6 +1297,16 @@ void DatabaseCatalog::loadMarkedAsDroppedTables() metadata_disk_list.insert(std::move(db_disk)); } + /// Startup may already have dropped tables, e.g. `SystemLog::prepareUnionTable` replaces union tables. + /// They are queued with their storage and their UUID mapping is still registered, so only drops of previous runs are loaded here. + auto is_drop_queued = [this](const DiskPtr & disk, const String & metadata_path) + { + std::lock_guard lock(tables_marked_dropped_mutex); + return std::ranges::any_of(tables_marked_dropped, [&](const TableMarkedAsDropped & dropped) + { + return dropped.metadata_path == metadata_path && dropped.db_disk->getName() == disk->getName(); + }); + }; /// /clickhouse_root/metadata_dropped/ contains files with metadata of tables, /// which where marked as dropped by Atomic databases. @@ -1337,6 +1347,12 @@ void DatabaseCatalog::loadMarkedAsDroppedTables() dropped_id.uuid = parse(sub_path_filename.substr(prev_dot_pos + 1, dot_pos - prev_dot_pos - 1)); String full_path = path + sub_path_filename; + if (is_drop_queued(db_disk, full_path)) + { + LOG_DEBUG(log, "Table {} was dropped during this startup and is already queued for removal", dropped_id.getNameForLogs()); + continue; + } + dropped_metadata.emplace(std::move(full_path), std::pair{std::move(dropped_id), db_disk}); } } diff --git a/src/Interpreters/InterpreterAlterQuery.cpp b/src/Interpreters/InterpreterAlterQuery.cpp index 016cda373bac0..74ad636323416 100644 --- a/src/Interpreters/InterpreterAlterQuery.cpp +++ b/src/Interpreters/InterpreterAlterQuery.cpp @@ -5,6 +5,7 @@ #include #include #include +#include #include #include #include @@ -16,6 +17,7 @@ #include #include #include +#include #include #include #include @@ -28,11 +30,16 @@ #include #include #include +#include #include #include +#include #include #include +#include +#include #include +#include #include #include #include @@ -69,15 +76,18 @@ namespace Setting extern const SettingsTimezone session_timezone; extern const SettingsUInt64 max_parser_depth; extern const SettingsUInt64 max_parser_backtracks; + extern const SettingsUInt64 distributed_ddl_entry_format_version; } namespace ServerSetting { extern const ServerSettingsBool disable_insertion_and_mutation; + extern const ServerSettingsBool distributed_ddl_use_initial_user_and_roles; } namespace ErrorCodes { + extern const int ACCESS_DENIED; extern const int LOGICAL_ERROR; extern const int NOT_IMPLEMENTED; extern const int TABLE_IS_PERMANENTLY_READ_ONLY; @@ -92,6 +102,72 @@ namespace ErrorCodes namespace { +/// A `Replicated`-database host applies the statement without the submitting user. `ON CLUSTER` does +/// not reach this: it is refused unless its hosts run as the initiating user. +void checkAddedIndexPermissions(const ASTAlterQuery & alter, const StoragePtr & table, const ContextPtr & context) +{ + StorageMetadataHandle metadata; + if (table) + metadata = table->getInMemoryMetadataPtr(context, /*bypass_metadata_cache=*/ false); + + bool adds_transform = false; + for (const auto & child : alter.command_list->children) + { + const auto & command = child->as(); + if (command.type != ASTAlterCommand::ADD_INDEX || !command.index_decl) + continue; + + if (const auto * declaration = command.index_decl->as()) + if (hasTextIndexPreOrPostProcessor(*declaration)) + adds_transform = true; + } + + /// Against the schema the whole statement produces: a command of the same statement can change what + /// an alias in a transform expands to. Transforms only, so other statements get no extra analysis. + if (!adds_transform || !metadata) + return; + + AlterCommands commands; + /// Runs of consecutive metadata commands; `runCommandSegments` commits each run on its own. + size_t metadata_segments = 0; + bool previous_is_metadata = false; + for (const auto & child : alter.command_list->children) + { + const auto * command_ast = child->as(); + std::optional command; + if (command_ast->type != ASTAlterCommand::EXECUTE_COMMAND) + command = AlterCommand::parse(command_ast); + + if (command) + { + if (!previous_is_metadata) + ++metadata_segments; + commands.emplace_back(std::move(*command)); + } + previous_is_metadata = command.has_value(); + } + + /// Applying every command at once would reject statements the executor, which commits segment by + /// segment, accepts. `AlterCommands::apply` authorizes each segment as the user anyway. + if (metadata_segments > 1) + return; + + /// The same sequence as `runCommandSegments`: `apply` relies on the checks before it. + commands.validate(table, context); + + bool share_nested = true; + if (auto * merge_tree = castStorage(table, DeferredTable::Load).get()) + share_nested = (*merge_tree->getSettings())[MergeTreeSetting::share_nested_offsets]; + + StorageInMemoryMetadata metadata_copy = *metadata; + commands.prepare(metadata_copy, share_nested); + table->checkAlterIsPossible(commands, context); + /// As the storages do before `apply`: an explicit `ADD STATISTICS` replaces an implicit one. + removeImplicitStatistics(metadata_copy.columns); + commands.apply(metadata_copy, context, share_nested); + +} + using CommandSegment = std::variant; using CommandSegments = std::vector; @@ -378,7 +454,9 @@ BlockIO runCommandSegments(CommandSegments & segments, const StoragePtr & table, if (settings[Setting::validate_mutation_query]) { MutationsInterpreter::Settings mutation_settings(false); - MutationsInterpreter(table, metadata_snapshot, *mutation_commands, context, mutation_settings).validate(); + auto validation_context = Context::createCopy(context); + validation_context->setAnalyzingBackgroundExpressions(true); + MutationsInterpreter(table, metadata_snapshot, *mutation_commands, validation_context, mutation_settings).validate(); } table->mutate(*mutation_commands, context); } @@ -406,6 +484,31 @@ BlockIO runCommandSegments(CommandSegments & segments, const StoragePtr & table, return res; } +/// A setting that sends the table's data elsewhere copies the data as it is stored, so nothing can filter the rows +/// that a row policy hides (as `StorageSet::checkNoRowPolicy`). It reads only the query and the access entities, +/// not the table, so it runs on the initiator. +void checkNoRowPolicyForExport(const ASTAlterQuery & alter, const String & database, const ContextPtr & context) +{ + for (const auto & child : alter.command_list->children) + { + const auto & command = child->as(); + if (command.type != ASTAlterCommand::MODIFY_SETTING || !command.settings_changes) + continue; + + const auto & changes = command.settings_changes->as().changes; + if (StorageFactory::instance().getExportAccessOfAnyEngine(changes).empty()) + continue; + + auto row_policy_filter = context->getRowPolicyFilter(database, alter.getTable(), RowPolicyFilterType::SELECT_FILTER); + if (row_policy_filter && !row_policy_filter->isAlwaysTrue()) + throw Exception( + ErrorCodes::ACCESS_DENIED, + "Cannot change the settings of table {}.{} that send its data elsewhere because a row policy applies to it. " + "The data is copied as it is stored, so its rows cannot be filtered", + backQuoteIfNeed(database), backQuoteIfNeed(alter.getTable())); + } +} + } InterpreterAlterQuery::InterpreterAlterQuery(const ASTPtr & query_ptr_, ContextMutablePtr context_) : WithMutableContext(context_), query_ptr(query_ptr_) @@ -486,15 +589,39 @@ BlockIO InterpreterAlterQuery::executeToTable(const ASTAlterQuery & alter) DDLQueryOnClusterParams params; params.access_to_check = getRequiredAccess(table); + /// The hosts run the entry without the user, so the row policy is checked here before the entry is enqueued. + checkNoRowPolicyForExport( + alter, alter.getDatabase().empty() ? getContext()->getCurrentDatabase() : alter.getDatabase(), getContext()); + + /// The hosts run the entry without the user, unless `distributed_ddl_use_initial_user_and_roles` is on + /// and the entry format carries the initiator's user, so a text index transform cannot be authorized there. + const bool workers_use_initiator_user + = getContext()->getServerSettings()[ServerSetting::distributed_ddl_use_initial_user_and_roles] + && getContext()->getSettingsRef()[Setting::distributed_ddl_entry_format_version] >= DDLLogEntry::INITIATOR_USER_VERSION; + if (!workers_use_initiator_user) + { + for (const auto & child : alter.command_list->children) + { + const auto & command = child->as(); + if (command.type != ASTAlterCommand::ADD_INDEX || !command.index_decl) + continue; + if (const auto * declaration = command.index_decl->as()) + rejectTextIndexTransformWithoutInitiatorUser(*declaration); + } + } + return executeDDLQueryOnCluster(query_ptr, getContext(), params); } getContext()->checkAccess(getRequiredAccess(table)); + checkNoRowPolicyForExport(alter, getContext()->resolveDatabase(alter.getDatabase()), getContext()); if (!table_id) throw Exception(ErrorCodes::UNKNOWN_DATABASE, "Database {} does not exist", backQuoteIfNeed(alter.getDatabase())); DatabasePtr database = DatabaseCatalog::instance().getDatabase(table_id.database_name); + checkAddedIndexPermissions(alter, table, getContext()); + if (database->shouldReplicateQuery(getContext(), query_ptr)) { auto guard = DatabaseCatalog::instance().getDDLGuard(table_id.database_name, table_id.table_name, database.get()); @@ -888,6 +1015,20 @@ AccessRightsElements InterpreterAlterQuery::getRequiredAccessForCommand( case ASTAlterCommand::MODIFY_SETTING: { required_access.emplace_back(AccessType::ALTER_SETTINGS, database, table); + + /// A setting that sends the table's data elsewhere, e.g. the move destination of a queue, is the same data flow as + /// `INSERT INTO FUNCTION s3(...) SELECT * FROM
`. The creator of the table supplied its source and credentials, + /// so the reading part is `SELECT` on the table. The grants do not depend on the engine of the table, so the initiator + /// of `ON CLUSTER` can check them without the table. + if (command.type == ASTAlterCommand::MODIFY_SETTING && command.settings_changes) + { + auto export_access = StorageFactory::instance().getExportAccessOfAnyEngine(command.settings_changes->as().changes); + if (!export_access.empty()) + { + required_access.append_range(std::move(export_access)); + required_access.emplace_back(AccessType::SELECT, database, table); + } + } break; } case ASTAlterCommand::ATTACH_PARTITION: diff --git a/src/Interpreters/InterpreterCreateQuery.cpp b/src/Interpreters/InterpreterCreateQuery.cpp index 47fad29cdac17..916e2d90f7098 100644 --- a/src/Interpreters/InterpreterCreateQuery.cpp +++ b/src/Interpreters/InterpreterCreateQuery.cpp @@ -39,6 +39,7 @@ #include #include #include +#include #include #include #include @@ -48,7 +49,9 @@ #include #include +#include #include +#include #include #include #include @@ -155,6 +158,7 @@ namespace Setting namespace ServerSetting { + extern const ServerSettingsBool distributed_ddl_use_initial_user_and_roles; extern const ServerSettingsBool ignore_empty_sql_security_in_create_view_query; } @@ -1065,6 +1069,14 @@ InterpreterCreateQuery::TableProperties InterpreterCreateQuery::getTableProperti if (!create.columns_list) create.set(create.columns_list, make_intrusive()); + /// Screened for every definition the user supplies now. A replay of stored metadata is not screened, + /// so such a table still attaches. + if (isFreshTableDefinition(mode, create.attach_short_syntax)) + { + for (const auto & index : properties.indices) + checkTextIndexPreAndPostProcessorPermissions(index, getContext()); + } + ASTPtr new_columns = formatColumns(properties.columns); ASTPtr new_indices = formatIndices(properties.indices); ASTPtr new_constraints = formatConstraints(properties.constraints); @@ -1976,6 +1988,11 @@ BlockIO InterpreterCreateQuery::createTable(ASTCreateQuery & create) /// Set and retrieve list of columns, indices and constraints. Set table engine if needed. Rewrite query in canonical way. TableProperties properties = getTablePropertiesAndNormalizeCreateQuery(create, mode); + /// Only here is an engine inherited through `AS` known, and nothing is enqueued or created yet. + /// A replay of a stored definition was checked when it was created. + if (isFreshTableDefinition(mode, create.attach_short_syntax)) + checkExportAccess(create); + DatabasePtr database; bool need_add_to_database = !create.isTemporary(); // In case of an ON CLUSTER query, the database may not be present on the initiator node @@ -3023,6 +3040,17 @@ BlockIO InterpreterCreateQuery::doCreateOrReplaceTemporaryTable(ASTCreateQuery & return fillTableIfNeeded(create); } +namespace +{ +/// The population's own `SETTINGS` are reapplied on top of its context under the names they were written with. +constexpr std::array settings_sending_population_to_other_servers +{ + "allow_experimental_parallel_reading_from_replicas", + "enable_parallel_replicas", + "parallel_distributed_insert_select", +}; +} + BlockIO InterpreterCreateQuery::fillTableIfNeeded(const ASTCreateQuery & create, bool skip_target_insert_access_check) { /// If the query is a CREATE SELECT, insert the data into the table. @@ -3038,10 +3066,22 @@ BlockIO InterpreterCreateQuery::fillTableIfNeeded(const ASTCreateQuery & create, else insert->select = create.select->clone(); + ContextMutablePtr insert_context = getContext(); + /// Every replica of a `Replicated` database executes this entry and fills its own copy of the table, so the population + /// must neither read through parallel replicas nor be sent to other servers as a distributed `INSERT`. + if (getContext()->getZooKeeperMetadataTransaction()) + { + auto local_context = Context::createCopy(getContext()); + local_context->setSetting("parallel_distributed_insert_select", Field{0}); + local_context->setSetting("enable_parallel_replicas", Field{0}); + removeSettingsFromQuery(insert->select, settings_sending_population_to_other_servers); + insert_context = local_context; + } + InterpreterInsertQuery interpreter( insert, - getContext(), - getContext()->getSettingsRef()[Setting::insert_allow_materialized_columns], + insert_context, + insert_context->getSettingsRef()[Setting::insert_allow_materialized_columns], /* no_squash */ false, /* no_destination */ false, /* async_isnert */ false); @@ -3491,6 +3531,19 @@ BlockIO InterpreterCreateQuery::executeQueryOnCluster(ASTCreateQuery & create) prepareOnClusterQuery(create, getContext(), create.cluster); DDLQueryOnClusterParams params; params.access_to_check = getRequiredAccess(); + + /// The hosts run the entry without the user, unless `distributed_ddl_use_initial_user_and_roles` is on + /// and the entry format carries the initiator's user, so a text index transform cannot be authorized there. + const bool workers_use_initiator_user + = getContext()->getServerSettings()[ServerSetting::distributed_ddl_use_initial_user_and_roles] + && getContext()->getSettingsRef()[Setting::distributed_ddl_entry_format_version] >= DDLLogEntry::INITIATOR_USER_VERSION; + if (!workers_use_initiator_user && create.columns_list && create.columns_list->indices) + { + for (const auto & index : create.columns_list->indices->children) + if (const auto * declaration = index->as()) + rejectTextIndexTransformWithoutInitiatorUser(*declaration); + } + return executeDDLQueryOnCluster(query_ptr, getContext(), params); } @@ -3542,6 +3595,50 @@ BlockIO InterpreterCreateQuery::execute() /// getRequiredAccess sees no engine, because this query holds none of its own if (inherited.storage && inherited.storage->engine) getContext()->checkAccess(AccessType::TABLE_ENGINE, inherited.storage->engine->name); + + /// setEngine copies only the outer engine of a `TimeSeries` source, and the worker copies + /// its inner engines while normalizing, so add the ones this query does not give itself + if (inherited.is_time_series_table) + { + auto as_create_ptr = DatabaseCatalog::instance().getDatabase(inherited.as_database)->getCreateTableQuery(inherited.as_table, getContext()); + const auto & as_create = as_create_ptr->as(); + if (as_create.targets) + for (const auto & target : as_create.targets->targets) + if (target.inner_engine && !inherited.getTargetInnerEngine(target.kind)) + inherited.setTargetInnerEngine(target.kind, target.inner_engine->clone()); + } + + /// the worker inherits the move destination of the source too, and runs with no user + checkExportAccess(inherited); + + /// A worker copies the source's indices with no user, so declare their transforms here. + if (inherited.storage && inherited.storage->engine && endsWith(inherited.storage->engine->name, "MergeTree")) + { + /// Before resolving it: an error below would otherwise disclose a source the user + /// cannot see. + getContext()->checkAccess(AccessType::SHOW_COLUMNS, create.as_database, create.as_table); + StoragePtr as_storage = resolveStorageProxyLoading( + DatabaseCatalog::instance().getTable({create.as_database, create.as_table}, getContext())); + if (const auto * alias = as_storage->as(); + alias && !alias->isTargetTableGranted(getContext(), AccessType::SHOW_COLUMNS, {})) + throw Exception(ErrorCodes::ACCESS_DENIED, "Not enough privileges to describe metadata exposed by {}", + StorageID{create.as_database, create.as_table}.getNameForLogs()); + /// The query dispatches `AS src`, so each host materializes its own source, which + /// may define the transform differently from the one authorized here. + const bool workers_use_initiator_user + = getContext()->getServerSettings()[ServerSetting::distributed_ddl_use_initial_user_and_roles] + && on_cluster_version >= DDLLogEntry::INITIATOR_USER_VERSION; + + auto as_storage_metadata = as_storage->getInMemoryMetadataPtr(getContext(), false); + for (const auto & index : as_storage_metadata->getSecondaryIndices()) + { + if (index.isImplicitlyCreated()) + continue; + if (!workers_use_initiator_user) + rejectTextIndexTransformWithoutInitiatorUser(index); + checkTextIndexPreAndPostProcessorPermissions(index, getContext()); + } + } } return executeQueryOnCluster(create); @@ -3626,9 +3723,28 @@ AccessRightsElements InterpreterCreateQuery::getRequiredAccess() const if (create.storage && create.storage->engine) required_access.emplace_back(AccessType::TABLE_ENGINE, create.storage->engine->name); + /// `ATTACH TABLE ... FROM ''` reads a directory in `user_files` as the table data and then moves it to the + /// data path of the new table. Reading it needs `READ` on the FILE source like the `file` function does, and + /// taking it away from `user_files` needs `WRITE`, like renaming files after processing does. Without this, + /// `CREATE TABLE` on a table of their own would let a user read or consume any data staged in `user_files`. + if (create.has_attach_from_path) + required_access.emplace_back(AccessType::READ | AccessType::WRITE, toStringSource(AccessTypeObjects::Source::FILE)); + + /// Settings that send the table's data elsewhere, e.g. the move destination of a queue, need the grants of that place. + required_access.append_range(StorageFactory::instance().getExportAccess(create)); + return required_access; } +void InterpreterCreateQuery::checkExportAccess(const ASTCreateQuery & create) const +{ + /// Internal queries (initiated by the server itself) always have access to everything. + if (internal) + return; + + getContext()->checkAccess(StorageFactory::instance().getExportAccess(create)); +} + void InterpreterCreateQuery::extendQueryLogElemImpl(QueryLogElement & elem, const ASTPtr &, ContextPtr) const { if (!as_table_saved.empty()) diff --git a/src/Interpreters/InterpreterCreateQuery.h b/src/Interpreters/InterpreterCreateQuery.h index 9adba483ee71b..9baa88d61bbd5 100644 --- a/src/Interpreters/InterpreterCreateQuery.h +++ b/src/Interpreters/InterpreterCreateQuery.h @@ -117,6 +117,10 @@ class InterpreterCreateQuery : public IInterpreter, WithMutableContext void setEngine(ASTCreateQuery & create) const; AccessRightsElements getRequiredAccess() const; + /// The part of `getRequiredAccess` for settings that send the table's data elsewhere, checked again once + /// `setEngine` has resolved an engine inherited through `AS`, which `getRequiredAccess` cannot see. + void checkExportAccess(const ASTCreateQuery & create) const; + /// Create IStorage and add it to database. If table already exists and IF NOT EXISTS specified, do nothing and return false. bool doCreateTable(ASTCreateQuery & create, const TableProperties & properties, DDLGuardPtr & ddl_guard, LoadingStrictnessLevel mode); BlockIO doCreateOrReplaceTable(ASTCreateQuery & create, const InterpreterCreateQuery::TableProperties & properties, LoadingStrictnessLevel mode); diff --git a/src/Interpreters/InterpreterDeleteQuery.cpp b/src/Interpreters/InterpreterDeleteQuery.cpp index 8d8b2b07e693c..c72f58120197e 100644 --- a/src/Interpreters/InterpreterDeleteQuery.cpp +++ b/src/Interpreters/InterpreterDeleteQuery.cpp @@ -127,7 +127,9 @@ BlockIO InterpreterDeleteQuery::execute() if (getContext()->getSettingsRef()[Setting::validate_mutation_query]) { MutationsInterpreter::Settings mutation_settings(false); - MutationsInterpreter(table, metadata_snapshot, mutation_commands, getContext(), mutation_settings).validate(); + auto validation_context = Context::createCopy(getContext()); + validation_context->setAnalyzingBackgroundExpressions(true); + MutationsInterpreter(table, metadata_snapshot, mutation_commands, validation_context, mutation_settings).validate(); } table->mutate(mutation_commands, getContext()); return {}; diff --git a/src/Interpreters/InterpreterInsertQuery.cpp b/src/Interpreters/InterpreterInsertQuery.cpp index a158eaf3ba07a..9402931bb6987 100644 --- a/src/Interpreters/InterpreterInsertQuery.cpp +++ b/src/Interpreters/InterpreterInsertQuery.cpp @@ -1126,11 +1126,13 @@ std::optional InterpreterInsertQuery::distributedWriteIntoReplica /// the forwarded query as a secondary query does not convert it again: it creates a plain storage that /// expands the globs and reads every file on its own instead of taking its share of the read tasks from /// the initiator, so N nodes insert the data N times. Rewrite the source the same way `IStorageCluster::read` - /// does for a `SELECT`: the function becomes its `*Cluster` variant with the cluster name argument, and the - /// structure and format arguments are added so that the nodes do not infer the schema again. + /// does for a `SELECT`: the function becomes its `*Cluster` variant with the cluster name argument. The other + /// arguments stay as written, so that the nodes take the structure from the insertion table with its `DEFAULT` + /// expressions unless the query gives one. { auto & select_to_send = query_to_send->as().select->as(); - src_storage_cluster->updateQueryToSendIfNeeded(select_to_send.list_of_selects->children.at(0), src_snapshot, local_context); + src_storage_cluster->updateQueryToSendIfNeeded( + select_to_send.list_of_selects->children.at(0), src_snapshot, local_context, /*add_structure_and_format=*/false); } String query_str; diff --git a/src/Interpreters/InterpreterSystemQuery.cpp b/src/Interpreters/InterpreterSystemQuery.cpp index a8b7f75ad33da..a152a634c05d3 100644 --- a/src/Interpreters/InterpreterSystemQuery.cpp +++ b/src/Interpreters/InterpreterSystemQuery.cpp @@ -574,8 +574,7 @@ BlockIO InterpreterSystemQuery::execute() getContext()->checkAccess(AccessType::SYSTEM_DROP_COMPILED_EXPRESSION_CACHE); if (auto * cache = CompiledExpressionCacheFactory::instance().tryGetCache()) cache->clear(); - /// Drop the static CHJIT slots so persistent LLVM state (`TargetMachine`, `Subtarget`, - /// `LLVMContext`-interned types/constants accumulated across compiles) is reclaimed too. + /// Drop the static CHJIT slots so persistent LLVM state (`TargetMachine`, `Subtarget`) is reclaimed too. /// Each in-flight compile and each cache holder retains its own `shared_ptr`, so /// concurrent operations are not affected: the actual CHJIT survives until every user /// has released its handle. After the cache->clear() above, the only remaining diff --git a/src/Interpreters/JIT/CHJIT.cpp b/src/Interpreters/JIT/CHJIT.cpp index 1f26d8e61d4b1..ace15933eb553 100644 --- a/src/Interpreters/JIT/CHJIT.cpp +++ b/src/Interpreters/JIT/CHJIT.cpp @@ -481,6 +481,11 @@ CHJIT::CompiledModule CHJIT::compileModule(std::function { std::lock_guard lock(jit_lock); + /// A fresh context per compile: `LLVMContext` never frees the constants it interns, and the optimizer + /// creates new ones for every distinct literal, so a long-lived context grows without bound. + /// Declared before `module`, which must be destroyed first. + auto context = JITAllocationScope::run([] { return std::make_unique(); }); + /// LLVM-touching work is wrapped in `JITAllocationScope` so its allocations land in the JIT /// arena. Caller-side ClickHouse code that runs between LLVM calls (error handling further /// down, symbol-name strings, result-map population) is intentionally left outside the scopes @@ -491,7 +496,7 @@ CHJIT::CompiledModule CHJIT::compileModule(std::function /// belong in the JIT arena alongside the module itself. auto module = JITAllocationScope::run([&] { - auto new_module = createModuleForCompilation(); + auto new_module = createModuleForCompilation(*context); compile_function(*new_module); return new_module; }); @@ -502,7 +507,7 @@ CHJIT::CompiledModule CHJIT::compileModule(std::function return module_info; } -std::unique_ptr CHJIT::createModuleForCompilation() +std::unique_ptr CHJIT::createModuleForCompilation(llvm::LLVMContext & context) { std::unique_ptr module = std::make_unique("jit" + std::to_string(current_module_key), context); module->setDataLayout(layout); diff --git a/src/Interpreters/JIT/CHJIT.h b/src/Interpreters/JIT/CHJIT.h index 3646451f6abf1..9296b100bb97c 100644 --- a/src/Interpreters/JIT/CHJIT.h +++ b/src/Interpreters/JIT/CHJIT.h @@ -89,7 +89,7 @@ class CHJIT private: - std::unique_ptr createModuleForCompilation(); + std::unique_ptr createModuleForCompilation(llvm::LLVMContext & context); CompiledModule compileModule(std::unique_ptr module); @@ -99,7 +99,6 @@ class CHJIT static std::unique_ptr getTargetMachine(); - llvm::LLVMContext context; std::unique_ptr machine; llvm::DataLayout layout; std::unique_ptr compiler; @@ -114,8 +113,7 @@ class CHJIT /// Drop the static CHJIT instances used for compiled expressions and compiled aggregate functions. /// On the next compile request, fresh instances are constructed (paying the one-time -/// `TargetMachine`/`Subtarget`/`LLVMContext` initialization cost again, ~few MiB), reclaiming any -/// LLVMContext-retained type/constant uniquing accumulated by previous compilations. +/// `TargetMachine`/`Subtarget` initialization cost again, ~few MiB). /// /// Concurrency: each in-flight compile and each cache holder retains its own `shared_ptr`, /// so the actual instance survives a reset until every user has released its handle. The reset diff --git a/src/Interpreters/convertColumnToType.cpp b/src/Interpreters/convertColumnToType.cpp index 56deff6b7e196..ea1632930e827 100644 --- a/src/Interpreters/convertColumnToType.cpp +++ b/src/Interpreters/convertColumnToType.cpp @@ -1,8 +1,10 @@ #include #include -#include #include +#include +#include +#include #include #include #include @@ -29,43 +31,70 @@ namespace ErrorCodes namespace { -/// Column-native conversion for the cases where CAST provably matches `convertFieldToType`: -/// plain native-numeric-to-native-numeric, in the default AND `strict` modes. `castColumnAccurateOrNull` -/// uses the same accurate numeric conversion (`accurate::convertNumeric`) as `convertFieldToType`'s -/// default path (out-of-range / inexact-narrowing -> NULL). For native numbers `strict` and default -/// agree with it too: `strict` only rejects additional cases for types that are already excluded here - -/// `Bool` (10 is not a valid `Bool`) and `Decimal` (scale reduction; `castColumnAccurateOrNull` would -/// round, so it must NOT be used for strict Decimal - and it isn't, `Decimal` is not a native number). -/// The strict native-number equivalence is pinned by `gtest_convert_column_to_type`. Returns: +/// Column-native conversion for native-numeric values: the same `accurate::convertNumeric` that +/// `convertFieldToType` applies, evaluated directly on the scalar instead of through a CAST function - +/// building and executing a CAST per single-row conversion dominated callers such as the `IN` set +/// builder, which converts every literal of the right-hand side separately. `exact` repeats the +/// predicate of `convertNumericTypeImpl`: an out-of-range value is never representable, and the +/// additional round-trip check that rejects an inexact conversion is relaxed only for a floating-point +/// target under `convert_inexact_floats` - conversions to an integer target stay exact in every mode, so +/// the flag must not disable the fast path wholesale (the `values` table function passes it for every +/// conversion). The equivalence with `convertFieldToType` across `strict` / `convert_inexact_floats` is +/// pinned by `gtest_convert_column_to_type`. Returns: /// - the converted size-1 column of `to` on success, /// - a null `ColumnPtr{}` when not representable, /// - std::nullopt when this fast path does not apply (caller falls back to the `Field` path). -/// Excluded: `convert_inexact_floats` mode (allows rounding, so `castColumnAccurateOrNull` differs), -/// `Bool` (clamp/validity semantics), and anything non-native-numeric +/// Excluded: `Bool` (clamp/validity semantics), and anything non-native-numeric /// (Decimal/Date/Enum/String/wide-int/wrappers/composite). std::optional tryConvertNumericColumnNative( const IColumn & value, const DataTypePtr & from, const DataTypePtr & to, + bool strict, bool convert_inexact_floats) { - if (convert_inexact_floats) - return std::nullopt; if (!isNativeNumber(from) || !isNativeNumber(to) || isBool(from) || isBool(to)) return std::nullopt; - /// `strict` is intentionally not a parameter: for native numbers it is equivalent to the default - /// accurate path (both reject non-representable values), so this fast path serves strict too. - - ColumnWithTypeAndName arg{value.getPtr(), from, ""}; - ColumnPtr casted = castColumnAccurateOrNull(arg, to); - /// `ExecutableFunctionCast` uses the default implementation for constants, so a `ColumnConst` - /// argument yields a `ColumnConst` result. Callers already pass a full column, but unwrap here too - /// so the `assert_cast` below is correct regardless of the argument's constness. - casted = casted->convertToFullColumnIfConst(); - const auto & nullable = assert_cast(*casted); - if (nullable.isNullAt(0)) - return ColumnPtr{}; - return nullable.getNestedColumnPtr(); + + ColumnPtr result; + auto convert = [&](const auto & types) + { + using Types = std::decay_t; + using From = typename Types::LeftType; + using To = typename Types::RightType; + + const From source = assert_cast &>(value).getData()[0]; + const bool exact = strict || !convert_inexact_floats || !is_floating_point; + + To converted{}; + const bool representable = exact + ? accurate::convertNumeric(source, converted) + : accurate::convertNumeric(source, converted); + + if (representable) + { + auto column = ColumnVector::create(); + column->insertValue(converted); + result = std::move(column); + } + return true; + }; + + if (!callOnBasicTypes(from->getTypeId(), to->getTypeId(), convert)) + return std::nullopt; + return result; +} + +/// `Bool` anywhere in the type tree (the shapes `retagBoolInField` normalizes, and any other nesting). +/// `Dynamic` and `JSON` count as containing `Bool`: their payload types are not children the type tree +/// exposes, yet a row of either may hold a `Bool`. +bool containsBool(const IDataType & type) +{ + auto is_bool_or_opaque = [](const IDataType & t) { return t.getName() == "Bool" || isDynamic(t) || isObject(t); }; + bool found = is_bool_or_opaque(type); + if (!found) + type.forEachChild([&](const IDataType & child) { found = found || is_bool_or_opaque(child); }); + return found; } /// `IColumn::get` reconstructs a `Field` using the storage column's `NearestFieldType`, which does not @@ -140,12 +169,25 @@ ColumnPtr convertColumnToTypeOrNull( chassert(value.size() == 1); /// Callers usually pass a `ColumnConst` (e.g. from `evaluateConstantExpressionAsColumn`); operate - /// on the underlying full column so the fast path's CAST returns a plain (non-const) column and the - /// `Field` fallback reads the value directly. - const ColumnPtr full = value.convertToFullColumnIfConst(); + /// on the underlying full column so both the native fast path and the `Field` fallback read the + /// value directly. + ColumnPtr full = value.convertToFullColumnIfConst(); const IColumn & unwrapped = *full; - if (auto native = tryConvertNumericColumnNative(unwrapped, from, to, convert_inexact_floats)) + /// Same as `convertFieldToType`, which returns the value untouched when `from` equals `to` - but + /// only for genuinely identical types that do not involve `Bool`. `IDataType::equals` ignores + /// custom names, so it also holds for `Bool` and its storage type `UInt8` (and for wrapper pairs + /// such as `Nullable(Bool)` and `Nullable(UInt8)`); like the identity check of `CAST`, require the + /// custom names to match too, comparing the full names so every nesting level is covered. And a + /// `Bool` column is a plain `ColumnUInt8` that may hold a raw byte such as 2 (e.g. from + /// `reinterpret`), which the `Field` path below normalizes to `true` through `retagBoolInField` + /// (and which `strict` rejects for a `Bool` target) - so a type containing `Bool` is never returned + /// untouched, not even for `Bool -> Bool`; `CAST` likewise treats `Bool -> Bool` as + /// `UInt8 -> Bool` normalization rather than identity. + if (from->equals(*to) && from->getName() == to->getName() && !containsBool(*from)) + return full; + + if (auto native = tryConvertNumericColumnNative(unwrapped, from, to, strict, convert_inexact_floats)) return std::move(*native); /// Fallback: materialize a `Field`, reuse `convertFieldToType`, rebuild a column. Column-native diff --git a/src/Interpreters/convertColumnToType.h b/src/Interpreters/convertColumnToType.h index 49f554739af54..18e00462153c0 100644 --- a/src/Interpreters/convertColumnToType.h +++ b/src/Interpreters/convertColumnToType.h @@ -18,9 +18,9 @@ namespace DB * column holding NULL — NOT as `ColumnPtr{}`. * * The purpose is to convert constants WITHOUT materializing a `Field`. Cases that can be done - * column-natively (currently: plain numeric-to-numeric in the default mode) go through - * `IColumn`/CAST; the rest still delegate to `convertFieldToType` (same behavior, just not yet - * `Field`-free). The behavior is pinned by `gtest_convert_column_to_type` against `convertFieldToType`, + * column-natively (currently: native numeric-to-numeric, in every `strict` / `convert_inexact_floats` + * mode) read the value straight from the column; the rest still delegate to `convertFieldToType` + * (same behavior, just not yet `Field`-free). The behavior is pinned by `gtest_convert_column_to_type` against `convertFieldToType`, * so more column-native fast paths can be added without changing results. * * The equivalence holds for scalar `Bool` and for `Bool` nested under the structural carriers diff --git a/src/Interpreters/tests/gtest_convert_column_to_type.cpp b/src/Interpreters/tests/gtest_convert_column_to_type.cpp index b67b2eb8f93ac..52cb1889f1f9f 100644 --- a/src/Interpreters/tests/gtest_convert_column_to_type.cpp +++ b/src/Interpreters/tests/gtest_convert_column_to_type.cpp @@ -192,6 +192,24 @@ TEST(ConvertColumnToType, MatchesConvertFieldToType) /// use `castColumnAccurateOrNull`, which would round `33.33` to `33.3` instead of rejecting it). {"Decimal64(2)", Field(DecimalField(Decimal64(3333), 2)), "Decimal64(1)", true}, // scale loss -> null {"Decimal64(1)", Field(DecimalField(Decimal64(333), 1)), "Decimal64(2)", true}, // widen -> 33.30 + + /// `convert_inexact_floats` native-number matrix. The flag relaxes the exactness check only for + /// a floating-point target, so the column-native fast path serves it for every native-number + /// pair rather than falling back to the `Field` path - which matters because the `values` table + /// function passes it for every single conversion it makes. + {"Float64", Field(Float64(0.1)), "Float32", true, true}, // strict wins over inexact -> null + {"Float64", Field(Float64(3.0)), "Int32", false, true}, // integer target, exact -> 3 + {"Float64", Field(Float64(3.5)), "Int32", false, true}, // integer target stays exact -> null + {"Int64", Field(Int64(9007199254740993ll)), "Float64", false, true}, // float target -> nearest + {"UInt64", Field(UInt64(5)), "UInt8", false, true}, // integer target stays exact -> 5 + {"UInt64", Field(UInt64(256)), "UInt8", false, true}, // out of range -> null + {"Int64", Field(Int64(-1)), "UInt8", false, true}, // negative -> null + + /// identical types: the value passes through untouched (including a genuine NULL) + {"UInt64", Field(UInt64(5)), "UInt64", true}, + {"String", Field(String("abc")), "String", true}, + {"Nullable(UInt8)", Field(), "Nullable(UInt8)", true}, + {"Decimal64(2)", Field(DecimalField(Decimal64(3333), 2)), "Decimal64(2)", true}, }; for (const auto & c : cases) diff --git a/src/Loggers/Loggers.cpp b/src/Loggers/Loggers.cpp index a44ea3d596a5b..708517eb61e9a 100644 --- a/src/Loggers/Loggers.cpp +++ b/src/Loggers/Loggers.cpp @@ -260,6 +260,9 @@ void Loggers::buildLoggers(Poco::Util::AbstractConfiguration & config, Poco::Log String text_log_level_str = config.getString("text_log.level", "trace"); int text_log_level = Poco::Logger::parseLevel(text_log_level_str); + if (config.has("text_log.level")) + max_log_level = std::max(text_log_level, max_log_level); + DB::SystemLogQueueSettings log_settings; log_settings.flush_interval_milliseconds = config.getUInt64("text_log.flush_interval_milliseconds", DB::TextLog::getDefaultFlushIntervalMilliseconds()); @@ -382,6 +385,12 @@ void Loggers::updateLevels(Poco::Util::AbstractConfiguration & config, Poco::Log } split->setLevel("Syslog", syslog_level); + if (allowTextLog() && config.has("text_log.level")) + { + int text_log_level = Poco::Logger::parseLevel(config.getString("text_log.level")); + max_log_level = std::max(text_log_level, max_log_level); + } + // Global logging level (it can be overridden for specific loggers). logger.setLevel(max_log_level); diff --git a/src/Parsers/FunctionSecretArgumentsFinder.cpp b/src/Parsers/FunctionSecretArgumentsFinder.cpp index edbf0a93d06b9..90d584a1d0047 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.cpp +++ b/src/Parsers/FunctionSecretArgumentsFinder.cpp @@ -66,6 +66,16 @@ namespace return AzureConnectionValue::PlainStorageAccountURL; } + /// The keys an Azure named collection accepts (`StorageAzureConfiguration`). An override with any + /// other key, which can name any credential, is rejected only after the statement is logged. + bool isAzureCollectionKey(std::string_view key) + { + static const std::unordered_set keys = {"blob_path", "container", "format", "compression", "structure", + "compression_method", "account_name", "account_key", "connection_string", "storage_account_url", "partition_strategy", + "partition_columns_in_data_file", "client_id", "tenant_id"}; + return keys.contains(key); + } + /// The backup engines whose locator names a destination with no credential in it, each with the /// argument count it accepts. `BackupFactory` registers exactly these plus `S3` and `AzureBlobStorage`. std::optional credentialFreeBackupEngineArity(const String & engine_name) @@ -626,43 +636,143 @@ void FunctionSecretArgumentsFinder::findAzureBlobStorageFunctionSecretArguments( /// azureBlobStorageCluster('cluster_name', 'conn_string/storage_account_url', ...) has 'conn_string/storage_account_url' as its second argument. size_t url_arg_idx = is_cluster_function ? 1 : 0; + maskAzureExtraCredentials(); + + /// The other values are hidden before a connection string is: hiding that replaces its whole argument. if (!is_cluster_function && isNamedCollectionName(0)) { /// azureBlobStorage(named_collection, ..., account_key = 'account_key', ...) + maskPositionalsAfterAzureCollection(1); + maskUnreadAzureCollectionOverrides(1); + findSecretNamedArgument("account_key", 1); if (maskAzureConnectionString(-1, true, 1)) return; - findSecretNamedArgument("account_key", 1); + maskUnshowableAzureConnectionOverrides(1); return; } if (is_cluster_function && isNamedCollectionName(1)) { /// azureBlobStorageCluster(cluster, named_collection, ..., account_key = 'account_key', ...) + maskPositionalsAfterAzureCollection(2); + maskUnreadAzureCollectionOverrides(2); + findSecretNamedArgument("account_key", 2); if (maskAzureConnectionString(-1, true, 2)) return; - findSecretNamedArgument("account_key", 2); + maskUnshowableAzureConnectionOverrides(2); return; } - if (maskAzureConnectionString(url_arg_idx)) + /// Slot `n` is the raw argument `arg(n)`; past the end it is an index no argument has. + const auto slot_arguments = azureSlotArguments(url_arg_idx); + auto arg = [&](size_t slot) { return slot < slot_arguments.size() ? slot_arguments[slot] : function->arguments->size(); }; + + if (const auto credential_arg_idx = azureCredentialArgument(slot_arguments, url_arg_idx)) + markSecretArgument(*credential_arg_idx); + + if (maskAzureConnectionString(arg(url_arg_idx))) return; - /// We should check other arguments first because we don't need to do any replacement in case of - /// azureBlobStorage(connection_string|storage_account_url, container_name, blobpath, format) -- in this case there is no account_key argument - /// azureBlobStorageCluster(cluster, connection_string|storage_account_url, container_name, blobpath, format) -- in this case there is no account_key argument - size_t count = function->arguments->size(); - if ((url_arg_idx + 4 <= count) && (count <= url_arg_idx + 7)) + if (arg(url_arg_idx) < function->arguments->size()) + maskUnshowableAzureConnectionValue(arg(url_arg_idx), *function->arguments->at(arg(url_arg_idx)), /* argument_is_named= */ false); +} + +void FunctionSecretArgumentsFinder::maskAzureExtraCredentials() +{ + for (size_t i = 0, size = function->arguments->size(); i < size; ++i) + { + if (const auto f = function->arguments->at(i)->getFunction(); f && f->name() == "extra_credentials") + markSecretArgument(i); + } +} + +std::optional FunctionSecretArgumentsFinder::tryGetAzureKey(const AbstractFunction & equals) +{ + String key; + if (equals.arguments && equals.arguments->size() == 2 && tryGetStringFromArgument(*equals.arguments->at(0), &key) + && isAzureCollectionKey(key)) + return key; + return {}; +} + +void FunctionSecretArgumentsFinder::maskUnreadAzureCollectionOverrides(size_t start) +{ + std::vector connection_overrides; + for (size_t i = start, size = function->arguments->size(); i < size; ++i) { - String fourth_arg; - if (tryGetStringFromArgument(url_arg_idx + 3, &fourth_arg)) + const auto f = function->arguments->at(i)->getFunction(); + if (!f || f->name() != "equals") + continue; + const auto key = tryGetAzureKey(*f); + if (!key) + markSecretArgument(i); + else if (!(tryGetStringFromArgument(*f->arguments->at(1), nullptr) || f->arguments->at(1)->tryGetLiteralText(nullptr))) + markSecretArgument(i, /* argument_is_named= */ true); + if (key == "connection_string" || key == "storage_account_url") + connection_overrides.push_back(i); + } + + if (connection_overrides.size() > 1) + for (const size_t i : connection_overrides) + markSecretArgument(i, /* argument_is_named= */ true); +} + +std::vector FunctionSecretArgumentsFinder::azureSlotArguments(size_t start) +{ + std::vector slots; + size_t extra_credentials = 0; + for (size_t i = 0, size = function->arguments->size(); i < size; ++i) + { + const auto argument = function->arguments->at(i); + if (argument->isSettings()) + continue; + const auto f = i < start ? nullptr : argument->getFunction(); + if (f && f->name() == "extra_credentials") + { + ++extra_credentials; + continue; + } + if (f && f->name() == "equals" && f->arguments && f->arguments->size() == 2) { - if (fourth_arg == "auto" || KnownFormatNames::instance().exists(fourth_arg)) - return; + const auto key = tryGetAzureKey(*f); + if (!key) + markSecretArgument(i); + else if ((key != "partition_strategy" && key != "partition_columns_in_data_file") || !f->arguments->at(1)->tryGetLiteralText(nullptr)) + markSecretArgument(i, /* argument_is_named= */ true); + continue; } + slots.push_back(i); } - /// We're going to replace 'account_key' with '[HIDDEN]' if account_key is used in the signature - if (url_arg_idx + 4 < count) - markSecretArgument(url_arg_idx + 4); + if (extra_credentials > 1) + { + maskEveryArgument(); + return {}; + } + return slots; +} + +std::optional FunctionSecretArgumentsFinder::azureCredentialArgument(const std::vector & slot_arguments, size_t url_arg_idx) const +{ + const size_t count = slot_arguments.size(); + if (count == url_arg_idx + 2) + return slot_arguments[url_arg_idx + 1]; + if (count <= url_arg_idx + 4) + return {}; + String fourth_arg; + if (count <= url_arg_idx + 7 && tryGetStringFromArgument(slot_arguments[url_arg_idx + 3], &fourth_arg) + && (fourth_arg == "auto" || KnownFormatNames::instance().exists(fourth_arg))) + return {}; + return slot_arguments[url_arg_idx + 4]; +} + +void FunctionSecretArgumentsFinder::maskPositionalsAfterAzureCollection(size_t start) +{ + for (size_t i = start, size = function->arguments->size(); i < size; ++i) + { + const auto f = function->arguments->at(i)->getFunction(); + if (!f || (f->name() != "equals" && f->name() != "extra_credentials")) + markSecretArgument(i); + } } bool FunctionSecretArgumentsFinder::maskAzureConnectionString(ssize_t url_arg_idx, bool argument_is_named, size_t start) @@ -682,30 +792,38 @@ bool FunctionSecretArgumentsFinder::maskAzureConnectionString(ssize_t url_arg_id return false; } - if (!url_arg.starts_with("http")) + if (!url_arg.starts_with("http") && maskAzureConnectionStringSecrets(url_arg)) { - if (maskConnectionStringKey(url_arg, "AccountKey=")) - { - chassert(result.count == 0); /// We shouldn't use replacement with masking other arguments - result.start = url_arg_idx; - result.are_named = argument_is_named; - result.count = 1; - result.replacement = url_arg; - return true; - } + chassert(result.count == 0); /// We shouldn't use replacement with masking other arguments + result.start = url_arg_idx; + result.are_named = argument_is_named; + result.count = 1; + result.replacement = url_arg; + return true; + } - if (maskConnectionStringKey(url_arg, "SharedAccessSignature=")) + return false; +} + +void FunctionSecretArgumentsFinder::maskUnshowableAzureConnectionValue( + size_t index, const AbstractFunction::Argument & value, bool argument_is_named) +{ + String text; + if (!value.tryGetString(&text, /* allow_identifier= */ false) || classifyAzureConnectionValue(text) == AzureConnectionValue::Unmaskable) + markSecretArgument(index, argument_is_named); +} + +void FunctionSecretArgumentsFinder::maskUnshowableAzureConnectionOverrides(size_t start) +{ + for (const auto * key : {"connection_string", "storage_account_url"}) + { + for (ssize_t i = findNamedArgument(nullptr, key, start); i >= 0; i = findNamedArgument(nullptr, key, static_cast(i) + 1)) { - chassert(result.count == 0); /// We shouldn't use replacement with masking other arguments - result.start = url_arg_idx; - result.are_named = argument_is_named; - result.count = 1; - result.replacement = url_arg; - return true; + const auto index = static_cast(i); + /// `findNamedArgument` returns only an `equals` function with two arguments. + maskUnshowableAzureConnectionValue(index, *function->arguments->at(index)->getFunction()->arguments->at(1), /* argument_is_named= */ true); } } - - return false; } bool FunctionSecretArgumentsFinder::azureCollectionArgumentsAreShowable(size_t start, size_t positional_limit) @@ -718,8 +836,10 @@ bool FunctionSecretArgumentsFinder::azureCollectionArgumentsAreShowable(size_t s { /// A key this rule cannot read hides which credential the override carries; a value that is /// no plain literal or identifier can nest one (`headers('Authorization' = '...')`). + String key; if (argument_function->arguments && argument_function->arguments->size() == 2 - && tryGetStringFromArgument(*argument_function->arguments->at(0), nullptr) + && tryGetStringFromArgument(*argument_function->arguments->at(0), &key) + && isAzureCollectionKey(key) && (tryGetStringFromArgument(*argument_function->arguments->at(1), nullptr) || argument_function->arguments->at(1)->tryGetLiteralText(nullptr))) continue; @@ -1145,6 +1265,8 @@ void FunctionSecretArgumentsFinder::findAzureBlobStorageTableEngineSecretArgumen /// AzureBlobStorage(connection_string|storage_account_url, container_name, blobpath, format, [account_name, account_key, ...]) size_t url_arg_idx = 0; + maskAzureExtraCredentials(); + if (isNamedCollectionName(url_arg_idx)) { /// AzureBlobStorage(named_collection, ..., account_key = 'account_key', ...) @@ -1159,28 +1281,15 @@ void FunctionSecretArgumentsFinder::findAzureBlobStorageTableEngineSecretArgumen return; } - /// We should check other arguments first because we don't need to do any replacement in case of - /// AzureBlobStorage(connection_string|storage_account_url, container_name, blobpath, format) -- in this case there is no account_key argument - size_t count = function->arguments->size(); - bool fourth_argument_is_format = false; - if ((url_arg_idx + 4 <= count) && (count <= url_arg_idx + 7)) - { - String fourth_arg; - if (tryGetStringFromArgument(url_arg_idx + 3, &fourth_arg)) - fourth_argument_is_format = fourth_arg == "auto" || KnownFormatNames::instance().exists(fourth_arg); - } - /// Which argument holds a credential: the two-argument shape takes a shared access signature beside - /// the url (`endpoint.sas_auth`), the longer ones an `account_key` - unless the fourth names a format. - std::optional credential_arg_idx; - if (count == url_arg_idx + 2) - credential_arg_idx = url_arg_idx + 1; - else if (!fourth_argument_is_format && (url_arg_idx + 4 < count)) - credential_arg_idx = url_arg_idx + 4; + /// Slot `n` is the raw argument `arg(n)`; past the end it is an index no argument has. + const auto slot_arguments = azureSlotArguments(url_arg_idx); + auto arg = [&](size_t slot) { return slot < slot_arguments.size() ? slot_arguments[slot] : function->arguments->size(); }; + const auto credential_arg_idx = azureCredentialArgument(slot_arguments, url_arg_idx); /// The engine reads this argument as a connection string or as a plain account url; a value of /// another shape is read by neither rule below, and a hidden connection string replaces it whole. String connection_value; - const auto shape = tryGetStringFromArgument(url_arg_idx, &connection_value) + const auto shape = tryGetStringFromArgument(arg(url_arg_idx), &connection_value) ? classifyAzureConnectionValue(connection_value) : AzureConnectionValue::Unmaskable; if (shape == AzureConnectionValue::Unmaskable || (shape == AzureConnectionValue::ConnectionString && credential_arg_idx)) @@ -1189,7 +1298,7 @@ void FunctionSecretArgumentsFinder::findAzureBlobStorageTableEngineSecretArgumen return; } - if (maskAzureConnectionString(url_arg_idx)) + if (maskAzureConnectionString(arg(url_arg_idx))) return; if (credential_arg_idx) diff --git a/src/Parsers/FunctionSecretArgumentsFinder.h b/src/Parsers/FunctionSecretArgumentsFinder.h index b72c87816017c..7e3baa828eb0f 100644 --- a/src/Parsers/FunctionSecretArgumentsFinder.h +++ b/src/Parsers/FunctionSecretArgumentsFinder.h @@ -191,12 +191,37 @@ class FunctionSecretArgumentsFinder void findXDBCSecretArguments(); void findS3FunctionSecretArguments(bool is_cluster_function); void findAzureBlobStorageFunctionSecretArguments(bool is_cluster_function); + /// Hides every `extra_credentials(...)` whole, its keys included: a key can be written as anything, a credential too. + void maskAzureExtraCredentials(); + /// The key of a `key = value` argument if it is one of the keys an Azure named collection accepts. Any other left + /// operand can be the credential itself (`'sp=r&sig=...' = 'x'`), so its argument is hidden whole. + static std::optional tryGetAzureKey(const AbstractFunction & equals); + /// Hides every override from `start` on that this rule cannot read: one whose key an Azure named collection does not + /// accept or whose value is no plain literal (it can nest a credential), and every connection override when there are + /// several: only one is read, and only one can be partly shown. + void maskUnreadAzureCollectionOverrides(size_t start); + /// The raw indexes of the arguments the Azure parser assigns slots to, the ones before `start` (the cluster name) as + /// written: it strips the `SETTINGS` clause and the first `extra_credentials(...)`, and a `key = value` takes no slot. + /// Also hides the value of a key the explicit form does not read (all of an argument whose key is no Azure key), and + /// every argument when a second `extra_credentials(...)` shifts the slots. + std::vector azureSlotArguments(size_t start); + /// The raw index of the argument holding a credential, if any: the two-argument shape takes a shared access signature + /// beside the url (`endpoint.sas_auth`), the longer ones an `account_key` - unless the fourth names a format. + std::optional azureCredentialArgument(const std::vector & slot_arguments, size_t url_arg_idx) const; + /// After the collection name every argument must be an override or `extra_credentials(...)`; a positional one is + /// invalid but logged before validation rejects it, and can be a credential. + void maskPositionalsAfterAzureCollection(size_t start); bool maskAzureConnectionString(ssize_t url_arg_idx, bool argument_is_named = false, size_t start = 0); /// Whether the arguments an `AzureBlobStorage(named_collection, ...)` destination or table takes /// from `start` can be shown: only an argument written here can carry a credential, and each has /// to be readable enough to tell that it does not. `positional_limit` bounds the plain literals /// read beside the overrides: one filename for a backup locator, none for a table engine. bool azureCollectionArgumentsAreShowable(size_t start, size_t positional_limit); + /// Hides a connection value that no rule here can show: a URL carrying a credential of its own + /// (`user:key@`, a SAS `?sig=...`), or a value that is not a plain string literal. + void maskUnshowableAzureConnectionValue(size_t index, const AbstractFunction::Argument & value, bool argument_is_named); + /// The same, for the `connection_string` and `storage_account_url` overrides of a named collection from `start` on. + void maskUnshowableAzureConnectionOverrides(size_t start); /// Masks the secrets of every URL form (`url`/`urlCluster` table functions, the `URL` table /// engine, and their named-collection variants): the userinfo password of the url positional or a /// named `url = ...` override, and the `headers(...)` values at any position. `url` is at diff --git a/src/Planner/Planner.cpp b/src/Planner/Planner.cpp index 0d438f4f8581b..b312f5c22898f 100644 --- a/src/Planner/Planner.cpp +++ b/src/Planner/Planner.cpp @@ -113,6 +113,7 @@ namespace DB namespace Setting { extern const SettingsMap additional_table_filters; + extern const SettingsBool use_query_cache; extern const SettingsUInt64 aggregation_in_order_max_block_bytes; extern const SettingsUInt64 aggregation_memory_efficient_merge_threads; extern const SettingsUInt64 allow_experimental_parallel_reading_from_replicas; @@ -2474,7 +2475,8 @@ static bool shouldUseQueryCacheForSubquery( for (const auto & change : query_node.getSettingsChanges()) { if (change.name == "use_query_cache") - return change.value.safeGet(); + /// The node's context holds the clause as clamped and applied; the literal may be a string such as `'true'`. + return query_node.getContext()->getSettingsRef()[Setting::use_query_cache]; } } diff --git a/src/Processors/Chunk.cpp b/src/Processors/Chunk.cpp index bd5e8027801c2..18e2314e0b259 100644 --- a/src/Processors/Chunk.cpp +++ b/src/Processors/Chunk.cpp @@ -70,13 +70,11 @@ void Chunk::checkNumRowsIsConsistent() MutableColumns Chunk::mutateColumns() { - size_t num_columns = columns.size(); - MutableColumns mutable_columns(num_columns); - for (size_t i = 0; i < num_columns; ++i) - mutable_columns[i] = IColumn::mutate(std::move(columns[i])); - - columns.clear(); - num_rows = 0; + MutableColumns mutable_columns(columns.size()); + /// Detach first: `IColumn::mutate` consumes the column even when it throws, and a chunk never holds a null column. + Columns detached_columns = detachColumns(); + for (size_t i = 0; i < detached_columns.size(); ++i) + mutable_columns[i] = IColumn::mutate(std::move(detached_columns[i])); return mutable_columns; } diff --git a/src/Processors/Executors/ExecutorTasks.cpp b/src/Processors/Executors/ExecutorTasks.cpp index fd87df1f1043c..20c5b596a820a 100644 --- a/src/Processors/Executors/ExecutorTasks.cpp +++ b/src/Processors/Executors/ExecutorTasks.cpp @@ -141,12 +141,16 @@ void ExecutorTasks::tryGetTask(ExecutionThreadContext & context) /// Enqueue thread into stack of waiting threads. threads_queue.push(context.thread_number); + + /// No queued or asynchronous tasks are left for the requested threads. + if (async_task_queue.empty() && threads_to_spawn.load(std::memory_order_relaxed)) + threads_to_spawn.store(0, std::memory_order_relaxed); } context.wait(finished); } -size_t ExecutorTasks::pushTasks(Queue & queue, Queue & async_queue, ExecutionThreadContext & context) +void ExecutorTasks::pushTasks(Queue & queue, Queue & async_queue, ExecutionThreadContext & context) { /// Take local task from queue if has one. if (!queue.empty() && !has_fast_tasks.load(std::memory_order_relaxed) @@ -162,13 +166,13 @@ size_t ExecutorTasks::pushTasks(Queue & queue, Queue & async_queue, ExecutionThr context.setTask(nullptr); } - if (!queue.empty() || !async_queue.empty()) + if (queue.empty() && async_queue.empty()) + return; // No new tasks -- no need for new threads + + SlotAllocationPtr slots; { std::unique_lock lock(mutex); - /// Measure the parallelism of the push: how many threads could run the new tasks - /// concurrently. Used by `PipelineExecutor` to size the slot-allocation ceiling via - /// `ISlotAllocation::setMax` on the pipeline's `cpu_slots` handle. const size_t pushed = queue.size() + async_queue.size(); #if defined(OS_LINUX) || defined(OS_DARWIN) @@ -186,25 +190,57 @@ size_t ExecutorTasks::pushTasks(Queue & queue, Queue & async_queue, ExecutionThr queue.pop(); } - /// Capture state under the lock — tryWakeUp may unlock it internally to wake a - /// thread, after which `total_slots` could change. - const size_t idle_before = threads_queue.size(); - const size_t total_slots_now = total_slots; - const size_t num_threads_now = num_threads; // const after init but read for clarity + /// A thread for every pushed task that idle threads can't cover, within the pipeline width. + /// Computed before `tryWakeUpAnyOtherThreadWithTasks`, which may unlock the mutex. + const size_t uncovered = pushed - std::min(pushed, threads_queue.size()); + const size_t requested = std::min(uncovered, widthLeft()); + addThreadsToSpawn(requested); + if (growDesiredThreads(requested)) + slots = cpu_slots; /// Wake up at least one thread that will wake up other threads if required tryWakeUpAnyOtherThreadWithTasks(context, lock); + } + + if (slots) + applyDesiredThreads(*slots); +} + +void ExecutorTasks::initDesiredThreads(bool lazy_allocation, size_t initial_parallel) +{ + /// The eager allocation already asks for all the threads. + if (!lazy_allocation) + { + desired_threads.store(num_threads, std::memory_order_relaxed); + return; + } - /// Return the number of additional pipeline threads the caller should target: pushed - /// tasks that idle threads can't cover, clamped to remaining pipeline-width headroom - /// (`num_threads - total_slots`). `PipelineExecutor` adds this delta to the - /// `cpu_slots->setMax(...)` target so the slot allocator grows in step with demand. - const size_t coverable = std::min(pushed, idle_before); - const size_t uncovered = pushed - coverable; - const size_t headroom = (num_threads_now > total_slots_now) ? (num_threads_now - total_slots_now) : 0; - return std::min(uncovered, headroom); + /// The initial tasks are not pushed by `pushTasks`, so ask for threads to run them in parallel here. + SlotAllocationPtr slots; + { + std::lock_guard lock(mutex); + if (initial_parallel > 1 && growDesiredThreads(initial_parallel - 1)) + slots = cpu_slots; } - return 0; // No new tasks -- no need for new threads + if (slots) + applyDesiredThreads(*slots); +} + +bool ExecutorTasks::growDesiredThreads(size_t threads) +{ + const size_t current = desired_threads.load(std::memory_order_relaxed); + const size_t target = std::min(num_threads, current + threads); + if (target <= current) + return false; + desired_threads.store(target, std::memory_order_relaxed); + return true; +} + +void ExecutorTasks::applyDesiredThreads(ISlotAllocation & slots) +{ + /// Each caller sets the latest target, so the last `setMax` is never below it. + std::lock_guard lock(set_max_mutex); + slots.setMax(desired_threads.load(std::memory_order_relaxed)); } void ExecutorTasks::init(size_t num_threads_, size_t use_threads_, const SlotAllocationPtr & cpu_slots_, bool profile_processors, bool trace_processors, const StepWallClockRegistry * step_to_wall_clock_registry, ReadProgressCallback * callback) @@ -278,14 +314,45 @@ size_t ExecutorTasks::upscale(size_t slot_id) ++total_slots; use_threads = std::max(use_threads, slot_id + 1); + /// Concurrent spawners may acquire more slots than requested; such slots are granted anyway, so they get a thread. + if (const size_t requested = threads_to_spawn.load(std::memory_order_relaxed)) + threads_to_spawn.store(requested - 1, std::memory_order_relaxed); + + if (num_threads <= total_slots) + { + if (threads_to_spawn.load(std::memory_order_relaxed)) + threads_to_spawn.store(0, std::memory_order_relaxed); + return 0; + } + // Too many idle threads can already cover incoming work — don't spawn more. if (threads_queue.size() > TOO_MANY_IDLE_THRESHOLD) return 0; // Remaining pipeline-width capacity. Callers spawn one at a time and re-call upscale. + return num_threads - total_slots; +} + +size_t ExecutorTasks::widthLeft() const +{ return num_threads > total_slots ? num_threads - total_slots : 0; } +void ExecutorTasks::addThreadsToSpawn(size_t threads) +{ + /// Requests that wait for a CPU slot are not dropped, but they can't exceed the pipeline width. + const size_t requested = threads_to_spawn.load(std::memory_order_relaxed); + const size_t new_requested = std::min(requested + threads, widthLeft()); + if (new_requested != requested) + threads_to_spawn.store(new_requested, std::memory_order_relaxed); +} + +void ExecutorTasks::requestThreads(size_t threads) +{ + std::lock_guard lock(mutex); + addThreadsToSpawn(threads); +} + void ExecutorTasks::downscale(size_t slot_id) { std::lock_guard lock(mutex); diff --git a/src/Processors/Executors/ExecutorTasks.h b/src/Processors/Executors/ExecutorTasks.h index 2e677cf53b0b8..19886cea89a90 100644 --- a/src/Processors/Executors/ExecutorTasks.h +++ b/src/Processors/Executors/ExecutorTasks.h @@ -58,9 +58,29 @@ class ExecutorTasks /// CPU slots for each thread. SlotAllocationPtr cpu_slots; + /// Requested threads (for queued tasks or in place of downscaled ones) that wait for a CPU slot. + /// Changed under `mutex`, read without it after every task, so it has its own cache line. + alignas(64) std::atomic threads_to_spawn = 0; + /// Threshold found by rolling dice. const static size_t TOO_MANY_IDLE_THRESHOLD = 4; + /// Pipeline width not taken by started threads. Requires `mutex`. + size_t widthLeft() const; + /// Add to `threads_to_spawn`, up to `widthLeft`. Requires `mutex`. + void addThreadsToSpawn(size_t threads); + + /// Current `setMax` target of the CPU slots. Only grows. Changed under `mutex`. + std::atomic desired_threads = 1; + + /// Orders `setMax` calls. Not `mutex`: the allocation takes its own locks, and it calls `resume` under them. + std::mutex set_max_mutex; + + /// Raise the `setMax` target by `threads`, up to `num_threads`. Returns whether it changed. Requires `mutex`. + bool growDesiredThreads(size_t threads); + /// Pass the current target to `setMax`. Must not be called under `mutex`. + void applyDesiredThreads(ISlotAllocation & slots); + public: using Stack = std::stack; /// This queue can grow a lot and lead to OOM. That is why we use non-default @@ -92,9 +112,13 @@ class ExecutorTasks // Local task optimization: the first regular task could be placed directly into thread to be executed next. // For async tasks proessor->schedule() is called. // If non-local tasks were added, wake up one thread to process them. - // Returns the number of additional threads the caller should try to spawn (0 if idle - // threads can cover the push, or no new tasks were added). - size_t pushTasks(Queue & queue, Queue & async_queue, ExecutionThreadContext & context); + // Requests a thread for every new task that idle threads can't cover, within the pipeline width not taken + // by started threads: adds them to `threads_to_spawn` and raises the `setMax` target of the CPU slots by as much. + void pushTasks(Queue & queue, Queue & async_queue, ExecutionThreadContext & context); + + /// Set the initial `setMax` target: the pipeline width for the eager allocation, + /// enough threads for the `initial_parallel` initial tasks for the lazy one. + void initDesiredThreads(bool lazy_allocation, size_t initial_parallel); void init(size_t num_threads_, size_t use_threads_, const SlotAllocationPtr & cpu_slots_, bool profile_processors, bool trace_processors, const StepWallClockRegistry * step_to_wall_clock_registry, ReadProgressCallback * callback); @@ -108,14 +132,22 @@ class ExecutorTasks void freeCPU(); /// Upscale to include slot_id. Updates use_threads to max(use_threads, slot_id + 1). + /// The new thread is one of `threads_to_spawn`. /// Returns the number of additional threads the caller may try to spawn next (0 means - /// "stop spawning": either pipeline width is saturated or enough idle threads can absorb - /// the work). Callers should still spawn one at a time and re-call upscale after each. + /// "stop spawning": either pipeline width is saturated, and the rest of `threads_to_spawn` is dropped, + /// or enough idle threads can absorb the work). Callers should still spawn one at a time and + /// re-call upscale after each. size_t upscale(size_t slot_id); /// Downscale by removing slot_id from active slots. Updates use_threads to highest active slot + 1 void downscale(size_t slot_id); + /// Request `threads` more threads (e.g. for the initial tasks), clamped to the pipeline width. + void requestThreads(size_t threads); + + /// Whether requested threads wait for a CPU slot. Lock-free, so it is checked after every task. + bool hasThreadsToSpawn() const { return threads_to_spawn.load(std::memory_order_relaxed) > 0; } + /// Temporarily release slot_id without downscale. Later either downscale() or resume() is called. void preempt(size_t slot_id); diff --git a/src/Processors/Executors/PipelineExecutor.cpp b/src/Processors/Executors/PipelineExecutor.cpp index e23ef59b1d300..528122b0b95eb 100644 --- a/src/Processors/Executors/PipelineExecutor.cpp +++ b/src/Processors/Executors/PipelineExecutor.cpp @@ -303,10 +303,8 @@ void PipelineExecutor::finalizeExecution() { single_thread_cpu_slot.reset(); tasks.freeCPU(); - { - std::lock_guard lock(spawn_mutex); - cpu_slots.reset(); - } + /// No thread spawns anymore: the pool is waited for, or there is no pool. + cpu_slots.reset(); checkTimeLimit(); @@ -398,7 +396,6 @@ void PipelineExecutor::executeStepImpl(size_t thread_num, WorkloadResources && r #endif /// Try to execute neighbour processor. - size_t spawn_count = 0; { Queue queue; Queue async_queue; @@ -410,7 +407,17 @@ void PipelineExecutor::executeStepImpl(size_t thread_num, WorkloadResources && r /// Push other tasks to global queue. if (status == ExecutingGraph::UpdateNodeStatus::Done) - spawn_count = tasks.pushTasks(queue, async_queue, context); + { + try + { + tasks.pushTasks(queue, async_queue, context); + } + catch (...) + { + cancel(ExecutionStatus::Exception); + throw; + } + } } #ifndef NDEBUG @@ -418,50 +425,21 @@ void PipelineExecutor::executeStepImpl(size_t thread_num, WorkloadResources && r #endif - /// Upscaling: `pushTasks` tells us how many additional threads would usefully - /// parallelize the tasks it just enqueued. Publish our spawn_count into the shared - /// `pending_demand` counter; the thread that actually holds `spawn_mutex` drains it - /// so no demand is lost when multiple threads push concurrently. - if (pool && spawn_count > 0) - pending_demand.fetch_add(spawn_count, std::memory_order_relaxed); - - // Drain pending_demand under `spawn_mutex`. A thread that fails `try_lock` - // must not exit while `pending_demand > 0` -- otherwise its add could race - // with the owner's final `exchange(0)` and be left stranded after the owner - // unlocks. The outer loop holds the thread until either it drains the demand - // itself or observes the counter return to zero (meaning the owner caught it). - while (pool && pending_demand.load(std::memory_order_acquire) > 0) + /// Upscaling: threads requested by `pushTasks` wait in `tasks` until their CPU slots are granted, + /// so a slot granted later (e.g. released by another query) still gets a thread. + if (pool && tasks.hasThreadsToSpawn()) { - if (!spawn_mutex.try_lock()) - { - std::this_thread::yield(); - continue; - } try { - std::lock_guard lock(spawn_mutex, std::adopt_lock); - while (true) - { - size_t accumulated = pending_demand.exchange(0, std::memory_order_acq_rel); - if (accumulated == 0) - break; - size_t target = std::min(max_pipeline_threads, desired_threads + accumulated); - if (target > desired_threads) - { - desired_threads = target; - cpu_slots->setMax(target); - } - } spawnThreads({}); } catch (...) { - /// spawnThreads can throw an exception, for example CANNOT_SCHEDULE_TASK. + /// spawnThreads can throw an exception, for example CANNOT_SCHEDULE_TASK or RESOURCE_ACCESS_DENIED. /// We should cancel execution properly before rethrow. cancel(ExecutionStatus::Exception); throw; } - break; } /// Preemption and downscaling. @@ -474,6 +452,8 @@ void PipelineExecutor::executeStepImpl(size_t thread_num, WorkloadResources && r if (!resources.renewCPULease()) { tasks.downscale(resources.getCPULeaseSlotId()); + /// The slot may be granted again later: start a thread for it then. + tasks.requestThreads(1); yield = true; break; // Downscaling. Unable to renew the lease - thread should stop (but could be rerun later). } @@ -620,21 +600,13 @@ void PipelineExecutor::initializeExecution(size_t num_threads, bool concurrency_ is_execution_initialized = true; tryUpdateExecutionStatus(ExecutionStatus::NotStarted, ExecutionStatus::Executing); - /// Capture the ceiling so the upscaling block can cap `setMax` at this value. - max_pipeline_threads = num_threads; - /// Read the flag ONCE. A config reload mid-initialization would otherwise split a /// single query between lazy and eager strategies (e.g. allocate lazy with max=1 but - /// then initialize desired_threads=num_threads, stranding subsequent setMax as a no-op). + /// then start from the full `setMax` target, stranding subsequent setMax as a no-op). const bool lazy_allocation = ConcurrencyControl::instance().getLazyAllocation(); cpu_slots = allocateCPU(num_threads, concurrency_control, lazy_allocation); - /// If rollback flag is off, we used eager allocate(1, num_threads) and will not call - /// setMax in the upscaling block (nothing to grow). Initialize desired_threads - /// to the full ceiling so the growth check in the block becomes a no-op. - desired_threads = lazy_allocation ? 1 : num_threads; - Queue queue; Queue async_queue; graph->initializeExecution(queue, async_queue); @@ -645,22 +617,12 @@ void PipelineExecutor::initializeExecution(size_t num_threads, bool concurrency_ tasks.init(num_threads, 1, cpu_slots, profile_processors, trace_processors, step_wall_clock_registry, read_progress_callback.get()); const size_t initial_parallel = tasks.fill(queue, async_queue); - /// Initial queued parallelism never routes through `pushTasks`, so size setMax here to - /// cover it. For multi-source pipelines (e.g. UNION ALL of N subqueries) this prevents - /// the pipeline from being stuck single-threaded if the master's first task happens not - /// to push more work. Only matters on the lazy path; the eager path already has the full - /// ceiling. `initial_parallel` is the total target — the master thread is one of its - /// consumers, no +1 needed. - if (lazy_allocation && initial_parallel > 1) - { - std::lock_guard lock(spawn_mutex); - const size_t target = std::min(max_pipeline_threads, initial_parallel); - if (target > desired_threads) - { - desired_threads = target; - cpu_slots->setMax(target); - } - } + /// For multi-source pipelines (e.g. UNION ALL of N subqueries) this prevents the pipeline from being + /// stuck single-threaded if the master's first task happens not to push more work. + tasks.initDesiredThreads(lazy_allocation, initial_parallel); + + /// The eager path spawns a thread for every granted slot, as many as the pipeline width allows. + tasks.requestThreads(lazy_allocation ? std::max(initial_parallel, 1) : num_threads); if (num_threads > 1) pool = std::make_unique(CurrentMetrics::QueryPipelineExecutorThreads, CurrentMetrics::QueryPipelineExecutorThreadsActive, CurrentMetrics::QueryPipelineExecutorThreadsScheduled, num_threads); @@ -668,7 +630,7 @@ void PipelineExecutor::initializeExecution(size_t num_threads, bool concurrency_ void PipelineExecutor::spawnThreads(AcquiredSlotPtr slot) { - while (cpu_slots) + while (true) { if (!slot) slot = cpu_slots->tryAcquire(); @@ -714,11 +676,8 @@ void PipelineExecutor::executeImpl(size_t num_threads, bool concurrency_control) { if (num_threads > 1) { - { - std::lock_guard lock(spawn_mutex); - // Start at least one thread, could block to acquire the first CPU slot - spawnThreads(cpu_slots->acquire()); - } + // Start at least one thread, could block to acquire the first CPU slot + spawnThreads(cpu_slots->acquire()); tasks.processAsyncTasks(); pool->wait(); } diff --git a/src/Processors/Executors/PipelineExecutor.h b/src/Processors/Executors/PipelineExecutor.h index efcde90f48cdf..b1dc4741a2072 100644 --- a/src/Processors/Executors/PipelineExecutor.h +++ b/src/Processors/Executors/PipelineExecutor.h @@ -85,22 +85,6 @@ class PipelineExecutor SlotAllocationPtr cpu_slots; AcquiredSlotPtr single_thread_cpu_slot; // cpu slot for single-thread mode to work using executeStep() std::unique_ptr pool; - std::mutex spawn_mutex; - - /// Pipeline's max thread count (captured from execute(num_threads)). - size_t max_pipeline_threads = 1; - - /// Current setMax target. Written by initializeExecution before any worker thread exists, - /// then read and updated only under `spawn_mutex` from the upscaling block. Non-atomic - /// because all mutations are serialized by spawn_mutex and initial write happens-before - /// any spawned worker via the ThreadPool barrier in initializeExecution. - size_t desired_threads = 1; - - /// Accumulates spawn_count from pushTasks calls that couldn't acquire `spawn_mutex` - /// (another thread was already handling the spawn). The thread that owns the mutex - /// drains this counter before deciding the new setMax target, so no demand is ever - /// silently dropped. - std::atomic pending_demand{0}; /// Flag that checks that initializeExecution was called. bool is_execution_initialized = false; @@ -138,7 +122,9 @@ class PipelineExecutor // Methods for CPU scheduling SlotAllocationPtr allocateCPU(size_t num_threads, bool concurrency_control, bool lazy_allocation); - void spawnThreads(AcquiredSlotPtr slot) TSA_REQUIRES(spawn_mutex); + /// Start a thread for `slot` (if any) and for every other granted slot, until `tasks` has no capacity left. + /// Lock-free: any thread calls it after a task while `tasks` has requested threads. + void spawnThreads(AcquiredSlotPtr slot); /// If execution_status == from, change it to desired. bool tryUpdateExecutionStatus(ExecutionStatus expected, ExecutionStatus desired); diff --git a/src/Processors/Formats/IInputFormat.h b/src/Processors/Formats/IInputFormat.h index 93dc66dd9b4c1..702784d97aeec 100644 --- a/src/Processors/Formats/IInputFormat.h +++ b/src/Processors/Formats/IInputFormat.h @@ -132,6 +132,8 @@ class IInputFormat : public ISource void needOnlyCount() { need_only_count = true; } + virtual void prefetch() {} + virtual std::optional, size_t>> getMatchedBuckets() const { return std::nullopt; } protected: diff --git a/src/Processors/Formats/Impl/ArrowBatchRowLimit.cpp b/src/Processors/Formats/Impl/ArrowBatchRowLimit.cpp new file mode 100644 index 0000000000000..8f76772b9ef62 --- /dev/null +++ b/src/Processors/Formats/Impl/ArrowBatchRowLimit.cpp @@ -0,0 +1,319 @@ +#include + +#if USE_ARROW || USE_PARQUET + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +namespace DB +{ + +namespace +{ + +/// The largest `n <= end - begin` for which the non-decreasing `offsets` grow by at most `limit` over +/// rows [begin, begin + n). +size_t maxRowsByOffsets(const IColumn::Offsets & offsets, size_t begin, size_t end, UInt64 limit) +{ + const UInt64 base = offsets[static_cast(begin) - 1]; + const auto * first = offsets.data() + begin; + const auto * it = std::upper_bound(first, offsets.data() + end, base + limit); + return static_cast(it - first); +} + +/// A writer either materializes the column or dictionary-encodes it; measuring it as materialized bounds +/// both, since a batch's dictionary holds at most the distinct values of its rows. `canBeInsideLowCardinality` +/// allows only fixed-width types, `String` and `FixedString`, and of those only the two string types reach +/// a 32-bit-offset buffer. +size_t maxRowsForLowCardinality( + const IColumn & column, + const DataTypePtr & type, + size_t begin, + size_t end, + bool fixed_string_as_fixed_byte_array, + UInt64 max_batch_bytes) +{ + const size_t num_rows = end - begin; + const DataTypePtr values_type = removeNullable(removeLowCardinality(type)); + const auto & low_cardinality = assert_cast(column); + const IColumn & values_column = *low_cardinality.getDictionary().getNestedNotNullableColumn(); + + if (isFixedString(values_type)) + { + if (fixed_string_as_fixed_byte_array) + return num_rows; + const size_t n = assert_cast(values_column).getN(); + return n == 0 ? num_rows : std::min(num_rows, max_batch_bytes / n); + } + + if (!isString(values_type)) + return num_rows; + + const auto & values = assert_cast(values_column); + const auto & value_offsets = values.getOffsets(); + + /// Scanning the indexes is only needed when the whole dictionary repeated on every row could overflow: + /// no single value is larger than all of them together. This is O(1), unlike finding the largest value, + /// so it does not add work proportional to the dictionary for every batch of a split chunk. + const UInt64 dictionary_bytes = values.empty() ? 0 : value_offsets.back(); + if (dictionary_bytes == 0 || num_rows <= max_batch_bytes / dictionary_bytes) + return num_rows; + + const auto & indexes = low_cardinality.getIndexes(); + UInt64 total = 0; + for (size_t row = begin; row != end; ++row) + { + const auto index = static_cast(indexes.getUInt(row)); + total += value_offsets[index] - value_offsets[index - 1]; + if (total > max_batch_bytes) + return row - begin; + } + return num_rows; +} + +/// A `Variant` becomes an Arrow dense union, whose own buffers are bounded by the row count; only its +/// children, holding one alternative's rows each, can overflow. +size_t maxRowsForVariant( + const IColumn & column, + const DataTypePtr & type, + size_t begin, + size_t end, + bool fixed_string_as_fixed_byte_array, + UInt64 max_batch_bytes) +{ + const size_t num_rows = end - begin; + const auto & variant_column = assert_cast(column); + const auto & variant_type = assert_cast(*type); + const size_t num_variants = variant_column.getNumVariants(); + const auto & local_discriminators = variant_column.getLocalDiscriminators(); + const auto & variant_offsets = variant_column.getOffsets(); + + /// Within a contiguous row range the offsets of each alternative are contiguous too (see + /// `ColumnVariant::updateHashWithValueRange`), so [begin, end) maps to the range + /// [nested_begin, nested_begin + nested_rows) of every alternative. + std::vector nested_begin(num_variants, 0); + std::vector nested_rows(num_variants, 0); + for (size_t row = begin; row != end; ++row) + { + const auto local_discr = local_discriminators[row]; + if (local_discr == ColumnVariant::NULL_DISCRIMINATOR) + continue; + if (nested_rows[local_discr] == 0) + nested_begin[local_discr] = variant_offsets[row]; + ++nested_rows[local_discr]; + } + + std::vector nested_rows_that_fit(num_variants); + bool all_fit = true; + for (size_t local_discr = 0; local_discr != num_variants; ++local_discr) + { + const auto global_discr + = variant_column.globalDiscriminatorByLocal(static_cast(local_discr)); + nested_rows_that_fit[local_discr] = maxRowsFittingOneArrowBatch( + variant_column.getVariantByLocalDiscriminator(local_discr), + variant_type.getVariant(global_discr), + nested_begin[local_discr], + nested_begin[local_discr] + nested_rows[local_discr], + fixed_string_as_fixed_byte_array, max_batch_bytes); + all_fit &= nested_rows_that_fit[local_discr] == nested_rows[local_discr]; + } + if (all_fit) + return num_rows; + + /// Some alternative cannot take all of its rows: cut at the row where it runs out. + std::vector nested_rows_taken(num_variants, 0); + for (size_t row = begin; row != end; ++row) + { + const auto local_discr = local_discriminators[row]; + if (local_discr == ColumnVariant::NULL_DISCRIMINATOR) + continue; + if (nested_rows_taken[local_discr] == nested_rows_that_fit[local_discr]) + return row - begin; + ++nested_rows_taken[local_discr]; + } + return num_rows; +} + +} + +size_t maxRowsFittingOneArrowBatch( + const IColumn & column, + const DataTypePtr & type, + size_t begin, + size_t end, + bool fixed_string_as_fixed_byte_array, + UInt64 max_batch_bytes) +{ + const size_t num_rows = end - begin; + if (num_rows == 0) + return 0; + + /// Arrow cannot address more than this however the caller was configured. + max_batch_bytes = std::min(max_batch_bytes, MAX_ARROW_BUFFER_SIZE); + + if (type->lowCardinality()) + return maxRowsForLowCardinality(column, type, begin, end, fixed_string_as_fixed_byte_array, max_batch_bytes); + + if (isVariant(type)) + return maxRowsForVariant(column, type, begin, end, fixed_string_as_fixed_byte_array, max_batch_bytes); + + /// A null row is written as a zero-length slot, so measuring the nested column only overestimates. + if (type->isNullable()) + { + return maxRowsFittingOneArrowBatch( + assert_cast(column).getNestedColumn(), + removeNullable(type), + begin, + end, + fixed_string_as_fixed_byte_array, max_batch_bytes); + } + + switch (WhichDataType(type).idx) + { + case TypeIndex::String: + { + return maxRowsByOffsets(assert_cast(column).getOffsets(), begin, end, max_batch_bytes); + } + case TypeIndex::FixedString: + { + if (fixed_string_as_fixed_byte_array) + return num_rows; + const size_t n = assert_cast(column).getN(); + return n == 0 ? num_rows : std::min(num_rows, max_batch_bytes / n); + } + case TypeIndex::Array: + { + const auto & array = assert_cast(column); + const auto & offsets = array.getOffsets(); + const size_t nested_begin = offsets[static_cast(begin) - 1]; + const size_t nested_rows = maxRowsFittingOneArrowBatch( + array.getData(), + assert_cast(*type).getNestedType(), + nested_begin, + offsets[end - 1], + fixed_string_as_fixed_byte_array, max_batch_bytes); + return maxRowsByOffsets(offsets, begin, end, std::min(nested_rows, max_batch_bytes)); + } + case TypeIndex::Map: + { + /// Both writers encode a Map exactly as its nested `Array(Tuple(key, value))`. + return maxRowsFittingOneArrowBatch( + assert_cast(column).getNestedColumn(), + assert_cast(*type).getNestedType(), + begin, + end, + fixed_string_as_fixed_byte_array, max_batch_bytes); + } + case TypeIndex::Tuple: + { + const auto & tuple = assert_cast(column); + const auto & elements = assert_cast(*type).getElements(); + size_t rows = num_rows; + for (size_t i = 0; i != elements.size(); ++i) + { + rows = std::min( + rows, + maxRowsFittingOneArrowBatch( + tuple.getColumn(i), elements[i], begin, begin + rows, fixed_string_as_fixed_byte_array, max_batch_bytes)); + } + return rows; + } + /// Fixed-width Arrow types: a single values buffer, no offsets, nothing to exceed. + case TypeIndex::UInt8: + case TypeIndex::UInt16: + case TypeIndex::UInt32: + case TypeIndex::UInt64: + case TypeIndex::UInt128: + case TypeIndex::UInt256: + case TypeIndex::Int8: + case TypeIndex::Int16: + case TypeIndex::Int32: + case TypeIndex::Int64: + case TypeIndex::Int128: + case TypeIndex::Int256: + case TypeIndex::Float32: + case TypeIndex::Float64: + case TypeIndex::Enum8: + case TypeIndex::Enum16: + case TypeIndex::Date: + case TypeIndex::Date32: + case TypeIndex::DateTime: + case TypeIndex::DateTime64: + case TypeIndex::Time: + case TypeIndex::Time64: + case TypeIndex::Decimal32: + case TypeIndex::Decimal64: + case TypeIndex::Decimal128: + case TypeIndex::Decimal256: + case TypeIndex::IPv4: + case TypeIndex::IPv6: + case TypeIndex::UUID: + case TypeIndex::Interval: + { + return num_rows; + } + default: + { + /// A type with no Arrow equivalent is written by serializing each value, so bounding its + /// encoded size would mean serializing twice. The encoder rejects a batch that overflows. + return num_rows; + } + } +} + +size_t maxRowsFittingOneArrowBatch( + const Columns & columns, + const DataTypes & types, + size_t begin, + size_t end, + bool fixed_string_as_fixed_byte_array, + UInt64 max_batch_bytes) +{ + chassert(columns.size() == types.size()); + const size_t num_rows = end - begin; + if (num_rows == 0) + return 0; + + /// Some limits (`Variant`, `LowCardinality`) take time linear in the rows they are given, so measuring + /// the whole remainder of a chunk for each of its batches would be quadratic. The window grows instead: + /// the limit of a prefix is the limit of the whole range capped at the prefix's length, so the first + /// window that does not fit entirely gives the answer, and the work stays proportional to the rows + /// emitted, plus the initial window. + static constexpr size_t initial_window = 8192; + size_t window = std::min(num_rows, initial_window); + while (true) + { + /// Two columns never share an Arrow buffer, so the smallest per-column limit applies. + size_t rows = window; + for (size_t i = 0; i < columns.size() && rows != 0; ++i) + rows = std::min( + rows, + maxRowsFittingOneArrowBatch( + *columns[i], types[i], begin, begin + rows, fixed_string_as_fixed_byte_array, max_batch_bytes)); + + if (rows < window || window == num_rows) + return rows; + window = std::min(num_rows, window * 2); + } +} + +} + +#endif diff --git a/src/Processors/Formats/Impl/ArrowBatchRowLimit.h b/src/Processors/Formats/Impl/ArrowBatchRowLimit.h new file mode 100644 index 0000000000000..c532138a8da0d --- /dev/null +++ b/src/Processors/Formats/Impl/ArrowBatchRowLimit.h @@ -0,0 +1,47 @@ +#pragma once + +#include "config.h" + +#if USE_ARROW || USE_PARQUET + +#include +#include +#include + +namespace DB +{ + +/// The largest `n` such that rows [begin, begin + n) of `column` stay within `max_batch_bytes`; writers +/// split an oversized chunk accordingly. Returns 0 only when the first row does not fit on its own, which +/// the writer then lets Arrow reject. `ColumnConst` and the sparse/replicated representations are not +/// handled; callers materialize first. +/// +/// `LowCardinality` is measured as materialized, which only over-splits when it is written as an Arrow +/// dictionary. A dictionary larger than one buffer is not covered at all: its bytes do not depend on the +/// row count, because `ColumnLowCardinality::insertRangeFrom` keeps a shared source dictionary whole, so +/// every slice re-emits it. Arrow rejects such a chunk. +/// +/// A type with no Arrow equivalent is not covered either, since only serializing its values reveals their +/// encoded size. Both uncovered shapes are rejected by the writer instead of being split. +size_t maxRowsFittingOneArrowBatch( + const IColumn & column, + const DataTypePtr & type, + size_t begin, + size_t end, + bool fixed_string_as_fixed_byte_array, + UInt64 max_batch_bytes = MAX_ARROW_BUFFER_SIZE); + +/// The largest `n` such that rows [begin, begin + n) of every column stay within `max_batch_bytes`: the +/// number of rows of the next record batch of a chunk. The work is proportional to the returned number of +/// rows rather than to `end - begin`, so splitting a chunk into many batches stays linear. +size_t maxRowsFittingOneArrowBatch( + const Columns & columns, + const DataTypes & types, + size_t begin, + size_t end, + bool fixed_string_as_fixed_byte_array, + UInt64 max_batch_bytes); + +} + +#endif diff --git a/src/Processors/Formats/Impl/ArrowBufferedStreams.cpp b/src/Processors/Formats/Impl/ArrowBufferedStreams.cpp index ac12c3baa4ffd..08b8015edb8fd 100644 --- a/src/Processors/Formats/Impl/ArrowBufferedStreams.cpp +++ b/src/Processors/Formats/Impl/ArrowBufferedStreams.cpp @@ -1,7 +1,7 @@ #pragma clang diagnostic ignored "-Wreserved-identifier" #include -#if USE_ARROW || USE_ORC || USE_PARQUET +#if USE_ARROW || USE_PARQUET #include #include #include diff --git a/src/Processors/Formats/Impl/ArrowBufferedStreams.h b/src/Processors/Formats/Impl/ArrowBufferedStreams.h index f2c29a832841d..de54491968800 100644 --- a/src/Processors/Formats/Impl/ArrowBufferedStreams.h +++ b/src/Processors/Formats/Impl/ArrowBufferedStreams.h @@ -3,7 +3,7 @@ #include -#if USE_ARROW || USE_ORC || USE_PARQUET +#if USE_ARROW || USE_PARQUET #include diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp index a5acaf6aaa625..c8e52e542a2ae 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.cpp @@ -1,7 +1,7 @@ #include #include -#if USE_ARROW || USE_ORC || USE_PARQUET +#if USE_ARROW || USE_PARQUET #include #include diff --git a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h index a11e7297ad7bb..f1817118c8299 100644 --- a/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h +++ b/src/Processors/Formats/Impl/ArrowColumnToCHColumn.h @@ -5,7 +5,7 @@ #include "config.h" -#if USE_ARROW || USE_ORC || USE_PARQUET +#if USE_ARROW || USE_PARQUET #include #include diff --git a/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.cpp index 6253eb086fd8d..39b8406f86c31 100644 --- a/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.cpp @@ -3,6 +3,7 @@ #if USE_ARROW #include +#include #include #include #include @@ -24,6 +25,7 @@ #include #include +#include #include namespace DB @@ -255,7 +257,7 @@ std::pair ArrowIPCBlockOutputFormat::encodeDictionaryCol if (!state.emitted || delta_size > 0) { ColumnPtr delta = state.values->cut(delta_start, delta_size); - auto dict_batch = encoder->encode({delta}, {value_type}, delta_size); + auto dict_batch = encoder->encode({delta}, {value_type}, 0, delta_size); auto written = writeBatchMessage(dict_batch, dict.id, /*is_delta=*/state.emitted); if (!stream) dictionary_blocks.push_back({written.offset, written.metadata_length, written.body_length}); @@ -351,6 +353,15 @@ std::pair ArrowIPCBlockOutputFormat::substituteDictionar return {column, type}; } +void ArrowIPCBlockOutputFormat::writeRecordBatch( + const Columns & columns, const DataTypes & types, size_t begin, size_t end) +{ + auto batch = encoder->encode(columns, types, begin, end); + auto written = writeBatchMessage(batch); + if (!stream) + record_blocks.push_back({written.offset, written.metadata_length, written.body_length}); +} + void ArrowIPCBlockOutputFormat::consume(Chunk chunk) { /// Not in `writeChunk`: a reader gets the schema on the first chunk even when batches are combined. @@ -400,10 +411,10 @@ void ArrowIPCBlockOutputFormat::writeChunk(Chunk chunk) const Columns & columns = chunk.getColumns(); /// Dictionary-encoded columns are replaced in the record batch by their integer index column; their - /// values go into separate DictionaryBatch messages (extended across batches via deltas). + /// values go into separate DictionaryBatch messages, written here so they precede every record batch + /// of this chunk (extended across chunks via deltas). Columns record_columns(columns.begin(), columns.end()); DataTypes record_types = column_types; - for (size_t i = 0; i < columns.size(); ++i) { auto [substituted_column, substituted_type] = substituteDictionaries(columns[i], column_types[i], column_dict_plans[i]); @@ -411,10 +422,25 @@ void ArrowIPCBlockOutputFormat::writeChunk(Chunk chunk) record_types[i] = std::move(substituted_type); } - auto batch = encoder->encode(record_columns, record_types, num_rows); - auto written = writeBatchMessage(batch); - if (!stream) - record_blocks.push_back({written.offset, written.metadata_length, written.body_length}); + /// One record batch per chunk, unless the chunk exceeds what a record batch can address; then it is + /// written as several batches over successive row ranges. The limit is measured on the chunk's own + /// columns, since a dictionary-encoded one is already reduced to fixed-width indexes by now. + size_t offset = 0; + do + { + size_t rows = maxRowsFittingOneArrowBatch( + columns, + column_types, + offset, + num_rows, + format_settings.arrow.output_fixed_string_as_fixed_byte_array, + format_settings.arrow.output_max_record_batch_bytes); + /// A row that does not fit on its own cannot be represented at all; let the encoder reject it. + rows = std::min(std::max(rows, 1), num_rows - offset); + + writeRecordBatch(record_columns, record_types, offset, offset + rows); + offset += rows; + } while (offset < num_rows); } void ArrowIPCBlockOutputFormat::finalizeImpl() diff --git a/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.h b/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.h index 2be27e359c1f9..965cd893deea8 100644 --- a/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/ArrowIPC/ArrowIPCBlockOutputFormat.h @@ -31,6 +31,8 @@ namespace DB /// metadata without the Apache Arrow C++ library. /// `LowCardinality` is written as its full column, or — when `output_format_arrow_low_cardinality_as_dictionary` /// is on — as an Arrow dictionary-encoded column (a single dictionary per id, extended across batches via deltas). +/// Each chunk becomes one record batch, except when it holds more data than the 32-bit offsets of the Arrow +/// IPC buffers can address, in which case it is split across several record batches. class ArrowIPCBlockOutputFormat final : public IOutputFormat { public: @@ -47,6 +49,8 @@ class ArrowIPCBlockOutputFormat final : public IOutputFormat void writeChunk(Chunk chunk); void writeSchemaIfNeeded(); + /// Writes rows [begin, end) of the already dictionary-substituted `columns` as one record batch. + void writeRecordBatch(const Columns & columns, const DataTypes & types, size_t begin, size_t end); /// Writes one encapsulated message for an encoded batch (a record batch, or a dictionary batch /// when `dictionary_id` is set), returning its location for recording an Arrow file `Block`. ArrowIPC::MessageWriter::WrittenMessage writeBatchMessage( diff --git a/src/Processors/Formats/Impl/ArrowIPC/RecordBatchDecoder.cpp b/src/Processors/Formats/Impl/ArrowIPC/RecordBatchDecoder.cpp index cac181c39f5b7..1f05e7aeb77c4 100644 --- a/src/Processors/Formats/Impl/ArrowIPC/RecordBatchDecoder.cpp +++ b/src/Processors/Formats/Impl/ArrowIPC/RecordBatchDecoder.cpp @@ -96,6 +96,25 @@ DataTypePtr matchColumnNullability(const DataTypePtr & type, const ColumnPtr & c return t; } +/// Marks as nullable the fields of `copy` (a copy of `original`) whose originals are in `fields`. +void declareNullable(ArrowField & copy, const ArrowField & original, const UnorderedSetWithMemoryTracking & fields) +{ + if (fields.contains(&original)) + copy.nullable = true; + /// Promotion depends on each batch's nulls, and a `Variant` type can't be converted to another one. + if (original.type.kind == TypeKind::Union) + return; + for (size_t i = 0; i < original.type.children.size(); ++i) + declareNullable(copy.type.children[i], original.type.children[i], fields); + /// The map entries and keys stay non-nullable, as `parseSchema` requires, so their nulls are rejected. + if (original.type.kind == TypeKind::Map) + { + ArrowField & entries = copy.type.children.at(0); + entries.nullable = false; + entries.type.children.at(0).nullable = false; + } +} + /// Expand an Arrow LSB-first bitmap into one byte per row (0 or 1). With `invert` each output byte is /// flipped, turning the validity bitmap (1 = valid) into a ClickHouse null map (1 = null). Unpacks 8 bits /// at a time via SWAR (the same trick as the Parquet decoder), scalar for the trailing < 8 bits. @@ -883,6 +902,10 @@ ColumnPtr RecordBatchDecoder::decodeDictionary( { const Slice indices_slice = nextBuffer(); ColumnPtr values = registry.get(field.dictionary->id); + /// The values of a field declared non-nullable are decoded non-nullable. When its indices contain nulls, + /// `decodeColumns` decodes a nullable copy of the field, so the values need a NULL key as well. + if (field.nullable && !values->isNullable() && values->canBeInsideNullable()) + values = makeNullable(values); const size_t dict_size = values->size(); const int bits = field.dictionary->index_bit_width; @@ -1221,6 +1244,38 @@ ColumnPtr RecordBatchDecoder::decodeField( return inner; } +bool RecordBatchDecoder::collectUndeclaredNullFields( + const ArrowField & field, size_t & node_offset, UnorderedSetWithMemoryTracking & fields) const +{ + const auto * nodes = current_batch->nodes(); + if (!nodes || node_index + node_offset >= nodes->size()) + return false; + const flatbuf::FieldNode & node = *nodes->Get(static_cast(node_index + node_offset)); + ++node_offset; + + /// Unions, `null`-typed and unsupported fields have no validity bitmap: `decodeField` rejects their nulls. + if (!field.nullable && node.null_count() != 0 + && (field.dictionary + || (field.type.kind != TypeKind::Union && field.type.kind != TypeKind::Null && field.type.kind != TypeKind::Unsupported))) + fields.insert(&field); + + /// The values of a dictionary-encoded field are in a separate DictionaryBatch. + if (field.dictionary) + return true; + + /// Mirrors the layouts `skipField` walks. The buffer layout of other unsupported types is unknown. + if (field.type.kind == TypeKind::Unsupported && field.type.skip_layout != ArrowType::SkipLayout::RunEndEncoded + && field.type.skip_layout != ArrowType::SkipLayout::ListView) + return false; + + for (const ArrowField & child : field.type.children) + { + if (!collectUndeclaredNullFields(child, node_offset, fields)) + return false; + } + return true; +} + void RecordBatchDecoder::skipField(const ArrowField & field) { /// Consume this field's FieldNode, mirroring decodeField. @@ -1353,7 +1408,8 @@ RecordBatchDecoder::DecodedColumns RecordBatchDecoder::decodeColumns( const flatbuf::RecordBatch & batch, const PODArray & body, const ArrowFields & fields, const UnorderedSetWithMemoryTracking * keep_top_level_fields, const UnorderedMapWithMemoryTracking * target_types_, - const VectorWithMemoryTracking * reachable_buffers) + const VectorWithMemoryTracking * reachable_buffers, + bool read_undeclared_nulls) { target_types = target_types_; current_batch = &batch; @@ -1401,15 +1457,33 @@ RecordBatchDecoder::DecodedColumns RecordBatchDecoder::decodeColumns( continue; } + /// Nulls in fields declared non-nullable are read as NULL, as the Apache Arrow library did. Decoding a + /// nullable copy of the field keeps the types derived from the schema consistent with the decoded columns. + /// Values of delta dictionary batches are merged into one column, so their types must not depend on each + /// batch's nulls: the dictionary path does not set `read_undeclared_nulls` and keeps rejecting them. + std::optional nullable_copy; + if (read_undeclared_nulls) + { + UnorderedSetWithMemoryTracking undeclared_null_fields; + size_t node_offset = 0; + collectUndeclaredNullFields(field, node_offset, undeclared_null_fields); + if (!undeclared_null_fields.empty()) + { + nullable_copy = field; + declareNullable(*nullable_copy, field, undeclared_null_fields); + } + } + const ArrowField & effective_field = nullable_copy ? *nullable_copy : field; + DecodedColumn decoded; decoded.name = field.name; - decoded.type = fieldToCHType(field, settings, field.nullable, /*allow_null_type=*/true); + decoded.type = fieldToCHType(effective_field, settings, effective_field.nullable, /*allow_null_type=*/true); /// A top-level dictionary-encoded field decodes into a LowCardinality column of its value type. - if (field.dictionary && decoded.type->canBeInsideLowCardinality()) + if (effective_field.dictionary && decoded.type->canBeInsideLowCardinality()) decoded.type = std::make_shared(decoded.type); /// `normalized_name` seeds the recursive `date32` numeric type hint (looked up in `target_types`); /// nested fields derive their hints from it as the decoder recurses. See decodeField/decodeInner. - decoded.column = decodeField(field, /*allow_low_cardinality=*/true, /*target_hint=*/nullptr, normalized_name); + decoded.column = decodeField(effective_field, /*allow_low_cardinality=*/true, /*target_hint=*/nullptr, normalized_name); /// `decodeField` may wrap a struct in Nullable based on the requested type hint even when /// `fieldToCHType` did not (setting off); reconcile the reported type with the decoded column so the /// column/type pair stays consistent for the subsequent cast. @@ -1607,7 +1681,8 @@ RecordBatchDecoder::decodeBatch( const UnorderedMapWithMemoryTracking * target_types_, const VectorWithMemoryTracking * reachable_buffers) { - return decodeColumns(batch, body, schema.fields, keep_top_level_fields, target_types_, reachable_buffers); + return decodeColumns( + batch, body, schema.fields, keep_top_level_fields, target_types_, reachable_buffers, /*read_undeclared_nulls=*/true); } VectorWithMemoryTracking RecordBatchDecoder::reachableTopLevelBuffers( diff --git a/src/Processors/Formats/Impl/ArrowIPC/RecordBatchDecoder.h b/src/Processors/Formats/Impl/ArrowIPC/RecordBatchDecoder.h index 5166ed393843f..700be74f67638 100644 --- a/src/Processors/Formats/Impl/ArrowIPC/RecordBatchDecoder.h +++ b/src/Processors/Formats/Impl/ArrowIPC/RecordBatchDecoder.h @@ -81,11 +81,14 @@ class RecordBatchDecoder const VectorWithMemoryTracking * reachable_buffers = nullptr); /// Decodes an explicit list of fields (used for dictionary batches, which carry one value column). + /// With `read_undeclared_nulls`, nulls in fields declared non-nullable are read as NULL (by decoding a + /// copy of the field that declares them nullable) instead of being rejected. DecodedColumns decodeColumns( const flatbuf::RecordBatch & batch, const PODArray & body, const ArrowFields & fields, const UnorderedSetWithMemoryTracking * keep_top_level_fields = nullptr, const UnorderedMapWithMemoryTracking * target_types = nullptr, - const VectorWithMemoryTracking * reachable_buffers = nullptr); + const VectorWithMemoryTracking * reachable_buffers = nullptr, + bool read_undeclared_nulls = false); /// The buffers (indices into `batch.buffers()`) referenced by the requested top-level fields, as a /// 0/1 mask of length `batch.buffers()->size()`. Computed by the same cursor walk decoding uses @@ -122,6 +125,11 @@ class RecordBatchDecoder /// reading or materializing its data. Used to skip an unrequested top-level column while keeping the /// flat node/buffer cursors aligned for the columns that follow. void skipField(const ArrowField & field); + /// Walks the FieldNodes of `field`'s subtree, starting `node_offset` nodes after the next one, without + /// consuming them, and collects the fields declared non-nullable whose nodes report nulls. Returns false + /// when the walk cannot continue (missing nodes or an unknown layout); decoding then reports the error. + bool collectUndeclaredNullFields( + const ArrowField & field, size_t & node_offset, UnorderedSetWithMemoryTracking & fields) const; ColumnPtr decodeInner(const ArrowField & field, size_t rows, const DataTypePtr & target_hint, const String & path); ColumnPtr decodeUnion(const ArrowField & field, size_t rows); ColumnPtr decodeDictionary( diff --git a/src/Processors/Formats/Impl/ArrowIPC/RecordBatchEncoder.cpp b/src/Processors/Formats/Impl/ArrowIPC/RecordBatchEncoder.cpp index ff53303c7921c..1b9ac495fad2a 100644 --- a/src/Processors/Formats/Impl/ArrowIPC/RecordBatchEncoder.cpp +++ b/src/Processors/Formats/Impl/ArrowIPC/RecordBatchEncoder.cpp @@ -2,6 +2,7 @@ #if USE_ARROW +#include #include #include #include @@ -47,6 +48,19 @@ namespace DB::ArrowIPC namespace { +/// The writer splits an oversized chunk across batches, so normally only a single row too large on its own +/// gets here; it is also the backstop for a shape the row limit does not cover, such as a `LowCardinality` +/// dictionary exceeding one buffer. +[[noreturn]] void throwValueDoesNotFitArrowBuffer(Int64 bytes) +{ + throw Exception( + ErrorCodes::TOO_LARGE_ARRAY_SIZE, + "Cannot write {} bytes of string data to Arrow IPC: `Utf8`/`Binary` offsets are 32-bit, so one record " + "batch cannot hold more than {} bytes", + bytes, MAX_ARROW_BUFFER_SIZE); +} + + /// Pack one bit per input byte into `out` (size (num_rows + 7) / 8, zero-initialised by the caller): /// bit = 1 where the byte is non-zero, optionally inverted. Arrow validity uses invert = true (1 = not-null), /// Bool uses invert = false. Processes 64 bytes per step via `bytes64MaskToBits64Mask`, scalar for the tail. @@ -92,7 +106,7 @@ void RecordBatchEncoder::appendEmptyBuffer() buffers.emplace_back(static_cast(body.size()), 0); } -Int64 RecordBatchEncoder::appendValidity(const IColumn * null_map_column, size_t num_rows) +Int64 RecordBatchEncoder::appendValidity(const IColumn * null_map_column, size_t begin, size_t end) { if (!null_map_column) { @@ -101,22 +115,32 @@ Int64 RecordBatchEncoder::appendValidity(const IColumn * null_map_column, size_t return 0; } + const size_t num_rows = end - begin; const auto & null_map = assert_cast(*null_map_column).getData(); PODArray bitmap((num_rows + 7) / 8, 0); - packBytesToBits(null_map.data(), num_rows, bitmap.data(), /*invert=*/true); /// Arrow validity bit: 1 = valid. + packBytesToBits(null_map.data() + begin, num_rows, bitmap.data(), /*invert=*/true); /// Arrow validity bit: 1 = valid. appendBuffer(bitmap.data(), bitmap.size()); - return countBytesInFilter(null_map.data(), 0, num_rows); + return countBytesInFilter(null_map.data(), begin, end); } -void RecordBatchEncoder::appendOffsets(const IColumn::Offsets & ch_offsets, size_t num_rows) +void RecordBatchEncoder::appendOffsets(const IColumn::Offsets & ch_offsets, size_t begin, size_t end) { + const size_t num_rows = end - begin; + const UInt64 base = ch_offsets[static_cast(begin) - 1]; PODArray arrow_offsets(num_rows + 1); arrow_offsets[0] = 0; for (size_t i = 0; i < num_rows; ++i) { - if (ch_offsets[i] > static_cast(std::numeric_limits::max())) - throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Arrow IPC offset {} exceeds 32 bits", ch_offsets[i]); - arrow_offsets[i + 1] = static_cast(ch_offsets[i]); + const UInt64 offset = ch_offsets[begin + i] - base; + /// The row limit bounds the element count of `Array`/`Map` exactly, so only a row too large on its + /// own gets here. + if (offset > MAX_ARROW_BUFFER_SIZE) + throw Exception( + ErrorCodes::TOO_LARGE_ARRAY_SIZE, + "Cannot write a row of {} elements to Arrow IPC: `List` offsets are 32-bit, so a single row " + "cannot hold more than {} elements", + offset, MAX_ARROW_BUFFER_SIZE); + arrow_offsets[i + 1] = static_cast(offset); } appendBuffer(arrow_offsets.data(), (num_rows + 1) * sizeof(Int32)); } @@ -126,11 +150,11 @@ namespace /// Appends a fixed-width column's contiguous data straight into the body (a single copy, no temporary). template -void appendFixedWidth(RecordBatchEncoder & self, const IColumn & column, size_t num_rows) +void appendFixedWidth(RecordBatchEncoder & self, const IColumn & column, size_t begin, size_t end) { using V = typename Col::ValueType; const auto & data = assert_cast(column).getData(); - self.appendBuffer(data.data(), num_rows * sizeof(V)); + self.appendBuffer(data.data() + begin, (end - begin) * sizeof(V)); } int arrowTimeUnitForScale(UInt32 scale) @@ -167,49 +191,50 @@ Int64 timeUnitsPerDay(UInt32 scale) } void RecordBatchEncoder::encodeValues( - const IColumn & column, const DataTypePtr & type, size_t num_rows, const IColumn * null_map_column) + const IColumn & column, const DataTypePtr & type, size_t begin, size_t end, const IColumn * null_map_column) { const WhichDataType which(type); + const size_t num_rows = end - begin; if (isBool(type)) { /// Pack the 0/1 UInt8 column into an Arrow bit-packed boolean buffer. const auto & data = assert_cast(column).getData(); PODArray bitmap((num_rows + 7) / 8, 0); - packBytesToBits(data.data(), num_rows, bitmap.data(), /*invert=*/false); + packBytesToBits(data.data() + begin, num_rows, bitmap.data(), /*invert=*/false); appendBuffer(bitmap.data(), bitmap.size()); return; } switch (which.idx) { - case TypeIndex::UInt8: appendFixedWidth(*this, column, num_rows); return; - case TypeIndex::UInt16: appendFixedWidth(*this, column, num_rows); return; - case TypeIndex::UInt32: appendFixedWidth(*this, column, num_rows); return; - case TypeIndex::UInt64: appendFixedWidth(*this, column, num_rows); return; - case TypeIndex::Int8: appendFixedWidth(*this, column, num_rows); return; - case TypeIndex::Enum8: appendFixedWidth(*this, column, num_rows); return; - case TypeIndex::Int16: appendFixedWidth(*this, column, num_rows); return; - case TypeIndex::Enum16: appendFixedWidth(*this, column, num_rows); return; - case TypeIndex::Int32: appendFixedWidth(*this, column, num_rows); return; - case TypeIndex::Int64: appendFixedWidth(*this, column, num_rows); return; - case TypeIndex::Float32: appendFixedWidth(*this, column, num_rows); return; - case TypeIndex::Float64: appendFixedWidth(*this, column, num_rows); return; - case TypeIndex::Date32: appendFixedWidth(*this, column, num_rows); return; - case TypeIndex::DateTime: appendFixedWidth(*this, column, num_rows); return; + case TypeIndex::UInt8: appendFixedWidth(*this, column, begin, end); return; + case TypeIndex::UInt16: appendFixedWidth(*this, column, begin, end); return; + case TypeIndex::UInt32: appendFixedWidth(*this, column, begin, end); return; + case TypeIndex::UInt64: appendFixedWidth(*this, column, begin, end); return; + case TypeIndex::Int8: appendFixedWidth(*this, column, begin, end); return; + case TypeIndex::Enum8: appendFixedWidth(*this, column, begin, end); return; + case TypeIndex::Int16: appendFixedWidth(*this, column, begin, end); return; + case TypeIndex::Enum16: appendFixedWidth(*this, column, begin, end); return; + case TypeIndex::Int32: appendFixedWidth(*this, column, begin, end); return; + case TypeIndex::Int64: appendFixedWidth(*this, column, begin, end); return; + case TypeIndex::Float32: appendFixedWidth(*this, column, begin, end); return; + case TypeIndex::Float64: appendFixedWidth(*this, column, begin, end); return; + case TypeIndex::Date32: appendFixedWidth(*this, column, begin, end); return; + case TypeIndex::DateTime: appendFixedWidth(*this, column, begin, end); return; case TypeIndex::Date: { if (settings.arrow.output_date_as_uint16) { /// Backwards-compatible mode: write Date as a plain uint16. - appendFixedWidth(*this, column, num_rows); + appendFixedWidth(*this, column, begin, end); return; } /// Date is UInt16 days; widen to Int32 to match Arrow date32. const auto & data = assert_cast(column).getData(); PODArray widened(num_rows); for (size_t i = 0; i < num_rows; ++i) - widened[i] = static_cast(data[i]); + widened[i] = static_cast(data[begin + i]); appendBuffer(widened.data(), num_rows * sizeof(Int32)); return; } @@ -229,14 +254,14 @@ void RecordBatchEncoder::encodeValues( /// A null row carries an arbitrary nested value that the validity bitmap already masks as /// NULL; do not rescale it (matching the Apache Arrow writer), so junk in a NULL row cannot /// raise DECIMAL_OVERFLOW for a value that is emitted as NULL anyway. - if (null_map && (*null_map)[i]) + if (null_map && (*null_map)[begin + i]) { values[i] = 0; continue; } /// Widening to a coarser Arrow unit multiplies the value; guard against silent wraparound, /// matching the Apache Arrow writer which raises `DECIMAL_OVERFLOW` for the same case. - if (common::mulOverflow(data[i].value, factor, values[i])) + if (common::mulOverflow(data[begin + i].value, factor, values[i])) throw Exception( ErrorCodes::DECIMAL_OVERFLOW, "Decimal value is too large to convert to the Arrow timestamp scale"); } @@ -254,12 +279,12 @@ void RecordBatchEncoder::encodeValues( = null_map_column ? &assert_cast(*null_map_column).getData() : nullptr; for (size_t i = 0; i < num_rows; ++i) { - if (null_map && (*null_map)[i]) + if (null_map && (*null_map)[begin + i]) continue; - if (data[i] < 0 || data[i] >= 86400) - throwArrowTimeOutOfRange(data[i], 86400); + if (data[begin + i] < 0 || data[begin + i] >= 86400) + throwArrowTimeOutOfRange(data[begin + i], 86400); } - appendFixedWidth(*this, column, num_rows); + appendFixedWidth(*this, column, begin, end); return; } case TypeIndex::Time64: @@ -287,12 +312,12 @@ void RecordBatchEncoder::encodeValues( { /// A null row's arbitrary value is masked by the validity bitmap; do not rescale or /// range-check it (matching the library writer's `AppendNull`). - if (null_map && (*null_map)[i]) + if (null_map && (*null_map)[begin + i]) { values[i] = 0; continue; } - Int64 value = data[i].value; + Int64 value = data[begin + i].value; if (value < 0 || value >= source_units_per_day) throwArrowTimeOutOfRange(value, source_units_per_day); if (common::mulOverflow(value, factor, value) @@ -307,14 +332,14 @@ void RecordBatchEncoder::encodeValues( PODArray values(num_rows); for (size_t i = 0; i < num_rows; ++i) { - if (null_map && (*null_map)[i]) + if (null_map && (*null_map)[begin + i]) { values[i] = 0; continue; } - if (data[i].value < 0 || data[i].value >= source_units_per_day) - throwArrowTimeOutOfRange(data[i].value, source_units_per_day); - if (common::mulOverflow(data[i].value, factor, values[i])) + if (data[begin + i].value < 0 || data[begin + i].value >= source_units_per_day) + throwArrowTimeOutOfRange(data[begin + i].value, source_units_per_day); + if (common::mulOverflow(data[begin + i].value, factor, values[i])) throw Exception(ErrorCodes::DECIMAL_OVERFLOW, "Decimal math overflow"); } appendBuffer(values.data(), num_rows * sizeof(Int64)); @@ -333,7 +358,7 @@ void RecordBatchEncoder::encodeValues( const auto & data = col.getData(); for (size_t i = 0; i < num_rows; ++i) { - const char * src = reinterpret_cast(&data[i]); + const char * src = reinterpret_cast(&data[begin + i]); char * dst = out.data() + i * arrow_width; memcpy(dst, src, ch_width); /// Sign-extend to the wider Arrow representation (little-endian two's complement). @@ -358,22 +383,40 @@ void RecordBatchEncoder::encodeValues( const auto & offs = cs.getOffsets(); const NullMap * null_map = null_map_column ? &assert_cast(*null_map_column).getData() : nullptr; + /// ClickHouse ColumnString stores no trailing '\0', so the offsets rebased against the row + /// before the range already are the Arrow ones. + const UInt64 base = offs[static_cast(begin) - 1]; + const UInt64 total_bytes = offs[static_cast(end) - 1] - base; PODArray arrow_offsets(num_rows + 1); - PODArray data; arrow_offsets[0] = 0; + + if (!null_map) + { + /// The bytes of the whole range are contiguous, so no full-size temporary is needed. + if (total_bytes > MAX_ARROW_BUFFER_SIZE) + throwValueDoesNotFitArrowBuffer(static_cast(total_bytes)); + for (size_t i = 0; i < num_rows; ++i) + arrow_offsets[i + 1] = static_cast(offs[begin + i] - base); + appendBuffer(arrow_offsets.data(), (num_rows + 1) * sizeof(Int32)); + appendBuffer(chars.data() + base, total_bytes); + return; + } + + PODArray data; + data.reserve(total_bytes); /// An upper bound: null rows contribute nothing. Int64 cur = 0; for (size_t i = 0; i < num_rows; ++i) { /// A null row carries an arbitrary nested value that the validity bitmap already masks as /// NULL; emit a zero-length slot for it (matching the Apache Arrow writer's `AppendNull`) /// so the bytes of a logically-NULL row never reach the IPC body. - if (!(null_map && (*null_map)[i])) + if (!(*null_map)[begin + i]) { - const size_t start = i == 0 ? 0 : offs[i - 1]; - const size_t len = offs[i] - start; /// ClickHouse ColumnString stores no trailing '\0' + const size_t start = offs[static_cast(begin + i) - 1]; + const size_t len = offs[begin + i] - start; cur += static_cast(len); - if (cur > std::numeric_limits::max()) - throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Arrow IPC string offset exceeds 32 bits"); + if (static_cast(cur) > MAX_ARROW_BUFFER_SIZE) + throwValueDoesNotFitArrowBuffer(cur); const size_t old = data.size(); data.resize(old + len); if (len) @@ -390,7 +433,7 @@ void RecordBatchEncoder::encodeValues( const auto & cfs = assert_cast(column); if (settings.arrow.output_fixed_string_as_fixed_byte_array) { - appendBuffer(cfs.getChars().data(), num_rows * cfs.getN()); + appendBuffer(cfs.getChars().data() + begin * cfs.getN(), num_rows * cfs.getN()); return; } /// `output_fixed_string_as_fixed_byte_array = 0`: the schema advertises a variable-width @@ -408,13 +451,13 @@ void RecordBatchEncoder::encodeValues( /// nested buffer can be copied in one shot. for (size_t i = 0; i < num_rows; ++i) { - const UInt64 end = static_cast(i + 1) * n; - if (end > static_cast(std::numeric_limits::max())) - throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Arrow IPC string offset exceeds 32 bits"); - arrow_offsets[i + 1] = static_cast(end); + const UInt64 slot_end = static_cast(i + 1) * n; + if (slot_end > MAX_ARROW_BUFFER_SIZE) + throwValueDoesNotFitArrowBuffer(static_cast(slot_end)); + arrow_offsets[i + 1] = static_cast(slot_end); } appendBuffer(arrow_offsets.data(), (num_rows + 1) * sizeof(Int32)); - appendBuffer(chars.data(), num_rows * n); + appendBuffer(chars.data() + begin * n, num_rows * n); return; } /// A null row carries an arbitrary nested value that the validity bitmap already masks as NULL; @@ -424,15 +467,15 @@ void RecordBatchEncoder::encodeValues( Int64 cur = 0; for (size_t i = 0; i < num_rows; ++i) { - if (!(*null_map)[i]) + if (!(*null_map)[begin + i]) { cur += static_cast(n); - if (cur > std::numeric_limits::max()) - throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Arrow IPC string offset exceeds 32 bits"); + if (static_cast(cur) > MAX_ARROW_BUFFER_SIZE) + throwValueDoesNotFitArrowBuffer(cur); const size_t old = data.size(); data.resize(old + n); if (n) - memcpy(data.data() + old, &chars[i * n], n); + memcpy(data.data() + old, &chars[(begin + i) * n], n); } arrow_offsets[i + 1] = static_cast(cur); } @@ -440,13 +483,13 @@ void RecordBatchEncoder::encodeValues( appendBuffer(data.data(), data.size()); return; } - case TypeIndex::IPv4: appendFixedWidth>(*this, column, num_rows); return; - case TypeIndex::Int128: appendFixedWidth>(*this, column, num_rows); return; - case TypeIndex::UInt128: appendFixedWidth>(*this, column, num_rows); return; - case TypeIndex::Int256: appendFixedWidth>(*this, column, num_rows); return; - case TypeIndex::UInt256: appendFixedWidth>(*this, column, num_rows); return; - case TypeIndex::IPv6: appendFixedWidth>(*this, column, num_rows); return; - case TypeIndex::Interval: appendFixedWidth>(*this, column, num_rows); return; + case TypeIndex::IPv4: appendFixedWidth>(*this, column, begin, end); return; + case TypeIndex::Int128: appendFixedWidth>(*this, column, begin, end); return; + case TypeIndex::UInt128: appendFixedWidth>(*this, column, begin, end); return; + case TypeIndex::Int256: appendFixedWidth>(*this, column, begin, end); return; + case TypeIndex::UInt256: appendFixedWidth>(*this, column, begin, end); return; + case TypeIndex::IPv6: appendFixedWidth>(*this, column, begin, end); return; + case TypeIndex::Interval: appendFixedWidth>(*this, column, begin, end); return; case TypeIndex::UUID: { /// fixed_size_binary(16) with the two 64-bit halves byte-reversed, matching the library writer. @@ -454,7 +497,7 @@ void RecordBatchEncoder::encodeValues( PODArray out(num_rows * 16); for (size_t i = 0; i < num_rows; ++i) { - UUID value = data[i]; + UUID value = data[begin + i]; auto * bytes = reinterpret_cast(&value); std::reverse(bytes, bytes + 8); std::reverse(bytes + 8, bytes + 16); @@ -466,9 +509,11 @@ void RecordBatchEncoder::encodeValues( case TypeIndex::Array: { const auto & ca = assert_cast(column); - appendOffsets(ca.getOffsets(), num_rows); + const auto & array_offsets = ca.getOffsets(); + appendOffsets(array_offsets, begin, end); const DataTypePtr & element_type = assert_cast(*type).getNestedType(); - encodeField(ca.getData(), element_type, ca.getData().size()); + encodeField( + ca.getData(), element_type, array_offsets[static_cast(begin) - 1], array_offsets[end - 1]); return; } case TypeIndex::Tuple: @@ -476,23 +521,25 @@ void RecordBatchEncoder::encodeValues( const auto & ct = assert_cast(column); const auto & elem_types = assert_cast(*type).getElements(); for (size_t i = 0; i < elem_types.size(); ++i) - encodeField(ct.getColumn(i), elem_types[i], num_rows); + encodeField(ct.getColumn(i), elem_types[i], begin, end); return; } case TypeIndex::Map: { const auto & cm = assert_cast(column); const auto & arr = cm.getNestedColumn(); /// ColumnArray(ColumnTuple(key, value)) - appendOffsets(arr.getOffsets(), num_rows); + const auto & array_offsets = arr.getOffsets(); + appendOffsets(array_offsets, begin, end); const auto & entries = assert_cast(arr.getData()); - const size_t entries_rows = entries.size(); + const size_t entries_begin = array_offsets[static_cast(begin) - 1]; + const size_t entries_end = array_offsets[end - 1]; /// The entries struct node (non-nullable), then its key and value children. - nodes.emplace_back(static_cast(entries_rows), 0); + nodes.emplace_back(static_cast(entries_end - entries_begin), 0); appendEmptyBuffer(); const auto & map_type = assert_cast(*type); - encodeField(entries.getColumn(0), map_type.getKeyType(), entries_rows); - encodeField(entries.getColumn(1), map_type.getValueType(), entries_rows); + encodeField(entries.getColumn(0), map_type.getKeyType(), entries_begin, entries_end); + encodeField(entries.getColumn(1), map_type.getValueType(), entries_begin, entries_end); return; } default: @@ -501,7 +548,7 @@ void RecordBatchEncoder::encodeValues( /// Arrow `Binary` column (read back as `String`); otherwise reject it. if (settings.arrow.output_unsupported_types_as_binary) { - encodeAsBinary(column, num_rows, null_map_column); + encodeAsBinary(column, begin, end, null_map_column); return; } throw Exception( @@ -512,8 +559,9 @@ void RecordBatchEncoder::encodeValues( } } -void RecordBatchEncoder::encodeAsBinary(const IColumn & column, size_t num_rows, const IColumn * null_map_column) +void RecordBatchEncoder::encodeAsBinary(const IColumn & column, size_t begin, size_t end, const IColumn * null_map_column) { + const size_t num_rows = end - begin; const NullMap * null_map = null_map_column ? &assert_cast(*null_map_column).getData() : nullptr; PODArray arrow_offsets(num_rows + 1); @@ -524,12 +572,12 @@ void RecordBatchEncoder::encodeAsBinary(const IColumn & column, size_t num_rows, { /// Skip the payload of a logically-NULL row (matching the Apache Arrow writer's `AppendNull`): /// emit a zero-length slot instead of the arbitrary bytes the null row may carry. - if (!(null_map && (*null_map)[i])) + if (!(null_map && (*null_map)[begin + i])) { - const std::string_view value = column.getDataAt(i); + const std::string_view value = column.getDataAt(begin + i); total += value.size(); - if (total > static_cast(std::numeric_limits::max())) - throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Arrow IPC binary offset exceeds 32 bits"); + if (total > MAX_ARROW_BUFFER_SIZE) + throwValueDoesNotFitArrowBuffer(static_cast(total)); data.insert(data.end(), value.data(), value.data() + value.size()); } arrow_offsets[i + 1] = static_cast(total); @@ -538,12 +586,14 @@ void RecordBatchEncoder::encodeAsBinary(const IColumn & column, size_t num_rows, appendBuffer(data.data(), data.size()); } -void RecordBatchEncoder::encodeField(const IColumn & column, const DataTypePtr & type, size_t num_rows) +void RecordBatchEncoder::encodeField(const IColumn & column, const DataTypePtr & type, size_t begin, size_t end) { + /// Materializing only the range keeps a split batch from expanding the whole chunk. `cut` is cheap for + /// these representations: it slices indexes or resizes a constant, it does not copy values. if (isColumnConst(column)) { - auto full = column.convertToFullColumnIfConst(); - encodeField(*full, type, num_rows); + auto full = column.cut(begin, end - begin)->convertToFullColumnIfConst(); + encodeField(*full, type, 0, end - begin); return; } @@ -554,23 +604,23 @@ void RecordBatchEncoder::encodeField(const IColumn & column, const DataTypePtr & /// library rather than a fix for an observed failure. if (column.isReplicated()) { - auto full = column.convertToFullColumnIfReplicated(); - encodeField(*full, type, num_rows); + auto full = column.cut(begin, end - begin)->convertToFullColumnIfReplicated(); + encodeField(*full, type, 0, end - begin); return; } if (type->lowCardinality()) { /// Materialize LowCardinality to its full column (matches output_format_arrow_low_cardinality_as_dictionary=false). - auto full = column.convertToFullColumnIfLowCardinality(); - encodeField(*full, removeLowCardinality(type), num_rows); + auto full = column.cut(begin, end - begin)->convertToFullColumnIfLowCardinality(); + encodeField(*full, removeLowCardinality(type), 0, end - begin); return; } /// Variant maps to an Arrow dense union, which (unlike every other type) has no validity buffer. if (isVariant(type)) { - encodeVariant(column, type, num_rows); + encodeVariant(column, type, begin, end); return; } @@ -586,12 +636,12 @@ void RecordBatchEncoder::encodeField(const IColumn & column, const DataTypePtr & } /// `appendValidity` packs the validity buffer and returns the null count in one pass over the null map. - const Int64 null_count = appendValidity(null_map_column, num_rows); - nodes.emplace_back(static_cast(num_rows), null_count); - encodeValues(*nested, nested_type, num_rows, null_map_column); + const Int64 null_count = appendValidity(null_map_column, begin, end); + nodes.emplace_back(static_cast(end - begin), null_count); + encodeValues(*nested, nested_type, begin, end, null_map_column); } -void RecordBatchEncoder::encodeVariant(const IColumn & column, const DataTypePtr & type, size_t num_rows) +void RecordBatchEncoder::encodeVariant(const IColumn & column, const DataTypePtr & type, size_t begin, size_t end) { const auto & variant_column = assert_cast(column); const auto & variant_type = assert_cast(*type); @@ -602,56 +652,83 @@ void RecordBatchEncoder::encodeVariant(const IColumn & column, const DataTypePtr throw Exception( ErrorCodes::NOT_IMPLEMENTED, "Native Arrow IPC writer does not support a Variant with {} alternatives (maximum is 127)", num_variants); + const size_t num_rows = end - begin; const auto & local_discriminators = variant_column.getLocalDiscriminators(); const auto & ch_offsets = variant_column.getOffsets(); + /// Each alternative's rows are contiguous within a contiguous row range (see + /// `ColumnVariant::updateHashWithValueRange`), so the range maps to one contiguous child range per + /// alternative, and the union offsets are rebased against that child's start. + const size_t num_local = variant_column.getNumVariants(); + std::vector nested_begin(num_local, 0); + std::vector nested_rows(num_local, 0); + for (size_t row = begin; row != end; ++row) + { + const auto local = local_discriminators[row]; + if (local == ColumnVariant::NULL_DISCRIMINATOR) + continue; + if (nested_rows[local] == 0) + nested_begin[local] = ch_offsets[row]; + ++nested_rows[local]; + } + /// The union node itself: no nulls are reported here (NULL rows use the dedicated null child). nodes.emplace_back(static_cast(num_rows), 0); PODArray type_ids(num_rows); PODArray value_offsets(num_rows); - for (size_t row = 0; row < num_rows; ++row) + for (size_t i = 0; i < num_rows; ++i) { - const auto local = local_discriminators[row]; + const auto local = local_discriminators[begin + i]; if (local == ColumnVariant::NULL_DISCRIMINATOR) { - type_ids[row] = static_cast(num_variants); - value_offsets[row] = 0; + type_ids[i] = static_cast(num_variants); + value_offsets[i] = 0; continue; } - type_ids[row] = static_cast(variant_column.globalDiscriminatorByLocal(local)); - const UInt64 off = ch_offsets[row]; - if (off > static_cast(std::numeric_limits::max())) + type_ids[i] = static_cast(variant_column.globalDiscriminatorByLocal(local)); + const UInt64 off = ch_offsets[begin + i] - nested_begin[local]; + if (off > MAX_ARROW_BUFFER_SIZE) throw Exception(ErrorCodes::TOO_LARGE_ARRAY_SIZE, "Arrow IPC dense union offset exceeds 32 bits"); - value_offsets[row] = static_cast(off); + value_offsets[i] = static_cast(off); } appendBuffer(type_ids.data(), num_rows * sizeof(Int8)); appendBuffer(value_offsets.data(), num_rows * sizeof(Int32)); /// Children in global discriminator order, then a trailing single-element null child for NULL rows. - for (size_t i = 0; i < num_variants; ++i) + for (size_t global_discr = 0; global_discr < num_variants; ++global_discr) { - const IColumn & child = variant_column.getVariantByGlobalDiscriminator(i); - encodeField(child, variant_type.getVariant(i), child.size()); + const auto local = variant_column.localDiscriminatorByGlobal(static_cast(global_discr)); + encodeField( + variant_column.getVariantByGlobalDiscriminator(global_discr), + variant_type.getVariant(global_discr), + nested_begin[local], + nested_begin[local] + nested_rows[local]); } nodes.emplace_back(1, 1); } -RecordBatchEncoder::EncodedBatch RecordBatchEncoder::encode(const Columns & columns, const DataTypes & types, size_t num_rows) +RecordBatchEncoder::EncodedBatch RecordBatchEncoder::encode( + const Columns & columns, const DataTypes & types, size_t begin, size_t end) { nodes.clear(); buffers.clear(); body.clear(); + const size_t num_rows = end - begin; + /// Reserve the body up front so `appendBuffer` does not repeatedly reallocate and copy as it grows. - /// The column byte sizes approximate the Arrow body size closely enough to serve as a capacity hint. + /// The column byte sizes approximate the Arrow body size closely enough to serve as a capacity hint, + /// scaled to the share of the chunk this batch covers. size_t body_hint = 0; for (const auto & column : columns) body_hint += column->byteSize(); + if (const size_t total_rows = columns.empty() ? 0 : columns.front()->size(); total_rows > num_rows) + body_hint = body_hint / total_rows * num_rows; body.reserve(body_hint); for (size_t i = 0; i < columns.size(); ++i) - encodeField(*columns[i], types[i], num_rows); + encodeField(*columns[i], types[i], begin, end); /// Pad the body so its total length is a multiple of 8. while (body.size() % 8 != 0) diff --git a/src/Processors/Formats/Impl/ArrowIPC/RecordBatchEncoder.h b/src/Processors/Formats/Impl/ArrowIPC/RecordBatchEncoder.h index b5f39701bc740..02173ca4a08dc 100644 --- a/src/Processors/Formats/Impl/ArrowIPC/RecordBatchEncoder.h +++ b/src/Processors/Formats/Impl/ArrowIPC/RecordBatchEncoder.h @@ -36,31 +36,35 @@ class RecordBatchEncoder std::optional codec; }; - EncodedBatch encode(const Columns & columns, const DataTypes & types, size_t num_rows); + /// Encodes rows [begin, end) of `columns`. A range rather than a whole column, so that a chunk too + /// large for one record batch is written as several without copying a slice of it per batch. + EncodedBatch encode(const Columns & columns, const DataTypes & types, size_t begin, size_t end); /// Appends a buffer to the body (8-byte aligned start) and records its {offset, length}. void appendBuffer(const void * data, size_t length); private: - void encodeField(const IColumn & column, const DataTypePtr & type, size_t num_rows); + void encodeField(const IColumn & column, const DataTypePtr & type, size_t begin, size_t end); /// `null_map_column` (the outer `ColumnNullable`'s null map, or null for a non-nullable column) lets /// leaf encoders skip work for null rows — e.g. `DateTime64` rescaling, which would otherwise overflow /// on the arbitrary value a null row may carry. - void encodeValues(const IColumn & column, const DataTypePtr & type, size_t num_rows, const IColumn * null_map_column = nullptr); + void encodeValues( + const IColumn & column, const DataTypePtr & type, size_t begin, size_t end, const IColumn * null_map_column = nullptr); /// Encodes a Variant column as an Arrow dense union (no validity buffer; a types and an offsets /// buffer, the variant children in global order, and a trailing single-element null child). - void encodeVariant(const IColumn & column, const DataTypePtr & type, size_t num_rows); + void encodeVariant(const IColumn & column, const DataTypePtr & type, size_t begin, size_t end); /// Writes a column with no first-class Arrow mapping as an Arrow `Binary` column (an int32 offsets /// buffer and the concatenated per-row `getDataAt` bytes), matching the Apache Arrow library writer's /// `output_format_arrow_unsupported_types_as_binary` fallback. Read back as `String`. `null_map_column` /// (when set) marks rows to emit as zero-length, so a NULL row's nested bytes are not written. - void encodeAsBinary(const IColumn & column, size_t num_rows, const IColumn * null_map_column = nullptr); + void encodeAsBinary(const IColumn & column, size_t begin, size_t end, const IColumn * null_map_column = nullptr); void appendEmptyBuffer(); /// Emits the validity buffer: a packed LSB-first bitmap (1 = valid) for nullable columns, or an /// empty buffer otherwise. Returns the null count. - Int64 appendValidity(const IColumn * null_map_column, size_t num_rows); - void appendOffsets(const IColumn::Offsets & ch_offsets, size_t num_rows); + Int64 appendValidity(const IColumn * null_map_column, size_t begin, size_t end); + /// Emits a `List`/`Map` offsets buffer, rebased so that the range's first row starts at 0. + void appendOffsets(const IColumn::Offsets & ch_offsets, size_t begin, size_t end); const FormatSettings & settings; VectorWithMemoryTracking nodes; diff --git a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp index 7b4c6afd92b31..1c3fe5395d67f 100644 --- a/src/Processors/Formats/Impl/AvroRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/AvroRowInputFormat.cpp @@ -1419,14 +1419,14 @@ NamesAndTypesList AvroSchemaReader::readSchema() DataTypePtr AvroSchemaReader::avroNodeToDataType(avro::NodePtr node, bool allow_nullable_tuple_type) { - checkStackSize(); - std::unordered_set seen_names; return avroNodeToDataTypeImpl(node, seen_names, allow_nullable_tuple_type); } DataTypePtr AvroSchemaReader::avroNodeToDataTypeImpl(const avro::NodePtr & node, std::unordered_set & seen_names, bool allow_nullable_tuple_type) { + checkStackSize(); + switch (node->type()) { case avro::Type::AVRO_INT: diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp index f5ea7197025ce..d14d65f2231a6 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.cpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include @@ -47,6 +48,7 @@ #include #include +#include #include #define FOR_INTERNAL_NUMERIC_TYPES(M) \ @@ -89,6 +91,7 @@ namespace DB extern const int ILLEGAL_COLUMN; extern const int UNKNOWN_TYPE; extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE; + extern const int TOO_LARGE_ARRAY_SIZE; } class ArrowUUIDExtensionType : public arrow::ExtensionType @@ -182,6 +185,18 @@ namespace DB return bitmap; } + /// Arrow's `Utf8`/`Binary` builders take an int32 length and their `Append` skips the copy when it is + /// negative, so a narrowed size would store an empty value instead of failing. + static void checkStringSizeFitsArrowOffsets(size_t size, const String & column_name, const String & format_name) + { + if (size > MAX_ARROW_BUFFER_SIZE) + throw Exception( + ErrorCodes::TOO_LARGE_ARRAY_SIZE, + "Cannot write a value of {} bytes of column {} to {}: `Utf8`/`Binary` offsets are 32-bit, so a single " + "value cannot be larger than {} bytes", + size, column_name, format_name, MAX_ARROW_BUFFER_SIZE); + } + static void fillArrowArrayWithRawColumnData( ColumnPtr write_column, const PaddedPODArray * null_bytemap, @@ -198,7 +213,11 @@ namespace DB if (null_bytemap && (*null_bytemap)[value_i]) status = builder.AppendNull(); else - status = builder.Append(write_column->getDataAt(value_i)); + { + const std::string_view value = write_column->getDataAt(value_i); + checkStringSizeFitsArrowOffsets(value.size(), write_column->getName(), format_name); + status = builder.Append(value); + } checkStatus(status, write_column->getName(), format_name); } } @@ -842,7 +861,9 @@ namespace DB const auto column_offsets = assert_cast(column_array->getOffsetsColumn()).getPtr(); size_t offsets_start = start > 0 ? start - 1 : 0; size_t offsets_view_start = start > 0 ? 1 : 0; - auto offsets = extractIndexes(column_offsets, offsets_start, end, false); + /// Keep the absolute offsets 64-bit: only the value rebased against `values_start` has to fit an + /// Arrow `List` offset, and for a range that does not start at row 0 the absolute one can be larger. + auto offsets = extractIndexes(column_offsets, offsets_start, end, false); size_t values_start = start == 0 ? 0 : offsets[0]; size_t values_end = offsets.empty() ? values_start : offsets.back(); @@ -856,7 +877,14 @@ namespace DB checkStatus(status, column_name, format_name); for (size_t i = offsets_view_start; i < offsets.size(); ++i) { - status = offsets_builder.Append(static_cast(offsets[i] - values_start)); + const Int64 rebased = offsets[i] - static_cast(values_start); + if (static_cast(rebased) > MAX_ARROW_BUFFER_SIZE) + throw Exception( + ErrorCodes::TOO_LARGE_ARRAY_SIZE, + "Cannot write a row of {} elements of column '{}' to {}: `List` offsets are 32-bit, so a " + "single row cannot hold more than {} elements", + rebased, column_name, format_name, MAX_ARROW_BUFFER_SIZE); + status = offsets_builder.Append(static_cast(rebased)); checkStatus(status, column_name, format_name); } @@ -1025,6 +1053,7 @@ namespace DB else { std::string_view string_ref = internal_column.getDataAt(string_i); + checkStringSizeFitsArrowOffsets(string_ref.size(), write_column->getName(), format_name); status = builder.Append(string_ref.data(), static_cast(string_ref.size())); } checkStatus(status, write_column->getName(), format_name); @@ -1035,6 +1064,7 @@ namespace DB for (size_t string_i = start; string_i < end; ++string_i) { std::string_view string_ref = internal_column.getDataAt(string_i); + checkStringSizeFitsArrowOffsets(string_ref.size(), write_column->getName(), format_name); status = builder.Append(string_ref.data(), static_cast(string_ref.size())); checkStatus(status, write_column->getName(), format_name); } @@ -1868,47 +1898,76 @@ namespace DB for (const auto & chunk : chunks) { - /// For arrow::Table creation + /// Normalize once per chunk: `fillArrowArray` converts a constant or replicated column on every + /// call, which would otherwise repeat for each row range below. + Columns columns(columns_num); + DataTypes column_types(columns_num); for (size_t column_i = 0; column_i < columns_num; ++column_i) { - const ColumnWithTypeAndName & header_column = header_columns[column_i]; - auto column_type = header_column.type; - auto column = chunk.getColumns()[column_i]; - + auto column = chunk.getColumns()[column_i]->convertToFullColumnIfConst()->convertToFullColumnIfReplicated(); + auto column_type = header_columns[column_i].type; if (!settings.low_cardinality_as_dictionary) { column = recursiveRemoveLowCardinality(column); column_type = recursiveRemoveLowCardinality(column_type); } + columns[column_i] = std::move(column); + column_types[column_i] = std::move(column_type); + } - // Generate the unwrapped builder schema (safe for MakeBuilder) - bool is_column_nullable = false; - auto builder_type = getArrowType( - column_type, column, header_column.name, format_name, settings, &is_column_nullable, true /* for_builder */); - - std::unique_ptr array_builder; - arrow::Status status = MakeBuilder(ArrowMemoryPool::instance(), builder_type, &array_builder); - checkStatus(status, column->getName(), format_name); + /// A chunk that exceeds the 32-bit Arrow offsets becomes several arrays of the ChunkedArray. + const size_t num_rows = chunk.getNumRows(); + size_t offset = 0; + do + { + size_t rows = maxRowsFittingOneArrowBatch( + columns, + column_types, + offset, + num_rows, + settings.output_fixed_string_as_fixed_byte_array, + settings.output_max_record_batch_bytes); + /// A row that does not fit on its own cannot be represented at all; the leaf encoders reject it. + rows = std::min(std::max(rows, 1), num_rows - offset); + + /// For arrow::Table creation + for (size_t column_i = 0; column_i < columns_num; ++column_i) + { + const ColumnWithTypeAndName & header_column = header_columns[column_i]; + const ColumnPtr & column = columns[column_i]; + const DataTypePtr & column_type = column_types[column_i]; + + // Generate the unwrapped builder schema (safe for MakeBuilder) + bool is_column_nullable = false; + auto builder_type = getArrowType( + column_type, column, header_column.name, format_name, settings, &is_column_nullable, true /* for_builder */); + + std::unique_ptr array_builder; + arrow::Status status = MakeBuilder(ArrowMemoryPool::instance(), builder_type, &array_builder); + checkStatus(status, column->getName(), format_name); + + std::shared_ptr arrow_array = fillArrowArray( + header_column.name, + column, + column_type, + nullptr, + array_builder.get(), + format_name, + offset, + offset + rows, + settings, + dictionary_values); - std::shared_ptr arrow_array = fillArrowArray( - header_column.name, - column, - column_type, - nullptr, - array_builder.get(), - format_name, - 0, - column->size(), - settings, - dictionary_values); + // Zero-copy cast to the extension-rich schema (handles infinite nesting) + auto target_type = schema->field(static_cast(column_i))->type(); + if (!arrow_array->type()->Equals(*target_type)) + arrow_array = checkResult(arrow_array->View(target_type), column->getName(), format_name); - // Zero-copy cast to the extension-rich schema (handles infinite nesting) - auto target_type = schema->field(static_cast(column_i))->type(); - if (!arrow_array->type()->Equals(*target_type)) - arrow_array = checkResult(arrow_array->View(target_type), column->getName(), format_name); + table_data.at(column_i).emplace_back(std::move(arrow_array)); + } - table_data.at(column_i).emplace_back(std::move(arrow_array)); - } + offset += rows; + } while (offset < num_rows); } std::vector> columns; diff --git a/src/Processors/Formats/Impl/CHColumnToArrowColumn.h b/src/Processors/Formats/Impl/CHColumnToArrowColumn.h index d3a3094e021b6..17cb5d82202da 100644 --- a/src/Processors/Formats/Impl/CHColumnToArrowColumn.h +++ b/src/Processors/Formats/Impl/CHColumnToArrowColumn.h @@ -4,6 +4,7 @@ #if USE_ARROW || USE_PARQUET #include +#include #include #include @@ -35,6 +36,8 @@ class CHColumnToArrowColumn bool output_date_as_uint16 = false; /// Output types having no conversion as raw binary data. If false - such types would raise UNKNOWN_TYPE exception. bool output_unsupported_types_as_binary = false; + /// Upper bound on one record batch; a chunk holding more is written as several. + UInt64 output_max_record_batch_bytes = MAX_ARROW_BUFFER_SIZE; }; static std::shared_ptr calculateArrowSchema( diff --git a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp index 8e630effe06df..6d1dd7232513e 100644 --- a/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/MsgPackRowInputFormat.cpp @@ -61,7 +61,7 @@ MsgPackRowInputFormat::MsgPackRowInputFormat(SharedHeader header_, std::unique_p void MsgPackRowInputFormat::resetParser() { IRowInputFormat::resetParser(); - visitor.reset(); + parser.init(); } void MsgPackRowInputFormat::setReadBuffer(ReadBuffer & in_) @@ -85,7 +85,7 @@ void MsgPackVisitor::set_info(IColumn & column, DataTypePtr type, UInt8 & read) info_stack.push(Info{column, type, false, std::nullopt, &read}); } -void MsgPackVisitor::reset() +void MsgPackVisitor::init() { info_stack = {}; } diff --git a/src/Processors/Formats/Impl/MsgPackRowInputFormat.h b/src/Processors/Formats/Impl/MsgPackRowInputFormat.h index 386ced3098ecc..3ed9fd4c3e513 100644 --- a/src/Processors/Formats/Impl/MsgPackRowInputFormat.h +++ b/src/Processors/Formats/Impl/MsgPackRowInputFormat.h @@ -52,7 +52,8 @@ class MsgPackVisitor : public msgpack::null_visitor /// Update info_stack void set_info(IColumn & column, DataTypePtr type, UInt8 & read); - void reset(); + /// Called by msgpack's parse_helper::init. + void init(); private: /// Stack is needed to process arrays and maps diff --git a/src/Processors/Formats/Impl/MySQLDumpRowInputFormat.cpp b/src/Processors/Formats/Impl/MySQLDumpRowInputFormat.cpp index 0c35e80bc0910..c60ec17480d7f 100644 --- a/src/Processors/Formats/Impl/MySQLDumpRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/MySQLDumpRowInputFormat.cpp @@ -377,8 +377,12 @@ static bool tryToExtractStructureFromCreateQuery(ReadBuffer & in, NamesAndTypesL if (!create_query) return false; + /// `CREATE TABLE ... LIKE` has no column list, and a column list with only indexes or constraints has no columns. + if (!create_query->columns_list) + return false; + const auto * create_defines = create_query->columns_list->as(); - if (!create_defines) + if (!create_defines || !create_defines->columns) return false; structure = getColumnsList(create_defines->columns); @@ -431,9 +435,10 @@ static void readFirstCreateAndInsertQueries(ReadBuffer & in, String & table_name if (type == MySQLQueryType::CREATE) { /// If we have CREATE query, we can extract columns names and types from it. + /// `tryToExtractStructureFromCreateQuery` reads the CREATE query up to and including its `;`, + /// so the next query must not be skipped: it can be the first INSERT query. if (tryToExtractStructureFromCreateQuery(in, structure_from_create)) column_names = structure_from_create.getNames(); - skipQuery(in); insert_query_present = skipToInsertQuery(table_name, in); } diff --git a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp index aa7115021e73a..21be2e8ba3859 100644 --- a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.cpp @@ -3,6 +3,7 @@ #if USE_ORC #include +#include #include #include #include @@ -47,8 +48,6 @@ #include #include #include -#include -#include #include #include #include @@ -57,7 +56,6 @@ #include #include -#include #include @@ -92,11 +90,6 @@ UInt64 ORCInputStream::getLength() const return file_size; } -UInt64 ORCInputStream::getNaturalReadSize() const -{ - return 128 * 1024; -} - void ORCInputStream::read(void * buf, UInt64 length, UInt64 offset) { if (use_offset_based_read) @@ -152,7 +145,7 @@ std::future ORCInputStream::readAsync(void * buf, uint64_t length, uint64_ } } -std::unique_ptr +std::unique_ptr asORCInputStream(ReadBuffer & in, const FormatSettings & settings, bool use_prefetch, std::atomic & is_cancelled) { bool has_file_size = isBufferWithFileSize(in); @@ -165,14 +158,15 @@ asORCInputStream(ReadBuffer & in, const FormatSettings & settings, bool use_pref return asORCInputStreamLoadIntoMemory(in, is_cancelled); } -std::unique_ptr asORCInputStreamLoadIntoMemory(ReadBuffer & in, std::atomic & is_cancelled) +std::unique_ptr asORCInputStreamLoadIntoMemory(ReadBuffer & in, std::atomic & is_cancelled) { - size_t magic_size = strlen(ORC_MAGIC_BYTES); + static constexpr std::string_view orc_magic_bytes = "ORC"; + size_t magic_size = orc_magic_bytes.size(); std::string file_data(magic_size, '\0'); /// Avoid loading the whole file if it doesn't seem to even be in the correct format. size_t bytes_read = in.read(file_data.data(), magic_size); - if (bytes_read < magic_size || file_data != ORC_MAGIC_BYTES) + if (bytes_read < magic_size || file_data != orc_magic_bytes) throw Exception(ErrorCodes::INCORRECT_DATA, "Not an ORC file"); { @@ -186,25 +180,8 @@ std::unique_ptr asORCInputStreamLoadIntoMemory(ReadBuffer & in /// Defined below, next to the read path that also uses it. The sargs path resolves predicate column /// names through the same recursive walk, so pruning and reading always agree on the column. -static const orc::Type * -traverseDownORCTypeByName(const std::string & target, const orc::Type * orc_type, DataTypePtr & type, bool ignore_case); - -/// The default ORC memory pool allocates with `std::malloc`, which returns a null pointer when it -/// fails - and the library dereferences it - and which is invisible to the memory tracker. A -/// malformed file can ask for an arbitrarily large buffer, so allocate through `operator new` -/// instead: it is accounted for and it throws instead of returning null. -class ORCMemoryPool : public orc::MemoryPool -{ -public: - char * malloc(uint64_t size) override { return new char[size]; } - void free(char * p) override { delete[] p; } -}; - -orc::MemoryPool & getORCMemoryPool() -{ - static ORCMemoryPool pool; - return pool; -} +static const ORC::Type * +traverseDownORCTypeByName(const std::string & target, const ORC::Type * orc_type, DataTypePtr & type, bool ignore_case); /// Resolves the CH type of `name` against `header`, following dots into tuple elements when the /// header carries only the parent column (which is the case for ORC, whose reader is not asked for @@ -298,28 +275,36 @@ static Block buildORCKeyConditionBlock(const Block & header, const ActionsDAG * return keys; } -static bool isDictionaryEncoded(const orc::StripeInformation * stripe_info, const orc::Type * orc_type) +static bool isDictionaryEncoded(const ORC::Proto::StripeFooter * stripe_footer, const ORC::Type * orc_type) { - if (!stripe_info) + if (!stripe_footer) return false; - auto encoding = stripe_info->getColumnEncoding(orc_type->getColumnId()); - return encoding == orc::ColumnEncodingKind_DICTIONARY || encoding == orc::ColumnEncodingKind_DICTIONARY_V2; + UInt64 column_id = orc_type->getColumnId(); + if (column_id >= stripe_footer->columns.size()) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "Failed to read ORC schema: the first stripe has no encoding for column {}, it has {} column encodings", + column_id, + stripe_footer->columns.size()); + + auto encoding = stripe_footer->columns[column_id].kind; + return encoding == ORC::Proto::ColumnEncodingKind::DICTIONARY || encoding == ORC::Proto::ColumnEncodingKind::DICTIONARY_V2; } static DataTypePtr parseORCType( - const orc::Type * orc_type, + const ORC::Type * orc_type, bool skip_columns_with_unsupported_types, bool dictionary_as_low_cardinality, - const orc::StripeInformation * stripe_info, + const ORC::Proto::StripeFooter * stripe_info, bool & skipped, size_t max_depth = DBMS_DEFAULT_MAX_PARSER_DEPTH, size_t depth = 0) { chassert(orc_type != nullptr); - /// ORC LIST/MAP/STRUCT types can be nested arbitrarily deep and the ORC library does not bound - /// the nesting, so reject deep nesting early (before building the type) with an explicit limit. + /// ORC LIST/MAP/STRUCT types can be nested deep (the ORC reader allows up to 2000 levels) + /// so reject deep nesting early (before building the type) with an explicit limit. /// This keeps schema inference cheap and interruptible instead of recursing over (and later /// walking) a pathologically deep type. checkStackSize is a last-resort backstop. /// max_depth == 0 means unlimited (matching the SQL parser), leaving only checkStackSize. @@ -339,7 +324,7 @@ static DataTypePtr parseORCType( /// branches with identical types cannot be represented as a Variant; reject them explicitly /// instead of silently squashing them. A branch that is itself a union is rejected too: it /// would map to a Variant, which Variant does not allow to nest. - if (orc_type->getKind() == orc::TypeKind::UNION) + if (orc_type->getKind() == ORC::TypeKind::UNION) { /// A union with more branches than Variant can hold (ColumnVariant::MAX_NESTED_COLUMNS) /// is valid ORC but not representable; reject it through the normal unsupported-type / @@ -370,7 +355,7 @@ static DataTypePtr parseORCType( /// here, before the outer DataTypeVariant is constructed - otherwise its constructor /// throws a confusing BAD_ARGUMENTS ("Nested Variant types are not allowed") and the /// skip setting is never consulted because the inner union has already been parsed. - if (subtype->getKind() == orc::TypeKind::UNION) + if (subtype->getKind() == ORC::TypeKind::UNION) { if (skip_columns_with_unsupported_types) { @@ -411,32 +396,32 @@ static DataTypePtr parseORCType( switch (orc_type->getKind()) { - case orc::TypeKind::BOOLEAN: + case ORC::TypeKind::BOOLEAN: return DataTypeFactory::instance().get("Bool"); - case orc::TypeKind::BYTE: + case ORC::TypeKind::BYTE: return std::make_shared(); - case orc::TypeKind::SHORT: + case ORC::TypeKind::SHORT: return std::make_shared(); - case orc::TypeKind::INT: + case ORC::TypeKind::INT: return std::make_shared(); - case orc::TypeKind::LONG: + case ORC::TypeKind::LONG: return std::make_shared(); - case orc::TypeKind::FLOAT: + case ORC::TypeKind::FLOAT: return std::make_shared(); - case orc::TypeKind::DOUBLE: + case ORC::TypeKind::DOUBLE: return std::make_shared(); - case orc::TypeKind::DATE: + case ORC::TypeKind::DATE: return std::make_shared(); - case orc::TypeKind::TIMESTAMP: + case ORC::TypeKind::TIMESTAMP: return std::make_shared(9); - case orc::TypeKind::TIMESTAMP_INSTANT: + case ORC::TypeKind::TIMESTAMP_INSTANT: return std::make_shared(9, "UTC"); - case orc::TypeKind::CHAR: - case orc::TypeKind::VARCHAR: - case orc::TypeKind::BINARY: - case orc::TypeKind::STRING: { + case ORC::TypeKind::CHAR: + case ORC::TypeKind::VARCHAR: + case ORC::TypeKind::BINARY: + case ORC::TypeKind::STRING: { DataTypePtr type; - if (orc_type->getKind() == orc::TypeKind::CHAR) + if (orc_type->getKind() == ORC::TypeKind::CHAR) type = std::make_shared(orc_type->getMaximumLength()); else type = std::make_shared(); @@ -447,7 +432,7 @@ static DataTypePtr parseORCType( return type; } - case orc::TypeKind::DECIMAL: { + case ORC::TypeKind::DECIMAL: { UInt64 precision = orc_type->getPrecision(); UInt64 scale = orc_type->getScale(); if (precision == 0) @@ -457,7 +442,7 @@ static DataTypePtr parseORCType( } return createDecimal(precision, scale); } - case orc::TypeKind::LIST: { + case ORC::TypeKind::LIST: { if (subtype_count != 1) throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid Orc List type {}", orc_type->toString()); @@ -468,7 +453,7 @@ static DataTypePtr parseORCType( return std::make_shared(nested_type); } - case orc::TypeKind::MAP: { + case ORC::TypeKind::MAP: { if (subtype_count != 2) throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid Orc Map type {}", orc_type->toString()); @@ -484,7 +469,7 @@ static DataTypePtr parseORCType( return std::make_shared(key_type, value_type); } - case orc::TypeKind::STRUCT: { + case ORC::TypeKind::STRUCT: { DataTypes nested_types; Strings nested_names; nested_types.reserve(subtype_count); @@ -519,30 +504,30 @@ static DataTypePtr parseORCType( } } -static std::optional convertORCTypeToPredicateType(const orc::Type & orc_type) +static std::optional convertORCTypeToPredicateType(const ORC::Type & orc_type) { switch (orc_type.getKind()) { - case orc::BOOLEAN: - return orc::PredicateDataType::BOOLEAN; - case orc::BYTE: - case orc::SHORT: - case orc::INT: - case orc::LONG: - return orc::PredicateDataType::LONG; - case orc::FLOAT: - case orc::DOUBLE: - return orc::PredicateDataType::FLOAT; - case orc::VARCHAR: - case orc::CHAR: - case orc::STRING: - return orc::PredicateDataType::STRING; - case orc::DATE: - return orc::PredicateDataType::DATE; - case orc::TIMESTAMP: - return orc::PredicateDataType::TIMESTAMP; - case orc::DECIMAL: - return orc::PredicateDataType::DECIMAL; + case ORC::BOOLEAN: + return ORC::PredicateDataType::BOOLEAN; + case ORC::BYTE: + case ORC::SHORT: + case ORC::INT: + case ORC::LONG: + return ORC::PredicateDataType::LONG; + case ORC::FLOAT: + case ORC::DOUBLE: + return ORC::PredicateDataType::FLOAT; + case ORC::VARCHAR: + case ORC::CHAR: + case ORC::STRING: + return ORC::PredicateDataType::STRING; + case ORC::DATE: + return ORC::PredicateDataType::DATE; + case ORC::TIMESTAMP: + return ORC::PredicateDataType::TIMESTAMP; + case ORC::DECIMAL: + return ORC::PredicateDataType::DECIMAL; default: return {}; } @@ -559,23 +544,23 @@ static String getColumnNameFromKeyCondition(const KeyCondition & key_condition, throw Exception(ErrorCodes::LOGICAL_ERROR, "Can't get column from KeyCondition with indice {}", indice); } -static std::optional -convertFieldToORCLiteral(const orc::Type & orc_type, const Field & field, DataTypePtr type_hint = nullptr) +static std::optional +convertFieldToORCLiteral(const ORC::Type & orc_type, const Field & field, DataTypePtr type_hint = nullptr) { try { /// We always fallback to return null if possible CH type hint not consistent with ORC type switch (orc_type.getKind()) { - case orc::BOOLEAN: { + case ORC::BOOLEAN: { /// May throw exception auto val = field.safeGet(); - return orc::Literal(val != 0); + return ORC::Literal(val != 0); } - case orc::BYTE: - case orc::SHORT: - case orc::INT: - case orc::LONG: { + case ORC::BYTE: + case ORC::SHORT: + case ORC::INT: + case ORC::LONG: { /// May throw exception. /// /// In particular, it'll throw if we request the column as unsigned, like this: @@ -583,30 +568,30 @@ convertFieldToORCLiteral(const orc::Type & orc_type, const Field & field, DataTy /// We have to reject this, otherwise it would miss values > 127 (because /// they're treated as negative by ORC). auto val = field.safeGet(); - return orc::Literal(val); + return ORC::Literal(val); } - case orc::FLOAT: - case orc::DOUBLE: { + case ORC::FLOAT: + case ORC::DOUBLE: { Float64 val = 0; if (field.tryGet(val)) - return orc::Literal(val); + return ORC::Literal(val); break; } - case orc::VARCHAR: - case orc::CHAR: - case orc::STRING: { + case ORC::VARCHAR: + case ORC::CHAR: + case ORC::STRING: { String str; if (field.tryGet(str)) - return orc::Literal(str.data(), str.size()); + return ORC::Literal(str.data(), str.size()); break; } - case orc::DATE: { + case ORC::DATE: { Int64 val = 0; if (field.tryGet(val)) - return orc::Literal(orc::PredicateDataType::DATE, val); + return ORC::Literal(ORC::PredicateDataType::DATE, val); break; } - case orc::TIMESTAMP: { + case ORC::TIMESTAMP: { if (type_hint && isDateTime64(type_hint)) { const auto * datetime64_type = typeid_cast(type_hint.get()); @@ -615,15 +600,23 @@ convertFieldToORCLiteral(const orc::Type & orc_type, const Field & field, DataTy } DecimalField ts; - if (field.tryGet(ts)) + if (field.tryGet(ts) && ts.getScale() <= 9) { - Int64 secs = (ts.getValue() / ts.getScaleMultiplier()).convertTo(); - Int32 nanos = (ts.getValue() - (ts.getValue() / ts.getScaleMultiplier()) * ts.getScaleMultiplier()).convertTo(); - return orc::Literal(secs, nanos); + /// The nanoseconds are not negative, as in the reference implementation and in the statistics. + const Int64 value = ts.getValue().value; + const Int64 multiplier = ts.getScaleMultiplier().value; + Int64 secs = value / multiplier; + Int64 rest = value % multiplier; + if (rest < 0) + { + rest += multiplier; + --secs; + } + return ORC::Literal(secs, static_cast(rest * (1'000'000'000 / multiplier))); } break; } - case orc::DECIMAL: { + case ORC::DECIMAL: { auto precision = orc_type.getPrecision(); if (precision == 0) precision = 38; @@ -634,8 +627,8 @@ convertFieldToORCLiteral(const orc::Type & orc_type, const Field & field, DataTy if (field.tryGet(val)) { Int64 right = val.getValue().convertTo(); - return orc::Literal( - orc::Int128(right), static_cast(orc_type.getPrecision()), static_cast(orc_type.getScale())); + return ORC::Literal( + Int128(right), static_cast(orc_type.getPrecision()), static_cast(orc_type.getScale())); } } else if (precision <= DecimalUtils::max_precision) @@ -644,8 +637,8 @@ convertFieldToORCLiteral(const orc::Type & orc_type, const Field & field, DataTy if (field.tryGet(val)) { Int64 right = val.getValue().convertTo(); - return orc::Literal( - orc::Int128(right), static_cast(orc_type.getPrecision()), static_cast(orc_type.getScale())); + return ORC::Literal( + Int128(right), static_cast(orc_type.getPrecision()), static_cast(orc_type.getScale())); } } else if (precision <= DecimalUtils::max_precision) @@ -653,10 +646,8 @@ convertFieldToORCLiteral(const orc::Type & orc_type, const Field & field, DataTy DecimalField val; if (field.tryGet(val)) { - Int64 high = val.getValue().value.items[1]; - UInt64 low = static_cast(val.getValue().value.items[0]); - return orc::Literal( - orc::Int128(high, low), static_cast(orc_type.getPrecision()), static_cast(orc_type.getScale())); + return ORC::Literal( + val.getValue().value, static_cast(orc_type.getPrecision()), static_cast(orc_type.getScale())); } } break; @@ -727,9 +718,9 @@ static bool evaluateRPNElement(const Field & field, const KeyCondition::RPNEleme static void buildORCSearchArgumentImpl( const KeyCondition & key_condition, const Block & header, - const orc::Type & schema, + const ORC::Type & schema, KeyCondition::RPN & rpn_stack, - orc::SearchArgumentBuilder & builder, + ORC::SearchArgumentBuilder & builder, const FormatSettings & format_settings) { if (rpn_stack.empty()) @@ -761,7 +752,7 @@ static void buildORCSearchArgumentImpl( /// Key filter expressions like "func(col) > 100" are not supported for ORC filter push down if (!curr.monotonic_functions_chain.empty()) { - builder.literal(orc::TruthValue::YES_NO_NULL); + builder.literal(ORC::TruthValue::YES_NO_NULL); break; } @@ -772,7 +763,7 @@ static void buildORCSearchArgumentImpl( { if (!set_index || set_index->getOrderedSet().size() != 1 || set_index->hasMonotonicFunctionsChain()) { - builder.literal(orc::TruthValue::YES_NO_NULL); + builder.literal(ORC::TruthValue::YES_NO_NULL); break; } } @@ -785,7 +776,7 @@ static void buildORCSearchArgumentImpl( auto column_type = findHeaderTypeByPath(header, column_name, ignore_case); if (!column_type) { - builder.literal(orc::TruthValue::YES_NO_NULL); + builder.literal(ORC::TruthValue::YES_NO_NULL); break; } @@ -796,7 +787,7 @@ static void buildORCSearchArgumentImpl( const auto * orc_type = traverseDownORCTypeByName(column_name, &schema, resolved_type, ignore_case); if (!orc_type) { - builder.literal(orc::TruthValue::YES_NO_NULL); + builder.literal(ORC::TruthValue::YES_NO_NULL); break; } @@ -813,7 +804,7 @@ static void buildORCSearchArgumentImpl( auto expect_type = makeNullableRecursively(parseORCType(orc_type, true, false, nullptr, skipped, format_settings.max_parser_depth), format_settings); if (!expect_type) { - builder.literal(orc::TruthValue::YES_NO_NULL); + builder.literal(ORC::TruthValue::YES_NO_NULL); break; } @@ -821,7 +812,7 @@ static void buildORCSearchArgumentImpl( auto expect_nested_type = removeNullable(expect_type); if (!nested_type->equals(*expect_nested_type)) { - builder.literal(orc::TruthValue::YES_NO_NULL); + builder.literal(ORC::TruthValue::YES_NO_NULL); break; } @@ -833,7 +824,7 @@ static void buildORCSearchArgumentImpl( bool match_if_default = evaluateRPNElement(column_type->getDefault(), curr); if (match_if_default != match_if_null) { - builder.literal(orc::TruthValue::YES_NO_NULL); + builder.literal(ORC::TruthValue::YES_NO_NULL); break; } } @@ -841,7 +832,7 @@ static void buildORCSearchArgumentImpl( auto predicate_type = convertORCTypeToPredicateType(*orc_type); if (!predicate_type.has_value()) { - builder.literal(orc::TruthValue::YES_NO_NULL); + builder.literal(ORC::TruthValue::YES_NO_NULL); break; } @@ -859,8 +850,8 @@ static void buildORCSearchArgumentImpl( bool has_right_bound = !range.right.isPositiveInfinity(); if (!has_left_bound && !has_right_bound) { - /// Transform whole range orc::TruthValue::YES_NULL - builder.literal(orc::TruthValue::YES_NULL); + /// Transform whole range ORC::TruthValue::YES_NULL + builder.literal(ORC::TruthValue::YES_NULL); } else if (has_left_bound && has_right_bound && range.left_included && range.right_included && range.left == range.right) { @@ -869,15 +860,15 @@ static void buildORCSearchArgumentImpl( if (literal.has_value()) builder.equals(orc_type->getColumnId(), *predicate_type, *literal); else - builder.literal(orc::TruthValue::YES_NO_NULL); + builder.literal(ORC::TruthValue::YES_NO_NULL); } else { - std::optional left_literal; + std::optional left_literal; if (has_left_bound) left_literal = convertFieldToORCLiteral(*orc_type, range.left); - std::optional right_literal; + std::optional right_literal; if (has_right_bound) right_literal = convertFieldToORCLiteral(*orc_type, range.right); @@ -897,7 +888,7 @@ static void buildORCSearchArgumentImpl( builder.end(); } else - builder.literal(orc::TruthValue::YES_NO_NULL); + builder.literal(ORC::TruthValue::YES_NO_NULL); } if (has_right_bound) @@ -910,7 +901,7 @@ static void buildORCSearchArgumentImpl( builder.lessThan(orc_type->getColumnId(), *predicate_type, *right_literal); } else - builder.literal(orc::TruthValue::YES_NO_NULL); + builder.literal(ORC::TruthValue::YES_NO_NULL); } if (has_left_bound && has_right_bound) @@ -924,7 +915,7 @@ static void buildORCSearchArgumentImpl( const auto & set_column = ordered_set[0]; bool fail = false; - std::vector literals; + std::vector literals; literals.reserve(set_column->size()); for (size_t i = 0; i < set_column->size(); ++i) { @@ -940,9 +931,9 @@ static void buildORCSearchArgumentImpl( /// set has zero element if (literals.empty()) - builder.literal(orc::TruthValue::YES); + builder.literal(ORC::TruthValue::YES); else if (fail) - builder.literal(orc::TruthValue::YES_NO_NULL); + builder.literal(ORC::TruthValue::YES_NO_NULL); else builder.in(orc_type->getColumnId(), *predicate_type, literals); } @@ -958,7 +949,7 @@ static void buildORCSearchArgumentImpl( case KeyCondition::RPNElement::FUNCTION_POINT_IN_POLYGON: case KeyCondition::RPNElement::FUNCTION_UNKNOWN: { - builder.literal(orc::TruthValue::YES_NO_NULL); + builder.literal(ORC::TruthValue::YES_NO_NULL); rpn_stack.pop_back(); break; } @@ -990,35 +981,35 @@ static void buildORCSearchArgumentImpl( } case KeyCondition::RPNElement::ALWAYS_FALSE: { - builder.literal(orc::TruthValue::NO); + builder.literal(ORC::TruthValue::NO); rpn_stack.pop_back(); break; } case KeyCondition::RPNElement::ALWAYS_TRUE: { - builder.literal(orc::TruthValue::YES); + builder.literal(ORC::TruthValue::YES); rpn_stack.pop_back(); break; } } } -std::unique_ptr buildORCSearchArgument( - const KeyCondition & key_condition, const Block & header, const orc::Type & schema, const FormatSettings & format_settings) +std::unique_ptr buildORCSearchArgument( + const KeyCondition & key_condition, const Block & header, const ORC::Type & schema, const FormatSettings & format_settings) { auto rpn_stack = key_condition.getRPN(); if (rpn_stack.empty()) return nullptr; - auto builder = orc::SearchArgumentFactory::newBuilder(); + auto builder = std::make_unique(); buildORCSearchArgumentImpl(key_condition, header, schema, rpn_stack, *builder, format_settings); return builder->build(); } -std::unique_ptr createORCReader(std::unique_ptr stream, const orc::ReaderOptions & options) +std::unique_ptr createORCReader(std::unique_ptr stream, const ORC::ReaderOptions & options) { - auto reader = orc::createReader(std::move(stream), options); - if (reader->getType().getKind() != orc::STRUCT) + auto reader = std::make_unique(std::move(stream), options); + if (reader->getType().getKind() != ORC::STRUCT) throw Exception( ErrorCodes::INCORRECT_DATA, "ORC files whose root type is not a struct are not supported, the file has root type {}", @@ -1028,7 +1019,7 @@ std::unique_ptr createORCReader(std::unique_ptr s static void getFileReader( ReadBuffer & in, - std::unique_ptr & file_reader, + std::unique_ptr & file_reader, const FormatSettings & format_settings, bool use_prefetch, size_t min_bytes_for_seek, @@ -1037,21 +1028,18 @@ static void getFileReader( if (is_stopped) return; - orc::ReaderOptions options; - options.setMemoryPool(getORCMemoryPool()); - /// ORC library requires rangeSizeLimit > holeSizeLimit. - static constexpr uint64_t default_range_size_limit = 10 * 1024 * 1024UL; - /// Clamp to avoid overflow when computing holeSizeLimit + 1. - uint64_t hole_size_limit = std::min(min_bytes_for_seek, std::numeric_limits::max() - 1); - uint64_t range_size_limit = std::max(default_range_size_limit, hole_size_limit + 1); - options.setCacheOptions(orc::CacheOptions{.holeSizeLimit = hole_size_limit, .rangeSizeLimit = range_size_limit}); + /// The ranges with the gaps up to `min_bytes_for_seek` are read with one request. + static constexpr UInt64 default_range_size_limit = 10 * 1024 * 1024UL; + ORC::ReaderOptions options; + options.hole_size_limit = min_bytes_for_seek; + options.range_size_limit = std::max(default_range_size_limit, min_bytes_for_seek); auto input_stream = asORCInputStream(in, format_settings, use_prefetch, is_stopped); file_reader = createORCReader(std::move(input_stream), options); } -static const orc::Type * -traverseDownORCTypeByName(const std::string & target, const orc::Type * orc_type, DataTypePtr & type, bool ignore_case) +static const ORC::Type * +traverseDownORCTypeByName(const std::string & target, const ORC::Type * orc_type, DataTypePtr & type, bool ignore_case) { /// Recurses the file-controlled ORC type tree. The matching CH type (and the requested column /// name) bound the depth in practice, but keep a stack backstop here too, since this runs in @@ -1061,7 +1049,7 @@ traverseDownORCTypeByName(const std::string & target, const orc::Type * orc_type if (target.empty()) return orc_type; - auto search_struct_field = [&](const std::string & target_, const orc::Type * type_) -> std::pair + auto search_struct_field = [&](const std::string & target_, const ORC::Type * type_) -> std::pair { auto target_copy = target_; if (ignore_case) @@ -1081,19 +1069,19 @@ traverseDownORCTypeByName(const std::string & target, const orc::Type * orc_type return {"", nullptr}; }; - if (orc::STRUCT == orc_type->getKind()) + if (ORC::STRUCT == orc_type->getKind()) { const auto [next_target, next_orc_type] = search_struct_field(target, orc_type); return next_orc_type ? traverseDownORCTypeByName(next_target, next_orc_type, type, ignore_case) : nullptr; } - if (orc::LIST == orc_type->getKind()) + if (ORC::LIST == orc_type->getKind()) { /// For cases in which header contains subcolumns flattened from nested columns. /// For example, "a Nested(x String, y Int64)" is flattened to "a.x Array(String), a.y Array(Int64)", and ORC file schema is still "a array>". /// In this case, we should skip possible array type and traverse down to its nested struct type. const auto * array_type = typeid_cast(removeNullable(type).get()); const auto * orc_nested_type = orc_type->getSubtype(0); - if (array_type && orc::STRUCT == orc_nested_type->getKind()) + if (array_type && ORC::STRUCT == orc_nested_type->getKind()) { auto next_type_and_target = search_struct_field(target, orc_nested_type); const auto & next_target = next_type_and_target.first; @@ -1112,12 +1100,17 @@ traverseDownORCTypeByName(const std::string & target, const orc::Type * orc_type /// Forward declarations: updateIncludeTypeIds needs the union branch-hint machinery (defined /// below, near readColumnFromORCColumn, which is where it is also used) to prune struct-branch /// fields the same way the read path does. See computeOrcUnionBranchHints. -static bool orcUnionBranchMatchesType(const orc::Type * orc_branch_type, const DataTypePtr & target_type, bool case_insensitive); -static bool orcUnionBranchPrefersType(const orc::Type * orc_branch_type, const DataTypePtr & target_type); -static DataTypes computeOrcUnionBranchHints(const orc::Type * orc_type, const DataTypePtr & type_hint, bool case_insensitive_matching); - -static void -updateIncludeTypeIds(DataTypePtr type, const orc::Type * orc_type, bool ignore_case, std::unordered_set & include_typeids) +static bool orcUnionBranchMatchesType(const ORC::Type * orc_branch_type, const DataTypePtr & target_type, bool case_insensitive); +static bool orcUnionBranchPrefersType(const ORC::Type * orc_branch_type, const DataTypePtr & target_type); +static DataTypes computeOrcUnionBranchHints(const ORC::Type * orc_type, const DataTypePtr & type_hint, bool case_insensitive_matching); + +/// Also collects the ORC structs that are requested as a `Tuple`. +static void updateIncludeTypeIds( + DataTypePtr type, + const ORC::Type * orc_type, + bool ignore_case, + std::unordered_set & include_typeids, + std::vector & requested_structs) { /// Recurses the file-controlled ORC type tree in lockstep with the (parser-bounded) CH type. /// Keep a stack backstop here too: this runs in prepareFileReader, before the @@ -1134,30 +1127,33 @@ updateIncludeTypeIds(DataTypePtr type, const orc::Type * orc_type, bool ignore_c auto non_nullable_type = removeNullable(type); switch (orc_type->getKind()) { - case orc::LIST: { + case ORC::LIST: { const auto * array_type = typeid_cast(non_nullable_type.get()); if (array_type) { - updateIncludeTypeIds(array_type->getNestedType(), orc_type->getSubtype(0), ignore_case, include_typeids); + updateIncludeTypeIds(array_type->getNestedType(), orc_type->getSubtype(0), ignore_case, include_typeids, requested_structs); + return; } - return; + break; } - case orc::MAP: { + case ORC::MAP: { const auto * map_type = typeid_cast(non_nullable_type.get()); if (map_type) { - updateIncludeTypeIds(map_type->getKeyType(), orc_type->getSubtype(0), ignore_case, include_typeids); - updateIncludeTypeIds(map_type->getValueType(), orc_type->getSubtype(1), ignore_case, include_typeids); + updateIncludeTypeIds(map_type->getKeyType(), orc_type->getSubtype(0), ignore_case, include_typeids, requested_structs); + updateIncludeTypeIds(map_type->getValueType(), orc_type->getSubtype(1), ignore_case, include_typeids, requested_structs); + return; } - return; + break; } - case orc::STRUCT: { + case ORC::STRUCT: { /// To make sure tuple field pruning work fine, we should include only the fields of ORC struct type which are also contained in CH tuple types, instead of all fields of ORC struct type. /// For example, CH tupe type in header is "x Tuple(a String)", ORC struct type is "x struct", then only type id of field "x.a" should be included. /// For tuple field pruning purpose, we should never include "x.b" for it is not required in format header. const auto * tuple_type = typeid_cast(non_nullable_type.get()); if (tuple_type) { + requested_structs.push_back(orc_type); if (tuple_type->hasExplicitNames()) { std::unordered_map orc_field_name_to_index; @@ -1184,19 +1180,21 @@ updateIncludeTypeIds(DataTypePtr type, const orc::Type * orc_type, bool ignore_c tuple_type->getElement(tuple_i), orc_type->getSubtype(orc_field_name_to_index[element_name]), ignore_case, - include_typeids); + include_typeids, + requested_structs); } } } else { for (size_t i = 0; i < tuple_type->getElements().size() && i < orc_type->getSubtypeCount(); ++i) - updateIncludeTypeIds(tuple_type->getElement(i), orc_type->getSubtype(i), ignore_case, include_typeids); + updateIncludeTypeIds(tuple_type->getElement(i), orc_type->getSubtype(i), ignore_case, include_typeids, requested_structs); } + return; } - return; + break; } - case orc::UNION: { + case ORC::UNION: { /// ORC union maps to the ClickHouse Variant type. A branch that gets a forced type hint /// (the same way readColumnFromORCColumn computes it - see computeOrcUnionBranchHints) /// is selected by recursing into the hint, so a STRUCT branch keeps the named-tuple @@ -1212,7 +1210,7 @@ updateIncludeTypeIds(DataTypePtr type, const orc::Type * orc_type, bool ignore_c { const auto * branch_orc_type = orc_type->getSubtype(i); if (branch_hints[i]) - updateIncludeTypeIds(branch_hints[i], branch_orc_type, ignore_case, include_typeids); + updateIncludeTypeIds(branch_hints[i], branch_orc_type, ignore_case, include_typeids, requested_structs); else include_typeids.insert(branch_orc_type->getColumnId()); } @@ -1221,6 +1219,50 @@ updateIncludeTypeIds(DataTypePtr type, const orc::Type * orc_type, bool ignore_c default: return; } + + /// A requested type of another kind reads the whole node, and the cast to the requested type converts it or throws. + include_typeids.insert(orc_type->getColumnId()); +} + +/// Whether a column of the subtree of `orc_type`, or one of its ancestors (which reads the subtree whole), is included. +static bool isORCTypeRead( + const ORC::Type & root, const ORC::Type & orc_type, const std::unordered_set & include_typeids) +{ + for (UInt64 id = orc_type.getColumnId(); id <= orc_type.getMaximumColumnId(); ++id) + { + if (include_typeids.contains(id)) + return true; + } + + const ORC::Type * node = &root; + while (node && node->getColumnId() < orc_type.getColumnId()) + { + if (include_typeids.contains(node->getColumnId())) + return true; + + const ORC::Type * child = nullptr; + for (UInt64 i = 0; i < node->getSubtypeCount() && !child; ++i) + { + if (node->getSubtype(i)->getMaximumColumnId() >= orc_type.getColumnId()) + child = node->getSubtype(i); + } + node = child; + } + return false; +} + +/// The field of an ORC struct with the fewest columns, or the struct itself if it has no fields. +static const ORC::Type & getNarrowestORCStructField(const ORC::Type & orc_struct) +{ + const ORC::Type * narrowest = &orc_struct; + for (UInt64 i = 0; i < orc_struct.getSubtypeCount(); ++i) + { + const auto * field = orc_struct.getSubtype(i); + if (narrowest == &orc_struct + || field->getMaximumColumnId() - field->getColumnId() < narrowest->getMaximumColumnId() - narrowest->getColumnId()) + narrowest = field; + } + return *narrowest; } NativeORCBlockInputFormat::NativeORCBlockInputFormat( @@ -1252,31 +1294,44 @@ void NativeORCBlockInputFormat::prepareFileReader() format_filter_info->filter_actions_dag.get(), format_settings.orc.case_insensitive_column_matching)); - std::unique_ptr stripe_info; - if (file_reader->getNumberOfStripes()) - stripe_info = file_reader->getStripe(0); - - orc_column_to_ch_column = std::make_unique( - getPort().getHeader(), - format_settings.orc.allow_missing_columns, - format_settings.null_as_default, - format_settings.orc.case_insensitive_column_matching, - format_settings.orc.dictionary_as_low_cardinality, - format_settings.date_time_overflow_behavior); - const bool ignore_case = format_settings.orc.case_insensitive_column_matching; const auto & header = getPort().getHeader(); const auto & file_schema = file_reader->getType(); std::unordered_set include_typeids; + std::vector requested_structs; for (const auto & column : header) { auto adjusted_type = column.type; const auto * orc_type = traverseDownORCTypeByName(column.name, &file_schema, adjusted_type, ignore_case); if (orc_type) - updateIncludeTypeIds(adjusted_type, orc_type, ignore_case, include_typeids); + updateIncludeTypeIds(adjusted_type, orc_type, ignore_case, include_typeids, requested_structs); + } + + /// A requested struct none of whose requested fields is in the file is still read, through its narrowest field, + /// for its presence and the sizes of the lists and maps around it; `matchORCTupleElementsByName` then fills the + /// requested fields with defaults. Inner structs first, as a read inner one makes its parents read. + std::ranges::sort(requested_structs, std::greater{}, &ORC::Type::getColumnId); + std::unordered_set count_only_fields; + for (const auto * orc_struct : requested_structs) + { + if (isORCTypeRead(file_schema, *orc_struct, include_typeids)) + continue; + const auto & narrowest = getNarrowestORCStructField(*orc_struct); + include_typeids.insert(narrowest.getColumnId()); + if (&narrowest != orc_struct) + count_only_fields.insert(narrowest.getColumnId()); } include_indices.assign(include_typeids.begin(), include_typeids.end()); + orc_column_to_ch_column = std::make_unique( + getPort().getHeader(), + format_settings.orc.allow_missing_columns, + format_settings.null_as_default, + format_settings.orc.case_insensitive_column_matching, + format_settings.orc.dictionary_as_low_cardinality, + format_settings.date_time_overflow_behavior, + std::move(count_only_fields)); + if (format_settings.orc.filter_push_down && format_filter_info && format_filter_info->key_condition && !sargs) sargs = buildORCSearchArgument(*format_filter_info->key_condition, header, file_schema, format_settings); @@ -1300,7 +1355,7 @@ void NativeORCBlockInputFormat::prefetchStripes() int stripe = selected_stripes[prefetch_iterator]; stripes.push_back(stripe); - total_stripe_size += file_reader->getStripe(stripe)->getLength(); + total_stripe_size += ORC::Reader::getStripeLength(file_reader->getStripe(stripe)); ++prefetch_iterator; } @@ -1337,26 +1392,25 @@ bool NativeORCBlockInputFormat::prepareStripeReader() return false; int current_stripe = selected_stripes[read_iterator]; - current_stripe_info = file_reader->getStripe(current_stripe); - if (!current_stripe_info->getNumberOfRows()) + current_stripe_info = &file_reader->getStripe(current_stripe); + if (!current_stripe_info->number_of_rows) throw Exception(ErrorCodes::INCORRECT_DATA, "ORC stripe {} has no rows", current_stripe); - orc::RowReaderOptions row_reader_options; - row_reader_options.setEnableLazyDecoding(format_settings.orc.dictionary_as_low_cardinality); - row_reader_options.includeTypes(include_indices); - row_reader_options.setTimezoneName(format_settings.orc.reader_time_zone_name); - row_reader_options.range(current_stripe_info->getOffset(), current_stripe_info->getLength()); + ORC::RowReaderOptions row_reader_options; + row_reader_options.enable_lazy_decoding = format_settings.orc.dictionary_as_low_cardinality; + row_reader_options.include_types = include_indices; + row_reader_options.timezone_name = format_settings.orc.reader_time_zone_name; + row_reader_options.range_offset = current_stripe_info->offset; + row_reader_options.range_length = ORC::Reader::getStripeLength(*current_stripe_info); if (format_settings.orc.filter_push_down && sargs) - { - row_reader_options.searchArgument(sargs); - } + row_reader_options.search_argument = sargs; stripe_reader = file_reader->createRowReader(row_reader_options); ++read_iterator; if (use_prefetch) { /// Release outdated buffer before boundary to avoid OOM - file_reader->releaseBuffer(current_stripe_info->getOffset()); + file_reader->releaseBuffer(current_stripe_info->offset); /// Prefetch next selected stripe prefetchStripes(); @@ -1379,7 +1433,7 @@ Chunk NativeORCBlockInputFormat::read() int current_stripe = selected_stripes[read_iterator]; ++read_iterator; - return getChunkForCount(file_reader->getStripe(current_stripe)->getNumberOfRows()); + return getChunkForCount(file_reader->getStripe(current_stripe).number_of_rows); } if (!stripe_reader) @@ -1391,25 +1445,12 @@ Chunk NativeORCBlockInputFormat::read() if (is_stopped) return {}; - /// TODO: figure out why reuse batch would cause asan fatals in https://s3.amazonaws.com/clickhouse-test-reports/55330/be39d23af2d7e27f5ec7f168947cf75aeaabf674/stateless_tests__asan__[4_4].html - /// Not sure if it is a false positive case. Notice that reusing batch will speed up reading ORC by 1.15x. + /// The values are decoded directly into the columns of the batch, which are then handed over to the chunk. Stopwatch time; - auto batch = stripe_reader->createRowBatch(format_settings.orc.row_batch_size); + auto batch = stripe_reader->createRowBatch(); while (true) { - bool ok = false; - try - { - ok = stripe_reader->next(*batch); - } - catch (const orc::ParseError & e) - { - /// The ORC library throws ParseError when the encoded data of a stripe is corrupt (for - /// example, a union tag that is out of range for the union's branches). Surface it as - /// INCORRECT_DATA instead of letting it propagate as a generic std::exception. - throw Exception(ErrorCodes::INCORRECT_DATA, "Failed to read ORC data: {}", e.what()); - } - if (ok) + if (stripe_reader->next(*batch, static_cast(format_settings.orc.row_batch_size))) break; /// No more rows to read in current stripe, continue to prepare reading next stripe @@ -1426,7 +1467,7 @@ Chunk NativeORCBlockInputFormat::read() orc_column_to_ch_column->orcTableToCHChunk(res, &schema, batch.get(), num_rows, &block_missing_values); LOG_TEST(getLogger("NativeORCBlockInputFormat"), "Convert {} rows take {} ms", num_rows, time.elapsedMilliseconds()); - approx_bytes_read_for_chunk = num_rows * current_stripe_info->getLength() / current_stripe_info->getNumberOfRows(); + approx_bytes_read_for_chunk = num_rows * ORC::Reader::getStripeLength(*current_stripe_info) / current_stripe_info->number_of_rows; return res; } @@ -1465,37 +1506,28 @@ NamesAndTypesList NativeORCSchemaReader::readSchema() const auto & schema = file_reader->getType(); Block header; - std::unique_ptr stripe_info; + std::optional first_stripe_footer; if (file_reader->getNumberOfStripes()) - stripe_info = file_reader->getStripe(0); + first_stripe_footer = file_reader->readStripeFooter(0); - try { for (size_t i = 0; i < schema.getSubtypeCount(); ++i) { const std::string & name = schema.getFieldName(i); - const orc::Type * orc_type = schema.getSubtype(i); + const ORC::Type * orc_type = schema.getSubtype(i); bool skipped = false; DataTypePtr type = parseORCType( orc_type, format_settings.orc.skip_columns_with_unsupported_types_in_schema_inference, format_settings.orc.dictionary_as_low_cardinality, - stripe_info.get(), + first_stripe_footer ? &*first_stripe_footer : nullptr, skipped, format_settings.max_parser_depth); if (!skipped) header.insert(ColumnWithTypeAndName{type, name}); } } - catch (const orc::ParseError & e) - { - /// The ORC library throws ParseError when the stripe footer is corrupt (for example, a - /// column id that has no matching column encoding, which is consulted here to detect - /// dictionary encoding). Surface it as INCORRECT_DATA instead of letting it propagate as a - /// generic std::exception. - throw Exception(ErrorCodes::INCORRECT_DATA, "Failed to read ORC schema: {}", e.what()); - } /// ORC doesn't have non-nullable data types. if (format_settings.schema_inference_make_columns_nullable != 0) @@ -1515,22 +1547,24 @@ ORCColumnToCHColumn::ORCColumnToCHColumn( bool null_as_default_, bool case_insensitive_matching_, bool dictionary_as_low_cardinality_, - FormatSettings::DateTimeOverflowBehavior date_time_overflow_behavior_) + FormatSettings::DateTimeOverflowBehavior date_time_overflow_behavior_, + std::unordered_set count_only_fields_) : header(header_) , allow_missing_columns(allow_missing_columns_) , null_as_default(null_as_default_) , case_insensitive_matching(case_insensitive_matching_) , dictionary_as_low_cardinality(dictionary_as_low_cardinality_) , date_time_overflow_behavior(date_time_overflow_behavior_) + , count_only_fields(std::move(count_only_fields_)) { } void ORCColumnToCHColumn::orcTableToCHChunk( - Chunk & res, const orc::Type * schema, const orc::ColumnVectorBatch * table, size_t num_rows, BlockMissingValues * block_missing_values) + Chunk & res, const ORC::Type * schema, const ORC::ColumnVectorBatch * table, size_t num_rows, BlockMissingValues * block_missing_values) { - const auto * struct_batch = dynamic_cast(table); + const auto * struct_batch = dynamic_cast(table); if (!struct_batch) - throw Exception(ErrorCodes::LOGICAL_ERROR, "ORC table must be StructVectorBatch but is {}", table->toString()); + throw Exception(ErrorCodes::LOGICAL_ERROR, "ORC table must be StructVectorBatch but is {}", demangle(typeid(*table).name())); if (schema->getSubtypeCount() != struct_batch->fields.size()) throw Exception( @@ -1555,7 +1589,7 @@ void ORCColumnToCHColumn::orcTableToCHChunk( } /// Creates a null bytemap from ORC's not-null bytemap -static ColumnPtr readByteMapFromORCColumn(const orc::ColumnVectorBatch * orc_column) +static ColumnPtr readByteMapFromORCColumn(const ORC::ColumnVectorBatch * orc_column) { if (!orc_column->hasNulls) return ColumnUInt8::create(orc_column->numElements, static_cast(0)); @@ -1598,70 +1632,39 @@ static ColumnPtr replaceNullPayloadRowsWithValidRow(const ColumnPtr & column, co } -static const orc::ColumnVectorBatch * getNestedORCColumn(const orc::ListVectorBatch * orc_column) +static const ORC::ColumnVectorBatch * getNestedORCColumn(const ORC::ListVectorBatch * orc_column) { return orc_column->elements.get(); } +/// The reader decodes the lengths directly into the end offsets, as in `ColumnArray`. template static ColumnPtr readOffsetsFromORCListColumn(const BatchType * orc_column) { - auto offsets_column = ColumnUInt64::create(); - ColumnArray::Offsets & offsets_data = assert_cast &>(*offsets_column).getData(); - offsets_data.reserve(orc_column->numElements); - - for (size_t i = 0; i < orc_column->numElements; ++i) - offsets_data.push_back(orc_column->offsets[i + 1]); - - return offsets_column; + return orc_column->offsets; } +/// The values of the primitive types are decoded directly into the columns (with zeros for the nulls). static ColumnWithTypeAndName -readColumnWithBooleanData(const orc::ColumnVectorBatch * orc_column, const String & column_name) +readColumnWithBooleanData(const ORC::ColumnVectorBatch * orc_column, const String & column_name) { - const auto * orc_bool_column = dynamic_cast(orc_column); - auto internal_type = DataTypeFactory::instance().get("Bool"); - auto internal_column = internal_type->createColumn(); - auto & column_data = assert_cast &>(*internal_column).getData(); - - column_data.reserve(orc_bool_column->numElements); - for (size_t i = 0; i < orc_bool_column->numElements; ++i) - { - if (!orc_bool_column->hasNulls || orc_bool_column->notNull[i]) - column_data.push_back(static_cast(orc_bool_column->data[i])); - else - column_data.push_back(static_cast(0)); - } - - return {std::move(internal_column), internal_type, column_name}; + const auto & orc_bool_column = typeid_cast(*orc_column); + return {orc_bool_column.column, DataTypeFactory::instance().get("Bool"), column_name}; } -template +template static ColumnWithTypeAndName -readColumnWithNumericData(const orc::ColumnVectorBatch * orc_column, const String & column_name) +readColumnWithNumericData(const ORC::ColumnVectorBatch * orc_column, const String & column_name) { - auto internal_type = std::make_shared>(); - auto internal_column = internal_type->createColumn(); - auto & column_data = static_cast &>(*internal_column).getData(); - const auto * orc_int_column = dynamic_cast(orc_column); - - column_data.reserve(orc_int_column->numElements); - for (size_t i = 0; i < orc_int_column->numElements; ++i) - { - if (!orc_int_column->hasNulls || orc_int_column->notNull[i]) - column_data.push_back(NumericType(orc_int_column->data[i])); - else - column_data.push_back(NumericType{}); - } - - return {std::move(internal_column), internal_type, column_name}; + const auto & orc_numeric_column = typeid_cast &>(*orc_column); + return {orc_numeric_column.column, std::make_shared>(), column_name}; } template static ColumnWithTypeAndName readColumnWithEncodedStringOrFixedStringData( - const orc::ColumnVectorBatch * orc_column, const orc::Type * orc_type, const String & column_name, bool nullable) + const ORC::ColumnVectorBatch * orc_column, const ORC::Type * orc_type, const String & column_name, bool nullable) { /// Fill CH holder_column with ORC dictionary /// Note that holder_column is always a ColumnString or ColumnFixedstring whether nullable is true or false, because ORC dictionary doesn't contain null values. @@ -1674,60 +1677,45 @@ static ColumnWithTypeAndName readColumnWithEncodedStringOrFixedStringData( DataTypePtr nested_type = nullable ? std::make_shared(holder_type) : holder_type; auto internal_type = std::make_shared(std::move(nested_type)); - const auto & orc_str_column = dynamic_cast(*orc_column); + const auto & orc_str_column = dynamic_cast(*orc_column); size_t rows = orc_str_column.numElements; - const auto & orc_dict = *orc_str_column.dictionary; - if (orc_dict.dictionaryOffset.size() <= 1) + const auto & orc_dict = assert_cast(*orc_str_column.dictionary->column); + size_t dict_size = orc_dict.size(); + if (dict_size == 0) { + /// Only nulls can be encoded with an empty dictionary. + for (size_t i = 0; i < rows; ++i) + { + if (!orc_str_column.hasNulls || orc_str_column.notNull[i]) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "ORC dictionary index {} at row {} is out of range [0, 0)", + orc_str_column.index[i], i); + } auto result_column = internal_type->createColumn(); result_column->insertManyDefaults(rows); return {std::move(result_column), internal_type, column_name}; } - size_t dict_size = orc_dict.dictionaryOffset.size() - 1; - auto holder_column = holder_type->createColumn(); + /// The dictionary of ORC is already a `ColumnString`, which is the holder for `String`. + ColumnPtr holder_column = orc_str_column.dictionary->column; if constexpr (fixed_string) { const size_t n = orc_type->getMaximumLength(); - auto & concrete_holder_column = assert_cast(*holder_column); - PaddedPODArray & column_chars = concrete_holder_column.getChars(); - size_t reserve_size = dict_size * n; - column_chars.resize_exact(reserve_size); - size_t curr_offset = 0; + auto fixed_holder_column = ColumnFixedString::create(n); + PaddedPODArray & column_chars = fixed_holder_column->getChars(); + column_chars.resize_fill(dict_size * n); for (size_t i = 0; i < dict_size; ++i) { - const auto * buf = orc_dict.dictionaryBlob.data() + orc_dict.dictionaryOffset[i]; - size_t buf_size = orc_dict.dictionaryOffset[i + 1] - orc_dict.dictionaryOffset[i]; - if (buf_size > n) + auto value = orc_dict.getDataAt(i); + if (value.size() > n) throw Exception( ErrorCodes::INCORRECT_DATA, "ORC dictionary entry {} has size {} that exceeds the declared FixedString length {}", - i, buf_size, n); - memcpy(&column_chars[curr_offset], buf, buf_size); - /// resize_exact does not zero-initialize, so pad shorter entries to keep FixedString values - /// deterministic and to avoid leaking uninitialized heap memory. - if (buf_size < n) - memset(&column_chars[curr_offset + buf_size], 0, n - buf_size); - curr_offset += n; - } - } - else - { - auto & concrete_holder_column = assert_cast(*holder_column); - PaddedPODArray & column_chars = concrete_holder_column.getChars(); - PaddedPODArray & column_offsets = concrete_holder_column.getOffsets(); - - column_chars.resize_exact(orc_dict.dictionaryBlob.size()); - column_offsets.resize_exact(dict_size); - size_t curr_offset = 0; - for (size_t i = 0; i < dict_size; ++i) - { - const auto * buf = orc_dict.dictionaryBlob.data() + orc_dict.dictionaryOffset[i]; - size_t buf_size = orc_dict.dictionaryOffset[i + 1] - orc_dict.dictionaryOffset[i]; - memcpy(&column_chars[curr_offset], buf, buf_size); - curr_offset += buf_size; - column_offsets[i] = curr_offset; + i, value.size(), n); + memcpy(&column_chars[i * n], value.data(), value.size()); } + holder_column = std::move(fixed_holder_column); } /// Insert CH dictionary_column from holder_column @@ -1791,57 +1779,22 @@ static ColumnWithTypeAndName readColumnWithEncodedStringOrFixedStringData( return {std::move(internal_column), internal_type, column_name}; } -static ColumnWithTypeAndName -readColumnWithStringData(const orc::ColumnVectorBatch * orc_column, const String & column_name) +static const ColumnString & getStringColumnFromBatch(const ORC::ColumnVectorBatch * orc_column, ColumnPtr & holder) { - auto internal_type = std::make_shared(); - auto internal_column = internal_type->createColumn(); - PaddedPODArray & column_chars = assert_cast(*internal_column).getChars(); - PaddedPODArray & column_offsets = assert_cast(*internal_column).getOffsets(); - - const auto * orc_str_column = dynamic_cast(orc_column); - size_t reserve_size = 0; - for (size_t i = 0; i < orc_str_column->numElements; ++i) - { - if (!orc_str_column->hasNulls || orc_str_column->notNull[i]) - reserve_size += orc_str_column->length[i]; - } - - column_chars.resize_exact(reserve_size); - column_offsets.resize_exact(orc_str_column->numElements); - - size_t curr_offset = 0; - if (!orc_str_column->hasNulls) - { - for (size_t i = 0; i < orc_str_column->numElements; ++i) - { - const auto * buf = orc_str_column->data[i]; - size_t buf_size = orc_str_column->length[i]; - memcpy(&column_chars[curr_offset], buf, buf_size); - curr_offset += buf_size; - column_offsets[i] = curr_offset; - } - } - else - { - for (size_t i = 0; i < orc_str_column->numElements; ++i) - { - if (orc_str_column->notNull[i]) - { - const auto * buf = orc_str_column->data[i]; - size_t buf_size = orc_str_column->length[i]; - memcpy(&column_chars[curr_offset], buf, buf_size); - curr_offset += buf_size; - } + holder = ORC::getStringColumn(dynamic_cast(*orc_column)); + return assert_cast(*holder); +} - column_offsets[i] = curr_offset; - } - } - return {std::move(internal_column), internal_type, column_name}; +static ColumnWithTypeAndName +readColumnWithStringData(const ORC::ColumnVectorBatch * orc_column, const String & column_name) +{ + ColumnPtr column; + getStringColumnFromBatch(orc_column, column); + return {column, std::make_shared(), column_name}; } static ColumnWithTypeAndName -readColumnWithFixedStringData(const orc::ColumnVectorBatch * orc_column, const orc::Type * orc_type, const String & column_name) +readColumnWithFixedStringData(const ORC::ColumnVectorBatch * orc_column, const ORC::Type * orc_type, const String & column_name) { size_t fixed_len = orc_type->getMaximumLength(); auto internal_type = std::make_shared(fixed_len); @@ -1849,25 +1802,22 @@ readColumnWithFixedStringData(const orc::ColumnVectorBatch * orc_column, const o PaddedPODArray & column_chars = assert_cast(*internal_column).getChars(); column_chars.reserve(orc_column->numElements * fixed_len); - const auto * orc_str_column = dynamic_cast(orc_column); - for (size_t i = 0; i < orc_str_column->numElements; ++i) + ColumnPtr holder; + const auto & strings = getStringColumnFromBatch(orc_column, holder); + for (size_t i = 0; i < orc_column->numElements; ++i) { - if (!orc_str_column->hasNulls || orc_str_column->notNull[i]) - { - const Int64 length = orc_str_column->length[i]; - if (length < 0 || static_cast(length) > fixed_len) - throw Exception( - ErrorCodes::INCORRECT_DATA, - "ORC string value at row {} has size {} that doesn't fit into FixedString({})", - i, length, fixed_len); + /// The null values are empty strings. + auto value = strings.getDataAt(i); + if (value.size() > fixed_len) + throw Exception( + ErrorCodes::INCORRECT_DATA, + "ORC string value at row {} has size {} that doesn't fit into FixedString({})", + i, value.size(), fixed_len); - column_chars.insert_assume_reserved(orc_str_column->data[i], orc_str_column->data[i] + length); - /// Zero-pad shorter values to the fixed width to keep the FixedString layout consistent. - if (static_cast(length) < fixed_len) - column_chars.resize_fill(column_chars.size() + (fixed_len - static_cast(length))); - } - else - column_chars.resize_fill(column_chars.size() + fixed_len); + column_chars.insert_assume_reserved(value.data(), value.data() + value.size()); + /// Zero-pad shorter values to the fixed width to keep the FixedString layout consistent. + if (value.size() < fixed_len) + column_chars.resize_fill(column_chars.size() + (fixed_len - value.size())); } return {std::move(internal_column), internal_type, column_name}; @@ -1876,10 +1826,10 @@ readColumnWithFixedStringData(const orc::ColumnVectorBatch * orc_column, const o template static ColumnWithTypeAndName readColumnWithDecimalDataCast( - const orc::ColumnVectorBatch * orc_column, const String & column_name, DataTypePtr internal_type) + const ORC::ColumnVectorBatch * orc_column, const String & column_name, DataTypePtr internal_type) { using NativeType = typename DecimalType::NativeType; - static_assert(std::is_same_v || std::is_same_v); + static_assert(std::is_same_v || std::is_same_v); auto internal_column = internal_type->createColumn(); auto & column_data = static_cast &>(*internal_column).getData(); @@ -1891,12 +1841,9 @@ static ColumnWithTypeAndName readColumnWithDecimalDataCast( if (!orc_decimal_column->hasNulls || orc_decimal_column->notNull[i]) { DecimalType decimal_value{}; - if constexpr (std::is_same_v) + if constexpr (std::is_same_v) { - Int128 int128_value; - int128_value.items[0] = orc_decimal_column->values[i].getLowBits(); - int128_value.items[1] = orc_decimal_column->values[i].getHighBits(); - decimal_value.value = static_cast(int128_value); + decimal_value.value = static_cast(orc_decimal_column->values[i]); } else decimal_value.value = static_cast(orc_decimal_column->values[i]); @@ -1911,26 +1858,30 @@ static ColumnWithTypeAndName readColumnWithDecimalDataCast( } static ColumnWithTypeAndName -readIPv6ColumnFromBinaryData(const orc::ColumnVectorBatch * orc_column, const String & column_name) +readIPv6ColumnFromBinaryData(const ORC::ColumnVectorBatch * orc_column, const String & column_name) { - const auto * orc_str_column = dynamic_cast(orc_column); + ColumnPtr holder; + const auto & strings = getStringColumnFromBatch(orc_column, holder); - for (size_t i = 0; i < orc_str_column->numElements; ++i) + for (size_t i = 0; i < orc_column->numElements; ++i) { /// If at least one value size is not 16 bytes, fallback to reading String column and further cast to IPv6. - if ((!orc_str_column->hasNulls || orc_str_column->notNull[i]) && orc_str_column->length[i] != sizeof(IPv6)) - return readColumnWithStringData(orc_column, column_name); + if ((!orc_column->hasNulls || orc_column->notNull[i]) && strings.getDataAt(i).size() != sizeof(IPv6)) + return {holder, std::make_shared(), column_name}; } auto internal_type = std::make_shared(); auto internal_column = internal_type->createColumn(); auto & ipv6_column = assert_cast(*internal_column); - ipv6_column.reserve(orc_str_column->numElements); + ipv6_column.reserve(orc_column->numElements); - for (size_t i = 0; i < orc_str_column->numElements; ++i) + for (size_t i = 0; i < orc_column->numElements; ++i) { - if (!orc_str_column->hasNulls || orc_str_column->notNull[i]) - ipv6_column.insertData(orc_str_column->data[i], orc_str_column->length[i]); + if (!orc_column->hasNulls || orc_column->notNull[i]) + { + auto value = strings.getDataAt(i); + ipv6_column.insertData(value.data(), value.size()); + } else ipv6_column.insertDefault(); } @@ -1939,49 +1890,46 @@ readIPv6ColumnFromBinaryData(const orc::ColumnVectorBatch * orc_column, const St } static ColumnWithTypeAndName -readIPv4ColumnWithInt32Data(const orc::ColumnVectorBatch * orc_column, const String & column_name) +readIPv4ColumnWithInt32Data(const ORC::ColumnVectorBatch * orc_column, const String & column_name) { - const auto * orc_int_column = dynamic_cast(orc_column); + const auto & data = typeid_cast(*orc_column).getData(); auto internal_type = std::make_shared(); auto internal_column = internal_type->createColumn(); auto & column_data = assert_cast(*internal_column).getData(); - column_data.reserve(orc_int_column->numElements); - - for (size_t i = 0; i < orc_int_column->numElements; ++i) - { - if (!orc_int_column->hasNulls || orc_int_column->notNull[i]) - column_data.push_back(static_cast(orc_int_column->data[i])); - else - column_data.push_back(0); - } + column_data.resize(data.size()); + /// The null values are zeros. + for (size_t i = 0; i < data.size(); ++i) + column_data[i] = static_cast(data[i]); return {std::move(internal_column), internal_type, column_name}; } template static ColumnWithTypeAndName readColumnWithBigNumberFromBinaryData( - const orc::ColumnVectorBatch * orc_column, const String & column_name, const DataTypePtr & column_type) + const ORC::ColumnVectorBatch * orc_column, const String & column_name, const DataTypePtr & column_type) { - const auto * orc_str_column = dynamic_cast(orc_column); + ColumnPtr holder; + const auto & strings = getStringColumnFromBatch(orc_column, holder); auto internal_column = column_type->createColumn(); auto & integer_column = assert_cast(*internal_column); - integer_column.reserve(orc_str_column->numElements); + integer_column.reserve(orc_column->numElements); - for (size_t i = 0; i < orc_str_column->numElements; ++i) + for (size_t i = 0; i < orc_column->numElements; ++i) { - if (!orc_str_column->hasNulls || orc_str_column->notNull[i]) + if (!orc_column->hasNulls || orc_column->notNull[i]) { - if (sizeof(typename ColumnType::ValueType) != orc_str_column->length[i]) + auto value = strings.getDataAt(i); + if (sizeof(typename ColumnType::ValueType) != value.size()) throw Exception( ErrorCodes::INCORRECT_DATA, "ValueType size {} of column {} is not equal to size of binary data {}", sizeof(typename ColumnType::ValueType), integer_column.getName(), - orc_str_column->length[i]); + value.size()); - integer_column.insertData(orc_str_column->data[i], orc_str_column->length[i]); + integer_column.insertData(value.data(), value.size()); } else { @@ -1992,7 +1940,7 @@ static ColumnWithTypeAndName readColumnWithBigNumberFromBinaryData( } static ColumnWithTypeAndName readColumnWithDateData( - const orc::ColumnVectorBatch * orc_column, + const ORC::ColumnVectorBatch * orc_column, const String & column_name, const DataTypePtr & raw_type_hint, FormatSettings::DateTimeOverflowBehavior date_time_overflow_behavior) @@ -2053,7 +2001,8 @@ static ColumnWithTypeAndName readColumnWithDateData( internal_type = std::make_shared(); } - const auto * orc_int_column = dynamic_cast(orc_column); + const auto * orc_int_column = dynamic_cast(orc_column); + const auto & orc_int_data = orc_int_column->getData(); auto internal_column = internal_type->createColumn(); PaddedPODArray & column_data = assert_cast &>(*internal_column).getData(); column_data.reserve(orc_int_column->numElements); @@ -2064,7 +2013,7 @@ static ColumnWithTypeAndName readColumnWithDateData( { /// Range-check the original Int64 day count before narrowing to Int32, /// otherwise a malformed value could wrap into an apparently valid date. - Int64 days_num = orc_int_column->data[i]; + Int64 days_num = orc_int_data[i]; if (check_date32_range && (days_num > DATE_LUT_MAX_EXTEND_DAY_NUM || days_num < DATE_LUT_MIN_EXTEND_DAY_NUM)) { if (date_time_overflow_behavior == FormatSettings::DateTimeOverflowBehavior::Saturate) @@ -2132,12 +2081,12 @@ static ColumnWithTypeAndName readColumnWithDateData( } static ColumnWithTypeAndName readColumnWithTimestampData( - const orc::ColumnVectorBatch * orc_column, + const ORC::ColumnVectorBatch * orc_column, const String & column_name, const DataTypePtr & type_hint, FormatSettings::DateTimeOverflowBehavior date_time_overflow_behavior) { - const auto * orc_ts_column = dynamic_cast(orc_column); + const auto * orc_ts_column = dynamic_cast(orc_column); /// ORC stores timestamps as (seconds, nanoseconds). ClickHouse used to convert them through a fixed /// DateTime64(9) intermediate (seconds * 1e9 + nanoseconds), which overflows Int64 for timestamps beyond @@ -2254,7 +2203,7 @@ static bool orcIntegerBranchWidensTo(const DataTypePtr & type, size_t orc_size) /// fields (the per-branch repair cast then projects/reorders the read tuple to exactly the target), /// while an unnamed target tuple is matched positionally with the same arity. `case_insensitive` /// mirrors `input_format_orc_case_insensitive_column_matching` for that name-based matching. -static bool orcUnionBranchMatchesType(const orc::Type * orc_branch_type, const DataTypePtr & target_type, bool case_insensitive) +static bool orcUnionBranchMatchesType(const ORC::Type * orc_branch_type, const DataTypePtr & target_type, bool case_insensitive) { checkStackSize(); @@ -2263,48 +2212,48 @@ static bool orcUnionBranchMatchesType(const orc::Type * orc_branch_type, const D switch (orc_branch_type->getKind()) { - case orc::TypeKind::BOOLEAN: + case ORC::TypeKind::BOOLEAN: /// A boolean holds only 0 and 1, so every integer type is a widening of it. return which.isUInt8() || which.isEnum8() || orcIntegerBranchWidensTo(type, 0); - case orc::TypeKind::BYTE: + case ORC::TypeKind::BYTE: return which.isInt8() || (which.isUInt8() && !isBool(type)) || which.isEnum8() || orcIntegerBranchWidensTo(type, 1); - case orc::TypeKind::SHORT: + case ORC::TypeKind::SHORT: return which.isInt16() || which.isUInt16() || which.isEnum16() || orcIntegerBranchWidensTo(type, 2); - case orc::TypeKind::INT: + case ORC::TypeKind::INT: return which.isInt32() || which.isUInt32() || which.isIPv4() || orcIntegerBranchWidensTo(type, 4); - case orc::TypeKind::LONG: + case ORC::TypeKind::LONG: return which.isInt64() || which.isUInt64() || orcIntegerBranchWidensTo(type, 8); - case orc::TypeKind::FLOAT: + case ORC::TypeKind::FLOAT: return which.isFloat32() || which.isFloat64(); - case orc::TypeKind::DOUBLE: + case ORC::TypeKind::DOUBLE: return which.isFloat64(); - case orc::TypeKind::DATE: + case ORC::TypeKind::DATE: return which.isDate32() || which.isDate(); - case orc::TypeKind::TIMESTAMP: + case ORC::TypeKind::TIMESTAMP: /// Permissive: matches any DateTime/DateTime64 alternative. The repair cast relabels the /// branch to the alternative's timezone, which is value-preserving for DateTime64. return which.isDateTime64() || which.isDateTime(); - case orc::TypeKind::TIMESTAMP_INSTANT: + case ORC::TypeKind::TIMESTAMP_INSTANT: /// Matches only an explicitly time-zoned target so it stays distinct from a TIMESTAMP /// branch in the same union (both read back as DateTime64(9)); see the comment above the /// function. return (which.isDateTime64() || which.isDateTime()) && orcTimestampTargetHasExplicitTimeZone(type); - case orc::TypeKind::DECIMAL: + case ORC::TypeKind::DECIMAL: return which.isDecimal(); - case orc::TypeKind::STRING: - case orc::TypeKind::VARCHAR: + case ORC::TypeKind::STRING: + case ORC::TypeKind::VARCHAR: return which.isStringOrFixedString(); - case orc::TypeKind::BINARY: + case ORC::TypeKind::BINARY: return which.isStringOrFixedString() || which.isIPv6() || which.isInt128() || which.isUInt128() || which.isInt256() || which.isUInt256() || which.isDecimal256(); - case orc::TypeKind::CHAR: + case ORC::TypeKind::CHAR: return which.isStringOrFixedString() || which.isInt128() || which.isUInt128() || which.isInt256() || which.isUInt256() || which.isDecimal256(); - case orc::TypeKind::LIST: + case ORC::TypeKind::LIST: return which.isArray() && orc_branch_type->getSubtypeCount() == 1 && orcUnionBranchMatchesType( orc_branch_type->getSubtype(0), assert_cast(*type).getNestedType(), case_insensitive); - case orc::TypeKind::MAP: + case ORC::TypeKind::MAP: { if (!which.isMap() || orc_branch_type->getSubtypeCount() != 2) return false; @@ -2312,7 +2261,7 @@ static bool orcUnionBranchMatchesType(const orc::Type * orc_branch_type, const D return orcUnionBranchMatchesType(orc_branch_type->getSubtype(0), map_type.getKeyType(), case_insensitive) && orcUnionBranchMatchesType(orc_branch_type->getSubtype(1), map_type.getValueType(), case_insensitive); } - case orc::TypeKind::STRUCT: + case ORC::TypeKind::STRUCT: { if (!which.isTuple()) return false; @@ -2324,7 +2273,7 @@ static bool orcUnionBranchMatchesType(const orc::Type * orc_branch_type, const D /// a subset of the ORC struct's fields, in any order. Each target field must map to /// an ORC field with a recursively matching type; extra ORC fields are projected out /// by the repair cast. Build a name -> ORC subtype map for the lookup. - std::unordered_map orc_field_by_name; + std::unordered_map orc_field_by_name; orc_field_by_name.reserve(orc_branch_type->getSubtypeCount()); for (size_t i = 0; i < orc_branch_type->getSubtypeCount(); ++i) { @@ -2355,10 +2304,10 @@ static bool orcUnionBranchMatchesType(const orc::Type * orc_branch_type, const D return false; return true; } - case orc::TypeKind::UNION: + case ORC::TypeKind::UNION: return which.isVariant(); - case orc::TypeKind::GEOMETRY: - case orc::TypeKind::GEOGRAPHY: + case ORC::TypeKind::GEOMETRY: + case ORC::TypeKind::GEOGRAPHY: /// Added in ORC 2.3. We do not read these, so no target type can match such a branch. return false; } @@ -2383,32 +2332,32 @@ static bool orcUnionBranchMatchesType(const orc::Type * orc_branch_type, const D /// schema `Variant(Int8, Int64)` leaves the `BYTE` branch with two candidates. Preferring the /// natural pairing keeps the exact-width alternative with its own branch and lets the wider /// alternative go to the branch that needs it. -static bool orcUnionBranchPrefersType(const orc::Type * orc_branch_type, const DataTypePtr & target_type) +static bool orcUnionBranchPrefersType(const ORC::Type * orc_branch_type, const DataTypePtr & target_type) { const DataTypePtr type = removeLowCardinality(removeNullableOrLowCardinalityNullable(target_type)); const WhichDataType which(type); switch (orc_branch_type->getKind()) { - case orc::TypeKind::BOOLEAN: + case ORC::TypeKind::BOOLEAN: return isBool(type); - case orc::TypeKind::BYTE: + case ORC::TypeKind::BYTE: return which.isInt8(); - case orc::TypeKind::SHORT: + case ORC::TypeKind::SHORT: return which.isInt16(); - case orc::TypeKind::INT: + case ORC::TypeKind::INT: return which.isInt32(); - case orc::TypeKind::LONG: + case ORC::TypeKind::LONG: return which.isInt64(); - case orc::TypeKind::FLOAT: + case ORC::TypeKind::FLOAT: return which.isFloat32(); - case orc::TypeKind::DOUBLE: + case ORC::TypeKind::DOUBLE: return which.isFloat64(); - case orc::TypeKind::CHAR: + case ORC::TypeKind::CHAR: return which.isFixedString() && assert_cast(*type).getN() == orc_branch_type->getMaximumLength(); - case orc::TypeKind::STRING: - case orc::TypeKind::VARCHAR: + case ORC::TypeKind::STRING: + case ORC::TypeKind::VARCHAR: return which.isString(); default: return false; @@ -2424,7 +2373,7 @@ static bool orcUnionBranchPrefersType(const orc::Type * orc_branch_type, const D /// hint (nullptr in the result), so no guess is ever made. Shared between /// readColumnFromORCColumn (drives the branch conversions) and updateIncludeTypeIds (prunes a /// hinted STRUCT branch's fields the same way the read path does), so the two stay in sync. -static DataTypes computeOrcUnionBranchHints(const orc::Type * orc_type, const DataTypePtr & type_hint, bool case_insensitive_matching) +static DataTypes computeOrcUnionBranchHints(const ORC::Type * orc_type, const DataTypePtr & type_hint, bool case_insensitive_matching) { const size_t num_children = orc_type->getSubtypeCount(); DataTypes branch_hints(num_children); @@ -2510,9 +2459,120 @@ static DataTypes computeOrcUnionBranchHints(const orc::Type * orc_type, const Da return branch_hints; } +/// Matches the elements of the tuples in `column` to those of the named tuples in `requested_type` by name, through +/// Nullable, Array and Map: a requested element that was not read is filled with defaults, and an element that is +/// not requested is dropped. A CAST would match tuples with no common element names by position instead. +static ColumnWithTypeAndName +matchORCTupleElementsByName(const ColumnWithTypeAndName & column, const DataTypePtr & requested_type, bool case_insensitive) +{ + if (column.type->equals(*requested_type)) + return column; + + if (const auto * nullable_type = typeid_cast(column.type.get())) + { + const auto & nullable_column = assert_cast(*column.column); + auto nested = matchORCTupleElementsByName( + {nullable_column.getNestedColumnPtr(), nullable_type->getNestedType(), column.name}, + removeNullable(requested_type), + case_insensitive); + return { + ColumnNullable::create(nested.column, nullable_column.getNullMapColumnPtr()), + std::make_shared(nested.type), + column.name}; + } + + const auto requested_non_nullable_type = removeNullable(requested_type); + + if (const auto * array_type = typeid_cast(column.type.get())) + { + const auto * requested_array_type = typeid_cast(requested_non_nullable_type.get()); + if (!requested_array_type) + return column; + const auto & array_column = assert_cast(*column.column); + auto nested = matchORCTupleElementsByName( + {array_column.getDataPtr(), array_type->getNestedType(), column.name}, requested_array_type->getNestedType(), case_insensitive); + return { + ColumnArray::create(nested.column, array_column.getOffsetsPtr()), std::make_shared(nested.type), column.name}; + } + + if (const auto * map_type = typeid_cast(column.type.get())) + { + const auto * requested_map_type = typeid_cast(requested_non_nullable_type.get()); + if (!requested_map_type) + return column; + const auto & map_column = assert_cast(*column.column); + auto keys = matchORCTupleElementsByName( + {map_column.getNestedData().getColumnPtr(0), map_type->getKeyType(), "key"}, requested_map_type->getKeyType(), case_insensitive); + auto values = matchORCTupleElementsByName( + {map_column.getNestedData().getColumnPtr(1), map_type->getValueType(), "value"}, + requested_map_type->getValueType(), + case_insensitive); + return { + ColumnMap::create(keys.column, values.column, map_column.getNestedColumn().getOffsetsPtr()), + std::make_shared(keys.type, values.type), + column.name}; + } + + const auto * tuple_type = typeid_cast(column.type.get()); + const auto * requested_tuple_type = typeid_cast(requested_non_nullable_type.get()); + if (!tuple_type || !requested_tuple_type) + return column; + + const auto & tuple_column = assert_cast(*column.column); + const size_t num_rows = tuple_column.size(); + Columns elements; + DataTypes types; + Strings names; + + /// An unnamed tuple is matched by position and keeps its elements, so the CAST rejects a different number of them. + if (!requested_tuple_type->hasExplicitNames()) + { + for (size_t i = 0; i < tuple_type->getElements().size(); ++i) + { + ColumnWithTypeAndName element{tuple_column.getColumnPtr(i), tuple_type->getElement(i), tuple_type->getNameByPosition(i + 1)}; + if (i < requested_tuple_type->getElements().size()) + element = matchORCTupleElementsByName(element, requested_tuple_type->getElement(i), case_insensitive); + elements.push_back(std::move(element.column)); + types.push_back(std::move(element.type)); + } + auto result_column = elements.empty() ? ColumnTuple::create(num_rows) : ColumnTuple::create(std::move(elements)); + return {std::move(result_column), std::make_shared(std::move(types)), column.name}; + } + + for (size_t i = 0; i < requested_tuple_type->getElements().size(); ++i) + { + const auto & requested_element_type = requested_tuple_type->getElement(i); + const auto & name = requested_tuple_type->getNameByPosition(i + 1); + + auto pos = tuple_type->tryGetPositionByName(name, /*case_insensitive=*/false); + if (!pos && case_insensitive) + pos = tuple_type->tryGetPositionByName(name, /*case_insensitive=*/true); + + if (pos) + { + auto element = matchORCTupleElementsByName( + {tuple_column.getColumnPtr(*pos), tuple_type->getElement(*pos), name}, requested_element_type, case_insensitive); + elements.push_back(std::move(element.column)); + types.push_back(std::move(element.type)); + } + else + { + auto element = requested_element_type->createColumn(); + requested_element_type->insertManyDefaultsInto(*element, num_rows); + elements.push_back(std::move(element)); + types.push_back(requested_element_type); + } + names.push_back(name); + } + + auto result_type = std::make_shared(std::move(types), std::move(names)); + auto result_column = elements.empty() ? ColumnTuple::create(num_rows) : ColumnTuple::create(std::move(elements)); + return {std::move(result_column), std::move(result_type), column.name}; +} + ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( - const orc::ColumnVectorBatch * orc_column, - const orc::Type * orc_type, + const ORC::ColumnVectorBatch * orc_column, + const ORC::Type * orc_type, const std::string & column_name, bool inside_nullable, DataTypePtr type_hint) const @@ -2524,7 +2584,7 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( bool skipped = false; if (!inside_nullable && (orc_column->hasNulls || (type_hint && isNullableOrLowCardinalityNullable(type_hint))) && !orc_column->isEncoded - && (orc_type->getKind() != orc::LIST && orc_type->getKind() != orc::MAP && orc_type->getKind() != orc::UNION)) + && (orc_type->getKind() != ORC::LIST && orc_type->getKind() != ORC::MAP && orc_type->getKind() != ORC::UNION)) { DataTypePtr nested_type_hint; if (type_hint) @@ -2540,9 +2600,9 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( /// ORC union maps to the ClickHouse Variant type. Handled before the switch below so the switch /// stays non-exhaustive (its default handles unsupported types). - if (orc_type->getKind() == orc::UNION) + if (orc_type->getKind() == ORC::UNION) { - const auto * orc_union_column = dynamic_cast(orc_column); + const auto * orc_union_column = dynamic_cast(orc_column); if (!orc_union_column) throw Exception(ErrorCodes::LOGICAL_ERROR, "ORC column for union type must be a UnionVectorBatch"); @@ -2586,7 +2646,7 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( /// forbids; reject it explicitly (matching the schema-inference path) instead of letting /// the DataTypeVariant below throw a confusing BAD_ARGUMENTS. Only reachable with an /// explicit structure, since schema inference already rejects nested unions. - if (orc_type->getSubtype(i)->getKind() == orc::UNION) + if (orc_type->getSubtype(i)->getKind() == ORC::UNION) throw Exception( ErrorCodes::UNKNOWN_TYPE, "ORC union type '{}' has a nested union branch, which is not supported, while reading column {}", @@ -2686,7 +2746,7 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( auto & offsets_data = offsets->getData(); offsets_data.resize_exact(num_rows); - const unsigned char * tags = orc_union_column->tags.data(); + const UInt8 * tags = orc_union_column->tags.data(); const uint64_t * orc_offsets = orc_union_column->offsets.data(); const char * not_null = orc_union_column->hasNulls ? orc_union_column->notNull.data() : nullptr; @@ -2738,9 +2798,9 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( switch (orc_type->getKind()) { - case orc::STRING: - case orc::BINARY: - case orc::VARCHAR: + case ORC::STRING: + case ORC::BINARY: + case ORC::VARCHAR: { if (type_hint) { @@ -2772,7 +2832,7 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( else return readColumnWithStringData(orc_column, column_name); } - case orc::CHAR: + case ORC::CHAR: { if (type_hint) { @@ -2798,32 +2858,32 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( else return readColumnWithFixedStringData(orc_column, orc_type, column_name); } - case orc::BOOLEAN: + case ORC::BOOLEAN: return readColumnWithBooleanData(orc_column, column_name); - case orc::BYTE: - return readColumnWithNumericData(orc_column, column_name); - case orc::SHORT: - return readColumnWithNumericData(orc_column, column_name); - case orc::INT: + case ORC::BYTE: + return readColumnWithNumericData(orc_column, column_name); + case ORC::SHORT: + return readColumnWithNumericData(orc_column, column_name); + case ORC::INT: { /// ORC format doesn't have unsigned integers and we output IPv4 as Int32. /// We should allow to read it back from Int32. if (type_hint && isIPv4(type_hint)) return readIPv4ColumnWithInt32Data(orc_column, column_name); - return readColumnWithNumericData(orc_column, column_name); + return readColumnWithNumericData(orc_column, column_name); } - case orc::LONG: - return readColumnWithNumericData(orc_column, column_name); - case orc::FLOAT: - return readColumnWithNumericData(orc_column, column_name); - case orc::DOUBLE: - return readColumnWithNumericData(orc_column, column_name); - case orc::DATE: + case ORC::LONG: + return readColumnWithNumericData(orc_column, column_name); + case ORC::FLOAT: + return readColumnWithNumericData(orc_column, column_name); + case ORC::DOUBLE: + return readColumnWithNumericData(orc_column, column_name); + case ORC::DATE: return readColumnWithDateData(orc_column, column_name, type_hint, date_time_overflow_behavior); - case orc::TIMESTAMP: [[fallthrough]]; - case orc::TIMESTAMP_INSTANT: + case ORC::TIMESTAMP: [[fallthrough]]; + case ORC::TIMESTAMP_INSTANT: return readColumnWithTimestampData(orc_column, column_name, type_hint, date_time_overflow_behavior); - case orc::DECIMAL: + case ORC::DECIMAL: { auto interal_type = parseORCType(orc_type, false, false, nullptr, skipped); @@ -2832,15 +2892,15 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( precision = 38; if (precision <= DecimalUtils::max_precision) - return readColumnWithDecimalDataCast(orc_column, column_name, interal_type); + return readColumnWithDecimalDataCast(orc_column, column_name, interal_type); if (precision <= DecimalUtils::max_precision) - return readColumnWithDecimalDataCast(orc_column, column_name, interal_type); + return readColumnWithDecimalDataCast(orc_column, column_name, interal_type); if (precision <= DecimalUtils::max_precision) - return readColumnWithDecimalDataCast(orc_column, column_name, interal_type); + return readColumnWithDecimalDataCast(orc_column, column_name, interal_type); throw Exception( ErrorCodes::ARGUMENT_OUT_OF_BOUND, "Decimal precision {} in ORC type {} is out of bound", precision, orc_type->toString()); } - case orc::MAP: + case ORC::MAP: { DataTypePtr key_type_hint; DataTypePtr value_type_hint; @@ -2854,13 +2914,18 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( } } - const auto * orc_map_column = dynamic_cast(orc_column); + const auto * orc_map_column = dynamic_cast(orc_column); const auto * orc_key_column = orc_map_column->keys.get(); const auto * orc_value_column = orc_map_column->elements.get(); const auto * orc_key_type = orc_type->getSubtype(0); const auto * orc_value_type = orc_type->getSubtype(1); auto key_column = readColumnFromORCColumn(orc_key_column, orc_key_type, "key", false, key_type_hint); + /// A map key cannot be Nullable. + if (!key_type_hint && !DataTypeMap::isValidKeyType(key_column.type)) + key_type_hint = removeNullableOrLowCardinalityNullable(key_column.type); + if (key_type_hint) + key_column = matchORCTupleElementsByName(key_column, key_type_hint, case_insensitive_matching); if (key_type_hint && !key_type_hint->equals(*key_column.type)) { /// Cast key column to target type, because it can happen @@ -2878,7 +2943,7 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( auto map_type = std::make_shared(key_column.type, value_column.type); return {map_column, map_type, column_name}; } - case orc::LIST: + case ORC::LIST: { DataTypePtr nested_type_hint; if (type_hint) @@ -2888,11 +2953,17 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( nested_type_hint = array_type_hint->getNestedType(); } - const auto * orc_list_column = dynamic_cast(orc_column); + const auto * orc_list_column = dynamic_cast(orc_column); const auto * orc_nested_column = getNestedORCColumn(orc_list_column); const auto * orc_nested_type = orc_type->getSubtype(0); auto nested_column = readColumnFromORCColumn(orc_nested_column, orc_nested_type, column_name, false, nested_type_hint); + /// A struct read only for its size has no elements, which `createNested` below rejects. Otherwise the elements + /// are matched after a subcolumn is extracted, as a hint for one can name a path, such as `x.v`. + const auto * read_tuple_type = typeid_cast(removeNullable(nested_column.type).get()); + if (type_hint && isNested(type_hint) && read_tuple_type && read_tuple_type->getElements().empty()) + nested_column = matchORCTupleElementsByName(nested_column, nested_type_hint, case_insensitive_matching); + auto offsets_column = readOffsetsFromORCListColumn(orc_list_column); DataTypePtr array_type; ColumnPtr array_data_column = nested_column.column; @@ -2917,17 +2988,21 @@ ColumnWithTypeAndName ORCColumnToCHColumn::readColumnFromORCColumn( auto array_column = ColumnArray::create(array_data_column, offsets_column); return {array_column, array_type, column_name}; } - case orc::STRUCT: + case ORC::STRUCT: { Columns tuple_elements; DataTypes tuple_types; std::vector tuple_names; const auto * tuple_type_hint = type_hint ? typeid_cast(type_hint.get()) : nullptr; - const auto * orc_struct_column = dynamic_cast(orc_column); + const auto * orc_struct_column = dynamic_cast(orc_column); for (size_t i = 0; i < orc_type->getSubtypeCount(); ++i) { + /// Read only for the size and the null map of the struct, so its values are not converted. + if (count_only_fields.contains(orc_type->getSubtype(i)->getColumnId())) + continue; + auto field_name = orc_type->getFieldName(i); DataTypePtr nested_type_hint; @@ -3044,6 +3119,7 @@ void ORCColumnToCHColumn::orcColumnsToCHChunk( orc_column_with_type.first, orc_column_with_type.second, header_column.name, false, header_column.type); } + column = matchORCTupleElementsByName(column, header_column.type, case_insensitive_matching); if (null_as_default) insertNullAsDefaultIfNeeded(column, header_column, column_i, block_missing_values); diff --git a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h index 49a29a7d85179..5596542988060 100644 --- a/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h +++ b/src/Processors/Formats/Impl/NativeORCBlockInputFormat.h @@ -11,7 +11,7 @@ #include #include #include -#include +#include #include namespace DB @@ -19,15 +19,14 @@ namespace DB class KeyCondition; -class ORCInputStream : public orc::InputStream +class ORCInputStream : public ORC::InputStream { public: ORCInputStream(SeekableReadBuffer & in_, size_t file_size_, bool use_prefetch); - uint64_t getLength() const override; - uint64_t getNaturalReadSize() const override; - void read(void * buf, uint64_t length, uint64_t offset) override; - std::future readAsync(void * buf, uint64_t length, uint64_t offset) override; + UInt64 getLength() const override; + void read(void * buf, UInt64 length, UInt64 offset) override; + std::future readAsync(void * buf, UInt64 length, UInt64 offset) override; const std::string & getName() const override { return name; } protected: @@ -52,22 +51,17 @@ class ORCInputStreamFromString : public ReadBufferFromOwnString, public ORCInput } }; -std::unique_ptr +std::unique_ptr asORCInputStream(ReadBuffer & in, const FormatSettings & settings, bool use_prefetch, std::atomic & is_cancelled); // Reads the whole file into a memory buffer, owned by the returned RandomAccessFile. -std::unique_ptr asORCInputStreamLoadIntoMemory(ReadBuffer & in, std::atomic & is_cancelled); - -/// The memory pool to set on the options of every ORC reader and writer we create: unlike the -/// default one of the library, it is accounted for by the memory tracker and it throws on failure -/// instead of returning a null pointer that the library dereferences. -orc::MemoryPool & getORCMemoryPool(); +std::unique_ptr asORCInputStreamLoadIntoMemory(ReadBuffer & in, std::atomic & is_cancelled); /// Creates an ORC file reader; throws INCORRECT_DATA if the root type of the file is not a struct. -std::unique_ptr createORCReader(std::unique_ptr stream, const orc::ReaderOptions & options); +std::unique_ptr createORCReader(std::unique_ptr stream, const ORC::ReaderOptions & options); -std::unique_ptr buildORCSearchArgument( - const KeyCondition & key_condition, const Block & header, const orc::Type & schema, const FormatSettings & format_settings); +std::unique_ptr buildORCSearchArgument( + const KeyCondition & key_condition, const Block & header, const ORC::Type & schema, const FormatSettings & format_settings); class ORCColumnToCHColumn; class NativeORCBlockInputFormat final : public IInputFormat @@ -101,14 +95,14 @@ class NativeORCBlockInputFormat final : public IInputFormat bool prepareStripeReader(); void prefetchStripes(); - std::unique_ptr file_reader; - std::unique_ptr stripe_reader; + std::unique_ptr file_reader; + std::unique_ptr stripe_reader; std::unique_ptr orc_column_to_ch_column; - std::shared_ptr sargs; + std::shared_ptr sargs; // indices of columns to read from ORC file - std::list include_indices; + std::vector include_indices; BlockMissingValues block_missing_values; size_t approx_bytes_read_for_chunk = 0; @@ -123,7 +117,7 @@ class NativeORCBlockInputFormat final : public IInputFormat size_t read_iterator{}; size_t prefetch_iterator{}; - std::unique_ptr current_stripe_info; + const ORC::Proto::StripeInformation * current_stripe_info = nullptr; std::atomic is_stopped{0}; }; @@ -140,7 +134,7 @@ class NativeORCSchemaReader final : public ISchemaReader void initializeIfNeeded(); const FormatSettings format_settings; - std::unique_ptr file_reader; + std::unique_ptr file_reader; std::atomic is_stopped{0}; bool initialized = false; }; @@ -148,8 +142,8 @@ class NativeORCSchemaReader final : public ISchemaReader class ORCColumnToCHColumn { public: - using ORCColumnPtr = const orc::ColumnVectorBatch *; - using ORCTypePtr = const orc::Type *; + using ORCColumnPtr = const ORC::ColumnVectorBatch *; + using ORCTypePtr = const ORC::Type *; using ORCColumnWithType = std::pair; using NameToColumnPtr = std::unordered_map; @@ -159,12 +153,13 @@ class ORCColumnToCHColumn bool null_as_default_, bool case_insensitive_matching_ = false, bool dictionary_as_low_cardinality_ = false, - FormatSettings::DateTimeOverflowBehavior date_time_overflow_behavior_ = FormatSettings::DateTimeOverflowBehavior::Ignore); + FormatSettings::DateTimeOverflowBehavior date_time_overflow_behavior_ = FormatSettings::DateTimeOverflowBehavior::Ignore, + std::unordered_set count_only_fields_ = {}); void orcTableToCHChunk( Chunk & res, - const orc::Type * schema, - const orc::ColumnVectorBatch * table, + const ORC::Type * schema, + const ORC::ColumnVectorBatch * table, size_t num_rows, BlockMissingValues * block_missing_values = nullptr); @@ -173,8 +168,8 @@ class ORCColumnToCHColumn private: ColumnWithTypeAndName readColumnFromORCColumn( - const orc::ColumnVectorBatch * orc_column, - const orc::Type * orc_type, + const ORC::ColumnVectorBatch * orc_column, + const ORC::Type * orc_type, const std::string & column_name, bool inside_nullable, DataTypePtr type_hint = nullptr) const; @@ -186,6 +181,8 @@ class ORCColumnToCHColumn bool case_insensitive_matching; bool dictionary_as_low_cardinality; FormatSettings::DateTimeOverflowBehavior date_time_overflow_behavior; + /// ORC column ids of the struct fields that are read only for the size and the null map of their struct. + const std::unordered_set count_only_fields; }; } #endif diff --git a/src/Processors/Formats/Impl/ORC/ColumnReader.cpp b/src/Processors/Formats/Impl/ORC/ColumnReader.cpp new file mode 100644 index 0000000000000..f792e8d980ad4 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/ColumnReader.cpp @@ -0,0 +1,1218 @@ +#include +#include + +#include +#include +#include +#include +#include +#include + +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int INCORRECT_DATA; + extern const int NOT_IMPLEMENTED; +} +} + +namespace DB::ORC +{ + +namespace +{ + +RleVersion getRleVersion(const Proto::ColumnEncoding & encoding) +{ + switch (encoding.kind) + { + case Proto::ColumnEncodingKind::DIRECT: + case Proto::ColumnEncodingKind::DICTIONARY: + return RleVersion::V1; + case Proto::ColumnEncodingKind::DIRECT_V2: + case Proto::ColumnEncodingKind::DICTIONARY_V2: + return RleVersion::V2; + } + throw Exception(ErrorCodes::INCORRECT_DATA, "Unknown ORC column encoding {}", static_cast(encoding.kind)); +} + +std::unique_ptr getRequiredStream(StripeStreams & stripe, UInt64 column_id, Proto::StreamKind kind, const char * stream_name, const char * column_kind) +{ + auto stream = stripe.getStream(column_id, kind); + if (!stream) + throw Exception(ErrorCodes::INCORRECT_DATA, "{} stream not found in ORC {} column {}", stream_name, column_kind, column_id); + return stream; +} + +/// Converts the bytes read into the beginning of a buffer of integers into the integers, in place. +/// BOOLEAN and BYTE: the values are decoded into the bytes of the column. +template +class ByteColumnReaderImpl final : public ColumnReader +{ +public: + ByteColumnReaderImpl(const Type & type, StripeStreams & stripe) + : ColumnReader(type, stripe) + , rle(getRequiredStream(stripe, column_id, Proto::StreamKind::DATA, "DATA", type.toString().c_str())) + { + } + + size_t skip(size_t num_values) override + { + num_values = ColumnReader::skip(num_values); + rle.skip(num_values); + return num_values; + } + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + ColumnReader::next(batch, num_values, incoming_mask); + auto & data = typeid_cast(batch).prepareColumn(num_values); + rle.next(reinterpret_cast(data.data()), num_values, batch.hasNulls ? batch.notNull.data() : nullptr); + } + + void seekToRowGroup(std::unordered_map & positions) override + { + ColumnReader::seekToRowGroup(positions); + rle.seek(positions.at(column_id)); + } + +private: + Decoder rle; +}; + +using BooleanColumnReader = ByteColumnReaderImpl; +using ByteColumnReader = ByteColumnReaderImpl; + +/// SHORT, INT, LONG and DATE: the values are decoded into a column of the matching width. +template +class IntegerColumnReader final : public ColumnReader +{ +public: + IntegerColumnReader(const Type & type, StripeStreams & stripe) + : ColumnReader(type, stripe) + { + rle = createIntRleDecoder( + getRequiredStream(stripe, column_id, Proto::StreamKind::DATA, "DATA", "integer"), + true, getRleVersion(stripe.getEncoding(column_id))); + } + + size_t skip(size_t num_values) override + { + num_values = ColumnReader::skip(num_values); + rle->skip(num_values); + return num_values; + } + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + ColumnReader::next(batch, num_values, incoming_mask); + auto & data = typeid_cast(batch).prepareColumn(num_values); + rle->next(data.data(), num_values, batch.hasNulls ? batch.notNull.data() : nullptr); + } + + void seekToRowGroup(std::unordered_map & positions) override + { + ColumnReader::seekToRowGroup(positions); + rle->seek(positions.at(column_id)); + } + +private: + std::unique_ptr rle; +}; + +/// FLOAT and DOUBLE: the values are stored as is (little endian), so they are copied into the column. +template +class FloatingColumnReader final : public ColumnReader +{ +public: + FloatingColumnReader(const Type & type, StripeStreams & stripe) + : ColumnReader(type, stripe) + , stream(getRequiredStream(stripe, column_id, Proto::StreamKind::DATA, "DATA", "floating point")) + { + } + + size_t skip(size_t num_values) override + { + num_values = ColumnReader::skip(num_values); + stream->skipBytes(num_values * sizeof(ItemType)); + return num_values; + } + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + ColumnReader::next(batch, num_values, incoming_mask); + const char * not_null = batch.hasNulls ? batch.notNull.data() : nullptr; + auto & data = typeid_cast &>(batch).prepareColumn(num_values); + ItemType * out = data.data(); + + /// The stream has only the non-null values: read them at once to the beginning of the column. + size_t num_present = num_values; + if (not_null) + num_present = std::count_if(not_null, not_null + num_values, [](char c) { return c != 0; }); + + stream->readBytes(reinterpret_cast(out), num_present * sizeof(ItemType)); + if constexpr (std::endian::native == std::endian::big) + for (size_t i = 0; i < num_present; ++i) + out[i] = std::bit_cast(std::byteswap(std::bit_cast(out[i]))); + + if (!not_null) + return; + + /// Move the values to their rows from the back: the k-th value goes to a row i >= k, so it is not overwritten before it is moved. + size_t next_value = num_present; + for (size_t i = num_values; i-- > 0;) + out[i] = not_null[i] ? out[--next_value] : 0; + } + + void seekToRowGroup(std::unordered_map & positions) override + { + ColumnReader::seekToRowGroup(positions); + stream->seek(positions.at(column_id)); + } + +private: + using Bits = std::conditional_t; + + std::unique_ptr stream; +}; + +/// Converts the lengths in `offsets` into the end offsets (as in `ColumnString` and `ColumnArray`), starting +/// from `total`, and returns the total length. The lengths of the null values are zeros. +UInt64 lengthsToEndOffsets(UInt64 * offsets, size_t num_values, UInt64 column_id, UInt64 total = 0) +{ + for (size_t i = 0; i < num_values; ++i) + { + /// The lengths are non-negative, a negative one is huge as unsigned. + if (offsets[i] > static_cast(std::numeric_limits::max()) || common::addOverflow(total, offsets[i], total) + || total > static_cast(std::numeric_limits::max())) + throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid length {} in ORC column {}", static_cast(offsets[i]), column_id); + offsets[i] = total; + } + return total; +} + +/// Reads the strings with their lengths in a separate stream into a `ColumnString`. +ColumnString::MutablePtr readStrings( + IntRleDecoder & length_rle, InputByteStream & blob_stream, size_t num_values, const char * not_null, UInt64 column_id) +{ + /// The buffers grow as the data is decoded: a malformed count or length must not allocate memory + /// before the streams prove to contain the data. + static constexpr size_t MIN_STEP = 64 * 1024; + auto column = ColumnString::create(); + auto & offsets = column->getOffsets(); + UInt64 total = 0; + while (offsets.size() < num_values) + { + size_t done = offsets.size(); + size_t step = std::min(num_values - done, std::max(MIN_STEP, done)); + offsets.resize(done + step); + length_rle.next(offsets.data() + done, step, not_null ? not_null + done : nullptr); + total = lengthsToEndOffsets(offsets.data() + done, step, column_id, total); + } + auto & chars = column->getChars(); + while (chars.size() < total) + { + size_t done = chars.size(); + size_t step = std::min(total - done, std::max(MIN_STEP, done)); + chars.resize(done + step); + blob_stream.readBytes(reinterpret_cast(chars.data() + done), step); + } + return column; +} + +std::shared_ptr loadStringDictionary(UInt64 column_id, StripeStreams & stripe) +{ + const auto & encoding = stripe.getEncoding(column_id); + /// The dictionary is not bounded by the number of rows of the stripe: a column nested in a list or a map + /// can have more values than the stripe has rows. + UInt32 dictionary_size = encoding.dictionary_size; + + auto dictionary = std::make_shared(); + if (dictionary_size == 0) + { + dictionary->column = ColumnString::create(); + return dictionary; + } + + auto length_rle = createIntRleDecoder( + getRequiredStream(stripe, column_id, Proto::StreamKind::LENGTH, "LENGTH", "string dictionary"), false, getRleVersion(encoding)); + auto blob_stream = stripe.getStream(column_id, Proto::StreamKind::DICTIONARY_DATA); + if (!blob_stream) + { + /// All entries can be empty strings, then the stream is not needed. + blob_stream = std::make_unique(std::string_view{}, CompressionKind::NONE, 0, "empty dictionary"); + } + dictionary->column = readStrings(*length_rle, *blob_stream, dictionary_size, nullptr, column_id); + return dictionary; +} + +class StringDictionaryColumnReader final : public ColumnReader +{ +public: + StringDictionaryColumnReader(const Type & type, StripeStreams & stripe) + : ColumnReader(type, stripe) + { + rle = createIntRleDecoder( + getRequiredStream(stripe, column_id, Proto::StreamKind::DATA, "DATA", "string dictionary"), + false, getRleVersion(stripe.getEncoding(column_id))); + dictionary = loadStringDictionary(column_id, stripe); + } + + size_t skip(size_t num_values) override + { + num_values = ColumnReader::skip(num_values); + rle->skip(num_values); + return num_values; + } + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + ColumnReader::next(batch, num_values, incoming_mask); + const char * not_null = batch.hasNulls ? batch.notNull.data() : nullptr; + auto & string_batch = dynamic_cast(batch); + index.resize(num_values); + rle->next(index.data(), num_values, not_null); + string_batch.isEncoded = false; + string_batch.column = decodeDictionary(assert_cast(*dictionary->column), index.data(), not_null, num_values); + } + + void nextEncoded(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + readIndex(dynamic_cast(batch), num_values, incoming_mask); + } + + void seekToRowGroup(std::unordered_map & positions) override + { + ColumnReader::seekToRowGroup(positions); + rle->seek(positions.at(column_id)); + } + +private: + void readIndex(EncodedStringVectorBatch & batch, size_t num_values, const char * incoming_mask) + { + ColumnReader::next(batch, num_values, incoming_mask); + batch.isEncoded = true; + batch.dictionary = dictionary; + batch.column = nullptr; + batch.index.resize(num_values); + rle->next(batch.index.data(), num_values, batch.hasNulls ? batch.notNull.data() : nullptr); + } + + std::shared_ptr dictionary; + std::unique_ptr rle; + PaddedPODArray index; +}; + +class StringDirectColumnReader final : public ColumnReader +{ +public: + StringDirectColumnReader(const Type & type, StripeStreams & stripe) + : ColumnReader(type, stripe) + { + length_rle = createIntRleDecoder( + getRequiredStream(stripe, column_id, Proto::StreamKind::LENGTH, "LENGTH", "string"), + false, getRleVersion(stripe.getEncoding(column_id))); + blob_stream = getRequiredStream(stripe, column_id, Proto::StreamKind::DATA, "DATA", "string"); + } + + size_t skip(size_t num_values) override + { + num_values = ColumnReader::skip(num_values); + static constexpr size_t BUFFER_SIZE = 1024; + UInt64 buffer[BUFFER_SIZE]; + size_t done = 0; + size_t total_bytes = 0; + while (done < num_values) + { + size_t step = std::min(BUFFER_SIZE, num_values - done); + length_rle->next(buffer, step, nullptr); + total_bytes += lengthsToEndOffsets(buffer, step, column_id); + done += step; + } + blob_stream->skipBytes(total_bytes); + return num_values; + } + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + ColumnReader::next(batch, num_values, incoming_mask); + auto & string_batch = dynamic_cast(batch); + string_batch.isEncoded = false; + string_batch.column = readStrings(*length_rle, *blob_stream, num_values, batch.hasNulls ? batch.notNull.data() : nullptr, column_id); + } + + void seekToRowGroup(std::unordered_map & positions) override + { + ColumnReader::seekToRowGroup(positions); + auto & provider = positions.at(column_id); + blob_stream->seek(provider); + length_rle->seek(provider); + } + +private: + std::unique_ptr length_rle; + std::unique_ptr blob_stream; +}; + +class TimestampColumnReader final : public ColumnReader +{ +public: + TimestampColumnReader(const Type & type, StripeStreams & stripe, bool is_instant) + : ColumnReader(type, stripe) + , writer_timezone(is_instant ? DateLUT::instance("UTC") : stripe.getWriterTimezone()) + , reader_timezone(is_instant ? DateLUT::instance("UTC") : stripe.getReaderTimezone()) + , same_timezone(&writer_timezone == &reader_timezone || writer_timezone.getTimeZone() == reader_timezone.getTimeZone()) + , writer_offset_of(writer_timezone) + , reader_offset_of(reader_timezone) + { + /// The base of the seconds is 2015-01-01 00:00:00 in the time zone of the writer. + static constexpr Int64 UTC_2015_01_01 = 1420070400; + epoch_offset = UTC_2015_01_01 - writer_timezone.timezoneOffset(UTC_2015_01_01); + + RleVersion version = getRleVersion(stripe.getEncoding(column_id)); + seconds_rle = createIntRleDecoder(getRequiredStream(stripe, column_id, Proto::StreamKind::DATA, "DATA", "timestamp"), true, version); + nanos_rle = createIntRleDecoder(getRequiredStream(stripe, column_id, Proto::StreamKind::SECONDARY, "SECONDARY", "timestamp"), false, version); + } + + size_t skip(size_t num_values) override + { + num_values = ColumnReader::skip(num_values); + seconds_rle->skip(num_values); + nanos_rle->skip(num_values); + return num_values; + } + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + ColumnReader::next(batch, num_values, incoming_mask); + const char * not_null = batch.hasNulls ? batch.notNull.data() : nullptr; + auto & timestamp_batch = typeid_cast(batch); + timestamp_batch.data.resize(num_values); + timestamp_batch.nanoseconds.resize(num_values); + Int64 * seconds = timestamp_batch.data.data(); + Int64 * nanos = timestamp_batch.nanoseconds.data(); + seconds_rle->next(seconds, num_values, not_null); + nanos_rle->next(nanos, num_values, not_null); + + for (size_t i = 0; i < num_values; ++i) + { + if (not_null && !not_null[i]) + continue; + + const Int64 encoded_nanos = nanos[i]; + /// The lowest 3 bits are the number of the trailing zeros (minus one) that are removed. + UInt64 zeros = nanos[i] & 0x7; + /// An arithmetic shift, as in the reference implementation: some writers (the Arrow + /// adapter) store negative nanoseconds for the timestamps before the epoch. + nanos[i] >>= 3; + if (zeros != 0 && common::mulOverflow(nanos[i], POWERS_OF_TEN_FOR_NANOS[zeros + 1], nanos[i])) + throwInvalidTimestamp(seconds[i], encoded_nanos); + if (nanos[i] < -999'999'999 || nanos[i] > 999'999'999) + throwInvalidTimestamp(seconds[i], encoded_nanos); + + Int64 writer_time = 0; + if (common::addOverflow(seconds[i], epoch_offset, writer_time)) + throwInvalidTimestamp(seconds[i], encoded_nanos); + if (writer_time < 0 && nanos[i] > 999999 && common::subOverflow(writer_time, Int64{1}, writer_time)) + throwInvalidTimestamp(seconds[i], encoded_nanos); + + if (!same_timezone) + { + /// Keep the same wall clock time when the writer and the reader time zones differ. + Int64 writer_offset = writer_offset_of(writer_time); + Int64 reader_offset = reader_offset_of(writer_time); + if (writer_offset != reader_offset) + { + /// If the adjustment crosses a DST boundary, the offset has to be reevaluated. + Int64 adjusted_time = 0; + if (common::addOverflow(writer_time, writer_offset - reader_offset, adjusted_time)) + throwInvalidTimestamp(seconds[i], encoded_nanos); + Int64 adjusted_reader_offset = reader_offset_of(adjusted_time); + if (common::addOverflow(writer_time, writer_offset - adjusted_reader_offset, writer_time)) + throwInvalidTimestamp(seconds[i], encoded_nanos); + } + } + seconds[i] = writer_time; + } + } + + void seekToRowGroup(std::unordered_map & positions) override + { + ColumnReader::seekToRowGroup(positions); + auto & provider = positions.at(column_id); + seconds_rle->seek(provider); + nanos_rle->seek(provider); + } + +private: + static constexpr Int64 POWERS_OF_TEN_FOR_NANOS[9] = {1, 10, 100, 1000, 10000, 100000, 1000000, 10000000, 100000000}; + + [[noreturn]] void throwInvalidTimestamp(Int64 seconds, Int64 encoded_nanos) const + { + throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid timestamp in ORC column {}: seconds={}, encoded nanoseconds={}", column_id, seconds, encoded_nanos); + } + + const DateLUTImpl & writer_timezone; + const DateLUTImpl & reader_timezone; + const bool same_timezone; + const TimezoneOffset writer_offset_of; + const TimezoneOffset reader_offset_of; + Int64 epoch_offset = 0; + std::unique_ptr seconds_rle; + std::unique_ptr nanos_rle; +}; + +constexpr UInt32 MAX_PRECISION_64 = 18; +constexpr UInt32 MAX_PRECISION_128 = 38; + +constexpr Int64 POWERS_OF_TEN[MAX_PRECISION_64 + 1] = { + 1LL, + 10LL, + 100LL, + 1000LL, + 10000LL, + 100000LL, + 1000000LL, + 10000000LL, + 100000000LL, + 1000000000LL, + 10000000000LL, + 100000000000LL, + 1000000000000LL, + 10000000000000LL, + 100000000000000LL, + 1000000000000000LL, + 10000000000000000LL, + 100000000000000000LL, + 1000000000000000000LL}; + +/// The DECIMAL columns: an unbounded signed varint per value in the DATA stream and the scale of +/// each value in the SECONDARY stream. The values are rescaled to the scale of the type. +class DecimalColumnReader : public ColumnReader +{ +public: + DecimalColumnReader(const Type & type, StripeStreams & stripe) + : ColumnReader(type, stripe) + , precision(static_cast(type.getPrecision())) + , scale(static_cast(type.getScale())) + { + if (type.getPrecision() > MAX_PRECISION_128 || type.getScale() > MAX_PRECISION_128) + throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid precision {} or scale {} of decimal ORC column {}", + type.getPrecision(), type.getScale(), column_id); + value_stream = getRequiredStream(stripe, column_id, Proto::StreamKind::DATA, "DATA", "decimal"); + scale_decoder = createIntRleDecoder( + getRequiredStream(stripe, column_id, Proto::StreamKind::SECONDARY, "SECONDARY", "decimal"), + true, getRleVersion(stripe.getEncoding(column_id))); + } + + size_t skip(size_t num_values) override + { + num_values = ColumnReader::skip(num_values); + size_t skipped = 0; + while (skipped < num_values) + if (!(value_stream->readByte() & 0x80)) + ++skipped; + scale_decoder->skip(num_values); + return num_values; + } + + void seekToRowGroup(std::unordered_map & positions) override + { + ColumnReader::seekToRowGroup(positions); + auto & provider = positions.at(column_id); + value_stream->seek(provider); + scale_decoder->seek(provider); + } + +protected: + /// Reads the zigzag-encoded varint of up to 128 bits. + Int128 readInt128() + { + UInt128 value = 0; + UInt32 shift = 0; + while (true) + { + UInt8 byte = value_stream->readByte(); + if (shift < 128) + value |= static_cast(byte & 0x7F) << shift; + shift += 7; + if (!(byte & 0x80)) + break; + } + return static_cast((value >> 1) ^ (UInt128(0) - (value & 1))); + } + + /// The scale of each value is read from the file. The valid scales are from 0 to 38, the bound + /// also keeps the differences of the scales and the rescaling work small. + Int32 getValueScale(Int64 value_scale) const + { + if (value_scale < -Int64(MAX_PRECISION_128) || value_scale > Int64(MAX_PRECISION_128)) + throw Exception(ErrorCodes::INCORRECT_DATA, "Decimal scale {} is out of range in ORC column {}", value_scale, column_id); + return static_cast(value_scale); + } + + static void scaleInt128(Int128 & value, Int32 target_scale, Int32 current_scale) + { + while (target_scale > current_scale) + { + Int32 adjust = std::min(MAX_PRECISION_64, target_scale - current_scale); + value = static_cast(static_cast(value) * static_cast(POWERS_OF_TEN[adjust])); + current_scale += adjust; + } + while (current_scale > target_scale) + { + Int32 adjust = std::min(MAX_PRECISION_64, current_scale - target_scale); + value /= POWERS_OF_TEN[adjust]; + current_scale -= adjust; + } + } + + const Int32 precision; + Int32 scale; + std::unique_ptr value_stream; + std::unique_ptr scale_decoder; + PaddedPODArray scales; +}; + +class Decimal64ColumnReader final : public DecimalColumnReader +{ +public: + using DecimalColumnReader::DecimalColumnReader; + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + ColumnReader::next(batch, num_values, incoming_mask); + const char * not_null = batch.hasNulls ? batch.notNull.data() : nullptr; + auto & decimal_batch = typeid_cast(batch); + scales.resize(num_values); + scale_decoder->next(scales.data(), num_values, not_null); + decimal_batch.precision = precision; + decimal_batch.scale = scale; + decimal_batch.values.resize(num_values); + Int64 * values = decimal_batch.values.data(); + for (size_t i = 0; i < num_values; ++i) + { + if (not_null && !not_null[i]) + continue; + values[i] = readInt64(getValueScale(scales[i])); + } + } + +private: + Int64 readInt64(Int32 current_scale) + { + UInt64 encoded = readVarint(*value_stream); + Int64 value = static_cast((encoded >> 1) ^ (~(encoded & 1) + 1)); + if (scale > current_scale && static_cast(scale - current_scale) <= MAX_PRECISION_64) + value = static_cast(static_cast(value) * static_cast(POWERS_OF_TEN[scale - current_scale])); + else if (scale < current_scale && static_cast(current_scale - scale) <= MAX_PRECISION_64) + value /= POWERS_OF_TEN[current_scale - scale]; + else if (scale != current_scale) + throw Exception(ErrorCodes::INCORRECT_DATA, "Decimal scale {} is out of range in ORC column {}", current_scale, column_id); + return value; + } +}; + +class Decimal128ColumnReader final : public DecimalColumnReader +{ +public: + using DecimalColumnReader::DecimalColumnReader; + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + ColumnReader::next(batch, num_values, incoming_mask); + const char * not_null = batch.hasNulls ? batch.notNull.data() : nullptr; + auto & decimal_batch = typeid_cast(batch); + scales.resize(num_values); + scale_decoder->next(scales.data(), num_values, not_null); + decimal_batch.precision = precision; + decimal_batch.scale = scale; + decimal_batch.values.resize(num_values); + Int128 * values = decimal_batch.values.data(); + for (size_t i = 0; i < num_values; ++i) + { + if (not_null && !not_null[i]) + continue; + values[i] = readInt128(); + scaleInt128(values[i], scale, getValueScale(scales[i])); + } + } +}; + +/// The DECIMAL columns with the precision up to 18 in the files of the format version 1.9999 +/// (`UNSTABLE_PRE_2_0`): the unscaled values are signed RLEv2 integers in the DATA stream, +/// all with the scale of the type, and there is no SECONDARY stream. +class Decimal64V2ColumnReader final : public ColumnReader +{ +public: + Decimal64V2ColumnReader(const Type & type, StripeStreams & stripe) + : ColumnReader(type, stripe) + , precision(static_cast(type.getPrecision())) + , scale(static_cast(type.getScale())) + { + if (type.getScale() > type.getPrecision()) + throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid precision {} or scale {} of decimal ORC column {}", + type.getPrecision(), type.getScale(), column_id); + /// As in the reference implementation, RLEv2 is used regardless of the column encoding. + value_decoder = createIntRleDecoder( + getRequiredStream(stripe, column_id, Proto::StreamKind::DATA, "DATA", "decimal"), true, RleVersion::V2); + } + + size_t skip(size_t num_values) override + { + num_values = ColumnReader::skip(num_values); + value_decoder->skip(num_values); + return num_values; + } + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + ColumnReader::next(batch, num_values, incoming_mask); + auto & decimal_batch = typeid_cast(batch); + decimal_batch.precision = precision; + decimal_batch.scale = scale; + decimal_batch.values.resize(num_values); + value_decoder->next(decimal_batch.values.data(), num_values, batch.hasNulls ? batch.notNull.data() : nullptr); + } + + void seekToRowGroup(std::unordered_map & positions) override + { + ColumnReader::seekToRowGroup(positions); + value_decoder->seek(positions.at(column_id)); + } + +private: + const Int32 precision; + const Int32 scale; + std::unique_ptr value_decoder; +}; + +/// The DECIMAL columns written by Hive 0.11, without the precision and the scale in the type. +/// As in the reference implementation, the values are read with the scale 6 and must have at most 38 digits. +class DecimalHive11ColumnReader final : public DecimalColumnReader +{ +public: + DecimalHive11ColumnReader(const Type & type, StripeStreams & stripe) + : DecimalColumnReader(type, stripe) + { + scale = 6; + } + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + ColumnReader::next(batch, num_values, incoming_mask); + const char * not_null = batch.hasNulls ? batch.notNull.data() : nullptr; + auto & decimal_batch = typeid_cast(batch); + scales.resize(num_values); + scale_decoder->next(scales.data(), num_values, not_null); + decimal_batch.precision = precision; + decimal_batch.scale = scale; + decimal_batch.values.resize(num_values); + Int128 * values = decimal_batch.values.data(); + + static const Int128 max_value = Int128(10000000000000000000ULL) * Int128(10000000000000000000ULL) - 1; + for (size_t i = 0; i < num_values; ++i) + { + if (not_null && !not_null[i]) + continue; + bool overflow = false; + UInt128 value = 0; + UInt32 shift = 0; + while (true) + { + UInt8 byte = value_stream->readByte(); + UInt32 bits = byte & 0x7F; + if (shift > 128 || (shift == 126 && bits > 3)) + overflow = true; + if (shift < 128) + value |= static_cast(bits) << shift; + shift += 7; + if (!(byte & 0x80)) + break; + } + Int128 decoded = static_cast((value >> 1) ^ (UInt128(0) - (value & 1))); + if (!overflow) + { + scaleInt128(decoded, scale, getValueScale(scales[i])); + overflow = decoded > max_value || decoded < -max_value; + } + if (overflow) + throw Exception(ErrorCodes::INCORRECT_DATA, "Hive 0.11 decimal was more than 38 digits in ORC column {}", column_id); + values[i] = decoded; + } + } +}; + +class StructColumnReader final : public ColumnReader +{ +public: + StructColumnReader(const Type & type, StripeStreams & stripe) + : ColumnReader(type, stripe) + { + const auto & encoding = stripe.getEncoding(column_id); + if (encoding.kind != Proto::ColumnEncodingKind::DIRECT) + throw Exception(ErrorCodes::INCORRECT_DATA, "Unknown encoding {} of ORC struct column {}", static_cast(encoding.kind), column_id); + const auto & selected = stripe.getSelectedColumns(); + for (UInt64 i = 0; i < type.getSubtypeCount(); ++i) + { + const Type & child = *type.getSubtype(i); + if (selected[child.getColumnId()]) + children.push_back(buildColumnReader(child, stripe)); + } + } + + size_t skip(size_t num_values) override + { + num_values = ColumnReader::skip(num_values); + for (auto & child : children) + child->skip(num_values); + return num_values; + } + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + nextImpl(batch, num_values, incoming_mask); + } + + void nextEncoded(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + nextImpl(batch, num_values, incoming_mask); + } + + void seekToRowGroup(std::unordered_map & positions) override + { + ColumnReader::seekToRowGroup(positions); + for (auto & child : children) + child->seekToRowGroup(positions); + } + +private: + template + void nextImpl(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) + { + ColumnReader::next(batch, num_values, incoming_mask); + const char * not_null = batch.hasNulls ? batch.notNull.data() : nullptr; + auto & struct_batch = typeid_cast(batch); + for (size_t i = 0; i < children.size(); ++i) + { + if constexpr (encoded) + children[i]->nextEncoded(*struct_batch.fields[i], num_values, not_null); + else + children[i]->next(*struct_batch.fields[i], num_values, not_null); + } + } + + std::vector> children; +}; + +/// Reads the lengths of the lists or maps as the end offsets, returns the total number of the children. +UInt64 readListOffsets(IntRleDecoder & rle, ColumnPtr & result, size_t num_values, const char * not_null, UInt64 column_id) +{ + auto column = ColumnUInt64::create(num_values); + auto & offsets = column->getData(); + rle.next(offsets.data(), num_values, not_null); + UInt64 total = lengthsToEndOffsets(offsets.data(), num_values, column_id); + result = std::move(column); + return total; +} + +UInt64 sumLengths(IntRleDecoder & rle, size_t num_values, UInt64 column_id) +{ + static constexpr size_t BUFFER_SIZE = 1024; + Int64 buffer[BUFFER_SIZE]; + UInt64 total = 0; + size_t read = 0; + while (read < num_values) + { + size_t chunk = std::min(num_values - read, BUFFER_SIZE); + rle.next(buffer, chunk, nullptr); + for (size_t i = 0; i < chunk; ++i) + { + if (buffer[i] < 0) + throw Exception(ErrorCodes::INCORRECT_DATA, "Negative length {} in ORC column {}", buffer[i], column_id); + total += static_cast(buffer[i]); + } + read += chunk; + } + return total; +} + +class ListColumnReader final : public ColumnReader +{ +public: + ListColumnReader(const Type & type, StripeStreams & stripe) + : ColumnReader(type, stripe) + { + rle = createIntRleDecoder( + getRequiredStream(stripe, column_id, Proto::StreamKind::LENGTH, "LENGTH", "list"), + false, getRleVersion(stripe.getEncoding(column_id))); + const Type & child_type = *type.getSubtype(0); + if (stripe.getSelectedColumns()[child_type.getColumnId()]) + child = buildColumnReader(child_type, stripe); + } + + size_t skip(size_t num_values) override + { + num_values = ColumnReader::skip(num_values); + if (child) + child->skip(sumLengths(*rle, num_values, column_id)); + else + rle->skip(num_values); + return num_values; + } + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + nextImpl(batch, num_values, incoming_mask); + } + + void nextEncoded(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + nextImpl(batch, num_values, incoming_mask); + } + + void seekToRowGroup(std::unordered_map & positions) override + { + ColumnReader::seekToRowGroup(positions); + rle->seek(positions.at(column_id)); + if (child) + child->seekToRowGroup(positions); + } + +private: + template + void nextImpl(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) + { + ColumnReader::next(batch, num_values, incoming_mask); + auto & list_batch = typeid_cast(batch); + const char * not_null = list_batch.hasNulls ? list_batch.notNull.data() : nullptr; + UInt64 total = readListOffsets(*rle, list_batch.offsets, num_values, not_null, column_id); + if (child) + { + if constexpr (encoded) + child->nextEncoded(*list_batch.elements, total, nullptr); + else + child->next(*list_batch.elements, total, nullptr); + } + } + + std::unique_ptr rle; + std::unique_ptr child; +}; + +class MapColumnReader final : public ColumnReader +{ +public: + MapColumnReader(const Type & type, StripeStreams & stripe) + : ColumnReader(type, stripe) + { + rle = createIntRleDecoder( + getRequiredStream(stripe, column_id, Proto::StreamKind::LENGTH, "LENGTH", "map"), + false, getRleVersion(stripe.getEncoding(column_id))); + const auto & selected = stripe.getSelectedColumns(); + const Type & key_type = *type.getSubtype(0); + if (selected[key_type.getColumnId()]) + key_reader = buildColumnReader(key_type, stripe); + const Type & value_type = *type.getSubtype(1); + if (selected[value_type.getColumnId()]) + value_reader = buildColumnReader(value_type, stripe); + } + + size_t skip(size_t num_values) override + { + num_values = ColumnReader::skip(num_values); + if (key_reader || value_reader) + { + UInt64 total = sumLengths(*rle, num_values, column_id); + if (key_reader) + key_reader->skip(total); + if (value_reader) + value_reader->skip(total); + } + else + rle->skip(num_values); + return num_values; + } + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + nextImpl(batch, num_values, incoming_mask); + } + + void nextEncoded(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + nextImpl(batch, num_values, incoming_mask); + } + + void seekToRowGroup(std::unordered_map & positions) override + { + ColumnReader::seekToRowGroup(positions); + rle->seek(positions.at(column_id)); + if (key_reader) + key_reader->seekToRowGroup(positions); + if (value_reader) + value_reader->seekToRowGroup(positions); + } + +private: + template + void nextImpl(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) + { + ColumnReader::next(batch, num_values, incoming_mask); + auto & map_batch = typeid_cast(batch); + const char * not_null = map_batch.hasNulls ? map_batch.notNull.data() : nullptr; + UInt64 total = readListOffsets(*rle, map_batch.offsets, num_values, not_null, column_id); + if (key_reader) + { + if constexpr (encoded) + key_reader->nextEncoded(*map_batch.keys, total, nullptr); + else + key_reader->next(*map_batch.keys, total, nullptr); + } + if (value_reader) + { + if constexpr (encoded) + value_reader->nextEncoded(*map_batch.elements, total, nullptr); + else + value_reader->next(*map_batch.elements, total, nullptr); + } + } + + std::unique_ptr rle; + std::unique_ptr key_reader; + std::unique_ptr value_reader; +}; + +class UnionColumnReader final : public ColumnReader +{ +public: + UnionColumnReader(const Type & type, StripeStreams & stripe) + : ColumnReader(type, stripe) + , rle(getRequiredStream(stripe, column_id, Proto::StreamKind::DATA, "DATA", "union")) + , num_children(type.getSubtypeCount()) + { + children.resize(num_children); + counts.resize(num_children); + const auto & selected = stripe.getSelectedColumns(); + for (size_t i = 0; i < num_children; ++i) + { + const Type & child = *type.getSubtype(i); + if (selected[child.getColumnId()]) + children[i] = buildColumnReader(child, stripe); + } + } + + size_t skip(size_t num_values) override + { + num_values = ColumnReader::skip(num_values); + std::fill(counts.begin(), counts.end(), 0); + static constexpr size_t BUFFER_SIZE = 1024; + char buffer[BUFFER_SIZE]; + size_t read = 0; + while (read < num_values) + { + size_t chunk = std::min(num_values - read, BUFFER_SIZE); + rle.next(buffer, chunk, nullptr); + for (size_t i = 0; i < chunk; ++i) + ++counts[checkedTag(static_cast(buffer[i]))]; + read += chunk; + } + for (size_t i = 0; i < num_children; ++i) + if (counts[i] && children[i]) + children[i]->skip(counts[i]); + return num_values; + } + + void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + nextImpl(batch, num_values, incoming_mask); + } + + void nextEncoded(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) override + { + nextImpl(batch, num_values, incoming_mask); + } + + void seekToRowGroup(std::unordered_map & positions) override + { + ColumnReader::seekToRowGroup(positions); + rle.seek(positions.at(column_id)); + for (auto & child : children) + if (child) + child->seekToRowGroup(positions); + } + +private: + size_t checkedTag(UInt8 tag) const + { + if (tag >= num_children) + throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid union tag {} for union with {} children in ORC column {}", static_cast(tag), num_children, column_id); + return tag; + } + + template + void nextImpl(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) + { + ColumnReader::next(batch, num_values, incoming_mask); + auto & union_batch = typeid_cast(batch); + const char * not_null = union_batch.hasNulls ? union_batch.notNull.data() : nullptr; + union_batch.tags.resize(num_values); + union_batch.offsets.resize(num_values); + UInt8 * tags = union_batch.tags.data(); + UInt64 * offsets = union_batch.offsets.data(); + memset(tags, 0, num_values); + rle.next(reinterpret_cast(tags), num_values, not_null); + std::fill(counts.begin(), counts.end(), 0); + for (size_t i = 0; i < num_values; ++i) + { + if (not_null && !not_null[i]) + continue; + offsets[i] = counts[checkedTag(tags[i])]++; + } + for (size_t i = 0; i < num_children; ++i) + { + if (!children[i]) + continue; + if constexpr (encoded) + children[i]->nextEncoded(*union_batch.children[i], counts[i], nullptr); + else + children[i]->next(*union_batch.children[i], counts[i], nullptr); + } + } + + ByteRleDecoder rle; + const size_t num_children; + std::vector> children; + std::vector counts; +}; + +} + + +ColumnReader::ColumnReader(const Type & type, StripeStreams & stripe) + : column_id(type.getColumnId()) +{ + if (auto stream = stripe.getStream(column_id, Proto::StreamKind::PRESENT)) + not_null_decoder = std::make_unique(std::move(stream)); +} + +void ColumnReader::next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) +{ + batch.reset(num_values); + + if (not_null_decoder) + { + char * not_null = batch.notNull.data(); + not_null_decoder->next(not_null, num_values, incoming_mask); + batch.hasNulls = memchr(not_null, 0, num_values) != nullptr; + return; + } + if (incoming_mask) + { + batch.hasNulls = true; + memcpy(batch.notNull.data(), incoming_mask, num_values); + return; + } + batch.hasNulls = false; +} + +size_t ColumnReader::skip(size_t num_values) +{ + if (!not_null_decoder) + return num_values; + + static constexpr size_t BUFFER_SIZE = 32768; + char buffer[BUFFER_SIZE]; + size_t remaining = num_values; + while (remaining) + { + size_t chunk = std::min(remaining, BUFFER_SIZE); + not_null_decoder->next(buffer, chunk, nullptr); + for (size_t i = 0; i < chunk; ++i) + if (!buffer[i]) + --num_values; + remaining -= chunk; + } + return num_values; +} + +void ColumnReader::seekToRowGroup(std::unordered_map & positions) +{ + if (not_null_decoder) + { + auto it = positions.find(column_id); + if (it == positions.end()) + throw Exception(ErrorCodes::INCORRECT_DATA, "No row index for ORC column {}", column_id); + not_null_decoder->seek(it->second); + } +} + +std::unique_ptr buildColumnReader(const Type & type, StripeStreams & stripe) +{ + checkStackSize(); + + switch (type.getKind()) + { + case BOOLEAN: + return std::make_unique(type, stripe); + case BYTE: + return std::make_unique(type, stripe); + case SHORT: + return std::make_unique>(type, stripe); + case INT: + return std::make_unique>(type, stripe); + case LONG: + case DATE: + return std::make_unique>(type, stripe); + case FLOAT: + return std::make_unique>(type, stripe); + case DOUBLE: + return std::make_unique>(type, stripe); + case STRING: + case BINARY: + case CHAR: + case VARCHAR: + case GEOMETRY: + case GEOGRAPHY: + switch (stripe.getEncoding(type.getColumnId()).kind) + { + case Proto::ColumnEncodingKind::DICTIONARY: + case Proto::ColumnEncodingKind::DICTIONARY_V2: + return std::make_unique(type, stripe); + case Proto::ColumnEncodingKind::DIRECT: + case Proto::ColumnEncodingKind::DIRECT_V2: + return std::make_unique(type, stripe); + } + break; + case TIMESTAMP: + return std::make_unique(type, stripe, false); + case TIMESTAMP_INSTANT: + return std::make_unique(type, stripe, true); + case DECIMAL: + if (type.getPrecision() == 0) + return std::make_unique(type, stripe); + if (type.getPrecision() <= MAX_PRECISION_64) + { + if (stripe.isDecimalAsLong()) + return std::make_unique(type, stripe); + return std::make_unique(type, stripe); + } + return std::make_unique(type, stripe); + case STRUCT: + return std::make_unique(type, stripe); + case LIST: + return std::make_unique(type, stripe); + case MAP: + return std::make_unique(type, stripe); + case UNION: + return std::make_unique(type, stripe); + } + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Reading ORC type {} is not supported", type.toString()); +} + +} diff --git a/src/Processors/Formats/Impl/ORC/ColumnReader.h b/src/Processors/Formats/Impl/ORC/ColumnReader.h new file mode 100644 index 0000000000000..1abaedb6fff58 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/ColumnReader.h @@ -0,0 +1,63 @@ +#pragma once + +#include +#include + +#include + +class DateLUTImpl; + +namespace DB::ORC +{ + +/// Access to the streams of one stripe for the column readers. +class StripeStreams +{ +public: + virtual ~StripeStreams() = default; + + virtual const Proto::ColumnEncoding & getEncoding(UInt64 column_id) const = 0; + /// Returns nullptr if the stripe does not have this stream. + virtual std::unique_ptr getStream(UInt64 column_id, Proto::StreamKind kind) const = 0; + virtual const std::vector & getSelectedColumns() const = 0; + /// The time zones for TIMESTAMP (not TIMESTAMP_INSTANT) columns. + virtual const DateLUTImpl & getWriterTimezone() const = 0; + virtual const DateLUTImpl & getReaderTimezone() const = 0; + virtual UInt64 getNumberOfRows() const = 0; + /// Whether the file has the format version 1.9999 (`UNSTABLE_PRE_2_0` in the reference implementation), + /// where DECIMAL columns with the precision up to 18 are written as signed RLEv2 integers in the DATA stream + /// without the SECONDARY stream of scales. + virtual bool isDecimalAsLong() const = 0; +}; + +class ColumnReader +{ +public: + ColumnReader(const Type & type, StripeStreams & stripe); + virtual ~ColumnReader() = default; + + /// Reads the next `num_values` values into `batch`. `incoming_mask` is the mask of the + /// non-null values of the parent (or nullptr). + virtual void next(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask); + + /// Same, but the string columns with dictionary encoding fill `EncodedStringVectorBatch` + /// without decoding the dictionary. + virtual void nextEncoded(ColumnVectorBatch & batch, size_t num_values, const char * incoming_mask) + { + next(batch, num_values, incoming_mask); + } + + /// Skips `num_values` values, returns the number of the skipped non-null values. + virtual size_t skip(size_t num_values); + + /// Moves to the beginning of a row group, by the positions from the row indexes of the columns. + virtual void seekToRowGroup(std::unordered_map & positions); + +protected: + const UInt64 column_id; + std::unique_ptr not_null_decoder; +}; + +std::unique_ptr buildColumnReader(const Type & type, StripeStreams & stripe); + +} diff --git a/src/Processors/Formats/Impl/ORC/Compression.cpp b/src/Processors/Formats/Impl/ORC/Compression.cpp new file mode 100644 index 0000000000000..cfacdc431ca8d --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Compression.cpp @@ -0,0 +1,437 @@ +#include + +#include + +#include +#include +#include +/// For `ZSTD_decompressBound`, which is not in the stable ABI of zstd. It is safe, as zstd is linked statically from contrib. +#define ZSTD_STATIC_LINKING_ONLY +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int INCORRECT_DATA; + extern const int CANNOT_COMPRESS; + extern const int NOT_IMPLEMENTED; +} +} + +namespace DB::ORC +{ + +namespace +{ + +[[noreturn]] void throwMalformedLzo(size_t offset) +{ + throw Exception(ErrorCodes::INCORRECT_DATA, "Malformed LZO-compressed data in ORC file at offset {}", offset); +} + +/// LZO1X decompressor, as written by the Hadoop codecs (and the ORC Java writer). A chunk can be a +/// concatenation of several LZO blocks, each terminated with the end-of-stream marker. +size_t decompressLzo(const char * src, size_t src_size, char * dst, size_t dst_capacity) +{ + const UInt8 * input = reinterpret_cast(src); + const UInt8 * const input_begin = input; + const UInt8 * const input_end = input + src_size; + char * output = dst; + char * const output_end = dst + dst_capacity; + + auto offset = [&] { return static_cast(input - input_begin); }; + + auto read_byte = [&]() -> UInt32 + { + if (input >= input_end) + throwMalformedLzo(offset()); + return *input++; + }; + + /// A run of zero bytes extends a length by 255 each, then the next byte is added. + auto read_variable_length = [&](UInt32 base) -> size_t + { + size_t length = base; + UInt32 next = 0; + while ((next = read_byte()) == 0) + { + length += 0xFF; + if (length > dst_capacity) + throwMalformedLzo(offset()); + } + return length + next; + }; + + while (input < input_end) + { + bool first_command = true; + size_t last_literal_length = 0; + + while (true) + { + UInt32 command = read_byte(); + if (command == 0x11) + break; + + size_t match_length = 0; + size_t match_offset = 0; + size_t literal_length = 0; + + if ((command & 0xF0) == 0) + { + if (last_literal_length == 0) + { + /// A long run of literals. + literal_length = command & 0xF; + if (literal_length == 0) + literal_length = read_variable_length(0xF); + literal_length += 3; + } + else if (last_literal_length <= 3) + { + match_length = 3; + match_offset = (command & 0xC) >> 2; + match_offset |= read_byte() << 2; + match_offset |= 0x800; + literal_length = command & 0x3; + } + else + { + match_length = 2; + match_offset = (command & 0xC) >> 2; + match_offset |= read_byte() << 2; + literal_length = command & 0x3; + } + } + else if (first_command) + { + literal_length = command - 17; + } + else if ((command & 0xF0) == 0x10) + { + match_length = command & 0x7; + if (match_length == 0) + match_length = read_variable_length(0x7); + match_length += 2; + UInt32 trailer = read_byte(); + trailer |= read_byte() << 8; + match_offset = trailer >> 2; + match_offset |= (command & 0x8) ? 0x8000 : 0x4000; + --match_offset; + literal_length = trailer & 0x3; + } + else if ((command & 0xE0) == 0x20) + { + match_length = command & 0x1F; + if (match_length == 0) + match_length = read_variable_length(0x1F); + match_length += 2; + UInt32 trailer = read_byte(); + trailer |= read_byte() << 8; + match_offset = trailer >> 2; + literal_length = trailer & 0x3; + } + else if ((command & 0xC0) != 0) + { + match_length = ((command & 0xE0) >> 5) + 1; + match_offset = (command & 0x1C) >> 2; + match_offset |= read_byte() << 3; + literal_length = command & 0x3; + } + else + throwMalformedLzo(offset()); + + first_command = false; + + if (match_length != 0) + { + /// LZO encodes the match offset minus one. + ++match_offset; + if (match_offset > static_cast(output - dst) || match_length > static_cast(output_end - output)) + throwMalformedLzo(offset()); + const char * match = output - match_offset; + /// The source can overlap the destination, so copy byte by byte. + for (size_t i = 0; i < match_length; ++i) + output[i] = match[i]; + output += match_length; + } + + if (literal_length > static_cast(output_end - output) || literal_length > static_cast(input_end - input)) + throwMalformedLzo(offset()); + memcpy(output, input, literal_length); + output += literal_length; + input += literal_length; + last_literal_length = literal_length; + } + + /// The end-of-stream marker is 0x11 0x00 0x00. + if (input_end - input < 2 || input[0] != 0 || input[1] != 0) + throwMalformedLzo(offset()); + input += 2; + } + + return output - dst; +} + +size_t decompressZlib(const char * src, size_t src_size, char * dst, size_t dst_capacity) +{ + z_stream stream{}; + if (inflateInit2(&stream, -15) != Z_OK) + throw Exception(ErrorCodes::INCORRECT_DATA, "Cannot initialize zlib decompression for ORC file"); + + stream.next_in = reinterpret_cast(const_cast(src)); + stream.avail_in = static_cast(src_size); + stream.next_out = reinterpret_cast(dst); + stream.avail_out = static_cast(dst_capacity); + + int ret = inflate(&stream, Z_FINISH); + size_t result = dst_capacity - stream.avail_out; + inflateEnd(&stream); + + if (ret != Z_STREAM_END) + { + if (ret == Z_BUF_ERROR && stream.avail_out == 0) + throw Exception(ErrorCodes::INCORRECT_DATA, + "A zlib-compressed chunk of ORC file decompresses to more than the {} bytes allowed for the chunk", dst_capacity); + throw Exception(ErrorCodes::INCORRECT_DATA, "Malformed zlib-compressed data in ORC file (zlib error {})", ret); + } + return result; +} + +size_t decompressSnappy(const char * src, size_t src_size, char * dst, size_t dst_capacity) +{ + size_t uncompressed_size = 0; + if (!snappy::GetUncompressedLength(src, src_size, &uncompressed_size)) + throw Exception(ErrorCodes::INCORRECT_DATA, "Malformed snappy-compressed data in ORC file"); + if (uncompressed_size > dst_capacity) + throw Exception(ErrorCodes::INCORRECT_DATA, + "A snappy-compressed chunk of ORC file decompresses to {} bytes, more than the {} bytes allowed for the chunk", + uncompressed_size, dst_capacity); + if (!snappy::RawUncompress(src, src_size, dst)) + throw Exception(ErrorCodes::INCORRECT_DATA, "Malformed snappy-compressed data in ORC file"); + return uncompressed_size; +} + +size_t decompressLz4(const char * src, size_t src_size, char * dst, size_t dst_capacity) +{ + if (src_size > static_cast(std::numeric_limits::max()) || dst_capacity > static_cast(std::numeric_limits::max())) + throw Exception(ErrorCodes::INCORRECT_DATA, "LZ4-compressed chunk of ORC file is too large"); + int res = LZ4_decompress_safe(src, dst, static_cast(src_size), static_cast(dst_capacity)); + if (res < 0) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Malformed LZ4-compressed data in ORC file, or it decompresses to more than the {} bytes allowed for the chunk", dst_capacity); + return static_cast(res); +} + +size_t decompressZstd(const char * src, size_t src_size, char * dst, size_t dst_capacity) +{ + size_t res = ZSTD_decompress(dst, dst_capacity, src, src_size); + if (ZSTD_isError(res)) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Malformed zstd-compressed data in ORC file, or it decompresses to more than the {} bytes allowed for the chunk: {}", + dst_capacity, ZSTD_getErrorName(res)); + return res; +} + + +class ZlibCompressor final : public Compressor +{ +public: + ZlibCompressor() + { + /// The level and the parameters are the same as in the C++ ORC library with its default + /// compression strategy, "speed". + if (deflateInit2(&stream, Z_BEST_SPEED + 1, Z_DEFLATED, -15, 8, Z_DEFAULT_STRATEGY) != Z_OK) + throw Exception(ErrorCodes::CANNOT_COMPRESS, "Cannot initialize zlib compression for ORC"); + } + + ~ZlibCompressor() override { deflateEnd(&stream); } + + size_t compress(const char * src, size_t src_size, std::string & out) override + { + if (deflateReset(&stream) != Z_OK) + throw Exception(ErrorCodes::CANNOT_COMPRESS, "Cannot reset zlib compression for ORC"); + + size_t old_size = out.size(); + /// Allow only the output smaller than the input. + out.resize(old_size + src_size); + stream.next_in = reinterpret_cast(const_cast(src)); + stream.avail_in = static_cast(src_size); + stream.next_out = reinterpret_cast(out.data() + old_size); + stream.avail_out = static_cast(src_size); + + int ret = deflate(&stream, Z_FINISH); + if (ret == Z_STREAM_END && stream.avail_out > 0) + { + size_t compressed = src_size - stream.avail_out; + out.resize(old_size + compressed); + return compressed; + } + out.resize(old_size); + if (ret != Z_STREAM_END && ret != Z_OK && ret != Z_BUF_ERROR) + throw Exception(ErrorCodes::CANNOT_COMPRESS, "zlib compression failed with error {}", ret); + return 0; + } + +private: + z_stream stream{}; +}; + +class SnappyCompressor final : public Compressor +{ +public: + size_t compress(const char * src, size_t src_size, std::string & out) override + { + size_t old_size = out.size(); + out.resize(old_size + snappy::MaxCompressedLength(src_size)); + size_t compressed = 0; + snappy::RawCompress(src, src_size, out.data() + old_size, &compressed); + return finish(out, old_size, compressed, src_size); + } + + static size_t finish(std::string & out, size_t old_size, size_t compressed, size_t src_size) + { + if (compressed >= src_size) + { + out.resize(old_size); + return 0; + } + out.resize(old_size + compressed); + return compressed; + } +}; + +class Lz4Compressor final : public Compressor +{ +public: + size_t compress(const char * src, size_t src_size, std::string & out) override + { + size_t old_size = out.size(); + int bound = LZ4_compressBound(static_cast(src_size)); + out.resize(old_size + bound); + int compressed = LZ4_compress_default(src, out.data() + old_size, static_cast(src_size), bound); + if (compressed <= 0) + throw Exception(ErrorCodes::CANNOT_COMPRESS, "LZ4 compression failed"); + return SnappyCompressor::finish(out, old_size, compressed, src_size); + } +}; + +class ZstdCompressor final : public Compressor +{ +public: + ZstdCompressor() : context(ZSTD_createCCtx()) + { + if (!context) + throw Exception(ErrorCodes::CANNOT_COMPRESS, "Cannot create zstd compression context"); + } + + ~ZstdCompressor() override { ZSTD_freeCCtx(context); } + + size_t compress(const char * src, size_t src_size, std::string & out) override + { + size_t old_size = out.size(); + size_t bound = ZSTD_compressBound(src_size); + out.resize(old_size + bound); + /// Level 1, as in the C++ ORC library with its default compression strategy, "speed". + size_t compressed = ZSTD_compressCCtx(context, out.data() + old_size, bound, src, src_size, 1); + if (ZSTD_isError(compressed)) + throw Exception(ErrorCodes::CANNOT_COMPRESS, "zstd compression failed: {}", ZSTD_getErrorName(compressed)); + return SnappyCompressor::finish(out, old_size, compressed, src_size); + } + +private: + ZSTD_CCtx * context; +}; + +} + + +size_t maxDecompressedSize(CompressionKind kind, const char * src, size_t src_size) +{ + /// The ratios are the theoretical maximums of the formats, rounded up, so that valid chunks are never rejected. + /// The sizes stored in the chunks are only trusted below these bounds. + static constexpr size_t SLACK = 1024; + switch (kind) + { + case CompressionKind::NONE: + return src_size; + case CompressionKind::SNAPPY: + { + size_t uncompressed_size = 0; + if (!snappy::GetUncompressedLength(src, src_size, &uncompressed_size)) + throw Exception(ErrorCodes::INCORRECT_DATA, "Malformed snappy-compressed data in ORC file"); + /// A copy of 64 bytes takes at least 3 bytes. + return std::min(uncompressed_size, src_size * 22 + SLACK); + } + case CompressionKind::ZSTD: + { + /// Sums the content sizes of all frames, or 128 KiB per block when a frame omits its size, as in `ArrowIPC/BufferCompression.cpp`. + UInt64 bound = ZSTD_decompressBound(src, src_size); + if (bound == ZSTD_CONTENTSIZE_ERROR) + throw Exception(ErrorCodes::INCORRECT_DATA, "Malformed zstd-compressed data in ORC file"); + /// A block of at most 128 KiB takes at least 4 bytes (an RLE block). + return std::min(bound, src_size * 32768 + SLACK); + } + /// A deflate match of 258 bytes takes at least 2 bits. + case CompressionKind::ZLIB: + return src_size * 1032 + SLACK; + /// Every byte of a match length extension adds at most 255 bytes. + case CompressionKind::LZ4: + case CompressionKind::LZO: + return src_size * 256 + SLACK; + default: + break; + } + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "ORC compression {} is not supported", compressionKindToString(kind)); +} + +size_t decompressChunk(CompressionKind kind, const char * src, size_t src_size, char * dst, size_t dst_capacity) +{ + switch (kind) + { + case CompressionKind::ZLIB: return decompressZlib(src, src_size, dst, dst_capacity); + case CompressionKind::SNAPPY: return decompressSnappy(src, src_size, dst, dst_capacity); + case CompressionKind::LZO: return decompressLzo(src, src_size, dst, dst_capacity); + case CompressionKind::LZ4: return decompressLz4(src, src_size, dst, dst_capacity); + case CompressionKind::ZSTD: return decompressZstd(src, src_size, dst, dst_capacity); + case CompressionKind::NONE: + case CompressionKind::BROTLI: + break; + } + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "ORC compression {} is not supported", compressionKindToString(kind)); +} + +std::unique_ptr createCompressor(CompressionKind kind) +{ + switch (kind) + { + case CompressionKind::NONE: return nullptr; + case CompressionKind::ZLIB: return std::make_unique(); + case CompressionKind::SNAPPY: return std::make_unique(); + case CompressionKind::LZ4: return std::make_unique(); + case CompressionKind::ZSTD: return std::make_unique(); + case CompressionKind::LZO: + case CompressionKind::BROTLI: + break; + } + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Writing ORC with compression {} is not supported", compressionKindToString(kind)); +} + +String compressionKindToString(CompressionKind kind) +{ + switch (kind) + { + case CompressionKind::NONE: return "NONE"; + case CompressionKind::ZLIB: return "ZLIB"; + case CompressionKind::SNAPPY: return "SNAPPY"; + case CompressionKind::LZO: return "LZO"; + case CompressionKind::LZ4: return "LZ4"; + case CompressionKind::ZSTD: return "ZSTD"; + case CompressionKind::BROTLI: return "BROTLI"; + } + return "UNKNOWN(" + std::to_string(static_cast(kind)) + ")"; +} + +} diff --git a/src/Processors/Formats/Impl/ORC/Compression.h b/src/Processors/Formats/Impl/ORC/Compression.h new file mode 100644 index 0000000000000..7e0ceddb72a79 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Compression.h @@ -0,0 +1,42 @@ +#pragma once + +#include + +#include +#include + +namespace DB::ORC +{ + +using Proto::CompressionKind; + +/// Every compressed chunk of an ORC stream is prefixed with a 3-byte little-endian header: +/// `(chunk_length << 1) | is_original`, where `is_original` means the chunk is stored as is. +static constexpr size_t COMPRESSION_HEADER_SIZE = 3; + +/// Decompresses one chunk into `dst`, which has room for `dst_capacity` bytes (the compression block +/// size of the file). Returns the decompressed size. Throws `INCORRECT_DATA` on malformed input, +/// including a chunk that decompresses to more than `dst_capacity` bytes. +size_t decompressChunk(CompressionKind kind, const char * src, size_t src_size, char * dst, size_t dst_capacity); + +/// An upper bound of the decompressed size of a chunk, from the maximum compression ratio of the codec +/// (and the size stored in the chunk), to avoid allocating the whole compression block for a small chunk. +size_t maxDecompressedSize(CompressionKind kind, const char * src, size_t src_size); + +/// Compresses chunks of an ORC stream. +class Compressor +{ +public: + virtual ~Compressor() = default; + + /// Appends the compressed representation of `src` to `out` and returns its size, + /// or appends nothing and returns 0 if it would not be smaller than the input. + virtual size_t compress(const char * src, size_t src_size, std::string & out) = 0; +}; + +/// Returns nullptr for `NONE`. +std::unique_ptr createCompressor(CompressionKind kind); + +String compressionKindToString(CompressionKind kind); + +} diff --git a/src/Processors/Formats/Impl/ORC/Proto.cpp b/src/Processors/Formats/Impl/ORC/Proto.cpp new file mode 100644 index 0000000000000..19e504dfd365d --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Proto.cpp @@ -0,0 +1,944 @@ +#include + +#include + +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int INCORRECT_DATA; +} +} + +namespace DB::ORC::Proto +{ + +void WireReader::check(size_t n) const +{ + if (static_cast(end - pos) < n) + throw Exception(ErrorCodes::INCORRECT_DATA, "Truncated protobuf message in ORC metadata"); +} + +UInt64 WireReader::readVarint() +{ + UInt64 result = 0; + for (size_t shift = 0; shift < 64; shift += 7) + { + check(1); + UInt8 byte = static_cast(*pos++); + result |= static_cast(byte & 0x7F) << shift; + if (!(byte & 0x80)) + return result; + } + throw Exception(ErrorCodes::INCORRECT_DATA, "Malformed varint in ORC metadata"); +} + +UInt64 WireReader::readFixed64() +{ + check(8); + UInt64 v = 0; + memcpy(&v, pos, 8); + pos += 8; + if constexpr (std::endian::native == std::endian::big) + v = std::byteswap(v); + return v; +} + +UInt32 WireReader::readFixed32() +{ + check(4); + UInt32 v = 0; + memcpy(&v, pos, 4); + pos += 4; + if constexpr (std::endian::native == std::endian::big) + v = std::byteswap(v); + return v; +} + +double WireReader::readDouble() +{ + return std::bit_cast(readFixed64()); +} + +std::string_view WireReader::readBytes() +{ + UInt64 size = readVarint(); + check(size); + std::string_view res(pos, size); + pos += size; + return res; +} + +bool WireReader::nextField(UInt32 & field_number, WireType & wire_type) +{ + if (pos == end) + return false; + UInt64 key = readVarint(); + field_number = static_cast(key >> 3); + wire_type = static_cast(key & 7); + if (field_number == 0 || (key >> 3) > std::numeric_limits::max()) + throw Exception(ErrorCodes::INCORRECT_DATA, "Malformed field key in ORC metadata"); + return true; +} + +void WireReader::skip(WireType wire_type) +{ + switch (wire_type) + { + case VARINT: readVarint(); return; + case FIXED64: check(8); pos += 8; return; + case LENGTH_DELIMITED: readBytes(); return; + case FIXED32: check(4); pos += 4; return; + case START_GROUP: + case END_GROUP: + break; + } + throw Exception(ErrorCodes::INCORRECT_DATA, "Unsupported protobuf wire type {} in ORC metadata", static_cast(wire_type)); +} + +bool WireReader::readRepeatedVarint(WireType wire_type, std::vector & out) +{ + if (wire_type == LENGTH_DELIMITED) + { + WireReader packed(readBytes()); + while (packed.pos != packed.end) + out.push_back(packed.readVarint()); + return true; + } + if (wire_type == VARINT) + { + out.push_back(readVarint()); + return true; + } + return false; +} + + +void WireWriter::appendRawVarint(std::string & s, UInt64 value) +{ + while (value >= 0x80) + { + s.push_back(static_cast((value & 0x7F) | 0x80)); + value >>= 7; + } + s.push_back(static_cast(value)); +} + +void WireWriter::writeVarint(UInt32 field, UInt64 value) +{ + writeKey(field, WireReader::VARINT); + appendRawVarint(out, value); +} + +void WireWriter::writeDouble(UInt32 field, double value) +{ + writeKey(field, WireReader::FIXED64); + UInt64 v = std::bit_cast(value); + if constexpr (std::endian::native == std::endian::big) + v = std::byteswap(v); + out.append(reinterpret_cast(&v), 8); +} + +void WireWriter::writeBytes(UInt32 field, std::string_view value) +{ + writeKey(field, WireReader::LENGTH_DELIMITED); + appendRawVarint(out, value.size()); + out.append(value); +} + +void WireWriter::writePackedVarint(UInt32 field, const std::vector & values) +{ + if (values.empty()) + return; + std::string packed; + for (UInt64 v : values) + appendRawVarint(packed, v); + writeBytes(field, packed); +} + +void WireWriter::writePackedFixed64(UInt32 field, const std::vector & values) +{ + if (values.empty()) + return; + std::string packed; + for (UInt64 v : values) + { + if constexpr (std::endian::native == std::endian::big) + v = std::byteswap(v); + packed.append(reinterpret_cast(&v), 8); + } + writeBytes(field, packed); +} + + +namespace +{ + +using WireType = WireReader::WireType; + +/// A field with an unexpected wire type is an unknown field, which the Protocol Buffers rules +/// require to skip. It is detected before anything is read, and handled in `parseMessage`. +struct WireTypeMismatch : std::exception +{ +}; + +void expectWireType(WireType actual, WireType expected) +{ + if (actual != expected) + throw WireTypeMismatch{}; +} + +UInt64 readVarintField(WireReader & r, WireType t) +{ + expectWireType(t, WireReader::VARINT); + return r.readVarint(); +} + +Int64 readSignedVarintField(WireReader & r, WireType t) +{ + expectWireType(t, WireReader::VARINT); + return r.readSignedVarint(); +} + +UInt32 readUInt32Field(WireReader & r, WireType t) +{ + /// Protobuf truncates the 64-bit varint for 32-bit fields. + return static_cast(readVarintField(r, t)); +} + +/// For `sint32` fields. +Int32 readSignedVarint32Field(WireReader & r, WireType t) +{ + expectWireType(t, WireReader::VARINT); + UInt32 v = static_cast(r.readVarint()); + return static_cast((v >> 1) ^ (~(v & 1) + 1)); +} + +double readDoubleField(WireReader & r, WireType t) +{ + expectWireType(t, WireReader::FIXED64); + return r.readDouble(); +} + +std::string_view readBytesField(WireReader & r, WireType t) +{ + expectWireType(t, WireReader::LENGTH_DELIMITED); + return r.readBytes(); +} + +void checkCount(size_t count, size_t max_count, std::string_view what) +{ + if (count >= max_count) + throw Exception(ErrorCodes::INCORRECT_DATA, "Too many {} in ORC metadata, at most {} expected", what, max_count); +} + +template +void parseMessage(std::string_view data, F && handle_field) +{ + WireReader r(data); + UInt32 field = 0; + WireType type = WireReader::VARINT; + while (r.nextField(field, type)) + { + bool handled = false; + try + { + handled = handle_field(r, field, type); + } + catch (const WireTypeMismatch &) + { + handled = false; + } + if (!handled) + r.skip(type); + } +} + +IntegerStatistics parseIntegerStatistics(std::string_view data) +{ + IntegerStatistics res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: res.minimum = readSignedVarintField(r, type); return true; + case 2: res.maximum = readSignedVarintField(r, type); return true; + case 3: res.sum = readSignedVarintField(r, type); return true; + default: return false; + } + }); + return res; +} + +DoubleStatistics parseDoubleStatistics(std::string_view data) +{ + DoubleStatistics res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: res.minimum = readDoubleField(r, type); return true; + case 2: res.maximum = readDoubleField(r, type); return true; + case 3: res.sum = readDoubleField(r, type); return true; + default: return false; + } + }); + return res; +} + +StringStatistics parseStringStatistics(std::string_view data) +{ + StringStatistics res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: res.minimum = std::string(readBytesField(r, type)); return true; + case 2: res.maximum = std::string(readBytesField(r, type)); return true; + case 3: res.sum = readSignedVarintField(r, type); return true; + case 4: res.lower_bound = std::string(readBytesField(r, type)); return true; + case 5: res.upper_bound = std::string(readBytesField(r, type)); return true; + default: return false; + } + }); + return res; +} + +BucketStatistics parseBucketStatistics(std::string_view data) +{ + BucketStatistics res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + if (field != 1) + return false; + return r.readRepeatedVarint(type, res.count); + }); + return res; +} + +DecimalStatistics parseDecimalStatistics(std::string_view data) +{ + DecimalStatistics res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: res.minimum = std::string(readBytesField(r, type)); return true; + case 2: res.maximum = std::string(readBytesField(r, type)); return true; + case 3: res.sum = std::string(readBytesField(r, type)); return true; + default: return false; + } + }); + return res; +} + +DateStatistics parseDateStatistics(std::string_view data) +{ + DateStatistics res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: res.minimum = readSignedVarint32Field(r, type); return true; + case 2: res.maximum = readSignedVarint32Field(r, type); return true; + default: return false; + } + }); + return res; +} + +TimestampStatistics parseTimestampStatistics(std::string_view data) +{ + TimestampStatistics res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: res.minimum = readSignedVarintField(r, type); return true; + case 2: res.maximum = readSignedVarintField(r, type); return true; + case 3: res.minimum_utc = readSignedVarintField(r, type); return true; + case 4: res.maximum_utc = readSignedVarintField(r, type); return true; + case 5: res.minimum_nanos = readVarintField(r, type); return true; + case 6: res.maximum_nanos = readVarintField(r, type); return true; + default: return false; + } + }); + return res; +} + +BinaryStatistics parseBinaryStatistics(std::string_view data) +{ + BinaryStatistics res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + if (field != 1) + return false; + res.sum = readSignedVarintField(r, type); + return true; + }); + return res; +} + +CollectionStatistics parseCollectionStatistics(std::string_view data) +{ + CollectionStatistics res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: res.min_children = readVarintField(r, type); return true; + case 2: res.max_children = readVarintField(r, type); return true; + case 3: res.total_children = readVarintField(r, type); return true; + default: return false; + } + }); + return res; +} + +Stream parseStream(std::string_view data) +{ + Stream res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: res.kind = readUInt32Field(r, type); return true; + case 2: res.column = readUInt32Field(r, type); return true; + case 3: res.length = readVarintField(r, type); return true; + default: return false; + } + }); + return res; +} + +ColumnEncoding parseColumnEncoding(std::string_view data) +{ + ColumnEncoding res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: + { + UInt32 kind = readUInt32Field(r, type); + if (kind > static_cast(ColumnEncodingKind::DICTIONARY_V2)) + throw Exception(ErrorCodes::INCORRECT_DATA, "Unknown ORC column encoding kind {}", kind); + res.kind = static_cast(kind); + return true; + } + case 2: res.dictionary_size = readUInt32Field(r, type); return true; + case 3: res.bloom_encoding = readUInt32Field(r, type); return true; + default: return false; + } + }); + return res; +} + +Type parseType(std::string_view data) +{ + Type res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: res.kind = readUInt32Field(r, type); return true; + case 2: return r.readRepeatedVarint(type, res.subtypes); + case 3: res.field_names.emplace_back(readBytesField(r, type)); return true; + case 4: res.maximum_length = readUInt32Field(r, type); return true; + case 5: res.precision = readUInt32Field(r, type); return true; + case 6: res.scale = readUInt32Field(r, type); return true; + case 7: + { + StringPair pair; + parseMessage(readBytesField(r, type), [&](WireReader & rr, UInt32 f, WireType t) + { + switch (f) + { + case 1: pair.key = std::string(readBytesField(rr, t)); return true; + case 2: pair.value = std::string(readBytesField(rr, t)); return true; + default: return false; + } + }); + res.attributes.push_back(std::move(pair)); + return true; + } + case 8: res.crs = std::string(readBytesField(r, type)); return true; + case 9: res.algorithm = readUInt32Field(r, type); return true; + default: return false; + } + }); + return res; +} + +StripeInformation parseStripeInformation(std::string_view data) +{ + StripeInformation res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: res.offset = readVarintField(r, type); return true; + case 2: res.index_length = readVarintField(r, type); return true; + case 3: res.data_length = readVarintField(r, type); return true; + case 4: res.footer_length = readVarintField(r, type); return true; + case 5: res.number_of_rows = readVarintField(r, type); return true; + default: return false; + } + }); + return res; +} + +UserMetadataItem parseUserMetadataItem(std::string_view data) +{ + UserMetadataItem res; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: res.name = std::string(readBytesField(r, type)); return true; + case 2: res.value = std::string(readBytesField(r, type)); return true; + default: return false; + } + }); + return res; +} + +std::string serializeIntegerStatistics(const IntegerStatistics & s) +{ + std::string out; + WireWriter w(out); + if (s.minimum) w.writeSignedVarint(1, *s.minimum); + if (s.maximum) w.writeSignedVarint(2, *s.maximum); + if (s.sum) w.writeSignedVarint(3, *s.sum); + return out; +} + +std::string serializeDoubleStatistics(const DoubleStatistics & s) +{ + std::string out; + WireWriter w(out); + if (s.minimum) w.writeDouble(1, *s.minimum); + if (s.maximum) w.writeDouble(2, *s.maximum); + if (s.sum) w.writeDouble(3, *s.sum); + return out; +} + +std::string serializeStringStatistics(const StringStatistics & s) +{ + std::string out; + WireWriter w(out); + if (s.minimum) w.writeBytes(1, *s.minimum); + if (s.maximum) w.writeBytes(2, *s.maximum); + if (s.sum) w.writeSignedVarint(3, *s.sum); + if (s.lower_bound) w.writeBytes(4, *s.lower_bound); + if (s.upper_bound) w.writeBytes(5, *s.upper_bound); + return out; +} + +std::string serializeDecimalStatistics(const DecimalStatistics & s) +{ + std::string out; + WireWriter w(out); + if (s.minimum) w.writeBytes(1, *s.minimum); + if (s.maximum) w.writeBytes(2, *s.maximum); + if (s.sum) w.writeBytes(3, *s.sum); + return out; +} + +std::string serializeDateStatistics(const DateStatistics & s) +{ + std::string out; + WireWriter w(out); + auto zigzag32 = [](Int32 v) { return static_cast((static_cast(v) << 1) ^ static_cast(v >> 31)); }; + if (s.minimum) w.writeVarint(1, zigzag32(*s.minimum)); + if (s.maximum) w.writeVarint(2, zigzag32(*s.maximum)); + return out; +} + +std::string serializeTimestampStatistics(const TimestampStatistics & s) +{ + std::string out; + WireWriter w(out); + if (s.minimum) w.writeSignedVarint(1, *s.minimum); + if (s.maximum) w.writeSignedVarint(2, *s.maximum); + if (s.minimum_utc) w.writeSignedVarint(3, *s.minimum_utc); + if (s.maximum_utc) w.writeSignedVarint(4, *s.maximum_utc); + /// `int32` fields: negative values are sign-extended to 64 bits. + if (s.minimum_nanos) w.writeVarint(5, *s.minimum_nanos); + if (s.maximum_nanos) w.writeVarint(6, *s.maximum_nanos); + return out; +} + +std::string serializeCollectionStatistics(const CollectionStatistics & s) +{ + std::string out; + WireWriter w(out); + if (s.min_children) w.writeVarint(1, *s.min_children); + if (s.max_children) w.writeVarint(2, *s.max_children); + if (s.total_children) w.writeVarint(3, *s.total_children); + return out; +} + +std::string serializeType(const Type & t) +{ + std::string out; + WireWriter w(out); + w.writeVarint(1, t.kind); + w.writePackedVarint(2, t.subtypes); + for (const auto & name : t.field_names) + w.writeBytes(3, name); + if (t.maximum_length) w.writeVarint(4, *t.maximum_length); + if (t.precision) w.writeVarint(5, *t.precision); + if (t.scale) w.writeVarint(6, *t.scale); + for (const auto & attr : t.attributes) + { + std::string pair; + WireWriter pw(pair); + pw.writeBytes(1, attr.key); + pw.writeBytes(2, attr.value); + w.writeBytes(7, pair); + } + if (t.crs) w.writeBytes(8, *t.crs); + if (t.algorithm) w.writeVarint(9, *t.algorithm); + return out; +} + +std::string serializeStripeInformation(const StripeInformation & s) +{ + std::string out; + WireWriter w(out); + w.writeVarint(1, s.offset); + w.writeVarint(2, s.index_length); + w.writeVarint(3, s.data_length); + w.writeVarint(4, s.footer_length); + w.writeVarint(5, s.number_of_rows); + return out; +} + +} + + +void PostScript::parse(std::string_view data) +{ + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: footer_length = readVarintField(r, type); return true; + case 2: + { + UInt64 kind = readVarintField(r, type); + if (kind > static_cast(CompressionKind::BROTLI)) + throw Exception(ErrorCodes::INCORRECT_DATA, "Unknown ORC compression kind {}", kind); + compression = static_cast(kind); + return true; + } + case 3: compression_block_size = readVarintField(r, type); return true; + case 4: return r.readRepeatedVarint(type, version); + case 5: metadata_length = readVarintField(r, type); return true; + case 6: writer_version = readUInt32Field(r, type); return true; + case 8000: magic = std::string(readBytesField(r, type)); return true; + default: return false; + } + }); +} + +std::string PostScript::serialize() const +{ + std::string out; + WireWriter w(out); + w.writeVarint(1, footer_length); + w.writeVarint(2, static_cast(compression)); + w.writeVarint(3, compression_block_size); + w.writePackedVarint(4, version); + w.writeVarint(5, metadata_length); + w.writeVarint(6, writer_version); + w.writeBytes(8000, magic); + return out; +} + +void ColumnStatistics::parse(std::string_view data) +{ + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: number_of_values = readVarintField(r, type); return true; + case 2: int_statistics = parseIntegerStatistics(readBytesField(r, type)); return true; + case 3: double_statistics = parseDoubleStatistics(readBytesField(r, type)); return true; + case 4: string_statistics = parseStringStatistics(readBytesField(r, type)); return true; + case 5: bucket_statistics = parseBucketStatistics(readBytesField(r, type)); return true; + case 6: decimal_statistics = parseDecimalStatistics(readBytesField(r, type)); return true; + case 7: date_statistics = parseDateStatistics(readBytesField(r, type)); return true; + case 8: binary_statistics = parseBinaryStatistics(readBytesField(r, type)); return true; + case 9: timestamp_statistics = parseTimestampStatistics(readBytesField(r, type)); return true; + case 10: has_null = readVarintField(r, type) != 0; return true; + case 11: bytes_on_disk = readVarintField(r, type); return true; + case 12: collection_statistics = parseCollectionStatistics(readBytesField(r, type)); return true; + default: return false; + } + }); +} + +std::string ColumnStatistics::serialize() const +{ + std::string out; + WireWriter w(out); + if (number_of_values) w.writeVarint(1, *number_of_values); + if (int_statistics) w.writeBytes(2, serializeIntegerStatistics(*int_statistics)); + if (double_statistics) w.writeBytes(3, serializeDoubleStatistics(*double_statistics)); + if (string_statistics) w.writeBytes(4, serializeStringStatistics(*string_statistics)); + if (bucket_statistics) + { + std::string bucket; + WireWriter(bucket).writePackedVarint(1, bucket_statistics->count); + w.writeBytes(5, bucket); + } + if (decimal_statistics) w.writeBytes(6, serializeDecimalStatistics(*decimal_statistics)); + if (date_statistics) w.writeBytes(7, serializeDateStatistics(*date_statistics)); + if (binary_statistics) + { + std::string binary; + if (binary_statistics->sum) + WireWriter(binary).writeSignedVarint(1, *binary_statistics->sum); + w.writeBytes(8, binary); + } + if (timestamp_statistics) w.writeBytes(9, serializeTimestampStatistics(*timestamp_statistics)); + if (has_null) w.writeVarint(10, *has_null); + if (bytes_on_disk) w.writeVarint(11, *bytes_on_disk); + if (collection_statistics) w.writeBytes(12, serializeCollectionStatistics(*collection_statistics)); + return out; +} + +void RowIndex::parse(std::string_view data, size_t max_entries) +{ + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + if (field != 1) + return false; + std::string_view entry_data = readBytesField(r, type); + checkCount(entry.size(), max_entries, "row index entries"); + auto & e = entry.emplace_back(); + parseMessage(entry_data, [&](WireReader & rr, UInt32 f, WireType t) + { + switch (f) + { + case 1: return rr.readRepeatedVarint(t, e.positions); + case 2: e.statistics.emplace().parse(readBytesField(rr, t)); return true; + default: return false; + } + }); + return true; + }); +} + +std::string RowIndex::serialize() const +{ + std::string out; + WireWriter w(out); + for (const auto & e : entry) + { + std::string entry_data; + WireWriter ew(entry_data); + ew.writePackedVarint(1, e.positions); + if (e.statistics) + ew.writeBytes(2, e.statistics->serialize()); + w.writeBytes(1, entry_data); + } + return out; +} + +void BloomFilterIndex::parse(std::string_view data, size_t max_entries) +{ + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + if (field != 1) + return false; + checkCount(bloom_filter.size(), max_entries, "bloom filters"); + BloomFilter bf; + parseMessage(readBytesField(r, type), [&](WireReader & rr, UInt32 f, WireType t) + { + switch (f) + { + case 1: bf.num_hash_functions = readUInt32Field(rr, t); return true; + case 2: + { + if (t == WireReader::LENGTH_DELIMITED) + { + /// Packed fixed64: a sequence of 8-byte little-endian values. + std::string_view bytes = rr.readBytes(); + if (bytes.size() % 8 != 0) + throw Exception(ErrorCodes::INCORRECT_DATA, "Malformed bloom filter bitset in ORC file"); + WireReader packed(bytes); + for (size_t i = 0; i < bytes.size() / 8; ++i) + bf.bitset.push_back(packed.readFixed64()); + } + else + { + expectWireType(t, WireReader::FIXED64); + bf.bitset.push_back(rr.readFixed64()); + } + return true; + } + case 3: bf.utf8bitset = std::string(readBytesField(rr, t)); return true; + default: return false; + } + }); + bloom_filter.push_back(std::move(bf)); + return true; + }); +} + +void StripeFooter::parse(std::string_view data, size_t max_columns) +{ + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: streams.push_back(parseStream(readBytesField(r, type))); return true; + case 2: + { + std::string_view encoding_data = readBytesField(r, type); + checkCount(columns.size(), max_columns, "column encodings"); + columns.push_back(parseColumnEncoding(encoding_data)); + return true; + } + case 3: writer_timezone = std::string(readBytesField(r, type)); return true; + case 4: readBytesField(r, type); ++encryption_variants; return true; + default: return false; + } + }); +} + +std::string StripeFooter::serialize() const +{ + std::string out; + WireWriter w(out); + for (const auto & s : streams) + { + std::string stream; + WireWriter sw(stream); + if (s.kind) + sw.writeVarint(1, *s.kind); + sw.writeVarint(2, s.column); + sw.writeVarint(3, s.length); + w.writeBytes(1, stream); + } + for (const auto & c : columns) + { + std::string encoding; + WireWriter ew(encoding); + ew.writeVarint(1, static_cast(c.kind)); + if (c.kind == ColumnEncodingKind::DICTIONARY || c.kind == ColumnEncodingKind::DICTIONARY_V2) + ew.writeVarint(2, c.dictionary_size); + w.writeBytes(2, encoding); + } + if (writer_timezone) + w.writeBytes(3, *writer_timezone); + return out; +} + +void Metadata::parse(std::string_view data, size_t max_stripes, size_t max_columns) +{ + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + if (field != 1) + return false; + std::string_view stripe_data = readBytesField(r, type); + checkCount(stripe_stats.size(), max_stripes, "stripe statistics"); + StripeStatistics stats; + parseMessage(stripe_data, [&](WireReader & rr, UInt32 f, WireType t) + { + if (f != 1) + return false; + std::string_view column_data = readBytesField(rr, t); + checkCount(stats.col_stats.size(), max_columns, "column statistics"); + stats.col_stats.emplace_back().parse(column_data); + return true; + }); + stripe_stats.push_back(std::move(stats)); + return true; + }); +} + +std::string Metadata::serialize() const +{ + std::string out; + WireWriter w(out); + for (const auto & stats : stripe_stats) + { + std::string stripe; + WireWriter sw(stripe); + for (const auto & col : stats.col_stats) + sw.writeBytes(1, col.serialize()); + w.writeBytes(1, stripe); + } + return out; +} + +void Footer::parse(std::string_view data) +{ + /// The statistics are parsed after the types, to limit their number by the number of columns. + std::vector statistics_data; + parseMessage(data, [&](WireReader & r, UInt32 field, WireType type) + { + switch (field) + { + case 1: header_length = readVarintField(r, type); return true; + case 2: content_length = readVarintField(r, type); return true; + case 3: stripes.push_back(parseStripeInformation(readBytesField(r, type))); return true; + case 4: types.push_back(parseType(readBytesField(r, type))); return true; + case 5: metadata.push_back(parseUserMetadataItem(readBytesField(r, type))); return true; + case 6: number_of_rows = readVarintField(r, type); return true; + case 7: statistics_data.push_back(readBytesField(r, type)); return true; + case 8: row_index_stride = readUInt32Field(r, type); return true; + case 9: writer = readUInt32Field(r, type); return true; + case 10: readBytesField(r, type); has_encryption = true; return true; + case 12: software_version = std::string(readBytesField(r, type)); return true; + default: return false; + } + }); + if (statistics_data.size() > types.size()) + throw Exception(ErrorCodes::INCORRECT_DATA, "Too many column statistics in ORC footer: {}, the file has {} columns", + statistics_data.size(), types.size()); + statistics.resize(statistics_data.size()); + for (size_t i = 0; i < statistics_data.size(); ++i) + statistics[i].parse(statistics_data[i]); +} + +std::string Footer::serialize() const +{ + std::string out; + WireWriter w(out); + w.writeVarint(1, header_length); + w.writeVarint(2, content_length); + for (const auto & s : stripes) + w.writeBytes(3, serializeStripeInformation(s)); + for (const auto & t : types) + w.writeBytes(4, serializeType(t)); + for (const auto & item : metadata) + { + std::string m; + WireWriter mw(m); + mw.writeBytes(1, item.name); + mw.writeBytes(2, item.value); + w.writeBytes(5, m); + } + w.writeVarint(6, number_of_rows); + for (const auto & s : statistics) + w.writeBytes(7, s.serialize()); + w.writeVarint(8, row_index_stride); + if (writer) + w.writeVarint(9, *writer); + if (software_version) + w.writeBytes(12, *software_version); + return out; +} + +} diff --git a/src/Processors/Formats/Impl/ORC/Proto.h b/src/Processors/Formats/Impl/ORC/Proto.h new file mode 100644 index 0000000000000..999839f6ab5df --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Proto.h @@ -0,0 +1,343 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include + +/// Structures of the ORC file metadata, which is serialized with Protocol Buffers. +/// See `orc_proto.proto` in the ORC specification: https://orc.apache.org/specification/ORCv1/ +/// +/// Only the messages and the fields that are used by the reader and the writer are represented. +/// The parser skips the unknown fields, as required by the Protocol Buffers rules, so the files +/// written by newer writers can be read. + +namespace DB::ORC::Proto +{ + +/// Low-level reader of the Protocol Buffers wire format over a contiguous buffer. +/// Throws `INCORRECT_DATA` on a malformed or truncated message. +class WireReader +{ +public: + enum WireType : UInt8 + { + VARINT = 0, + FIXED64 = 1, + LENGTH_DELIMITED = 2, + START_GROUP = 3, + END_GROUP = 4, + FIXED32 = 5, + }; + + WireReader(const char * begin_, const char * end_) : pos(begin_), end(end_) {} + explicit WireReader(std::string_view data) : pos(data.data()), end(data.data() + data.size()) {} + + /// Reads the next field key. Returns false at the end of the message. + bool nextField(UInt32 & field_number, WireType & wire_type); + + UInt64 readVarint(); + Int64 readSignedVarint() { UInt64 v = readVarint(); return static_cast((v >> 1) ^ (~(v & 1) + 1)); } + UInt64 readFixed64(); + UInt32 readFixed32(); + double readDouble(); + std::string_view readBytes(); + + /// Skips a field of the given wire type. + void skip(WireType wire_type); + + /// Reads a scalar field that can be either packed or not packed. + /// Returns false (and reads nothing) if the wire type is not suitable. + [[nodiscard]] bool readRepeatedVarint(WireType wire_type, std::vector & out); + +private: + void check(size_t n) const; + + const char * pos; + const char * end; +}; + +/// Low-level writer of the Protocol Buffers wire format. +class WireWriter +{ +public: + explicit WireWriter(std::string & out_) : out(out_) {} + + void writeVarint(UInt32 field, UInt64 value); + void writeSignedVarint(UInt32 field, Int64 value) { writeVarint(field, (static_cast(value) << 1) ^ static_cast(value >> 63)); } + void writeDouble(UInt32 field, double value); + void writeBytes(UInt32 field, std::string_view value); + void writePackedVarint(UInt32 field, const std::vector & values); + void writePackedFixed64(UInt32 field, const std::vector & values); + + static void appendRawVarint(std::string & s, UInt64 value); + +private: + void writeKey(UInt32 field, UInt8 wire_type) { appendRawVarint(out, (static_cast(field) << 3) | wire_type); } + + std::string & out; +}; + + +enum class CompressionKind : UInt8 +{ + NONE = 0, + ZLIB = 1, + SNAPPY = 2, + LZO = 3, + LZ4 = 4, + ZSTD = 5, + BROTLI = 6, +}; + +struct PostScript +{ + UInt64 footer_length = 0; + CompressionKind compression = CompressionKind::NONE; + /// The default value in the specification. + UInt64 compression_block_size = 256 * 1024; + std::vector version; + UInt64 metadata_length = 0; + UInt32 writer_version = 0; + std::string magic; + + void parse(std::string_view data); + std::string serialize() const; +}; + +struct IntegerStatistics +{ + std::optional minimum; + std::optional maximum; + std::optional sum; +}; + +struct DoubleStatistics +{ + std::optional minimum; + std::optional maximum; + std::optional sum; +}; + +struct StringStatistics +{ + std::optional minimum; + std::optional maximum; + std::optional sum; + std::optional lower_bound; + std::optional upper_bound; +}; + +struct BucketStatistics +{ + std::vector count; +}; + +struct DecimalStatistics +{ + std::optional minimum; + std::optional maximum; + std::optional sum; +}; + +struct DateStatistics +{ + std::optional minimum; + std::optional maximum; +}; + +struct TimestampStatistics +{ + /// In milliseconds, in the local time zone of the writer (deprecated). + std::optional minimum; + std::optional maximum; + /// In milliseconds since the epoch in UTC. + std::optional minimum_utc; + std::optional maximum_utc; + /// The sub-millisecond part of the nanoseconds plus one, if not zero. Unvalidated when read from a file. + std::optional minimum_nanos; + std::optional maximum_nanos; +}; + +struct BinaryStatistics +{ + std::optional sum; +}; + +struct CollectionStatistics +{ + std::optional min_children; + std::optional max_children; + std::optional total_children; +}; + +struct ColumnStatistics +{ + std::optional number_of_values; + std::optional int_statistics; + std::optional double_statistics; + std::optional string_statistics; + std::optional bucket_statistics; + std::optional decimal_statistics; + std::optional date_statistics; + std::optional binary_statistics; + std::optional timestamp_statistics; + std::optional has_null; + std::optional bytes_on_disk; + std::optional collection_statistics; + + void parse(std::string_view data); + std::string serialize() const; +}; + +struct RowIndexEntry +{ + std::vector positions; + std::optional statistics; +}; + +struct RowIndex +{ + std::vector entry; + + /// The limits of the number of the repeated objects keep a small malformed message from allocating much memory. + void parse(std::string_view data, size_t max_entries); + std::string serialize() const; +}; + +struct BloomFilter +{ + UInt32 num_hash_functions = 0; + std::vector bitset; + std::optional utf8bitset; +}; + +struct BloomFilterIndex +{ + std::vector bloom_filter; + + void parse(std::string_view data, size_t max_entries); +}; + +enum class StreamKind : UInt32 +{ + PRESENT = 0, + DATA = 1, + LENGTH = 2, + DICTIONARY_DATA = 3, + DICTIONARY_COUNT = 4, + SECONDARY = 5, + ROW_INDEX = 6, + BLOOM_FILTER = 7, + BLOOM_FILTER_UTF8 = 8, + ENCRYPTED_INDEX = 9, + ENCRYPTED_DATA = 10, + STRIPE_STATISTICS = 100, + FILE_STATISTICS = 101, +}; + +struct Stream +{ + std::optional kind; + UInt32 column = 0; + UInt64 length = 0; +}; + +enum class ColumnEncodingKind : UInt32 +{ + DIRECT = 0, + DICTIONARY = 1, + DIRECT_V2 = 2, + DICTIONARY_V2 = 3, +}; + +struct ColumnEncoding +{ + ColumnEncodingKind kind = ColumnEncodingKind::DIRECT; + UInt32 dictionary_size = 0; + /// 1 means the bloom filters use UTF-8 for the strings and UTC for the timestamps. + std::optional bloom_encoding; +}; + +struct StripeFooter +{ + std::vector streams; + std::vector columns; + std::optional writer_timezone; + /// Non-zero if the stripe has encrypted columns, which are not supported. + size_t encryption_variants = 0; + + void parse(std::string_view data, size_t max_columns); + std::string serialize() const; +}; + +struct StringPair +{ + std::string key; + std::string value; +}; + +struct Type +{ + UInt32 kind = 0; + std::vector subtypes; + std::vector field_names; + std::optional maximum_length; + std::optional precision; + std::optional scale; + std::vector attributes; + /// Only for GEOMETRY and GEOGRAPHY. + std::optional crs; + std::optional algorithm; +}; + +struct StripeInformation +{ + UInt64 offset = 0; + UInt64 index_length = 0; + UInt64 data_length = 0; + UInt64 footer_length = 0; + UInt64 number_of_rows = 0; +}; + +struct UserMetadataItem +{ + std::string name; + std::string value; +}; + +struct StripeStatistics +{ + std::vector col_stats; +}; + +struct Metadata +{ + std::vector stripe_stats; + + void parse(std::string_view data, size_t max_stripes, size_t max_columns); + std::string serialize() const; +}; + +struct Footer +{ + UInt64 header_length = 0; + UInt64 content_length = 0; + std::vector stripes; + std::vector types; + std::vector metadata; + UInt64 number_of_rows = 0; + std::vector statistics; + UInt32 row_index_stride = 0; + std::optional writer; + bool has_encryption = false; + std::optional software_version; + + void parse(std::string_view data); + std::string serialize() const; +}; + +} diff --git a/src/Processors/Formats/Impl/ORC/RLE.cpp b/src/Processors/Formats/Impl/ORC/RLE.cpp new file mode 100644 index 0000000000000..7b9762e32de33 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/RLE.cpp @@ -0,0 +1,1129 @@ +#include + +#include + +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int INCORRECT_DATA; +} +} + +namespace DB::ORC +{ + +namespace +{ + +inline Int64 unZigZag(UInt64 value) +{ + return static_cast((value >> 1) ^ (~(value & 1) + 1)); +} + +inline UInt64 zigZag(Int64 value) +{ + return (static_cast(value) << 1) ^ static_cast(value >> 63); +} + +/// The 5-bit codes of the bit widths in RLE v2. +constexpr UInt8 code_to_bit_width[32] = { + 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 26, 28, 30, 32, 40, 48, 56, 64}; + +/// The smallest bit width representable in RLE v2 that fits the given number of bits. +constexpr UInt8 closest_fixed_bits[65] = { + 1, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, + 22, 23, 24, 26, 26, 28, 28, 30, 30, 32, 32, 40, 40, 40, 40, 40, 40, 40, 40, 48, 48, 48, + 48, 48, 48, 48, 48, 56, 56, 56, 56, 56, 56, 56, 56, 64, 64, 64, 64, 64, 64, 64, 64}; + +inline UInt32 decodeBitWidth(UInt32 code) +{ + return code_to_bit_width[code & 0x1F]; +} + +inline UInt32 getClosestFixedBits(UInt32 n) +{ + return n <= 64 ? closest_fixed_bits[n] : 64; +} + +inline UInt32 encodeBitWidth(UInt32 width) +{ + width = getClosestFixedBits(width); + if (width <= 24) + return width - 1; + switch (width) + { + case 26: return 24; + case 28: return 25; + case 30: return 26; + case 32: return 27; + case 40: return 28; + case 48: return 29; + case 56: return 30; + default: return 31; + } +} + +/// The number of bits required to represent the value, rounded to a representable bit width. +/// Negative values require 64 bits. +inline UInt32 findClosestNumBits(Int64 value) +{ + if (value < 0) + return 64; + UInt32 count = 64 - static_cast(std::countl_zero(static_cast(value))); + return getClosestFixedBits(count); +} + +} + + +UInt64 readVarint(InputByteStream & stream) +{ + UInt64 result = 0; + for (size_t shift = 0; shift < 64; shift += 7) + { + UInt8 byte = stream.readByte(); + result |= static_cast(byte & 0x7F) << shift; + if (!(byte & 0x80)) + return result; + } + throw Exception(ErrorCodes::INCORRECT_DATA, "Malformed varint in ORC stream {}", stream.getName()); +} + +Int64 readSignedVarint(InputByteStream & stream) +{ + return unZigZag(readVarint(stream)); +} + + +void ByteRleDecoder::readHeader() +{ + Int8 header = static_cast(stream->readByte()); + if (header >= 0) + { + repeating = true; + remaining_values = static_cast(header) + 3; + value = static_cast(stream->readByte()); + } + else + { + repeating = false; + remaining_values = static_cast(-static_cast(header)); + } +} + +void ByteRleDecoder::next(char * data, size_t num_values, const char * not_null) +{ + if (not_null) + memset(data, 0, num_values); + size_t position = 0; + while (position < num_values) + { + if (not_null) + { + while (position < num_values && !not_null[position]) + ++position; + if (position == num_values) + return; + } + + if (remaining_values == 0) + readHeader(); + + size_t count = std::min(num_values - position, remaining_values); + size_t consumed = 0; + if (repeating) + { + if (not_null) + { + for (size_t i = 0; i < count; ++i) + { + if (not_null[position + i]) + { + data[position + i] = value; + ++consumed; + } + } + } + else + { + memset(data + position, value, count); + consumed = count; + } + } + else + { + if (not_null) + { + for (size_t i = 0; i < count; ++i) + { + if (not_null[position + i]) + { + data[position + i] = static_cast(stream->readByte()); + ++consumed; + } + } + } + else + { + stream->readBytes(data + position, count); + consumed = count; + } + } + remaining_values -= consumed; + position += count; + } +} + +void ByteRleDecoder::skip(size_t num_values) +{ + while (num_values) + { + if (remaining_values == 0) + readHeader(); + size_t count = std::min(num_values, remaining_values); + remaining_values -= count; + num_values -= count; + if (!repeating) + stream->skipBytes(count); + } +} + +void ByteRleDecoder::seek(PositionProvider & positions) +{ + stream->seek(positions); + remaining_values = 0; + skip(positions.next()); +} + + +void BooleanRleDecoder::next(char * data, size_t num_values, const char * not_null) +{ + for (size_t i = 0; i < num_values; ++i) + { + if (not_null && !not_null[i]) + { + /// The PRESENT stream of a child column is read with the mask of its parent, + /// and the masked positions must be null. + data[i] = 0; + continue; + } + if (remaining_bits == 0) + { + char byte = 0; + byte_decoder.next(&byte, 1, nullptr); + last_byte = static_cast(byte); + remaining_bits = 8; + } + --remaining_bits; + data[i] = (last_byte >> remaining_bits) & 1; + } +} + +void BooleanRleDecoder::skip(size_t num_values) +{ + if (num_values <= remaining_bits) + { + remaining_bits -= num_values; + return; + } + num_values -= remaining_bits; + byte_decoder.skip(num_values / 8); + if (num_values % 8) + { + char byte = 0; + byte_decoder.next(&byte, 1, nullptr); + last_byte = static_cast(byte); + remaining_bits = 8 - num_values % 8; + } + else + remaining_bits = 0; +} + +void BooleanRleDecoder::seek(PositionProvider & positions) +{ + byte_decoder.seek(positions); + UInt64 consumed = positions.next(); + remaining_bits = 0; + if (consumed > 8) + throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid seek position {} of a boolean stream in ORC file", consumed); + if (consumed) + { + char byte = 0; + byte_decoder.next(&byte, 1, nullptr); + last_byte = static_cast(byte); + remaining_bits = 8 - consumed; + } +} + + +namespace +{ + +class IntRleDecoderV1 final : public IntRleDecoder +{ +public: + IntRleDecoderV1(std::unique_ptr stream_, bool is_signed_) : stream(std::move(stream_)), is_signed(is_signed_) {} + + void next(Int64 * data, size_t num_values, const char * not_null) override { nextImpl(data, num_values, not_null); } + void next(UInt64 * data, size_t num_values, const char * not_null) override { nextImpl(data, num_values, not_null); } + void next(Int32 * data, size_t num_values, const char * not_null) override { nextImpl(data, num_values, not_null); } + void next(Int16 * data, size_t num_values, const char * not_null) override { nextImpl(data, num_values, not_null); } + + template + void nextImpl(T * data, size_t num_values, const char * not_null) + { + if (not_null) + memset(static_cast(data), 0, num_values * sizeof(T)); + size_t position = 0; + while (position < num_values) + { + if (not_null) + { + while (position < num_values && !not_null[position]) + ++position; + if (position == num_values) + return; + } + + if (remaining_values == 0) + readHeader(); + + size_t count = std::min(num_values - position, remaining_values); + size_t consumed = 0; + for (size_t i = 0; i < count; ++i) + { + if (not_null && !not_null[position + i]) + continue; + if (repeating) + { + data[position + i] = static_cast(value); + value = static_cast(static_cast(value) + static_cast(delta)); + } + else + data[position + i] = static_cast(readValue()); + ++consumed; + } + remaining_values -= consumed; + position += count; + } + } + + void skip(size_t num_values) override + { + while (num_values) + { + if (remaining_values == 0) + readHeader(); + size_t count = std::min(num_values, remaining_values); + remaining_values -= count; + num_values -= count; + if (repeating) + value = static_cast(static_cast(value) + static_cast(delta) * count); + else + for (size_t i = 0; i < count; ++i) + readVarint(*stream); + } + } + + void seek(PositionProvider & positions) override + { + stream->seek(positions); + remaining_values = 0; + skip(positions.next()); + } + +private: + Int64 readValue() + { + return is_signed ? readSignedVarint(*stream) : static_cast(readVarint(*stream)); + } + + void readHeader() + { + Int8 header = static_cast(stream->readByte()); + if (header >= 0) + { + repeating = true; + remaining_values = static_cast(header) + 3; + delta = static_cast(stream->readByte()); + value = readValue(); + } + else + { + repeating = false; + remaining_values = static_cast(-static_cast(header)); + } + } + + std::unique_ptr stream; + const bool is_signed; + size_t remaining_values = 0; + bool repeating = false; + Int64 value = 0; + Int64 delta = 0; +}; + + +class IntRleDecoderV2 final : public IntRleDecoder +{ +public: + IntRleDecoderV2(std::unique_ptr stream_, bool is_signed_) : stream(std::move(stream_)), is_signed(is_signed_) {} + + void next(Int64 * data, size_t num_values, const char * not_null) override { nextImpl(data, num_values, not_null); } + void next(UInt64 * data, size_t num_values, const char * not_null) override { nextImpl(data, num_values, not_null); } + void next(Int32 * data, size_t num_values, const char * not_null) override { nextImpl(data, num_values, not_null); } + void next(Int16 * data, size_t num_values, const char * not_null) override { nextImpl(data, num_values, not_null); } + + template + void nextImpl(T * data, size_t num_values, const char * not_null) + { + if (not_null) + memset(static_cast(data), 0, num_values * sizeof(T)); + size_t position = 0; + while (position < num_values) + { + if (not_null) + { + while (position < num_values && !not_null[position]) + ++position; + if (position == num_values) + return; + } + + if (run_read == run_length) + readRun(); + + size_t count = std::min(num_values - position, run_length - run_read); + if (not_null) + { + for (size_t i = position; i < position + count; ++i) + if (not_null[i]) + data[i] = static_cast(literals[run_read++]); + } + else if constexpr (sizeof(T) == sizeof(Int64)) + { + memcpy(static_cast(data + position), literals.data() + run_read, count * sizeof(Int64)); + run_read += count; + } + else + { + for (size_t i = 0; i < count; ++i) + data[position + i] = static_cast(literals[run_read + i]); + run_read += count; + } + position += count; + } + } + + void skip(size_t num_values) override + { + while (num_values) + { + if (run_read == run_length) + readRun(); + size_t count = std::min(num_values, run_length - run_read); + run_read += count; + num_values -= count; + } + } + + void seek(PositionProvider & positions) override + { + stream->seek(positions); + run_read = run_length = 0; + skip(positions.next()); + } + +private: + enum EncodingType : UInt8 + { + SHORT_REPEAT = 0, + DIRECT = 1, + PATCHED_BASE = 2, + DELTA = 3, + }; + + Int64 readLongBE(size_t num_bytes) + { + UInt64 result = 0; + for (size_t i = 0; i < num_bytes; ++i) + result = (result << 8) | stream->readByte(); + return static_cast(result); + } + + template + static void decodeBigEndian(const UInt8 * in, Int64 * out, size_t count) + { + for (size_t i = 0; i < count; ++i, in += num_bytes) + { + UInt64 value = 0; + for (size_t j = 0; j < num_bytes; ++j) + value = (value << 8) | in[j]; + out[i] = static_cast(value); + } + } + + static void decodeBigEndian(const UInt8 * in, Int64 * out, size_t count, size_t num_bytes) + { + switch (num_bytes) + { + case 1: decodeBigEndian<1>(in, out, count); return; + case 2: decodeBigEndian<2>(in, out, count); return; + case 3: decodeBigEndian<3>(in, out, count); return; + case 4: decodeBigEndian<4>(in, out, count); return; + case 5: decodeBigEndian<5>(in, out, count); return; + case 6: decodeBigEndian<6>(in, out, count); return; + case 7: decodeBigEndian<7>(in, out, count); return; + default: decodeBigEndian<8>(in, out, count); return; + } + } + + /// The only bit widths that are not a multiple of 8 are at most 30. + template + static void decodeBitPacked(Int64 * out, size_t count, UInt32 bit_width, ReadByte && read_byte) + { + UInt64 buffer = 0; + UInt32 bits_in_buffer = 0; + const UInt64 mask = (1ULL << bit_width) - 1; + for (size_t i = 0; i < count; ++i) + { + while (bits_in_buffer < bit_width) + { + buffer = (buffer << 8) | read_byte(); + bits_in_buffer += 8; + } + bits_in_buffer -= bit_width; + out[i] = static_cast((buffer >> bits_in_buffer) & mask); + } + } + + /// Reads `count` bit-packed values of `bit_width` bits, starting and ending at a byte boundary. + void readInts(Int64 * out, size_t count, UInt32 bit_width) + { + if (bit_width % 8 == 0) + { + const size_t num_bytes = bit_width / 8; + while (count) + { + /// Decode the values that are entirely in the current chunk directly from it. + size_t in_chunk = std::min(count, static_cast(stream->end - stream->pos) / num_bytes); + if (in_chunk == 0) + { + *out = readLongBE(num_bytes); + ++out; + --count; + continue; + } + decodeBigEndian(reinterpret_cast(stream->pos), out, in_chunk, num_bytes); + stream->pos += in_chunk * num_bytes; + out += in_chunk; + count -= in_chunk; + } + return; + } + + const size_t total_bytes = (count * bit_width + 7) / 8; + if (static_cast(stream->end - stream->pos) >= total_bytes) + { + const UInt8 * in = reinterpret_cast(stream->pos); + decodeBitPacked(out, count, bit_width, [&] { return *in++; }); + stream->pos += total_bytes; + } + else + { + decodeBitPacked(out, count, bit_width, [&] { return stream->readByte(); }); + } + } + + void readRun() + { + UInt8 first_byte = stream->readByte(); + auto encoding = static_cast(first_byte >> 6); + run_read = 0; + switch (encoding) + { + case SHORT_REPEAT: + { + size_t num_bytes = ((first_byte >> 3) & 0x07) + 1; + run_length = (first_byte & 0x07) + 3; + Int64 value = readLongBE(num_bytes); + if (is_signed) + value = unZigZag(static_cast(value)); + std::fill(literals.begin(), literals.begin() + run_length, value); + break; + } + case DIRECT: + { + UInt32 bit_width = decodeBitWidth((first_byte >> 1) & 0x1F); + run_length = ((static_cast(first_byte & 0x01) << 8) | stream->readByte()) + 1; + readInts(literals.data(), run_length, bit_width); + if (is_signed) + for (size_t i = 0; i < run_length; ++i) + literals[i] = unZigZag(static_cast(literals[i])); + break; + } + case PATCHED_BASE: + readPatchedBase(first_byte); + break; + case DELTA: + readDelta(first_byte); + break; + } + } + + void readPatchedBase(UInt8 first_byte) + { + UInt32 bit_width = decodeBitWidth((first_byte >> 1) & 0x1F); + run_length = ((static_cast(first_byte & 0x01) << 8) | stream->readByte()) + 1; + + UInt8 third_byte = stream->readByte(); + size_t base_bytes = ((third_byte >> 5) & 0x07) + 1; + UInt32 patch_bit_width = decodeBitWidth(third_byte & 0x1F); + + UInt8 fourth_byte = stream->readByte(); + UInt32 patch_gap_width = ((fourth_byte >> 5) & 0x07) + 1; + size_t patch_list_length = fourth_byte & 0x1F; + if (patch_list_length == 0) + throw Exception(ErrorCodes::INCORRECT_DATA, "Corrupt PATCHED_BASE encoded data in ORC stream {} (empty patch list)", stream->getName()); + + Int64 base = readLongBE(base_bytes); + Int64 sign_mask = static_cast(1ULL << (base_bytes * 8 - 1)); + if (base & sign_mask) + { + base &= ~sign_mask; + base = -base; + } + + readInts(literals.data(), run_length, bit_width); + + if (patch_bit_width + patch_gap_width > 64) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Corrupt PATCHED_BASE encoded data in ORC stream {} (patch width + patch gap width > 64)", stream->getName()); + + Int64 patches[32]; + readInts(patches, patch_list_length, getClosestFixedBits(patch_bit_width + patch_gap_width)); + + const Int64 patch_mask = patch_bit_width >= 64 ? -1 : static_cast((1ULL << patch_bit_width) - 1); + size_t patch_index = 0; + Int64 gap = 0; + Int64 patch = 0; + + auto adjust_gap_and_patch = [&] + { + UInt64 current_gap = static_cast(patches[patch_index]) >> patch_bit_width; + patch = patches[patch_index] & patch_mask; + Int64 actual_gap = 0; + /// A gap of 255 with a zero patch means that the gap continues in the next entry. + while (current_gap == 255 && patch == 0) + { + actual_gap += 255; + ++patch_index; + if (patch_index >= patch_list_length) + throw Exception(ErrorCodes::INCORRECT_DATA, "Corrupt PATCHED_BASE encoded data in ORC stream {} (unterminated gap)", stream->getName()); + current_gap = static_cast(patches[patch_index]) >> patch_bit_width; + patch = patches[patch_index] & patch_mask; + } + gap = actual_gap + static_cast(current_gap); + }; + + adjust_gap_and_patch(); + for (size_t i = 0; i < run_length; ++i) + { + if (static_cast(i) != gap) + { + literals[i] = static_cast(static_cast(literals[i]) + static_cast(base)); + } + else + { + UInt64 patched = static_cast(literals[i]) | (bit_width < 64 ? static_cast(patch) << bit_width : 0); + literals[i] = static_cast(static_cast(base) + patched); + ++patch_index; + if (patch_index < patch_list_length) + { + adjust_gap_and_patch(); + /// The next gap is relative to the current position. + gap += static_cast(i); + } + } + } + } + + void readDelta(UInt8 first_byte) + { + UInt32 code = (first_byte >> 1) & 0x1F; + UInt32 bit_width = code ? decodeBitWidth(code) : 0; + run_length = ((static_cast(first_byte & 0x01) << 8) | stream->readByte()) + 1; + + Int64 first_value = is_signed ? readSignedVarint(*stream) : static_cast(readVarint(*stream)); + literals[0] = first_value; + Int64 delta_base = readSignedVarint(*stream); + + if (bit_width == 0) + { + /// Fixed delta. + for (size_t i = 1; i < run_length; ++i) + literals[i] = static_cast(static_cast(literals[i - 1]) + static_cast(delta_base)); + return; + } + + if (run_length < 2) + throw Exception(ErrorCodes::INCORRECT_DATA, "Illegal run length {} for delta encoding in ORC stream {}", run_length, stream->getName()); + + Int64 previous = static_cast(static_cast(first_value) + static_cast(delta_base)); + literals[1] = previous; + readInts(literals.data() + 2, run_length - 2, bit_width); + for (size_t i = 2; i < run_length; ++i) + { + if (delta_base < 0) + previous = static_cast(static_cast(previous) - static_cast(literals[i])); + else + previous = static_cast(static_cast(previous) + static_cast(literals[i])); + literals[i] = previous; + } + } + + std::unique_ptr stream; + const bool is_signed; + std::array literals{}; + size_t run_length = 0; + size_t run_read = 0; +}; + +} + +std::unique_ptr createIntRleDecoder(std::unique_ptr stream, bool is_signed, RleVersion version) +{ + if (version == RleVersion::V1) + return std::make_unique(std::move(stream), is_signed); + return std::make_unique(std::move(stream), is_signed); +} + + +void ByteRleEncoder::writeValues() +{ + if (num_literals == 0) + return; + if (repeat) + { + stream.writeByte(static_cast(num_literals - MIN_REPEAT)); + stream.writeByte(static_cast(literals[0])); + } + else + { + stream.writeByte(static_cast(-static_cast(num_literals))); + stream.write(literals, num_literals); + } + repeat = false; + tail_run_length = 0; + num_literals = 0; +} + +void ByteRleEncoder::write(char value) +{ + if (num_literals == 0) + { + literals[num_literals++] = value; + tail_run_length = 1; + } + else if (repeat) + { + if (value == literals[0]) + { + ++num_literals; + if (num_literals == MAX_REPEAT_SIZE) + writeValues(); + } + else + { + writeValues(); + literals[num_literals++] = value; + tail_run_length = 1; + } + } + else + { + if (value == literals[num_literals - 1]) + ++tail_run_length; + else + tail_run_length = 1; + + if (tail_run_length == MIN_REPEAT) + { + if (num_literals + 1 == MIN_REPEAT) + { + repeat = true; + ++num_literals; + } + else + { + num_literals -= MIN_REPEAT - 1; + writeValues(); + literals[0] = value; + repeat = true; + num_literals = MIN_REPEAT; + } + } + else + { + literals[num_literals++] = value; + if (num_literals == MAX_LITERAL_SIZE) + writeValues(); + } + } +} + +void ByteRleEncoder::recordPosition(std::vector & positions) const +{ + stream.recordPosition(positions); + positions.push_back(num_literals); +} + +void ByteRleEncoder::flush() +{ + writeValues(); +} + + +void BooleanRleEncoder::write(bool value) +{ + current = static_cast((current << 1) | (value ? 1 : 0)); + ++bits_in_current; + if (bits_in_current == 8) + { + byte_encoder.write(static_cast(current)); + current = 0; + bits_in_current = 0; + } +} + +void BooleanRleEncoder::recordPosition(std::vector & positions) const +{ + byte_encoder.recordPosition(positions); + positions.push_back(bits_in_current); +} + +void BooleanRleEncoder::flush() +{ + if (bits_in_current) + { + byte_encoder.write(static_cast(current << (8 - bits_in_current))); + current = 0; + bits_in_current = 0; + } + byte_encoder.flush(); +} + + +void IntRleEncoderV2::recordPosition(std::vector & positions) const +{ + stream.recordPosition(positions); + positions.push_back(num_literals); +} + +void IntRleEncoderV2::flush() +{ + size_t literal_start = 0; + size_t i = 0; + while (i < num_literals) + { + size_t j = i + 1; + while (j < num_literals && literals[j] == literals[i]) + ++j; + if (j - i >= 3) + { + if (literal_start < i) + encodeLiterals(literals + literal_start, i - literal_start); + encodeRepeat(literals[i], j - i); + literal_start = j; + } + i = j; + } + if (literal_start < num_literals) + encodeLiterals(literals + literal_start, num_literals - literal_start); + num_literals = 0; +} + +void IntRleEncoderV2::writeVarint(UInt64 value) +{ + while (value >= 0x80) + { + stream.writeByte(static_cast((value & 0x7F) | 0x80)); + value >>= 7; + } + stream.writeByte(static_cast(value)); +} + +void IntRleEncoderV2::writeInts(const Int64 * values, size_t count, UInt32 bit_width) +{ + if (bit_width % 8 == 0) + { + size_t num_bytes = bit_width / 8; + for (size_t i = 0; i < count; ++i) + { + UInt64 v = static_cast(values[i]); + for (size_t b = num_bytes; b > 0; --b) + stream.writeByte(static_cast(v >> ((b - 1) * 8))); + } + return; + } + + UInt64 buffer = 0; + UInt32 bits_in_buffer = 0; + const UInt64 mask = (1ULL << bit_width) - 1; + for (size_t i = 0; i < count; ++i) + { + buffer = (buffer << bit_width) | (static_cast(values[i]) & mask); + bits_in_buffer += bit_width; + while (bits_in_buffer >= 8) + { + bits_in_buffer -= 8; + stream.writeByte(static_cast(buffer >> bits_in_buffer)); + } + } + if (bits_in_buffer) + stream.writeByte(static_cast(buffer << (8 - bits_in_buffer))); +} + +void IntRleEncoderV2::encodeRepeat(Int64 value, size_t count) +{ + UInt64 encoded = is_signed ? zigZag(value) : static_cast(value); + if (count <= 10) + { + /// SHORT_REPEAT + UInt32 bits = findClosestNumBits(static_cast(encoded)); + UInt32 num_bytes = (bits + 7) / 8; + stream.writeByte(static_cast(((num_bytes - 1) << 3) | (count - 3))); + for (UInt32 b = num_bytes; b > 0; --b) + stream.writeByte(static_cast(encoded >> ((b - 1) * 8))); + return; + } + + /// DELTA with zero fixed delta. + size_t length = count - 1; + stream.writeByte(static_cast((3 << 6) | ((length >> 8) & 1))); + stream.writeByte(static_cast(length & 0xFF)); + writeVarint(encoded); + writeSignedVarint(0); +} + +void IntRleEncoderV2::writeDirect(const Int64 * values, size_t count) +{ + Int64 encoded[MAX_LITERAL_SIZE]; + UInt32 bit_width = 1; + for (size_t i = 0; i < count; ++i) + { + encoded[i] = is_signed ? static_cast(zigZag(values[i])) : values[i]; + bit_width = std::max(bit_width, findClosestNumBits(encoded[i])); + } + size_t length = count - 1; + stream.writeByte(static_cast((1 << 6) | (encodeBitWidth(bit_width) << 1) | ((length >> 8) & 1))); + stream.writeByte(static_cast(length & 0xFF)); + writeInts(encoded, count, bit_width); +} + +namespace +{ + +/// The bit width that is enough for the given fraction `p` of the values (as in the reference implementation). +UInt32 percentileBits(const Int64 * values, size_t count, double p) +{ + UInt32 histogram[32] = {}; + for (size_t i = 0; i < count; ++i) + ++histogram[encodeBitWidth(findClosestNumBits(values[i]))]; + + Int64 per_len = static_cast(static_cast(count) * (1.0 - p)); + for (Int32 i = 31; i >= 0; --i) + { + per_len -= histogram[i]; + if (per_len < 0) + return decodeBitWidth(i); + } + return 0; +} + +} + +void IntRleEncoderV2::encodeLiterals(const Int64 * values, size_t count) +{ + if (count <= 3) + { + writeDirect(values, count); + return; + } + + Int64 min = values[0]; + Int64 max = values[0]; + bool is_increasing = true; + bool is_decreasing = true; + bool is_fixed_delta = true; + bool deltas_overflow = false; + Int64 initial_delta = 0; + if (__builtin_sub_overflow(values[1], values[0], &initial_delta)) + deltas_overflow = true; + + /// Absolute values of the deltas, starting from the second one. + Int64 abs_deltas[MAX_LITERAL_SIZE]; + UInt64 max_abs_delta = 0; + + for (size_t i = 1; i < count; ++i) + { + min = std::min(min, values[i]); + max = std::max(max, values[i]); + is_increasing &= values[i - 1] <= values[i]; + is_decreasing &= values[i - 1] >= values[i]; + + Int64 delta = 0; + if (__builtin_sub_overflow(values[i], values[i - 1], &delta)) + { + deltas_overflow = true; + is_fixed_delta = false; + continue; + } + is_fixed_delta &= (delta == initial_delta); + if (i > 1) + { + UInt64 abs_delta = delta < 0 ? (0 - static_cast(delta)) : static_cast(delta); + abs_deltas[i - 2] = static_cast(abs_delta); + max_abs_delta = std::max(max_abs_delta, abs_delta); + } + } + + Int64 range = 0; + if (__builtin_sub_overflow(max, min, &range)) + { + writeDirect(values, count); + return; + } + + if (!deltas_overflow && is_fixed_delta) + { + size_t length = count - 1; + stream.writeByte(static_cast((3 << 6) | ((length >> 8) & 1))); + stream.writeByte(static_cast(length & 0xFF)); + if (is_signed) + writeSignedVarint(values[0]); + else + writeVarint(static_cast(values[0])); + writeSignedVarint(initial_delta); + return; + } + + if (!deltas_overflow && initial_delta != 0 && (is_increasing || is_decreasing) + && max_abs_delta <= static_cast(std::numeric_limits::max())) + { + UInt32 bit_width = findClosestNumBits(static_cast(max_abs_delta)); + /// The bit width code 0 means a fixed delta, so the width 1 is not representable. + if (bit_width == 1) + bit_width = 2; + size_t length = count - 1; + stream.writeByte(static_cast((3 << 6) | (encodeBitWidth(bit_width) << 1) | ((length >> 8) & 1))); + stream.writeByte(static_cast(length & 0xFF)); + if (is_signed) + writeSignedVarint(values[0]); + else + writeVarint(static_cast(values[0])); + writeSignedVarint(initial_delta); + writeInts(abs_deltas, count - 2, bit_width); + return; + } + + /// DIRECT or PATCHED_BASE. + Int64 encoded[MAX_LITERAL_SIZE]; + for (size_t i = 0; i < count; ++i) + encoded[i] = is_signed ? static_cast(zigZag(values[i])) : values[i]; + + UInt32 bits_100p = percentileBits(encoded, count, 1.0); + UInt32 bits_90p = percentileBits(encoded, count, 0.9); + + static constexpr Int64 BASE_VALUE_LIMIT = 1LL << 56; + if (bits_100p - bits_90p <= 1 || min <= -BASE_VALUE_LIMIT || min >= BASE_VALUE_LIMIT) + { + writeDirect(values, count); + return; + } + + Int64 base_reduced[MAX_LITERAL_SIZE]; + for (size_t i = 0; i < count; ++i) + base_reduced[i] = values[i] - min; + + UInt32 br_bits_95p = percentileBits(base_reduced, count, 0.95); + UInt32 br_bits_100p = percentileBits(base_reduced, count, 1.0); + if (br_bits_100p == br_bits_95p) + { + writeDirect(values, count); + return; + } + + UInt32 patch_width = getClosestFixedBits(br_bits_100p - br_bits_95p); + if (patch_width == 64) + { + patch_width = 56; + br_bits_95p = 8; + } + const Int64 mask = static_cast((1ULL << br_bits_95p) - 1); + + size_t gaps[MAX_LITERAL_SIZE]; + Int64 patches[MAX_LITERAL_SIZE]; + size_t num_patches = 0; + size_t previous = 0; + size_t max_gap = 0; + for (size_t i = 0; i < count; ++i) + { + if (base_reduced[i] > mask) + { + size_t gap = i - previous; + max_gap = std::max(max_gap, gap); + previous = i; + gaps[num_patches] = gap; + patches[num_patches] = base_reduced[i] >> br_bits_95p; + ++num_patches; + base_reduced[i] &= mask; + } + } + + UInt32 patch_gap_width = (max_gap == 0) ? 1 : findClosestNumBits(static_cast(max_gap)); + patch_gap_width = std::min(patch_gap_width, 8u); + + Int64 patch_list[MAX_LITERAL_SIZE]; + size_t patch_list_length = 0; + for (size_t p = 0; p < num_patches; ++p) + { + size_t gap = gaps[p]; + while (gap > 255) + { + patch_list[patch_list_length++] = static_cast(255ULL << patch_width); + gap -= 255; + } + patch_list[patch_list_length++] = static_cast((static_cast(gap) << patch_width) | static_cast(patches[p])); + } + + /// The length of the patch list is stored in 5 bits. + if (patch_list_length > 31) + { + writeDirect(values, count); + return; + } + + bool is_negative = min < 0; + UInt64 base = is_negative ? static_cast(-min) : static_cast(min); + UInt32 base_width = findClosestNumBits(static_cast(base)) + 1; + UInt32 base_bytes = (base_width + 7) / 8; + if (is_negative) + base |= 1ULL << (base_bytes * 8 - 1); + + size_t length = count - 1; + stream.writeByte(static_cast((2 << 6) | (encodeBitWidth(br_bits_95p) << 1) | ((length >> 8) & 1))); + stream.writeByte(static_cast(length & 0xFF)); + stream.writeByte(static_cast(((base_bytes - 1) << 5) | encodeBitWidth(patch_width))); + stream.writeByte(static_cast(((patch_gap_width - 1) << 5) | patch_list_length)); + for (UInt32 b = base_bytes; b > 0; --b) + stream.writeByte(static_cast(base >> ((b - 1) * 8))); + writeInts(base_reduced, count, getClosestFixedBits(br_bits_95p)); + writeInts(patch_list, patch_list_length, getClosestFixedBits(patch_gap_width + patch_width)); +} + +} diff --git a/src/Processors/Formats/Impl/ORC/RLE.h b/src/Processors/Formats/Impl/ORC/RLE.h new file mode 100644 index 0000000000000..c5212bf5fbf6e --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/RLE.h @@ -0,0 +1,151 @@ +#pragma once + +#include + +#include +#include + +/// Run-length encodings of ORC: https://orc.apache.org/specification/ORCv1/#run-length-encoding +/// +/// All decoders fill the positions where `not_null` is non-zero (or all positions if it is +/// nullptr), consuming one value from the stream for each of them; the other positions are set to zero. + +namespace DB::ORC +{ + +/// Byte run-length encoding, used for the bytes and for the booleans (packed into bytes). +class ByteRleDecoder +{ +public: + explicit ByteRleDecoder(std::unique_ptr stream_) : stream(std::move(stream_)) {} + + void next(char * data, size_t num_values, const char * not_null); + void skip(size_t num_values); + void seek(PositionProvider & positions); + +private: + void readHeader(); + + std::unique_ptr stream; + size_t remaining_values = 0; + bool repeating = false; + char value = 0; +}; + +/// Booleans packed into bytes, most significant bit first, encoded with the byte run-length encoding. +class BooleanRleDecoder +{ +public: + explicit BooleanRleDecoder(std::unique_ptr stream_) : byte_decoder(std::move(stream_)) {} + + /// Writes 0 or 1. + void next(char * data, size_t num_values, const char * not_null); + void skip(size_t num_values); + void seek(PositionProvider & positions); + +private: + ByteRleDecoder byte_decoder; + size_t remaining_bits = 0; + UInt8 last_byte = 0; +}; + +/// Integer run-length encodings, version 1 and 2. +class IntRleDecoder +{ +public: + virtual ~IntRleDecoder() = default; + /// The values are converted to the type of the output, which can be narrower than the encoded values. + virtual void next(Int64 * data, size_t num_values, const char * not_null) = 0; + virtual void next(UInt64 * data, size_t num_values, const char * not_null) = 0; + virtual void next(Int32 * data, size_t num_values, const char * not_null) = 0; + virtual void next(Int16 * data, size_t num_values, const char * not_null) = 0; + virtual void skip(size_t num_values) = 0; + virtual void seek(PositionProvider & positions) = 0; +}; + +enum class RleVersion : UInt8 +{ + V1, + V2, +}; + +std::unique_ptr createIntRleDecoder(std::unique_ptr stream, bool is_signed, RleVersion version); + +/// Reads a base 128 varint, as used by the DECIMAL columns. +UInt64 readVarint(InputByteStream & stream); +Int64 readSignedVarint(InputByteStream & stream); + + +class ByteRleEncoder +{ +public: + explicit ByteRleEncoder(OutputByteStream & stream_) : stream(stream_) {} + + void write(char value); + void recordPosition(std::vector & positions) const; + /// Writes out the buffered values. + void flush(); + +private: + void writeValues(); + + static constexpr size_t MIN_REPEAT = 3; + static constexpr size_t MAX_LITERAL_SIZE = 128; + static constexpr size_t MAX_REPEAT_SIZE = 127 + MIN_REPEAT; + + OutputByteStream & stream; + char literals[MAX_LITERAL_SIZE]{}; + size_t num_literals = 0; + size_t tail_run_length = 0; + bool repeat = false; +}; + +class BooleanRleEncoder +{ +public: + explicit BooleanRleEncoder(OutputByteStream & stream_) : byte_encoder(stream_) {} + + void write(bool value); + void recordPosition(std::vector & positions) const; + void flush(); + +private: + ByteRleEncoder byte_encoder; + UInt8 current = 0; + size_t bits_in_current = 0; +}; + +/// Integer run-length encoding, version 2. +class IntRleEncoderV2 +{ +public: + IntRleEncoderV2(OutputByteStream & stream_, bool is_signed_) : stream(stream_), is_signed(is_signed_) {} + + void write(Int64 value) + { + literals[num_literals++] = value; + if (num_literals == MAX_LITERAL_SIZE) + flush(); + } + + void recordPosition(std::vector & positions) const; + /// Writes out the buffered values. + void flush(); + + static constexpr size_t MAX_LITERAL_SIZE = 512; + +private: + void encodeLiterals(const Int64 * values, size_t count); + void encodeRepeat(Int64 value, size_t count); + void writeDirect(const Int64 * values, size_t count); + void writeVarint(UInt64 value); + void writeSignedVarint(Int64 value) { writeVarint((static_cast(value) << 1) ^ static_cast(value >> 63)); } + void writeInts(const Int64 * values, size_t count, UInt32 bit_width); + + OutputByteStream & stream; + const bool is_signed; + Int64 literals[MAX_LITERAL_SIZE]{}; + size_t num_literals = 0; +}; + +} diff --git a/src/Processors/Formats/Impl/ORC/Reader.cpp b/src/Processors/Formats/Impl/ORC/Reader.cpp new file mode 100644 index 0000000000000..76c6a432d1b06 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Reader.cpp @@ -0,0 +1,842 @@ +#include + +#include +#include +#include +#include + +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int INCORRECT_DATA; + extern const int NOT_IMPLEMENTED; +} +} + +namespace DB::ORC +{ + +std::future InputStream::readAsync(void * buf, UInt64 length, UInt64 offset) +{ + read(buf, length, offset); + std::promise promise; + promise.set_value(); + return promise.get_future(); +} + + +Reader::Reader(std::unique_ptr stream_, const ReaderOptions & options_) + : stream(std::move(stream_)), options(options_) +{ + readTail(); +} + +Reader::~Reader() +{ + /// The background reads write into the buffers of the cache. + for (auto & range : cache) + if (!range.ready && range.future.valid()) + range.future.wait(); +} + +void Reader::readTail() +{ + const UInt64 file_size = stream->getLength(); + /// The header "ORC" and at least the length of the postscript. + if (file_size < 4) + throw Exception(ErrorCodes::INCORRECT_DATA, "Not an ORC file: {} is too small ({} bytes)", stream->getName(), file_size); + + static constexpr UInt64 DIRECTORY_SIZE_GUESS = 16 * 1024; + UInt64 tail_size = std::min(file_size, DIRECTORY_SIZE_GUESS); + std::string tail(tail_size, '\0'); + stream->read(tail.data(), tail_size, file_size - tail_size); + + postscript_length = static_cast(tail.back()); + if (postscript_length + 1 > tail_size) + throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid postscript length {} in ORC file {}", postscript_length, stream->getName()); + + postscript.parse(std::string_view(tail).substr(tail_size - 1 - postscript_length, postscript_length)); + + if (postscript.magic != "ORC") + { + /// Very old files do not have the magic in the postscript, but they have it at the beginning. + char header[3]; + stream->read(header, 3, 0); + if (memcmp(header, "ORC", 3) != 0) + throw Exception(ErrorCodes::INCORRECT_DATA, "Not an ORC file: {}", stream->getName()); + } + + if (postscript.compression != CompressionKind::NONE) + { + if (postscript.compression_block_size == 0 || postscript.compression_block_size > (1ULL << 30)) + throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid compression block size {} in ORC file {}", postscript.compression_block_size, stream->getName()); + if (postscript.compression == CompressionKind::BROTLI) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "BROTLI compression in ORC files is not supported"); + } + + UInt64 tail_length = 0; + if (__builtin_add_overflow(postscript.footer_length, postscript.metadata_length, &tail_length) + || __builtin_add_overflow(tail_length, postscript_length + 1, &tail_length) + || tail_length > file_size) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Invalid footer length {} and metadata length {} in ORC file {} of size {}", + postscript.footer_length, postscript.metadata_length, stream->getName(), file_size); + + if (tail_length > tail_size) + { + tail.resize(tail_length); + tail_size = tail_length; + stream->read(tail.data(), tail_size, file_size - tail_size); + } + + std::string_view footer_data = std::string_view(tail).substr(tail_size - 1 - postscript_length - postscript.footer_length, postscript.footer_length); + footer.parse(decompress(footer_data, "footer")); + + if (footer.has_encryption) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Encrypted ORC files are not supported"); + + schema = Type::fromProto(footer.types); + + /// Validate the stripes once, so that the sums of their offsets and lengths cannot overflow later. + if (footer.stripes.size() > static_cast(std::numeric_limits::max())) + throw Exception(ErrorCodes::INCORRECT_DATA, "Too many stripes {} in ORC file {}", footer.stripes.size(), stream->getName()); + for (size_t i = 0; i < footer.stripes.size(); ++i) + { + const auto & stripe = footer.stripes[i]; + UInt64 stripe_end = 0; + if (common::addOverflow(stripe.offset, stripe.index_length, stripe_end) + || common::addOverflow(stripe_end, stripe.data_length, stripe_end) + || common::addOverflow(stripe_end, stripe.footer_length, stripe_end) + || stripe_end > file_size) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Malformed stripe information at stripe index {} in ORC file {}: offset={}, indexLength={}, dataLength={}, footerLength={}, fileLength={}", + i, stream->getName(), stripe.offset, stripe.index_length, stripe.data_length, stripe.footer_length, file_size); + } + + std::string_view metadata_data = std::string_view(tail).substr( + tail_size - 1 - postscript_length - postscript.footer_length - postscript.metadata_length, postscript.metadata_length); + if (!metadata_data.empty()) + metadata.emplace().parse(decompress(metadata_data, "metadata"), footer.stripes.size(), footer.types.size()); + else + metadata.emplace(); +} + +const Proto::Metadata & Reader::getMetadata() +{ + return *metadata; +} + +std::string Reader::decompress(std::string_view data, const std::string & name) const +{ + if (postscript.compression == CompressionKind::NONE) + return std::string(data); + + std::string result; + InputByteStream in(data, postscript.compression, postscript.compression_block_size, name); + while (in.nextChunk()) + result.append(in.pos, in.end); + return result; +} + +Proto::StripeFooter Reader::readStripeFooter(size_t stripe_index) +{ + const auto & stripe = getStripe(stripe_index); + UInt64 offset = stripe.offset + stripe.index_length + stripe.data_length; + UInt64 end = 0; + if (__builtin_add_overflow(offset, stripe.footer_length, &end) || end > stream->getLength() || offset < stripe.offset) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Malformed stripe information at stripe index {} in ORC file {}: offset={}, indexLength={}, dataLength={}, footerLength={}, fileLength={}", + stripe_index, stream->getName(), stripe.offset, stripe.index_length, stripe.data_length, stripe.footer_length, stream->getLength()); + + auto slices = readRanges({{offset, stripe.footer_length}}); + Proto::StripeFooter result; + result.parse(decompress(slices[0].data, "stripe footer"), footer.types.size()); + if (result.encryption_variants) + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Encrypted ORC files are not supported"); + return result; +} + +std::vector> Reader::getStripeRanges( + size_t stripe_index, const Proto::StripeFooter & stripe_footer, const std::vector & selected) const +{ + const auto & stripe = getStripe(stripe_index); + std::vector> ranges; + UInt64 offset = stripe.offset; + const UInt64 data_end = stripe.offset + stripe.index_length + stripe.data_length; + for (const auto & s : stripe_footer.streams) + { + /// Every stream is checked: the offsets of the following streams depend on its length. + UInt64 stream_end = 0; + if (common::addOverflow(offset, s.length, stream_end) || stream_end > data_end) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Malformed stream at offset {} of length {} in stripe {} of ORC file {}", offset, s.length, stripe_index, stream->getName()); + if (s.kind && s.column < selected.size() && selected[s.column] && s.length) + ranges.emplace_back(offset, s.length); + offset = stream_end; + } + return ranges; +} + +std::vector Reader::readRanges(const std::vector> & ranges) +{ + std::vector result(ranges.size()); + + std::vector order(ranges.size()); + for (size_t i = 0; i < order.size(); ++i) + order[i] = i; + std::sort(order.begin(), order.end(), [&](size_t a, size_t b) { return ranges[a].first < ranges[b].first; }); + + size_t i = 0; + while (i < order.size()) + { + UInt64 begin = ranges[order[i]].first; + UInt64 end = begin + ranges[order[i]].second; + size_t j = i + 1; + while (j < order.size()) + { + UInt64 next_begin = ranges[order[j]].first; + UInt64 next_end = next_begin + ranges[order[j]].second; + UInt64 new_end = std::max(end, next_end); + if (next_begin > end && next_begin - end > options.hole_size_limit) + break; + if (new_end - begin > options.range_size_limit) + break; + end = new_end; + ++j; + } + + std::shared_ptr> holder; + const char * base = nullptr; + UInt64 base_offset = begin; + { + std::lock_guard lock(cache_mutex); + for (auto & cached : cache) + { + if (cached.offset <= begin && end <= cached.offset + cached.length) + { + if (!cached.ready) + { + cached.ready = true; + cached.future.get(); + } + holder = cached.buffer; + base = cached.buffer->data(); + base_offset = cached.offset; + break; + } + } + } + + if (!holder) + { + if (end > stream->getLength()) + throw Exception(ErrorCodes::INCORRECT_DATA, "A range at {} of length {} goes beyond the end of ORC file {}", begin, end - begin, stream->getName()); + auto buffer = std::make_shared>(end - begin); + if (end > begin) + stream->read(buffer->data(), end - begin, begin); + base = buffer->data(); + holder = std::move(buffer); + } + + for (size_t k = i; k < j; ++k) + { + const auto & range = ranges[order[k]]; + result[order[k]] = Slice{holder, std::string_view(base + (range.first - base_offset), range.second)}; + } + i = j; + } + return result; +} + +void Reader::preBuffer(const std::vector & stripes, const std::vector & include_types) +{ + auto selected = selectColumnsByTypeIds(*schema, include_types); + for (UInt32 stripe_index : stripes) + { + const auto & stripe = getStripe(stripe_index); + auto stripe_footer = readStripeFooter(stripe_index); + auto ranges = getStripeRanges(stripe_index, stripe_footer, selected); + ranges.emplace_back(stripe.offset + stripe.index_length + stripe.data_length, stripe.footer_length); + std::sort(ranges.begin(), ranges.end()); + + size_t i = 0; + while (i < ranges.size()) + { + UInt64 begin = ranges[i].first; + UInt64 end = begin + ranges[i].second; + size_t j = i + 1; + while (j < ranges.size()) + { + UInt64 next_begin = ranges[j].first; + UInt64 new_end = std::max(end, next_begin + ranges[j].second); + if ((next_begin > end && next_begin - end > options.hole_size_limit) || new_end - begin > options.range_size_limit) + break; + end = new_end; + ++j; + } + + auto buffer = std::make_shared>(end - begin); + { + /// The entry must be owned by the cache before the background read starts, so that + /// the destructor waits for the read even if an exception is thrown in between. + std::lock_guard lock(cache_mutex); + auto & cached = cache.emplace_back(CachedRange{.offset = begin, .length = end - begin, .buffer = std::move(buffer), .future = {}}); + try + { + cached.future = stream->readAsync(cached.buffer->data(), end - begin, begin); + } + catch (...) + { + /// Nothing has been scheduled, so the entry can be removed safely. + cache.pop_back(); + throw; + } + } + i = j; + } + } +} + +void Reader::releaseBuffer(UInt64 boundary) +{ + std::lock_guard lock(cache_mutex); + for (auto it = cache.begin(); it != cache.end();) + { + if (it->offset + it->length <= boundary) + { + /// The buffer can be destroyed only after the background read finishes. The data has + /// not been used, so an error of reading it does not matter. + if (!it->ready && it->future.valid()) + it->future.wait(); + it = cache.erase(it); + } + else + ++it; + } +} + +std::unique_ptr Reader::createRowReader(const RowReaderOptions & row_options) +{ + return std::make_unique(*this, row_options); +} + + +class RowReader::StripeStreamsImpl final : public StripeStreams +{ +public: + StripeStreamsImpl( + Reader & reader_, + size_t stripe_index_, + Proto::StripeFooter footer_, + const std::vector & selected_columns_, + const DateLUTImpl & writer_timezone_, + const DateLUTImpl & reader_timezone_) + : reader(reader_) + , stripe_index(stripe_index_) + , footer(std::move(footer_)) + , selected_columns(selected_columns_) + , writer_timezone(writer_timezone_) + , reader_timezone(reader_timezone_) + { + const auto & stripe = reader.getStripe(stripe_index); + UInt64 offset = stripe.offset; + const UInt64 data_end = stripe.offset + stripe.index_length + stripe.data_length; + + for (const auto & s : footer.streams) + { + /// Every stream is checked: the offsets of the following streams depend on its length. + UInt64 stream_end = 0; + if (common::addOverflow(offset, s.length, stream_end) || stream_end > data_end) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Malformed stream meta in stripe {} of ORC file: streamOffset={}, streamLength={}, stripeOffset={}, stripeIndexLength={}, stripeDataLength={}", + stripe_index, offset, s.length, stripe.offset, stripe.index_length, stripe.data_length); + if (s.kind && s.column < selected_columns.size() && selected_columns[s.column]) + { + /// Use the first stream if there are several of the same kind, as the reference implementation. + stream_ranges.try_emplace({static_cast(s.column), *s.kind}, offset, s.length); + } + else if (s.kind && s.column >= selected_columns.size()) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Malformed stream meta in stripe {} of ORC file: column {} is out of range, the file has {} columns", + stripe_index, s.column, selected_columns.size()); + offset = stream_end; + } + } + + static bool isIndexStream(UInt32 kind) + { + return kind == static_cast(Proto::StreamKind::ROW_INDEX) + || kind == static_cast(Proto::StreamKind::BLOOM_FILTER) + || kind == static_cast(Proto::StreamKind::BLOOM_FILTER_UTF8); + } + + /// Reads the streams into memory at once, only the index streams or all of them. The streams + /// that are not read are loaded on demand by `getStream`. + void loadStreams(bool only_index) + { + std::vector> ranges; + std::vector> keys; + for (const auto & [key, range] : stream_ranges) + { + if (streams.contains(key) || (only_index && !isIndexStream(key.second))) + continue; + keys.push_back(key); + ranges.push_back(range); + } + + auto new_slices = reader.readRanges(ranges); + for (size_t i = 0; i < keys.size(); ++i) + streams.emplace(keys[i], new_slices[i].data); + slices.insert(slices.end(), std::make_move_iterator(new_slices.begin()), std::make_move_iterator(new_slices.end())); + } + + const Proto::ColumnEncoding & getEncoding(UInt64 column_id) const override + { + if (column_id >= footer.columns.size()) + throw Exception(ErrorCodes::INCORRECT_DATA, + "ORC stripe {} has no encoding for column {}, it has {} column encodings", stripe_index, column_id, footer.columns.size()); + return footer.columns[column_id]; + } + + std::unique_ptr getStream(UInt64 column_id, Proto::StreamKind kind) const override + { + std::pair key{column_id, static_cast(kind)}; + auto name = fmt::format("column {} kind {} in stripe {}", column_id, static_cast(kind), stripe_index); + if (auto it = streams.find(key); it != streams.end()) + return std::make_unique(it->second, reader.getCompression(), reader.getCompressionBlockSize(), std::move(name)); + + auto it = stream_ranges.find(key); + if (it == stream_ranges.end()) + return nullptr; + auto loader = [&file_reader = reader, stream_offset = it->second.first](UInt64 offset, UInt64 length) + { + auto loaded = file_reader.readRanges({{stream_offset + offset, length}}); + return LoadedRange{std::move(loaded[0].holder), loaded[0].data}; + }; + return std::make_unique( + std::move(loader), it->second.second, reader.getCompression(), reader.getCompressionBlockSize(), std::move(name)); + } + + std::optional getStreamData(UInt64 column_id, Proto::StreamKind kind) const + { + auto it = streams.find({column_id, static_cast(kind)}); + if (it == streams.end()) + return {}; + return it->second; + } + + const std::vector & getSelectedColumns() const override { return selected_columns; } + const DateLUTImpl & getWriterTimezone() const override { return writer_timezone; } + const DateLUTImpl & getReaderTimezone() const override { return reader_timezone; } + UInt64 getNumberOfRows() const override { return reader.getStripe(stripe_index).number_of_rows; } + bool isDecimalAsLong() const override + { + const auto & version = reader.getPostScript().version; + return version.size() == 2 && version[0] == 1 && version[1] == 9999; + } + + const Proto::StripeFooter & getFooter() const { return footer; } + +private: + Reader & reader; + size_t stripe_index; + Proto::StripeFooter footer; + const std::vector & selected_columns; + const DateLUTImpl & writer_timezone; + const DateLUTImpl & reader_timezone; + /// The offset in the file and the length of each selected stream. + std::map, std::pair> stream_ranges; + /// The streams read into memory. + std::vector slices; + std::map, std::string_view> streams; +}; + + +RowReader::RowReader(Reader & reader_, const RowReaderOptions & options_) + : reader(reader_) + , options(options_) + , reader_timezone(DateLUT::instance(options.timezone_name)) +{ + const Type & schema = reader.getType(); + if (options.include_types) + selected_columns = selectColumnsByTypeIds(schema, *options.include_types); + else + selected_columns.assign(schema.getMaximumColumnId() + 1, true); + selected_type = Type::buildSelectedType(schema, selected_columns); + + UInt64 range_end = options.range_offset + std::min(options.range_length, std::numeric_limits::max() - options.range_offset); + const auto & stripes = reader.getFooter().stripes; + current_stripe = stripes.size(); + last_stripe = 0; + for (size_t i = 0; i < stripes.size(); ++i) + { + if (stripes[i].offset >= options.range_offset && stripes[i].offset < range_end) + { + current_stripe = std::min(current_stripe, i); + last_stripe = std::max(last_stripe, i + 1); + } + } + if (current_stripe >= last_stripe) + current_stripe = last_stripe = 0; + + /// The search argument can be applied only if there are row indexes. + if (reader.getFooter().row_index_stride == 0) + options.search_argument.reset(); +} + +RowReader::~RowReader() = default; + +std::unique_ptr RowReader::createRowBatch() const +{ + return ORC::createRowBatch(*selected_type, options.enable_lazy_decoding); +} + +bool Reader::hasBadBloomFilters() const +{ + /// Same as in the reference implementation. Only the C++ writer (id 1) of some old releases wrote bad bloom filters. + if (footer.writer.value_or(0) != 1) + return false; + /// `software_version` is written since 1.5.13, 1.6.11 and 1.7.0, and the C++ writer supports bloom filters since 1.6.0, + /// so the files of the C++ writer without `software_version` have bad bloom filters. + if (!footer.software_version) + return true; + /// Strip the suffix of snapshot versions, like 1.6.12-SNAPSHOT. + std::string_view version = *footer.software_version; + version = version.substr(0, version.find('-')); + static constexpr std::string_view bad_versions[] = { + "1.6.0", "1.6.1", "1.6.2", "1.6.3", "1.6.4", "1.6.5", "1.6.6", "1.6.7", "1.6.8", "1.6.9", "1.6.10", "1.6.11", "1.7.0"}; + return std::find(std::begin(bad_versions), std::end(bad_versions), version) != std::end(bad_versions); +} + +const DateLUTImpl & RowReader::getWriterTimezone(const Proto::StripeFooter & stripe_footer) const +{ + if (stripe_footer.writer_timezone) + return DateLUT::instance(*stripe_footer.writer_timezone); + /// The reference implementation uses the local time zone of the reader if the file does not specify it. + return DateLUT::serverTimezoneInstance(); +} + +bool RowReader::evaluateStatistics(const std::vector & stats) const +{ + const auto & leaves = options.search_argument->getLeaves(); + std::vector values(leaves.size(), TruthValue::YES_NO_NULL); + UInt32 writer_version = reader.getPostScript().writer_version; + for (size_t i = 0; i < leaves.size(); ++i) + { + UInt64 column = leaves[i].getColumnId(); + if (column < stats.size()) + values[i] = leaves[i].evaluate(writer_version, stats[column], nullptr); + } + return isNeeded(options.search_argument->evaluate(values)); +} + +UInt64 RowReader::numberOfRowGroups() const +{ + const UInt64 stride = reader.getFooter().row_index_stride; + return rows_in_current_stripe / stride + (rows_in_current_stripe % stride != 0); +} + +void RowReader::loadStripeIndex() +{ + row_indexes.clear(); + bloom_filters.clear(); + /// One extra entry is accepted from lenient writers. + const UInt64 max_entries = std::min(numberOfRowGroups(), std::numeric_limits::max() - 1) + 1; + const bool use_bloom_filters = !reader.hasBadBloomFilters(); + const auto & stripe_footer = stripe_streams->getFooter(); + for (const auto & s : stripe_footer.streams) + { + if (!s.kind || s.column >= selected_columns.size() || !selected_columns[s.column]) + continue; + auto kind = static_cast(*s.kind); + if (kind == Proto::StreamKind::ROW_INDEX) + { + if (row_indexes.contains(s.column)) + continue; + auto data = stripe_streams->getStreamData(s.column, kind); + row_indexes[s.column].parse(reader.decompress(data.value_or(std::string_view{}), "row index"), max_entries); + } + else if (kind == Proto::StreamKind::BLOOM_FILTER_UTF8 && use_bloom_filters) + { + if (bloom_filters.contains(s.column)) + continue; + const auto & encoding = stripe_streams->getEncoding(s.column); + Proto::BloomFilterIndex index; + auto data = stripe_streams->getStreamData(s.column, kind); + index.parse(reader.decompress(data.value_or(std::string_view{}), "bloom filter"), max_entries); + auto & filters = bloom_filters[s.column]; + for (const auto & bf : index.bloom_filter) + { + /// Only the bloom filters with the UTF-8 strings and UTC timestamps can be used. + if (encoding.bloom_encoding.value_or(0) == 1 && bf.num_hash_functions && bf.utf8bitset) + filters.push_back(std::make_shared(bf)); + else + filters.push_back(nullptr); + } + } + } +} + +void RowReader::pickRowGroups() +{ + const UInt64 stride = reader.getFooter().row_index_stride; + const UInt64 groups = numberOfRowGroups(); + const auto & leaves = options.search_argument->getLeaves(); + + /// The number of the row groups comes from the file. Check it against the row indexes, which are already + /// read, before allocating memory for it. + bool has_leaf_index = false; + for (const auto & leaf : leaves) + { + auto it = row_indexes.find(leaf.getColumnId()); + if (it == row_indexes.end()) + continue; + has_leaf_index = true; + if (it->second.entry.size() < groups) + throw Exception(ErrorCodes::INCORRECT_DATA, + "The row index of ORC column {} has {} entries, less than the {} row groups of stripe {}", + it->first, it->second.entry.size(), groups, current_stripe); + } + + if (!has_leaf_index) + { + /// Without the row indexes all row groups are read. + next_skipped_rows.clear(); + return; + } + + next_skipped_rows.assign(groups, 0); + std::vector values(leaves.size(), TruthValue::YES_NO_NULL); + const UInt32 writer_version = reader.getPostScript().writer_version; + const Proto::ColumnStatistics empty_statistics; + + UInt64 next_skipped_group = groups; + for (UInt64 group = groups; group > 0;) + { + --group; + for (size_t i = 0; i < leaves.size(); ++i) + { + UInt64 column = leaves[i].getColumnId(); + auto it = row_indexes.find(column); + if (it == row_indexes.end()) + { + values[i] = TruthValue::YES_NO_NULL; + continue; + } + if (group >= it->second.entry.size()) + throw Exception(ErrorCodes::INCORRECT_DATA, + "The row index of ORC column {} has {} entries, less than the {} row groups of stripe {}", + column, it->second.entry.size(), groups, current_stripe); + const auto & entry = it->second.entry[group]; + const BloomFilter * bloom_filter = nullptr; + if (auto bf = bloom_filters.find(column); bf != bloom_filters.end() && group < bf->second.size()) + bloom_filter = bf->second[group].get(); + values[i] = leaves[i].evaluate(writer_version, entry.statistics ? *entry.statistics : empty_statistics, bloom_filter); + } + + bool needed = isNeeded(options.search_argument->evaluate(values)); + if (!needed) + { + next_skipped_rows[group] = 0; + next_skipped_group = group; + } + else + next_skipped_rows[group] = (next_skipped_group == groups) ? rows_in_current_stripe : next_skipped_group * stride; + } +} + +UInt64 RowReader::advanceToNextRowGroup(UInt64 current_row) const +{ + const UInt64 groups = next_skipped_rows.size(); + if (groups == 0) + return std::min(current_row, rows_in_current_stripe); + + const UInt64 stride = reader.getFooter().row_index_stride; + UInt64 group = current_row / stride; + if (group >= groups) + return rows_in_current_stripe; + if (next_skipped_rows[group] != 0) + return current_row; + while (group < groups && next_skipped_rows[group] == 0) + ++group; + if (group < groups) + return group * stride; + return rows_in_current_stripe; +} + +UInt64 RowReader::computeBatchSize(UInt64 requested, UInt64 current_row) const +{ + UInt64 end_row = rows_in_current_stripe; + const UInt64 groups = next_skipped_rows.size(); + if (groups > 0) + { + const UInt64 stride = reader.getFooter().row_index_stride; + UInt64 group = current_row / stride; + if (group >= groups) + return 0; + UInt64 next_skipped_row = next_skipped_rows[group]; + if (next_skipped_row == 0) + return 0; + end_row = next_skipped_row; + } + return std::min(requested, end_row - current_row); +} + +void RowReader::seekToRowGroup(size_t row_group) +{ + std::unordered_map positions; + for (const auto & [column, index] : row_indexes) + { + if (row_group >= index.entry.size()) + throw Exception(ErrorCodes::INCORRECT_DATA, + "The row index of ORC column {} has {} entries, the row group {} is requested", column, index.entry.size(), row_group); + positions.emplace(column, PositionProvider(index.entry[row_group].positions)); + } + column_reader->seekToRowGroup(positions); +} + +void RowReader::startNextStripe() +{ + column_reader.reset(); + stripe_streams.reset(); + row_indexes.clear(); + bloom_filters.clear(); + next_skipped_rows.clear(); + + const auto & search_argument = options.search_argument; + const auto & footer = reader.getFooter(); + + if (search_argument && !file_statistics_evaluated) + { + file_statistics_result = footer.statistics.empty() || evaluateStatistics(footer.statistics); + file_statistics_evaluated = true; + } + if (search_argument && !file_statistics_result) + { + current_stripe = last_stripe; + return; + } + + do + { + const auto & stripe = footer.stripes[current_stripe]; + rows_in_current_stripe = stripe.number_of_rows; + + bool is_stripe_needed = true; + const auto & stripe_statistics = reader.getMetadata().stripe_stats; + if (search_argument && stripe_statistics.size() == footer.stripes.size()) + { + const auto & stats = stripe_statistics[current_stripe].col_stats; + is_stripe_needed = stats.empty() || evaluateStatistics(stats); + } + + if (is_stripe_needed) + { + auto stripe_footer = reader.readStripeFooter(current_stripe); + const DateLUTImpl & writer_timezone = getWriterTimezone(stripe_footer); + stripe_streams = std::make_unique( + reader, current_stripe, std::move(stripe_footer), selected_columns, writer_timezone, reader_timezone); + if (!search_argument) + stripe_streams->loadStreams(/* only_index */ false); + + if (search_argument) + { + stripe_streams->loadStreams(/* only_index */ true); + loadStripeIndex(); + pickRowGroups(); + bool has_selected = next_skipped_rows.empty(); + for (UInt64 group = current_row_in_stripe / footer.row_index_stride; !has_selected && group < next_skipped_rows.size(); ++group) + has_selected = next_skipped_rows[group] != 0; + if (has_selected) + { + /// If some row groups are skipped, the data is loaded on demand, only around the + /// selected row groups. Otherwise the whole stripe is read at once. + bool has_skipped = std::find(next_skipped_rows.begin(), next_skipped_rows.end(), 0) != next_skipped_rows.end(); + if (!has_skipped) + stripe_streams->loadStreams(/* only_index */ false); + break; + } + is_stripe_needed = false; + stripe_streams.reset(); + } + } + + if (!is_stripe_needed) + { + ++current_stripe; + current_row_in_stripe = 0; + } + } while (search_argument && current_stripe < last_stripe); + + if (current_stripe >= last_stripe) + return; + + column_reader = buildColumnReader(reader.getType(), *stripe_streams); + + if (search_argument) + { + current_row_in_stripe = advanceToNextRowGroup(current_row_in_stripe); + if (current_row_in_stripe > 0) + seekToRowGroup(current_row_in_stripe / footer.row_index_stride); + } +} + +bool RowReader::next(ColumnVectorBatch & batch, size_t max_rows) +{ + if (current_stripe >= last_stripe) + { + batch.numElements = 0; + return false; + } + + if (current_row_in_stripe == 0) + startNextStripe(); + + if (current_stripe >= last_stripe) + { + batch.numElements = 0; + return false; + } + + UInt64 rows_to_read = std::min(max_rows, rows_in_current_stripe - current_row_in_stripe); + if (options.search_argument && rows_to_read > 0) + rows_to_read = computeBatchSize(rows_to_read, current_row_in_stripe); + + batch.numElements = rows_to_read; + if (rows_to_read == 0) + { + current_stripe = last_stripe; + return false; + } + + if (options.enable_lazy_decoding) + column_reader->nextEncoded(batch, rows_to_read, nullptr); + else + column_reader->next(batch, rows_to_read, nullptr); + + current_row_in_stripe += rows_to_read; + + if (options.search_argument) + { + UInt64 next_row = advanceToNextRowGroup(current_row_in_stripe); + if (current_row_in_stripe != next_row) + { + current_row_in_stripe = next_row; + if (current_row_in_stripe < rows_in_current_stripe) + seekToRowGroup(current_row_in_stripe / reader.getFooter().row_index_stride); + } + } + + if (current_row_in_stripe >= rows_in_current_stripe) + { + ++current_stripe; + current_row_in_stripe = 0; + } + return true; +} + +} diff --git a/src/Processors/Formats/Impl/ORC/Reader.h b/src/Processors/Formats/Impl/ORC/Reader.h new file mode 100644 index 0000000000000..981ab8c5c7996 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Reader.h @@ -0,0 +1,188 @@ +#pragma once + +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +/// The reader of the ORC files: https://orc.apache.org/specification/ORCv1/ + +namespace DB::ORC +{ + +/// Random access to the file. +class InputStream +{ +public: + virtual ~InputStream() = default; + + virtual UInt64 getLength() const = 0; + /// Reads exactly `length` bytes at `offset`. Must be thread-safe if `readAsync` is used. + virtual void read(void * buf, UInt64 length, UInt64 offset) = 0; + /// Reads in the background. The default implementation reads synchronously. + virtual std::future readAsync(void * buf, UInt64 length, UInt64 offset); + virtual const std::string & getName() const = 0; +}; + +struct ReaderOptions +{ + /// Two ranges are read with one request if the gap between them is at most this size... + UInt64 hole_size_limit = 8192; + /// ... and the combined range is at most this size. + UInt64 range_size_limit = 32 * 1024 * 1024; +}; + +struct RowReaderOptions +{ + /// The ids of the columns to read, with the same semantics as in the reference implementation + /// (see `selectColumnsByTypeIds`). If not set, all columns are read. + std::optional> include_types; + /// Read only the stripes that start in `[range_offset, range_offset + range_length)`. + UInt64 range_offset = 0; + UInt64 range_length = std::numeric_limits::max(); + /// For the dictionary-encoded string columns, fill `EncodedStringVectorBatch`. + bool enable_lazy_decoding = false; + /// The time zone to show the values of the TIMESTAMP (not TIMESTAMP_INSTANT) columns in. + std::string timezone_name = "GMT"; + std::shared_ptr search_argument; +}; + +class RowReader; + +class Reader +{ +public: + Reader(std::unique_ptr stream_, const ReaderOptions & options_); + ~Reader(); + + const Type & getType() const { return *schema; } + UInt64 getNumberOfRows() const { return footer.number_of_rows; } + UInt64 getNumberOfStripes() const { return footer.stripes.size(); } + const Proto::StripeInformation & getStripe(size_t index) const { return footer.stripes.at(index); } + static UInt64 getStripeLength(const Proto::StripeInformation & stripe) { return stripe.index_length + stripe.data_length + stripe.footer_length; } + + const Proto::Footer & getFooter() const { return footer; } + const Proto::PostScript & getPostScript() const { return postscript; } + CompressionKind getCompression() const { return postscript.compression; } + /// Whether the bloom filters of the file were written by a version of the C++ writer + /// that hashed the integers incorrectly (ORC-1024), so they must not be used. + bool hasBadBloomFilters() const; + UInt64 getCompressionBlockSize() const { return postscript.compression_block_size; } + + /// The statistics of the whole file, one per column. + const std::vector & getStatistics() const { return footer.statistics; } + /// The statistics of the stripes, read on the first call. Can be empty. + const Proto::Metadata & getMetadata(); + + /// Reads the stripe footer. Not thread-safe. + Proto::StripeFooter readStripeFooter(size_t stripe_index); + + std::unique_ptr createRowReader(const RowReaderOptions & options); + + /// Starts reading the data of the selected columns of the stripes in the background. + void preBuffer(const std::vector & stripes, const std::vector & include_types); + /// Releases the prefetched data that ends before `boundary`. + void releaseBuffer(UInt64 boundary); + + /// A piece of the file in memory. + struct Slice + { + std::shared_ptr> holder; + std::string_view data; + }; + + /// Reads the ranges `(offset, length)` of the file, from the prefetched data if possible. + /// The adjacent ranges are coalesced according to the options. + std::vector readRanges(const std::vector> & ranges); + + /// Decompresses a whole metadata stream (a footer, an index). + std::string decompress(std::string_view data, const std::string & name) const; + +private: + void readTail(); + + struct CachedRange + { + UInt64 offset; + UInt64 length; + std::shared_ptr> buffer; + std::future future; + bool ready = false; + }; + + /// Returns the ranges of the streams of the selected columns in the stripe, and the stripe footer. + std::vector> getStripeRanges(size_t stripe_index, const Proto::StripeFooter & stripe_footer, const std::vector & selected) const; + + std::unique_ptr stream; + ReaderOptions options; + Proto::PostScript postscript; + UInt64 postscript_length = 0; + Proto::Footer footer; + std::optional metadata; + std::unique_ptr schema; + + std::mutex cache_mutex; + std::list cache; +}; + +/// Reads the rows of the selected stripes and columns in batches. +class RowReader +{ +public: + RowReader(Reader & reader_, const RowReaderOptions & options_); + ~RowReader(); + + std::unique_ptr createRowBatch() const; + + /// Fills the batch with the next rows, at most `max_rows`. Returns false at the end. + bool next(ColumnVectorBatch & batch, size_t max_rows); + + /// The type of the file with only the selected columns. The batches have this structure. + const Type & getSelectedType() const { return *selected_type; } + +private: + class StripeStreamsImpl; + + void startNextStripe(); + void loadStripeIndex(); + UInt64 numberOfRowGroups() const; + void pickRowGroups(); + void seekToRowGroup(size_t row_group); + UInt64 advanceToNextRowGroup(UInt64 current_row) const; + UInt64 computeBatchSize(UInt64 requested, UInt64 current_row) const; + bool evaluateStatistics(const std::vector & stats) const; + const DateLUTImpl & getWriterTimezone(const Proto::StripeFooter & stripe_footer) const; + + Reader & reader; + RowReaderOptions options; + std::vector selected_columns; + std::unique_ptr selected_type; + const DateLUTImpl & reader_timezone; + + UInt64 current_stripe = 0; + UInt64 last_stripe = 0; + UInt64 rows_in_current_stripe = 0; + UInt64 current_row_in_stripe = 0; + bool file_statistics_evaluated = false; + bool file_statistics_result = true; + + std::unique_ptr stripe_streams; + std::unique_ptr column_reader; + + /// For the predicate pushdown, by the column id. + std::unordered_map row_indexes; + std::unordered_map>> bloom_filters; + /// For each row group of the current stripe: zero if it is skipped, otherwise the row (in the + /// stripe) of the next skipped row group or the number of rows in the stripe. + std::vector next_skipped_rows; +}; + +} diff --git a/src/Processors/Formats/Impl/ORC/SearchArgument.cpp b/src/Processors/Formats/Impl/ORC/SearchArgument.cpp new file mode 100644 index 0000000000000..e0543230c614a --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/SearchArgument.cpp @@ -0,0 +1,1075 @@ +#include + +#include +#include +#include + +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; + extern const int INCORRECT_DATA; +} +} + +namespace DB::ORC +{ + +TruthValue operator||(TruthValue left, TruthValue right) +{ + if (right == TruthValue::YES || left == TruthValue::YES) + return TruthValue::YES; + if (right == TruthValue::YES_NULL || left == TruthValue::YES_NULL) + return TruthValue::YES_NULL; + if (right == TruthValue::NO) + return left; + if (left == TruthValue::NO) + return right; + if (left == TruthValue::IS_NULL) + { + if (right == TruthValue::NO_NULL || right == TruthValue::IS_NULL) + return TruthValue::IS_NULL; + return TruthValue::YES_NULL; + } + if (right == TruthValue::IS_NULL) + { + if (left == TruthValue::NO_NULL) + return TruthValue::IS_NULL; + return TruthValue::YES_NULL; + } + if (left == TruthValue::NO_NULL && right == TruthValue::NO_NULL) + return TruthValue::NO_NULL; + return TruthValue::YES_NO_NULL; +} + +TruthValue operator&&(TruthValue left, TruthValue right) +{ + if (right == TruthValue::NO || left == TruthValue::NO) + return TruthValue::NO; + if (right == TruthValue::NO_NULL || left == TruthValue::NO_NULL) + return TruthValue::NO_NULL; + if (right == TruthValue::YES) + return left; + if (left == TruthValue::YES) + return right; + if (left == TruthValue::IS_NULL) + { + if (right == TruthValue::YES_NULL || right == TruthValue::IS_NULL) + return TruthValue::IS_NULL; + return TruthValue::NO_NULL; + } + if (right == TruthValue::IS_NULL) + { + if (left == TruthValue::YES_NULL) + return TruthValue::IS_NULL; + return TruthValue::NO_NULL; + } + if (left == TruthValue::YES_NULL && right == TruthValue::YES_NULL) + return TruthValue::YES_NULL; + return TruthValue::YES_NO_NULL; +} + +TruthValue operator!(TruthValue value) +{ + switch (value) + { + case TruthValue::NO: return TruthValue::YES; + case TruthValue::YES: return TruthValue::NO; + case TruthValue::IS_NULL: + case TruthValue::YES_NO: + case TruthValue::YES_NO_NULL: + return value; + case TruthValue::NO_NULL: return TruthValue::YES_NULL; + case TruthValue::YES_NULL: return TruthValue::NO_NULL; + } + return value; +} + +bool isNeeded(TruthValue value) +{ + switch (value) + { + case TruthValue::NO: + case TruthValue::IS_NULL: + case TruthValue::NO_NULL: + return false; + default: + return true; + } +} + + +bool Literal::Decimal::parse(std::string_view s) +{ + bool negative = false; + size_t i = 0; + if (i < s.size() && (s[i] == '-' || s[i] == '+')) + { + negative = s[i] == '-'; + ++i; + } + Int256 result = 0; + bool has_digits = false; + bool has_point = false; + Int32 result_scale = 0; + for (; i < s.size(); ++i) + { + char c = s[i]; + if (c == '.' && !has_point) + { + has_point = true; + continue; + } + if (c < '0' || c > '9') + return false; + has_digits = true; + result = result * 10 + (c - '0'); + if (has_point) + ++result_scale; + /// More than 38 digits do not fit into the 128-bit value. + if (result > Int256(std::numeric_limits::max())) + return false; + } + if (!has_digits) + return false; + value = static_cast(negative ? -result : result); + scale = result_scale; + return true; +} + +std::string Literal::Decimal::toString() const +{ + bool negative = value < 0; + std::string digits = DB::toString(negative ? -Int256(value) : Int256(value)); + std::string result; + if (scale <= 0) + result = digits; + else + { + if (digits.size() <= static_cast(scale)) + digits = std::string(scale - digits.size() + 1, '0') + digits; + result = digits.substr(0, digits.size() - scale) + "." + digits.substr(digits.size() - scale); + size_t last = result.find_last_not_of('0'); + if (result[last] == '.') + result.resize(last); + else + result.resize(last + 1); + } + return negative ? "-" + result : result; +} + +namespace +{ + +int compareDecimals(const Literal::Decimal & a, const Literal::Decimal & b) +{ + Int256 left = a.value; + Int256 right = b.value; + for (Int32 s = a.scale; s < b.scale; ++s) + left *= 10; + for (Int32 s = b.scale; s < a.scale; ++s) + right *= 10; + return left < right ? -1 : (left == right ? 0 : 1); +} + +} + +Literal::Literal(bool value) : type(PredicateDataType::BOOLEAN), int_value(value) {} +Literal::Literal(Int64 value) : type(PredicateDataType::LONG), int_value(value) {} +Literal::Literal(double value) : type(PredicateDataType::FLOAT), float_value(value) {} +Literal::Literal(const char * data, size_t size) : type(PredicateDataType::STRING), string_value(data, size) {} + +Literal::Literal(PredicateDataType type_, Int64 value) : type(type_), int_value(value) +{ + if (type != PredicateDataType::DATE) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Only DATE literals can be created from a type and an integer"); +} + +Literal::Literal(Int64 second, Int32 nanos) : type(PredicateDataType::TIMESTAMP), timestamp_value{second, nanos} {} + +Literal::Literal(Int128 value, Int32 /*precision*/, Int32 scale) : type(PredicateDataType::DECIMAL), decimal_value{value, scale} {} + + +namespace +{ + +/// Where a point is relative to the range [min, max]. +enum class Location : UInt8 +{ + BEFORE, + MIN, + MIDDLE, + MAX, + AFTER, +}; + +template +Location compareToRange(const T & point, const T & min, const T & max, Less less, Equal equal) +{ + if (less(point, min)) + return Location::BEFORE; + if (equal(point, min)) + return Location::MIN; + if (less(max, point)) + return Location::AFTER; + if (equal(point, max)) + return Location::MAX; + return Location::MIDDLE; +} + +template +TruthValue evaluatePredicateRange( + PredicateLeaf::Operator op, const std::vector & values, const T & min_value, const T & max_value, bool has_null, Less less, Equal equal) +{ + auto loc_of = [&](const T & v) { return compareToRange(v, min_value, max_value, less, equal); }; + bool min_equals_max = equal(min_value, max_value); + + switch (op) + { + case PredicateLeaf::Operator::NULL_SAFE_EQUALS: + { + auto loc = loc_of(values.at(0)); + if (loc == Location::BEFORE || loc == Location::AFTER) + return TruthValue::NO; + return TruthValue::YES_NO; + } + case PredicateLeaf::Operator::EQUALS: + { + auto loc = loc_of(values.at(0)); + if (min_equals_max && loc == Location::MIN) + return has_null ? TruthValue::YES_NULL : TruthValue::YES; + if (loc == Location::BEFORE || loc == Location::AFTER) + return has_null ? TruthValue::NO_NULL : TruthValue::NO; + return has_null ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; + } + case PredicateLeaf::Operator::LESS_THAN: + { + auto loc = loc_of(values.at(0)); + if (loc == Location::AFTER) + return has_null ? TruthValue::YES_NULL : TruthValue::YES; + if (loc == Location::BEFORE || loc == Location::MIN) + return has_null ? TruthValue::NO_NULL : TruthValue::NO; + return has_null ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; + } + case PredicateLeaf::Operator::LESS_THAN_EQUALS: + { + auto loc = loc_of(values.at(0)); + if (loc == Location::AFTER || loc == Location::MAX || (loc == Location::MIN && min_equals_max)) + return has_null ? TruthValue::YES_NULL : TruthValue::YES; + if (loc == Location::BEFORE) + return has_null ? TruthValue::NO_NULL : TruthValue::NO; + return has_null ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; + } + case PredicateLeaf::Operator::IN: + { + if (min_equals_max) + { + for (const auto & value : values) + if (loc_of(value) == Location::MIN) + return has_null ? TruthValue::YES_NULL : TruthValue::YES; + return has_null ? TruthValue::NO_NULL : TruthValue::NO; + } + for (const auto & value : values) + { + auto loc = loc_of(value); + if (loc == Location::MIN || loc == Location::MIDDLE || loc == Location::MAX) + return has_null ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; + } + return has_null ? TruthValue::NO_NULL : TruthValue::NO; + } + case PredicateLeaf::Operator::BETWEEN: + { + if (values.empty()) + return TruthValue::YES_NO; + auto loc = loc_of(values.at(0)); + if (loc == Location::BEFORE || loc == Location::MIN) + { + auto loc2 = loc_of(values.at(1)); + if (loc2 == Location::AFTER || loc2 == Location::MAX) + return has_null ? TruthValue::YES_NULL : TruthValue::YES; + if (loc2 == Location::BEFORE) + return has_null ? TruthValue::NO_NULL : TruthValue::NO; + return has_null ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; + } + if (loc == Location::AFTER) + return has_null ? TruthValue::NO_NULL : TruthValue::NO; + return has_null ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; + } + case PredicateLeaf::Operator::IS_NULL: + return has_null ? TruthValue::YES_NO : TruthValue::NO; + } + return has_null ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; +} + +template +TruthValue evaluateRange(PredicateLeaf::Operator op, const std::vector & values, const T & min_value, const T & max_value, bool has_null) +{ + return evaluatePredicateRange(op, values, min_value, max_value, has_null, + [](const T & a, const T & b) { return a < b; }, + [](const T & a, const T & b) { return a == b; }); +} + +TruthValue evaluateBoolPredicate(PredicateLeaf::Operator op, const std::vector & literals, const Proto::ColumnStatistics & stats) +{ + bool has_null = stats.has_null.value_or(false); + if (!stats.bucket_statistics || stats.bucket_statistics->count.empty()) + return has_null ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; + + UInt64 true_count = stats.bucket_statistics->count[0]; + UInt64 false_count = stats.number_of_values.value_or(0) - true_count; + switch (op) + { + case PredicateLeaf::Operator::IS_NULL: + return has_null ? TruthValue::YES_NO : TruthValue::NO; + case PredicateLeaf::Operator::NULL_SAFE_EQUALS: + { + if (literals.at(0).getBool()) + { + if (true_count == 0) + return TruthValue::NO; + if (false_count == 0) + return TruthValue::YES; + } + else + { + if (false_count == 0) + return TruthValue::NO; + if (true_count == 0) + return TruthValue::YES; + } + return TruthValue::YES_NO; + } + case PredicateLeaf::Operator::EQUALS: + { + if (literals.at(0).getBool()) + { + if (true_count == 0) + return has_null ? TruthValue::NO_NULL : TruthValue::NO; + if (false_count == 0) + return has_null ? TruthValue::YES_NULL : TruthValue::YES; + } + else + { + if (false_count == 0) + return has_null ? TruthValue::NO_NULL : TruthValue::NO; + if (true_count == 0) + return has_null ? TruthValue::YES_NULL : TruthValue::YES; + } + return has_null ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; + } + default: + return has_null ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; + } +} + +template +std::vector literalsTo(const std::vector & literals, Get get) +{ + std::vector result; + for (const auto & literal : literals) + if (!literal.isNull()) + result.push_back(get(literal)); + return result; +} + +/// From the reference implementation: the writer version that fixed the timestamp statistics. +constexpr UInt32 WRITER_VERSION_ORC_135 = 6; + +} + +TruthValue PredicateLeaf::evaluatePredicateMinMax(const Proto::ColumnStatistics & stats) const +{ + TruthValue result = TruthValue::YES_NO_NULL; + bool has_null = stats.has_null.value_or(false); + + switch (type) + { + case PredicateDataType::LONG: + { + if (stats.int_statistics && stats.int_statistics->minimum && stats.int_statistics->maximum) + result = evaluateRange(op, literalsTo(literals, [](const Literal & l) { return l.getLong(); }), + *stats.int_statistics->minimum, *stats.int_statistics->maximum, has_null); + break; + } + case PredicateDataType::FLOAT: + { + if (stats.double_statistics && stats.double_statistics->minimum && stats.double_statistics->maximum) + { + if (!std::isfinite(stats.double_statistics->sum.value_or(0))) + result = has_null ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; + else + result = evaluateRange(op, literalsTo(literals, [](const Literal & l) { return l.getFloat(); }), + *stats.double_statistics->minimum, *stats.double_statistics->maximum, has_null); + } + break; + } + case PredicateDataType::STRING: + { + if (stats.string_statistics && stats.string_statistics->minimum && stats.string_statistics->maximum) + result = evaluateRange(op, literalsTo(literals, [](const Literal & l) { return l.getString(); }), + *stats.string_statistics->minimum, *stats.string_statistics->maximum, has_null); + break; + } + case PredicateDataType::DATE: + { + if (stats.date_statistics && stats.date_statistics->minimum && stats.date_statistics->maximum) + result = evaluateRange(op, literalsTo(literals, [](const Literal & l) { return l.getDate(); }), + static_cast(*stats.date_statistics->minimum), static_cast(*stats.date_statistics->maximum), has_null); + break; + } + case PredicateDataType::TIMESTAMP: + { + const auto & ts = stats.timestamp_statistics; + /// The stored nanoseconds are the sub-millisecond part plus one; malformed ones disable the pruning. + auto valid_nanos = [](const std::optional & nanos) { return !nanos || (*nanos >= 1 && *nanos <= 1'000'000); }; + if (ts && ts->minimum_utc && ts->maximum_utc && valid_nanos(ts->minimum_nanos) && valid_nanos(ts->maximum_nanos)) + { + Int32 min_nanos = ts->minimum_nanos ? static_cast(*ts->minimum_nanos) - 1 : 0; + Int32 max_nanos = ts->maximum_nanos ? static_cast(*ts->maximum_nanos) - 1 : 999999; + /// The milliseconds are rounded down, so the nanoseconds are not negative, as in the literals. + auto to_timestamp = [](Int64 millis, Int32 nanos) + { + Int64 second = millis / 1000; + Int64 rest = millis % 1000; + if (rest < 0) + { + rest += 1000; + --second; + } + return Literal::Timestamp{second, static_cast(rest * 1000000) + nanos}; + }; + Literal::Timestamp min = to_timestamp(*ts->minimum_utc, min_nanos); + Literal::Timestamp max = to_timestamp(*ts->maximum_utc, max_nanos); + result = evaluateRange(op, literalsTo(literals, [](const Literal & l) { return l.getTimestamp(); }), + min, max, has_null); + } + break; + } + case PredicateDataType::DECIMAL: + { + const auto & ds = stats.decimal_statistics; + if (ds && ds->minimum && ds->maximum) + { + Literal::Decimal min; + Literal::Decimal max; + if (min.parse(*ds->minimum) && max.parse(*ds->maximum)) + result = evaluatePredicateRange(op, + literalsTo(literals, [](const Literal & l) { return l.getDecimal(); }), + min, max, has_null, + [](const Literal::Decimal & a, const Literal::Decimal & b) { return compareDecimals(a, b) < 0; }, + [](const Literal::Decimal & a, const Literal::Decimal & b) { return compareDecimals(a, b) == 0; }); + } + break; + } + case PredicateDataType::BOOLEAN: + { + if (stats.bucket_statistics) + result = evaluateBoolPredicate(op, literals, stats); + break; + } + } + + /// Respect a null literal for IN. + if (op == Operator::IN && has_null) + for (const auto & literal : literals) + if (literal.isNull()) + return TruthValue::YES_NO_NULL; + + return result; +} + +namespace +{ + +TruthValue checkInBloomFilter(PredicateDataType type, const Literal & literal, const BloomFilter & bf, bool has_null) +{ + TruthValue result = has_null ? TruthValue::NO_NULL : TruthValue::NO; + if (literal.isNull()) + result = has_null ? TruthValue::YES_NO_NULL : TruthValue::NO; + else + { + bool found = true; + switch (type) + { + case PredicateDataType::LONG: found = bf.testLong(literal.getLong()); break; + case PredicateDataType::FLOAT: found = bf.testDouble(literal.getFloat()); break; + case PredicateDataType::STRING: found = bf.testBytes(literal.getString().data(), literal.getString().size()); break; + case PredicateDataType::DECIMAL: + { + std::string s = literal.getDecimal().toString(); + found = bf.testBytes(s.data(), s.size()); + break; + } + case PredicateDataType::TIMESTAMP: + { + /// The writers round the negative milliseconds differently: the Java one down, the C++ one toward zero. + const auto timestamp = literal.getTimestamp(); + const Int64 millis_down = timestamp.getMillis(); + const bool has_fraction = timestamp.second < 0 && timestamp.nanos % 1000000 != 0; + found = bf.testLong(millis_down) || (has_fraction && bf.testLong(millis_down + 1)); + break; + } + case PredicateDataType::DATE: found = bf.testLong(literal.getDate()); break; + case PredicateDataType::BOOLEAN: found = true; break; + } + if (found) + result = TruthValue::YES_NO_NULL; + } + + if (result == TruthValue::YES_NO_NULL && !has_null) + result = TruthValue::YES_NO; + return result; +} + +} + +TruthValue PredicateLeaf::evaluatePredicateBloomFilter(const BloomFilter & bloom_filter, bool has_null) const +{ + switch (op) + { + case Operator::NULL_SAFE_EQUALS: + /// Null-safe equals does not return the `*_NULL` variants. + return checkInBloomFilter(type, literals.front(), bloom_filter, false); + case Operator::EQUALS: + return checkInBloomFilter(type, literals.front(), bloom_filter, has_null); + case Operator::IN: + for (const auto & literal : literals) + { + TruthValue result = checkInBloomFilter(type, literal, bloom_filter, has_null); + if (result == TruthValue::YES_NO_NULL || result == TruthValue::YES_NO) + return result; + } + return has_null ? TruthValue::NO_NULL : TruthValue::NO; + default: + return has_null ? TruthValue::YES_NO_NULL : TruthValue::YES_NO; + } +} + +TruthValue PredicateLeaf::evaluate(UInt32 writer_version, const Proto::ColumnStatistics & stats, const BloomFilter * bloom_filter) const +{ + /// The files written before ORC-135 store the timestamps in the local time zone of the writer. + if (type == PredicateDataType::TIMESTAMP && writer_version < WRITER_VERSION_ORC_135) + return TruthValue::YES_NO_NULL; + + /// The files written by some writers do not have the `has_null` field. + if (!stats.has_null) + return TruthValue::YES_NO_NULL; + + bool has_null = *stats.has_null; + bool all_null = has_null && stats.number_of_values.value_or(0) == 0; + if (op == Operator::IS_NULL || ((op == Operator::EQUALS || op == Operator::NULL_SAFE_EQUALS) && literals.at(0).isNull())) + return all_null ? TruthValue::YES : (has_null ? TruthValue::YES_NO : TruthValue::NO); + if (all_null) + return TruthValue::IS_NULL; + + TruthValue result = evaluatePredicateMinMax(stats); + if (bloom_filter && result != TruthValue::NO_NULL && result != TruthValue::NO + && (op == Operator::EQUALS || op == Operator::NULL_SAFE_EQUALS || op == Operator::IN)) + return evaluatePredicateBloomFilter(*bloom_filter, has_null); + return result; +} + + +class ExpressionTree +{ +public: + enum class Operator : UInt8 + { + OR, + AND, + NOT, + LEAF, + CONSTANT, + }; + + explicit ExpressionTree(Operator op_) : op(op_) {} + explicit ExpressionTree(TruthValue constant_) : op(Operator::CONSTANT), constant(constant_) {} + explicit ExpressionTree(size_t leaf_) : op(Operator::LEAF), leaf(leaf_) {} + ExpressionTree(Operator op_, std::vector children_) : op(op_), children(std::move(children_)) {} + + TruthValue evaluate(const std::vector & leaves) const + { + switch (op) + { + case Operator::OR: + { + TruthValue result = children.at(0)->evaluate(leaves); + for (size_t i = 1; i < children.size() && !isNeeded(result); ++i) + result = children[i]->evaluate(leaves) || result; + return result; + } + case Operator::AND: + { + TruthValue result = children.at(0)->evaluate(leaves); + for (size_t i = 1; i < children.size() && isNeeded(result); ++i) + result = children[i]->evaluate(leaves) && result; + return result; + } + case Operator::NOT: + return !children.at(0)->evaluate(leaves); + case Operator::LEAF: + return leaves.at(leaf); + case Operator::CONSTANT: + return constant; + } + return TruthValue::YES_NO_NULL; + } + + Operator op; + TruthValue constant = TruthValue::YES_NO_NULL; + size_t leaf = 0; + std::vector children; +}; + + +SearchArgument::SearchArgument(ExpressionTreePtr root_, std::vector leaves_) + : root(std::move(root_)), leaves(std::move(leaves_)) +{ +} + +TruthValue SearchArgument::evaluate(const std::vector & leaf_values) const +{ + return root ? root->evaluate(leaf_values) : TruthValue::YES_NO_NULL; +} + + +namespace +{ + +using Op = ExpressionTree::Operator; + +/// Pushes the negations down to the leaves and removes the double negations. +ExpressionTreePtr pushDownNot(ExpressionTreePtr root) +{ + if (root->op == Op::NOT) + { + ExpressionTreePtr child = root->children.at(0); + switch (child->op) + { + case Op::NOT: + return pushDownNot(child->children.at(0)); + case Op::CONSTANT: + return std::make_shared(!child->constant); + case Op::AND: + case Op::OR: + { + auto result = std::make_shared(child->op == Op::AND ? Op::OR : Op::AND); + for (const auto & kid : child->children) + result->children.push_back(pushDownNot(std::make_shared(Op::NOT, std::vector{kid}))); + return result; + } + case Op::LEAF: + break; + } + } + else + { + for (auto & kid : root->children) + kid = pushDownNot(kid); + } + return root; +} + +/// Removes the MAYBE (YES_NO_NULL) constants: they are dropped from AND and they replace OR. +ExpressionTreePtr foldMaybe(ExpressionTreePtr expr) +{ + for (auto & kid : expr->children) + { + ExpressionTreePtr child = foldMaybe(kid); + if (child->op == Op::CONSTANT && child->constant == TruthValue::YES_NO_NULL) + { + if (expr->op == Op::AND) + kid = nullptr; + else if (expr->op == Op::OR) + return child; + else + throw Exception(ErrorCodes::LOGICAL_ERROR, "Got a maybe as a child of a NOT in ORC search argument"); + } + else + kid = child; + } + + if (!expr->children.empty()) + { + std::erase(expr->children, nullptr); + if (expr->children.empty()) + return std::make_shared(TruthValue::YES_NO_NULL); + } + return expr; +} + +/// Converts multi-level ANDs and ORs into single level ones. +ExpressionTreePtr flatten(ExpressionTreePtr root) +{ + std::vector nodes; + for (auto & kid : root->children) + { + ExpressionTreePtr child = flatten(kid); + if (child->op == root->op && child->op != Op::NOT) + nodes.insert(nodes.end(), child->children.begin(), child->children.end()); + else + nodes.push_back(child); + } + root->children = std::move(nodes); + if ((root->op == Op::OR || root->op == Op::AND) && root->children.size() == 1) + return root->children[0]; + return root; +} + +ExpressionTreePtr copyTree(const ExpressionTreePtr & node) +{ + auto result = std::make_shared(*node); + return result; +} + +void generateAllCombinations( + std::vector & result, const std::vector & and_list, size_t and_index, const std::vector & non_and_list) +{ + const auto & kids = and_list[and_index]->children; + if (result.empty()) + { + for (const auto & kid : kids) + { + auto node = std::make_shared(Op::OR); + for (const auto & non_and : non_and_list) + node->children.push_back(copyTree(non_and)); + node->children.push_back(kid); + result.push_back(node); + } + } + else + { + std::vector work = std::move(result); + result.clear(); + for (const auto & kid : kids) + { + for (const auto & node : work) + { + auto copy = copyTree(node); + copy->children.push_back(kid); + result.push_back(copy); + } + } + } + if (and_index + 1 < and_list.size()) + generateAllCombinations(result, and_list, and_index + 1, non_and_list); +} + +constexpr size_t CNF_COMBINATIONS_THRESHOLD = 256; + +/// Converts the expression into the conjunctive normal form, if it is not too large. +ExpressionTreePtr convertToCNF(ExpressionTreePtr root) +{ + for (auto & kid : root->children) + kid = convertToCNF(kid); + + if (root->op != Op::OR) + return root; + + std::vector non_and_list; + std::vector and_list; + for (const auto & child : root->children) + { + if (child->op == Op::AND) + and_list.push_back(child); + else if (child->op == Op::OR) + non_and_list.insert(non_and_list.end(), child->children.begin(), child->children.end()); + else + non_and_list.push_back(child); + } + + if (and_list.empty()) + return root; + + size_t num_combinations = 1; + for (const auto & node : and_list) + { + num_combinations *= node->children.size(); + if (num_combinations > CNF_COMBINATIONS_THRESHOLD) + return std::make_shared(TruthValue::YES_NO_NULL); + } + + auto result = std::make_shared(Op::AND); + generateAllCombinations(result->children, and_list, 0, non_and_list); + return result; +} + +void collectUsedLeaves(const ExpressionTreePtr & node, std::vector & reorder, size_t & next) +{ + if (node->op == Op::LEAF) + { + if (reorder[node->leaf] == std::numeric_limits::max()) + reorder[node->leaf] = next++; + return; + } + for (const auto & kid : node->children) + collectUsedLeaves(kid, reorder, next); +} + +/// The trees can share nodes after the CNF conversion, so rewrite every leaf node exactly once. +void rewriteLeaves(const ExpressionTreePtr & root, const std::vector & reorder) +{ + std::unordered_set leaves; + std::deque nodes{root.get()}; + while (!nodes.empty()) + { + ExpressionTree * node = nodes.front(); + nodes.pop_front(); + if (node->op == Op::LEAF) + leaves.insert(node); + else + for (const auto & kid : node->children) + nodes.push_back(kid.get()); + } + for (auto * leaf : leaves) + leaf->leaf = reorder[leaf->leaf]; +} + +} + + +SearchArgumentBuilder::SearchArgumentBuilder() +{ + root = std::make_shared(Op::AND); + stack.push_back(root); +} + +SearchArgumentBuilder & SearchArgumentBuilder::start(UInt8 op) +{ + auto node = std::make_shared(static_cast(op)); + stack.back()->children.push_back(node); + stack.push_back(node); + return *this; +} + +SearchArgumentBuilder & SearchArgumentBuilder::startOr() { return start(static_cast(Op::OR)); } +SearchArgumentBuilder & SearchArgumentBuilder::startAnd() { return start(static_cast(Op::AND)); } +SearchArgumentBuilder & SearchArgumentBuilder::startNot() { return start(static_cast(Op::NOT)); } + +SearchArgumentBuilder & SearchArgumentBuilder::end() +{ + auto current = stack.back(); + stack.pop_back(); + if (stack.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unbalanced end() in ORC search argument builder"); + if (current->children.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot create an expression with no children in ORC search argument builder"); + if (current->op == Op::NOT && current->children.size() != 1) + throw Exception(ErrorCodes::LOGICAL_ERROR, "A NOT expression must have exactly one child in ORC search argument builder"); + return *this; +} + +SearchArgumentBuilder & SearchArgumentBuilder::literal(TruthValue value) +{ + stack.back()->children.push_back(std::make_shared(value)); + return *this; +} + +SearchArgumentBuilder & SearchArgumentBuilder::addLeaf(PredicateLeaf leaf) +{ + size_t index = leaves.size(); + leaves.push_back(std::move(leaf)); + stack.back()->children.push_back(std::make_shared(index)); + return *this; +} + +SearchArgumentBuilder & SearchArgumentBuilder::lessThan(UInt64 column_id, PredicateDataType type, Literal value) +{ + return addLeaf(PredicateLeaf(PredicateLeaf::Operator::LESS_THAN, type, column_id, {std::move(value)})); +} + +SearchArgumentBuilder & SearchArgumentBuilder::lessThanEquals(UInt64 column_id, PredicateDataType type, Literal value) +{ + return addLeaf(PredicateLeaf(PredicateLeaf::Operator::LESS_THAN_EQUALS, type, column_id, {std::move(value)})); +} + +SearchArgumentBuilder & SearchArgumentBuilder::equals(UInt64 column_id, PredicateDataType type, Literal value) +{ + return addLeaf(PredicateLeaf(PredicateLeaf::Operator::EQUALS, type, column_id, {std::move(value)})); +} + +SearchArgumentBuilder & SearchArgumentBuilder::nullSafeEquals(UInt64 column_id, PredicateDataType type, Literal value) +{ + return addLeaf(PredicateLeaf(PredicateLeaf::Operator::NULL_SAFE_EQUALS, type, column_id, {std::move(value)})); +} + +SearchArgumentBuilder & SearchArgumentBuilder::in(UInt64 column_id, PredicateDataType type, const std::vector & values) +{ + if (values.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Cannot create an IN expression with no arguments in ORC search argument builder"); + return addLeaf(PredicateLeaf(PredicateLeaf::Operator::IN, type, column_id, values)); +} + +SearchArgumentBuilder & SearchArgumentBuilder::isNull(UInt64 column_id, PredicateDataType type) +{ + return addLeaf(PredicateLeaf(PredicateLeaf::Operator::IS_NULL, type, column_id, {})); +} + +SearchArgumentBuilder & SearchArgumentBuilder::between(UInt64 column_id, PredicateDataType type, Literal lower, Literal upper) +{ + return addLeaf(PredicateLeaf(PredicateLeaf::Operator::BETWEEN, type, column_id, {std::move(lower), std::move(upper)})); +} + +std::unique_ptr SearchArgumentBuilder::build() +{ + if (stack.size() != 1) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Failed to end {} operations in ORC search argument builder", stack.size() - 1); + if (root->children.empty()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Empty ORC search argument"); + + root = pushDownNot(root); + root = foldMaybe(root); + root = flatten(root); + root = convertToCNF(root); + root = flatten(root); + + std::vector reorder(leaves.size(), std::numeric_limits::max()); + size_t num_used = 0; + collectUsedLeaves(root, reorder, num_used); + rewriteLeaves(root, reorder); + + std::vector used_leaves; + used_leaves.reserve(num_used); + std::vector by_new_index(num_used); + for (size_t i = 0; i < leaves.size(); ++i) + if (reorder[i] != std::numeric_limits::max()) + by_new_index[reorder[i]] = &leaves[i]; + for (const auto * leaf : by_new_index) + used_leaves.push_back(*leaf); + + return std::make_unique(root, std::move(used_leaves)); +} + + +namespace +{ + +constexpr UInt32 MURMUR3_DEFAULT_SEED = 104729; + +inline UInt64 rotl64(UInt64 x, int r) +{ + return (x << r) | (x >> (64 - r)); +} + +inline UInt64 fmix64(UInt64 value) +{ + value ^= (value >> 33); + value *= 0xff51afd7ed558ccdULL; + value ^= (value >> 33); + value *= 0xc4ceb9fe1a85ec53ULL; + value ^= (value >> 33); + return value; +} + +/// The 64-bit variant of Murmur3 used by ORC (the lower half of the 128-bit hash with one state). +UInt64 murmur3Hash64(const UInt8 * data, size_t size) +{ + UInt64 h = MURMUR3_DEFAULT_SEED; + const UInt64 c1 = 0x87c37b91114253d5ULL; + const UInt64 c2 = 0x4cf5ad432745937fULL; + size_t blocks = size >> 3; + for (size_t i = 0; i < blocks; ++i) + { + UInt64 k = 0; + memcpy(&k, data + i * 8, 8); + if constexpr (std::endian::native == std::endian::big) + k = std::byteswap(k); + k *= c1; + k = rotl64(k, 31); + k *= c2; + h ^= k; + h = rotl64(h, 27); + h = h * 5 + 0x52dce729; + } + + UInt64 k = 0; + size_t idx = blocks << 3; + size_t tail = size - idx; + for (size_t i = tail; i > 0; --i) + k ^= static_cast(data[idx + i - 1]) << ((i - 1) * 8); + if (tail) + { + k *= c1; + k = rotl64(k, 31); + k *= c2; + h ^= k; + } + + h ^= static_cast(size); + return fmix64(h); +} + +/// Thomas Wang's 64-bit integer hash, as in the reference implementation. +Int64 getLongHash(Int64 key) +{ + UInt64 k = static_cast(key); + k = (~k) + (k << 21); + k = k ^ static_cast(static_cast(k) >> 24); + k = (k + (k << 3)) + (k << 8); + k = k ^ static_cast(static_cast(k) >> 14); + k = (k + (k << 2)) + (k << 4); + k = k ^ static_cast(static_cast(k) >> 28); + k = k + (k << 31); + return static_cast(k); +} + +} + +BloomFilter::BloomFilter(const Proto::BloomFilter & proto) : num_hash_functions(proto.num_hash_functions) +{ + /// The writers use a few hash functions (the optimal number for the false positive probability), + /// a huge number from a corrupt file would make every probe extremely slow. + static constexpr UInt32 MAX_HASH_FUNCTIONS = 256; + if (num_hash_functions == 0 || num_hash_functions > MAX_HASH_FUNCTIONS) + throw Exception(ErrorCodes::INCORRECT_DATA, "Invalid number of hash functions {} of a bloom filter in ORC file, must be from 1 to {}", + num_hash_functions, MAX_HASH_FUNCTIONS); + const std::string & bitset = proto.utf8bitset.value(); + if (bitset.empty() || bitset.size() % 8 != 0) + throw Exception(ErrorCodes::INCORRECT_DATA, "The size of a bloom filter in ORC file must be a positive multiple of 8 bytes"); + bits.resize(bitset.size() / 8); + memcpy(bits.data(), bitset.data(), bitset.size()); + if constexpr (std::endian::native == std::endian::big) + for (auto & word : bits) + word = std::byteswap(word); +} + +bool BloomFilter::testHash(Int64 hash) const +{ + Int32 hash1 = static_cast(hash & 0xffffffff); + Int32 hash2 = static_cast(static_cast(hash) >> 32); + UInt64 num_bits = bits.size() * 64; + for (UInt32 i = 1; i <= num_hash_functions; ++i) + { + Int32 combined = static_cast(static_cast(hash1) + i * static_cast(hash2)); + if (combined < 0) + combined = ~combined; + UInt64 pos = static_cast(combined) % num_bits; + if (!(bits[pos >> 6] & (1ULL << (pos % 64)))) + return false; + } + return true; +} + +bool BloomFilter::testLong(Int64 value) const +{ + return testHash(getLongHash(value)); +} + +bool BloomFilter::testDouble(double value) const +{ + return testLong(std::bit_cast(value)); +} + +bool BloomFilter::testBytes(const char * data, size_t size) const +{ + return testHash(static_cast(murmur3Hash64(reinterpret_cast(data), size))); +} + +} diff --git a/src/Processors/Formats/Impl/ORC/SearchArgument.h b/src/Processors/Formats/Impl/ORC/SearchArgument.h new file mode 100644 index 0000000000000..da19904275e75 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/SearchArgument.h @@ -0,0 +1,198 @@ +#pragma once + +#include + +#include +#include +#include + +/// Predicates for skipping stripes and row groups by the column statistics and the bloom filters. +/// The semantics (the truth values, the normalization of the expression, the evaluation of each +/// predicate against the statistics) are the same as in the reference implementation. + +namespace DB::ORC +{ + +/// The possible outcomes of a predicate over a set of rows: whether any row can satisfy it (YES), +/// fail it (NO), or evaluate it to NULL. +enum class TruthValue : UInt8 +{ + YES, + NO, + IS_NULL, + YES_NULL, + NO_NULL, + YES_NO, + YES_NO_NULL, +}; + +TruthValue operator||(TruthValue left, TruthValue right); +TruthValue operator&&(TruthValue left, TruthValue right); +TruthValue operator!(TruthValue value); +/// Whether the rows have to be read. +bool isNeeded(TruthValue value); + +enum class PredicateDataType : UInt8 +{ + LONG, + FLOAT, + STRING, + DATE, + DECIMAL, + TIMESTAMP, + BOOLEAN, +}; + +class Literal +{ +public: + struct Timestamp + { + Int64 second = 0; + Int32 nanos = 0; + + /// Rounded down, as the nanoseconds are not negative. + Int64 getMillis() const { return second * 1000 + nanos / 1000000; } + auto operator<=>(const Timestamp &) const = default; + }; + + struct Decimal + { + Int128 value = 0; + Int32 scale = 0; + + /// Parses a decimal number like `-123.45`. Returns false if it is malformed. + bool parse(std::string_view s); + /// Without the trailing zeros in the fraction, as in the bloom filters. + std::string toString() const; + }; + + explicit Literal(bool value); + explicit Literal(Int64 value); + explicit Literal(double value); + Literal(const char * data, size_t size); + /// Only for DATE: the number of days since the epoch. + Literal(PredicateDataType type, Int64 value); + /// TIMESTAMP: the seconds since the epoch in UTC and the nanoseconds. + Literal(Int64 second, Int32 nanos); + Literal(Int128 value, Int32 precision, Int32 scale); + + PredicateDataType getType() const { return type; } + bool isNull() const { return is_null; } + + bool getBool() const { return int_value != 0; } + Int64 getLong() const { return int_value; } + Int64 getDate() const { return int_value; } + double getFloat() const { return float_value; } + const std::string & getString() const { return string_value; } + Timestamp getTimestamp() const { return timestamp_value; } + Decimal getDecimal() const { return decimal_value; } + +private: + PredicateDataType type; + bool is_null = false; + Int64 int_value = 0; + double float_value = 0; + std::string string_value; + Timestamp timestamp_value; + Decimal decimal_value; +}; + +class BloomFilter; + +class PredicateLeaf +{ +public: + enum class Operator : UInt8 + { + EQUALS, + NULL_SAFE_EQUALS, + LESS_THAN, + LESS_THAN_EQUALS, + IN, + BETWEEN, + IS_NULL, + }; + + PredicateLeaf(Operator op_, PredicateDataType type_, UInt64 column_id_, std::vector literals_) + : op(op_), type(type_), column_id(column_id_), literals(std::move(literals_)) {} + + UInt64 getColumnId() const { return column_id; } + + /// `writer_version` is from the postscript of the file. `bloom_filter` can be nullptr. + TruthValue evaluate(UInt32 writer_version, const Proto::ColumnStatistics & stats, const BloomFilter * bloom_filter) const; + +private: + TruthValue evaluatePredicateMinMax(const Proto::ColumnStatistics & stats) const; + TruthValue evaluatePredicateBloomFilter(const BloomFilter & bloom_filter, bool has_null) const; + + Operator op; + PredicateDataType type; + UInt64 column_id; + std::vector literals; +}; + +class ExpressionTree; +using ExpressionTreePtr = std::shared_ptr; + +class SearchArgument +{ +public: + SearchArgument(ExpressionTreePtr root_, std::vector leaves_); + + const std::vector & getLeaves() const { return leaves; } + TruthValue evaluate(const std::vector & leaf_values) const; + +private: + ExpressionTreePtr root; + std::vector leaves; +}; + +class SearchArgumentBuilder +{ +public: + SearchArgumentBuilder(); + + SearchArgumentBuilder & startOr(); + SearchArgumentBuilder & startAnd(); + SearchArgumentBuilder & startNot(); + SearchArgumentBuilder & end(); + + SearchArgumentBuilder & literal(TruthValue value); + SearchArgumentBuilder & lessThan(UInt64 column_id, PredicateDataType type, Literal value); + SearchArgumentBuilder & lessThanEquals(UInt64 column_id, PredicateDataType type, Literal value); + SearchArgumentBuilder & equals(UInt64 column_id, PredicateDataType type, Literal value); + SearchArgumentBuilder & nullSafeEquals(UInt64 column_id, PredicateDataType type, Literal value); + SearchArgumentBuilder & in(UInt64 column_id, PredicateDataType type, const std::vector & values); + SearchArgumentBuilder & isNull(UInt64 column_id, PredicateDataType type); + SearchArgumentBuilder & between(UInt64 column_id, PredicateDataType type, Literal lower, Literal upper); + + std::unique_ptr build(); + +private: + SearchArgumentBuilder & start(UInt8 op); + SearchArgumentBuilder & addLeaf(PredicateLeaf leaf); + + std::vector stack; + ExpressionTreePtr root; + std::vector leaves; +}; + +/// A bloom filter of a row group (or a stripe), from the BLOOM_FILTER_UTF8 stream. +class BloomFilter +{ +public: + explicit BloomFilter(const Proto::BloomFilter & proto); + + bool testLong(Int64 value) const; + bool testDouble(double value) const; + bool testBytes(const char * data, size_t size) const; + +private: + bool testHash(Int64 hash) const; + + UInt32 num_hash_functions; + std::vector bits; +}; + +} diff --git a/src/Processors/Formats/Impl/ORC/Statistics.cpp b/src/Processors/Formats/Impl/ORC/Statistics.cpp new file mode 100644 index 0000000000000..0b1b4c86cd9b3 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Statistics.cpp @@ -0,0 +1,462 @@ +#include +#include + +#include + +#include + + +namespace DB::ORC +{ + +namespace +{ + +int compareDecimals(Int128 a, Int32 a_scale, Int128 b, Int32 b_scale) +{ + Int256 left = a; + Int256 right = b; + for (Int32 s = a_scale; s < b_scale; ++s) + left *= 10; + for (Int32 s = b_scale; s < a_scale; ++s) + right *= 10; + return left < right ? -1 : (left == right ? 0 : 1); +} + +std::string decimalToString(Int128 value, Int32 scale) +{ + return Literal::Decimal{value, scale}.toString(); +} + +} + +void StatisticsBuilder::updateInteger(Int64 value) +{ + if (!has_minimum) + { + has_minimum = true; + int_minimum = int_maximum = value; + } + else + { + int_minimum = std::min(int_minimum, value); + int_maximum = std::max(int_maximum, value); + } + if (has_int_sum && __builtin_add_overflow(int_sum, value, &int_sum)) + has_int_sum = false; +} + +void StatisticsBuilder::updateDouble(double value) +{ + if (!has_minimum) + { + has_minimum = true; + double_minimum = double_maximum = value; + } + else + { + double_minimum = std::min(double_minimum, value); + double_maximum = std::max(double_maximum, value); + } + double_sum += value; +} + +void StatisticsBuilder::updateString(std::string_view value) +{ + if (!has_minimum) + { + has_minimum = true; + string_minimum = value; + string_maximum = value; + } + else + { + if (value < std::string_view(string_minimum)) + string_minimum = value; + if (value > std::string_view(string_maximum)) + string_maximum = value; + } + total_length += value.size(); +} + +void StatisticsBuilder::updateBinary(size_t length) +{ + total_length += length; +} + +void StatisticsBuilder::updateBoolean(bool value) +{ + if (value) + ++true_count; +} + +void StatisticsBuilder::updateDate(Int32 value) +{ + updateInteger(value); +} + +void StatisticsBuilder::updateDecimal(Int128 value, Int32 scale) +{ + if (!has_minimum) + { + has_minimum = true; + decimal_minimum = decimal_maximum = value; + decimal_minimum_scale = decimal_maximum_scale = scale; + } + else + { + if (compareDecimals(value, scale, decimal_minimum, decimal_minimum_scale) < 0) + { + decimal_minimum = value; + decimal_minimum_scale = scale; + } + if (compareDecimals(value, scale, decimal_maximum, decimal_maximum_scale) > 0) + { + decimal_maximum = value; + decimal_maximum_scale = scale; + } + } + + if (has_decimal_sum) + { + Int256 sum = decimal_sum; + Int256 addend = value; + Int32 result_scale = std::max(decimal_sum_scale, scale); + for (Int32 s = decimal_sum_scale; s < result_scale; ++s) + sum *= 10; + for (Int32 s = scale; s < result_scale; ++s) + addend *= 10; + sum += addend; + if (sum > Int256(std::numeric_limits::max()) || sum < Int256(std::numeric_limits::min())) + has_decimal_sum = false; + else + { + decimal_sum = static_cast(sum); + decimal_sum_scale = result_scale; + } + } +} + +void StatisticsBuilder::updateTimestamp(Int64 millis, Int32 nanos) +{ + if (!has_minimum) + { + has_minimum = true; + int_minimum = int_maximum = millis; + minimum_nanos = maximum_nanos = nanos; + return; + } + if (millis <= int_minimum) + { + if (millis < int_minimum || nanos < minimum_nanos) + minimum_nanos = nanos; + int_minimum = millis; + } + if (millis >= int_maximum) + { + if (millis > int_maximum || nanos > maximum_nanos) + maximum_nanos = nanos; + int_maximum = millis; + } +} + +void StatisticsBuilder::updateCollection(UInt64 children) +{ + Int64 value = static_cast(children); + if (!has_minimum) + { + has_minimum = true; + int_minimum = int_maximum = value; + } + else + { + int_minimum = std::min(int_minimum, value); + int_maximum = std::max(int_maximum, value); + } + total_children += children; +} + +void StatisticsBuilder::merge(const StatisticsBuilder & other) +{ + number_of_values += other.number_of_values; + has_null |= other.has_null; + + switch (kind) + { + case BYTE: + case SHORT: + case INT: + case LONG: + case DATE: + if (other.has_minimum) + { + if (!has_minimum) + { + int_minimum = other.int_minimum; + int_maximum = other.int_maximum; + } + else + { + int_minimum = std::min(int_minimum, other.int_minimum); + int_maximum = std::max(int_maximum, other.int_maximum); + } + has_minimum = true; + } + has_int_sum = has_int_sum && other.has_int_sum; + if (has_int_sum && __builtin_add_overflow(int_sum, other.int_sum, &int_sum)) + has_int_sum = false; + break; + case FLOAT: + case DOUBLE: + if (other.has_minimum) + { + if (!has_minimum) + { + double_minimum = other.double_minimum; + double_maximum = other.double_maximum; + } + else + { + double_minimum = std::min(double_minimum, other.double_minimum); + double_maximum = std::max(double_maximum, other.double_maximum); + } + has_minimum = true; + } + double_sum += other.double_sum; + break; + case STRING: + case CHAR: + case VARCHAR: + if (other.has_minimum) + { + if (!has_minimum) + { + string_minimum = other.string_minimum; + string_maximum = other.string_maximum; + } + else + { + if (other.string_minimum < string_minimum) + string_minimum = other.string_minimum; + if (other.string_maximum > string_maximum) + string_maximum = other.string_maximum; + } + has_minimum = true; + } + total_length += other.total_length; + break; + case BINARY: + total_length += other.total_length; + break; + case BOOLEAN: + true_count += other.true_count; + break; + case DECIMAL: + if (other.has_minimum) + { + if (!has_minimum) + { + decimal_minimum = other.decimal_minimum; + decimal_minimum_scale = other.decimal_minimum_scale; + decimal_maximum = other.decimal_maximum; + decimal_maximum_scale = other.decimal_maximum_scale; + } + else + { + if (compareDecimals(other.decimal_minimum, other.decimal_minimum_scale, decimal_minimum, decimal_minimum_scale) < 0) + { + decimal_minimum = other.decimal_minimum; + decimal_minimum_scale = other.decimal_minimum_scale; + } + if (compareDecimals(other.decimal_maximum, other.decimal_maximum_scale, decimal_maximum, decimal_maximum_scale) > 0) + { + decimal_maximum = other.decimal_maximum; + decimal_maximum_scale = other.decimal_maximum_scale; + } + } + has_minimum = true; + } + has_decimal_sum = has_decimal_sum && other.has_decimal_sum; + if (has_decimal_sum) + { + /// Reuse the update of the sum without touching the minimum and the maximum. + bool saved_has_minimum = has_minimum; + Int128 saved_min = decimal_minimum; + Int32 saved_min_scale = decimal_minimum_scale; + Int128 saved_max = decimal_maximum; + Int32 saved_max_scale = decimal_maximum_scale; + updateDecimal(other.decimal_sum, other.decimal_sum_scale); + has_minimum = saved_has_minimum; + decimal_minimum = saved_min; + decimal_minimum_scale = saved_min_scale; + decimal_maximum = saved_max; + decimal_maximum_scale = saved_max_scale; + } + break; + case TIMESTAMP: + case TIMESTAMP_INSTANT: + if (other.has_minimum) + { + if (!has_minimum) + { + int_minimum = other.int_minimum; + int_maximum = other.int_maximum; + minimum_nanos = other.minimum_nanos; + maximum_nanos = other.maximum_nanos; + has_minimum = true; + } + else + { + if (other.int_minimum <= int_minimum) + { + if (other.int_minimum < int_minimum || other.minimum_nanos < minimum_nanos) + minimum_nanos = other.minimum_nanos; + int_minimum = other.int_minimum; + } + if (other.int_maximum >= int_maximum) + { + if (other.int_maximum > int_maximum || other.maximum_nanos > maximum_nanos) + maximum_nanos = other.maximum_nanos; + int_maximum = other.int_maximum; + } + } + } + break; + case LIST: + case MAP: + if (other.has_minimum) + { + if (!has_minimum) + { + int_minimum = other.int_minimum; + int_maximum = other.int_maximum; + } + else + { + int_minimum = std::min(int_minimum, other.int_minimum); + int_maximum = std::max(int_maximum, other.int_maximum); + } + has_minimum = true; + } + total_children += other.total_children; + break; + default: + break; + } +} + +void StatisticsBuilder::reset() +{ + *this = StatisticsBuilder(kind); +} + +Proto::ColumnStatistics StatisticsBuilder::toProto() const +{ + Proto::ColumnStatistics res; + res.has_null = has_null; + res.number_of_values = number_of_values; + + switch (kind) + { + case BYTE: + case SHORT: + case INT: + case LONG: + { + auto & stats = res.int_statistics.emplace(); + if (has_minimum) + { + stats.minimum = int_minimum; + stats.maximum = int_maximum; + } + if (has_int_sum) + stats.sum = int_sum; + break; + } + case DATE: + { + auto & stats = res.date_statistics.emplace(); + if (has_minimum) + { + stats.minimum = static_cast(int_minimum); + stats.maximum = static_cast(int_maximum); + } + break; + } + case FLOAT: + case DOUBLE: + { + auto & stats = res.double_statistics.emplace(); + if (has_minimum) + { + stats.minimum = double_minimum; + stats.maximum = double_maximum; + } + stats.sum = double_sum; + break; + } + case STRING: + case CHAR: + case VARCHAR: + { + auto & stats = res.string_statistics.emplace(); + if (has_minimum) + { + stats.minimum = string_minimum; + stats.maximum = string_maximum; + } + stats.sum = static_cast(total_length); + break; + } + case BINARY: + res.binary_statistics.emplace().sum = static_cast(total_length); + break; + case BOOLEAN: + res.bucket_statistics.emplace().count.push_back(true_count); + break; + case DECIMAL: + { + auto & stats = res.decimal_statistics.emplace(); + if (has_minimum) + { + stats.minimum = decimalToString(decimal_minimum, decimal_minimum_scale); + stats.maximum = decimalToString(decimal_maximum, decimal_maximum_scale); + } + if (has_decimal_sum) + stats.sum = decimalToString(decimal_sum, decimal_sum_scale); + break; + } + case TIMESTAMP: + case TIMESTAMP_INSTANT: + { + auto & stats = res.timestamp_statistics.emplace(); + if (has_minimum) + { + stats.minimum_utc = int_minimum; + stats.maximum_utc = int_maximum; + if (minimum_nanos != 0) + stats.minimum_nanos = static_cast(minimum_nanos) + 1; + if (maximum_nanos != 999999) + stats.maximum_nanos = static_cast(maximum_nanos) + 1; + } + break; + } + case LIST: + case MAP: + { + auto & stats = res.collection_statistics.emplace(); + if (has_minimum) + { + stats.min_children = static_cast(int_minimum); + stats.max_children = static_cast(int_maximum); + } + stats.total_children = total_children; + break; + } + default: + break; + } + return res; +} + +} diff --git a/src/Processors/Formats/Impl/ORC/Statistics.h b/src/Processors/Formats/Impl/ORC/Statistics.h new file mode 100644 index 0000000000000..c00c00e07b944 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Statistics.h @@ -0,0 +1,71 @@ +#pragma once + +#include + +#include + +namespace DB::ORC +{ + +/// Accumulates the statistics of a column for a row group, a stripe or the whole file, +/// in the same way as the reference implementation. +class StatisticsBuilder +{ +public: + explicit StatisticsBuilder(TypeKind kind_) : kind(kind_) {} + + void increase(UInt64 count) { number_of_values += count; } + void setHasNull() { has_null = true; } + + void updateInteger(Int64 value); + void updateDouble(double value); + void updateString(std::string_view value); + void updateBinary(size_t length); + void updateBoolean(bool value); + void updateDate(Int32 value); + void updateDecimal(Int128 value, Int32 scale); + /// `millis` since the epoch in UTC, `nanos` is the sub-millisecond part. + void updateTimestamp(Int64 millis, Int32 nanos); + void updateCollection(UInt64 children); + + void merge(const StatisticsBuilder & other); + void reset(); + + Proto::ColumnStatistics toProto() const; + +private: + TypeKind kind; + UInt64 number_of_values = 0; + bool has_null = false; + bool has_minimum = false; + + Int64 int_minimum = 0; + Int64 int_maximum = 0; + Int64 int_sum = 0; + bool has_int_sum = true; + + double double_minimum = 0; + double double_maximum = 0; + double double_sum = 0; + + std::string string_minimum; + std::string string_maximum; + UInt64 total_length = 0; + + UInt64 true_count = 0; + + Int128 decimal_minimum = 0; + Int32 decimal_minimum_scale = 0; + Int128 decimal_maximum = 0; + Int32 decimal_maximum_scale = 0; + Int128 decimal_sum = 0; + Int32 decimal_sum_scale = 0; + bool has_decimal_sum = true; + + Int32 minimum_nanos = 0; + Int32 maximum_nanos = 999999; + + UInt64 total_children = 0; +}; + +} diff --git a/src/Processors/Formats/Impl/ORC/Stream.cpp b/src/Processors/Formats/Impl/ORC/Stream.cpp new file mode 100644 index 0000000000000..275e75915dca2 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Stream.cpp @@ -0,0 +1,248 @@ +#include + +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int INCORRECT_DATA; +} +} + +namespace DB::ORC +{ + +UInt64 PositionProvider::next() +{ + if (index >= positions.size()) + throw Exception(ErrorCodes::INCORRECT_DATA, "Not enough positions in a row index entry of ORC file"); + return positions[index++]; +} + + +InputByteStream::InputByteStream(std::string_view data_, CompressionKind compression_, size_t block_size_, std::string name_) + : data(data_), length(data_.size()), compression(compression_), block_size(block_size_), name(std::move(name_)) +{ +} + +InputByteStream::InputByteStream(RangeLoader loader_, UInt64 length_, CompressionKind compression_, size_t block_size_, std::string name_) + : length(length_), loader(std::move(loader_)), compression(compression_), block_size(block_size_), name(std::move(name_)) +{ +} + +std::string_view InputByteStream::getRange(UInt64 offset, UInt64 size) +{ + if (!loader) + return data.substr(offset, size); + + if (offset < window_offset || offset + size > window_offset + data.size()) + { + UInt64 load_size = std::min(length - offset, std::max(size, LAZY_READ_SIZE)); + auto range = loader(offset, load_size); + if (range.data.size() != load_size) + throw Exception(ErrorCodes::INCORRECT_DATA, "Loaded {} bytes instead of {} from ORC stream {}", range.data.size(), load_size, name); + window_holder = std::move(range.holder); + data = range.data; + window_offset = offset; + } + return data.substr(offset - window_offset, size); +} + +bool InputByteStream::nextChunk() +{ + while (next_chunk_offset < length) + { + if (compression == CompressionKind::NONE) + { + UInt64 size = length - next_chunk_offset; + if (loader) + size = std::min(size, LAZY_READ_SIZE); + pos = getRange(next_chunk_offset, size).data(); + end = pos + size; + next_chunk_offset += size; + return true; + } + + if (length - next_chunk_offset < COMPRESSION_HEADER_SIZE) + throw Exception(ErrorCodes::INCORRECT_DATA, "Truncated compression chunk header in ORC stream {}", name); + + const UInt8 * header = reinterpret_cast(getRange(next_chunk_offset, COMPRESSION_HEADER_SIZE).data()); + UInt32 header_value = header[0] | (header[1] << 8) | (header[2] << 16); + bool is_original = header_value & 1; + size_t chunk_length = header_value >> 1; + size_t chunk_offset = next_chunk_offset + COMPRESSION_HEADER_SIZE; + + if (chunk_length > length - chunk_offset) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Compression chunk of size {} at offset {} goes beyond the end of ORC stream {} of size {}", + chunk_length, next_chunk_offset, name, length); + + next_chunk_offset = chunk_offset + chunk_length; + const char * chunk = getRange(chunk_offset, chunk_length).data(); + + if (is_original) + { + pos = chunk; + end = chunk + chunk_length; + } + else + { + /// The chunk length is at most 8 MiB, so the bound does not overflow. + size_t capacity = std::min(block_size, maxDecompressedSize(compression, chunk, chunk_length)); + if (decompressed.size() < capacity) + decompressed.resize(capacity); + size_t size = decompressChunk(compression, chunk, chunk_length, decompressed.data(), capacity); + pos = decompressed.data(); + end = decompressed.data() + size; + } + + if (pos != end) + return true; + } + + pos = end = nullptr; + return false; +} + +void InputByteStream::readBytes(char * to, size_t n) +{ + while (n) + { + if (pos == end && !nextChunk()) + throwUnexpectedEnd(); + size_t count = std::min(n, static_cast(end - pos)); + memcpy(to, pos, count); + pos += count; + to += count; + n -= count; + } +} + +void InputByteStream::skipBytes(size_t n) +{ + while (n) + { + if (pos == end && !nextChunk()) + throwUnexpectedEnd(); + size_t count = std::min(n, static_cast(end - pos)); + pos += count; + n -= count; + } +} + +void InputByteStream::seek(PositionProvider & positions) +{ + if (compression == CompressionKind::NONE) + { + UInt64 offset = positions.next(); + if (offset > length) + throw Exception(ErrorCodes::INCORRECT_DATA, "Seek position {} is beyond the end of ORC stream {} of size {}", offset, name, length); + next_chunk_offset = offset; + pos = end = nullptr; + nextChunk(); + return; + } + + UInt64 chunk_offset = positions.next(); + UInt64 uncompressed_offset = positions.next(); + if (chunk_offset > length) + throw Exception(ErrorCodes::INCORRECT_DATA, "Seek position {} is beyond the end of ORC stream {} of size {}", chunk_offset, name, length); + + next_chunk_offset = chunk_offset; + pos = end = nullptr; + if (!nextChunk()) + { + if (uncompressed_offset != 0) + throw Exception(ErrorCodes::INCORRECT_DATA, "Seek position {} is beyond the end of ORC stream {}", uncompressed_offset, name); + return; + } + + if (uncompressed_offset > static_cast(end - pos)) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Seek position {} is beyond the end of the compression chunk of size {} in ORC stream {}", + uncompressed_offset, end - pos, name); + pos += uncompressed_offset; +} + +void InputByteStream::throwUnexpectedEnd() const +{ + throw Exception(ErrorCodes::INCORRECT_DATA, "Unexpected end of ORC stream {}", name); +} + + +OutputByteStream::OutputByteStream(Compressor * compressor_, size_t block_size_) + : compressor(compressor_), block_size(block_size_) +{ + buffer.reserve(std::min(block_size, 64 * 1024)); +} + +void OutputByteStream::write(const char * data, size_t size) +{ + while (size) + { + if (buffer.size() == block_size) + flushChunk(); + size_t count = std::min(size, block_size - buffer.size()); + buffer.append(data, count); + data += count; + size -= count; + } +} + +void OutputByteStream::recordPosition(std::vector & positions) const +{ + if (compressor) + { + positions.push_back(output.size()); + positions.push_back(buffer.size()); + } + else + positions.push_back(output.size() + buffer.size()); +} + +void OutputByteStream::flushChunk() +{ + if (buffer.empty()) + return; + + if (!compressor) + { + output.append(buffer); + buffer.clear(); + return; + } + + size_t header_offset = output.size(); + output.append(COMPRESSION_HEADER_SIZE, '\0'); + size_t compressed_size = compressor->compress(buffer.data(), buffer.size(), output); + + UInt32 header_value = 0; + if (compressed_size) + header_value = static_cast(compressed_size << 1); + else + { + output.append(buffer); + header_value = static_cast((buffer.size() << 1) | 1); + } + + output[header_offset] = static_cast(header_value & 0xFF); + output[header_offset + 1] = static_cast((header_value >> 8) & 0xFF); + output[header_offset + 2] = static_cast((header_value >> 16) & 0xFF); + buffer.clear(); +} + +size_t OutputByteStream::finish() +{ + flushChunk(); + return output.size(); +} + +void OutputByteStream::reset() +{ + buffer.clear(); + output.clear(); +} + +} diff --git a/src/Processors/Formats/Impl/ORC/Stream.h b/src/Processors/Formats/Impl/ORC/Stream.h new file mode 100644 index 0000000000000..44d209d38321e --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Stream.h @@ -0,0 +1,140 @@ +#pragma once + +#include +#include + +#include +#include +#include +#include +#include + +namespace DB::ORC +{ + +/// A sequence of positions from a row index entry, consumed by the streams and the decoders of a +/// column in their order. +class PositionProvider +{ +public: + explicit PositionProvider(const std::vector & positions_) : positions(positions_) {} + UInt64 next(); + +private: + const std::vector & positions; + size_t index = 0; +}; + +/// A piece of a stream loaded from the file. `data` is valid while `holder` is alive. +struct LoadedRange +{ + std::shared_ptr holder; + std::string_view data; +}; + +/// Loads the range `(offset, length)` of a stream from the file. +using RangeLoader = std::function; + +/// A stream of an ORC column, such as PRESENT or DATA, decompressed chunk by chunk. The decoders +/// read from the buffer `[pos, end)` of the current chunk directly. +class InputByteStream +{ +public: + /// The whole stream in memory. `data` must outlive the stream. + InputByteStream(std::string_view data_, CompressionKind compression_, size_t block_size_, std::string name_); + /// The stream of `length_` bytes is loaded from the file on demand, by `loader_`, a window of + /// at least `LAZY_READ_SIZE` bytes at a time. Used when only some row groups are read. + InputByteStream(RangeLoader loader_, UInt64 length_, CompressionKind compression_, size_t block_size_, std::string name_); + + static constexpr size_t LAZY_READ_SIZE = 1024 * 1024; + + /// Loads the next chunk. Returns false at the end of the stream. + bool nextChunk(); + + bool eof() + { + return pos == end && !nextChunk(); + } + + UInt8 readByte() + { + if (pos == end && !nextChunk()) + throwUnexpectedEnd(); + return static_cast(*pos++); + } + + void readBytes(char * to, size_t n); + void skipBytes(size_t n); + + /// Moves to the position from the row index: one value (the byte offset) for an uncompressed + /// stream, two values (the offset of the compressed chunk and the offset in its decompressed + /// data) for a compressed one. + void seek(PositionProvider & positions); + + [[noreturn]] void throwUnexpectedEnd() const; + + const std::string & getName() const { return name; } + + const char * pos = nullptr; + const char * end = nullptr; + +private: + /// Returns the range `[offset, offset + length)` of the stream. It is valid until the next call. + std::string_view getRange(UInt64 offset, UInt64 length); + + /// The stream in memory, or the currently loaded window of a lazily loaded stream. + std::string_view data; + UInt64 length; + RangeLoader loader; + /// For a lazily loaded stream: the offset of `data` in the stream, and its owner. + UInt64 window_offset = 0; + std::shared_ptr window_holder; + + CompressionKind compression; + size_t block_size; + std::string name; + + /// The offset of the next chunk to read in `data`. + size_t next_chunk_offset = 0; + PODArray decompressed; +}; + +/// Accumulates a stream of an ORC column, compressing it chunk by chunk. +class OutputByteStream +{ +public: + /// `compressor` can be nullptr, which means no compression. + OutputByteStream(Compressor * compressor_, size_t block_size_); + + void writeByte(UInt8 byte) + { + if (buffer.size() == block_size) + flushChunk(); + buffer.push_back(static_cast(byte)); + } + + void write(const char * data, size_t size); + + /// Appends the current position to `positions`, in the format that `InputByteStream::seek` expects. + void recordPosition(std::vector & positions) const; + + /// Finishes the stream and returns its size in bytes. + size_t finish(); + + const std::string & getOutput() const { return output; } + + /// Clears the stream to start a new stripe. + void reset(); + + size_t estimateSize() const { return output.size() + buffer.size(); } + +private: + void flushChunk(); + + Compressor * compressor; + size_t block_size; + std::string buffer; + std::string output; +}; + +} diff --git a/src/Processors/Formats/Impl/ORC/Timezone.cpp b/src/Processors/Formats/Impl/ORC/Timezone.cpp new file mode 100644 index 0000000000000..50c4b80e67c01 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Timezone.cpp @@ -0,0 +1,90 @@ +#include + +#include +#include + +#include + +/// The embedded time zone files, see `DateLUTImpl.cpp`. +std::string_view getTimeZone(const char * name); /// NOLINT(misc-use-internal-linkage) + +namespace DB +{ +namespace ErrorCodes +{ + extern const int LOGICAL_ERROR; +} +} + +namespace DB::ORC +{ + +TimezoneOffset::TimezoneOffset(const DateLUTImpl & lut_) + : lut(lut_) +{ + const std::string & name = lut.getTimeZone(); + const std::string_view file = getTimeZone(name.c_str()); + /// `Fixed/UTC±HH:MM:SS` zones are synthesized without a file and have no transitions. + if (file.empty()) + { + if (!name.starts_with("Fixed/")) + throw Exception(ErrorCodes::LOGICAL_ERROR, "No time zone file for {}", name); + return; + } + + /// Raw records, as `cctz::time_zone::next_transition` skips no-op ones (Europe/Lisbon has one in 1884). + static constexpr size_t header_size = 44; + size_t pos = 0; + auto check = [&](size_t size) + { + if (size > file.size() - pos) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Malformed time zone file for {}", name); + }; + auto read_count = [&](size_t index) { return unalignedLoadBigEndian(file.data() + pos + 20 + index * 4); }; + + check(header_size); + if (!file.starts_with("TZif")) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Malformed time zone file for {}", name); + size_t time_size = 4; + /// Version 2 and later repeat the data with 64-bit times after the 32-bit section. + if (file[4] != '\0') + { + const size_t v1_size = read_count(3) * 5 + read_count(4) * 6 + read_count(5) + read_count(2) * 8 + read_count(1) + read_count(0); + check(header_size + v1_size); + pos += header_size + v1_size; + check(header_size); + time_size = 8; + } + const size_t time_count = read_count(3); + const size_t type_count = read_count(4); + pos += header_size; + check(time_count * (time_size + 1) + type_count * 6); + if (time_count == 0) + return; + + const char * times = file.data() + pos; + const auto * type_indices = reinterpret_cast(times + time_count * time_size); + const char * types = times + time_count * (time_size + 1); + auto type_record = [&](size_t type) + { + if (type >= type_count) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Malformed time zone file for {}", name); + return types + type * 6; + }; + + first_transition = time_size == 8 ? unalignedLoadBigEndian(times) : unalignedLoadBigEndian(times); + /// The oldest standard time, or the first type if all of them are daylight saving time. + const char * ancient = type_record(0); + for (size_t i = 0; i < time_count; ++i) + { + const char * record = type_record(type_indices[i]); + if (record[4] == 0) + { + ancient = record; + break; + } + } + ancient_offset = unalignedLoadBigEndian(ancient); +} + +} diff --git a/src/Processors/Formats/Impl/ORC/Timezone.h b/src/Processors/Formats/Impl/ORC/Timezone.h new file mode 100644 index 0000000000000..8a7dbd7361fd6 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Timezone.h @@ -0,0 +1,25 @@ +#pragma once + +#include + +#include + +namespace DB::ORC +{ + +/// The UTC offset as in the reference implementation: before the first transition record it is the oldest +/// standard time, not the local mean time (Asia/Shanghai +08:00, not +08:05:43). Shared by reader and writer. +class TimezoneOffset +{ +public: + explicit TimezoneOffset(const DateLUTImpl & lut_); + + Int64 operator()(Int64 t) const { return t < first_transition ? ancient_offset : lut.timezoneOffset(t); } + +private: + const DateLUTImpl & lut; + Int64 first_transition = std::numeric_limits::min(); + Int64 ancient_offset = 0; +}; + +} diff --git a/src/Processors/Formats/Impl/ORC/Type.cpp b/src/Processors/Formats/Impl/ORC/Type.cpp new file mode 100644 index 0000000000000..b716b8c9ccb1e --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Type.cpp @@ -0,0 +1,380 @@ +#include + +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int INCORRECT_DATA; + extern const int UNKNOWN_TYPE; + extern const int LOGICAL_ERROR; +} +} + +namespace DB::ORC +{ + +namespace +{ + +bool isUnquotedFieldName(const std::string & name) +{ + for (char c : name) + if (!isalnum(static_cast(c)) && c != '_') + return false; + return true; +} + +std::string algorithmToString(UInt32 algorithm) +{ + switch (algorithm) + { + case 0: return "spherical"; + case 1: return "vincenty"; + case 2: return "thomas"; + case 3: return "andoyer"; + case 4: return "karney"; + default: return std::to_string(algorithm); + } +} + +/// The same limit as in the reference implementation. +constexpr size_t MAX_TYPE_NESTING_DEPTH = 2000; + +std::unique_ptr convertType(const std::vector & types, UInt64 index, UInt64 & next_id, size_t depth) +{ + checkStackSize(); + + if (depth > MAX_TYPE_NESTING_DEPTH) + throw Exception(ErrorCodes::INCORRECT_DATA, "ORC file footer is corrupt: type nesting depth exceeds the limit of {}", MAX_TYPE_NESTING_DEPTH); + + /// The types are listed in pre-order and the column id of each type is its index in the list. + if (index != next_id || index >= types.size()) + throw Exception(ErrorCodes::INCORRECT_DATA, + "ORC file footer is corrupt: the types are not a tree listed in pre-order (type {} is referenced at position {})", + index, next_id); + ++next_id; + + const auto & proto = types[index]; + if (proto.kind > GEOGRAPHY) + throw Exception(ErrorCodes::UNKNOWN_TYPE, "Unknown ORC type kind {}", proto.kind); + + auto kind = static_cast(proto.kind); + auto result = std::make_unique(kind); + + switch (kind) + { + case LIST: + if (proto.subtypes.size() != 1) + throw Exception(ErrorCodes::INCORRECT_DATA, "Illegal ORC LIST type that doesn't contain one subtype"); + break; + case MAP: + if (proto.subtypes.size() != 2) + throw Exception(ErrorCodes::INCORRECT_DATA, "Illegal ORC MAP type that doesn't contain two subtypes"); + break; + case UNION: + if (proto.subtypes.empty()) + throw Exception(ErrorCodes::INCORRECT_DATA, "Illegal ORC UNION type that doesn't contain any subtypes"); + /// The tags of a union are bytes. + if (proto.subtypes.size() > 256) + throw Exception(ErrorCodes::INCORRECT_DATA, "Illegal ORC UNION type with {} subtypes, more than 256", proto.subtypes.size()); + break; + case STRUCT: + if (proto.subtypes.size() > proto.field_names.size()) + throw Exception(ErrorCodes::INCORRECT_DATA, "Illegal ORC STRUCT type that contains less field names than subtypes"); + break; + default: + if (!proto.subtypes.empty()) + throw Exception(ErrorCodes::INCORRECT_DATA, "Illegal ORC type {} with subtypes", proto.kind); + break; + } + + if (kind == CHAR || kind == VARCHAR) + result->setMaximumLength(proto.maximum_length.value_or(0)); + if (kind == DECIMAL) + result->setPrecisionAndScale(proto.precision.value_or(0), proto.scale.value_or(0)); + if (kind == GEOMETRY || kind == GEOGRAPHY) + result->setGeospatial(proto.crs.value_or("OGC:CRS84"), proto.algorithm.value_or(0)); + + for (size_t i = 0; i < proto.subtypes.size(); ++i) + { + auto child = convertType(types, proto.subtypes[i], next_id, depth + 1); + if (kind == STRUCT) + result->addStructField(proto.field_names[i], std::move(child)); + else + result->addChild(std::move(child)); + } + + for (const auto & attribute : proto.attributes) + result->setAttribute(attribute.key, attribute.value); + + return result; +} + +void selectChildren(std::vector & selected, const Type & type) +{ + for (UInt64 c = type.getColumnId(); c <= type.getMaximumColumnId(); ++c) + selected[c] = true; +} + +bool selectParents(std::vector & selected, const Type & type) +{ + size_t id = type.getColumnId(); + bool result = selected[id]; + size_t num_selected_subtypes = 0; + for (UInt64 c = 0; c < type.getSubtypeCount(); ++c) + { + if (selectParents(selected, *type.getSubtype(c))) + { + result = true; + ++num_selected_subtypes; + } + } + selected[id] = result; + + /// The branches of a union are either all selected or all not selected. + if (type.getKind() == UNION && result && num_selected_subtypes > 0 && num_selected_subtypes < type.getSubtypeCount()) + for (UInt64 c = 0; c < type.getSubtypeCount(); ++c) + selectChildren(selected, *type.getSubtype(c)); + + return result; +} + +} + + +std::string Type::toString() const +{ + switch (kind) + { + case BOOLEAN: return "boolean"; + case BYTE: return "tinyint"; + case SHORT: return "smallint"; + case INT: return "int"; + case LONG: return "bigint"; + case FLOAT: return "float"; + case DOUBLE: return "double"; + case STRING: return "string"; + case BINARY: return "binary"; + case TIMESTAMP: return "timestamp"; + case TIMESTAMP_INSTANT: return "timestamp with local time zone"; + case LIST: return "array<" + (subtypes.empty() ? std::string("void") : subtypes[0]->toString()) + ">"; + case MAP: + return "map<" + (subtypes.empty() ? std::string("void") : subtypes[0]->toString()) + "," + + (subtypes.size() < 2 ? std::string("void") : subtypes[1]->toString()) + ">"; + case STRUCT: + { + std::string result = "struct<"; + for (size_t i = 0; i < subtypes.size(); ++i) + { + if (i != 0) + result += ","; + if (isUnquotedFieldName(field_names[i])) + result += field_names[i]; + else + { + result += "`"; + for (char c : field_names[i]) + { + if (c == '`') + result += "``"; + else + result += c; + } + result += "`"; + } + result += ":"; + result += subtypes[i]->toString(); + } + result += ">"; + return result; + } + case UNION: + { + std::string result = "uniontype<"; + for (size_t i = 0; i < subtypes.size(); ++i) + { + if (i != 0) + result += ","; + result += subtypes[i]->toString(); + } + result += ">"; + return result; + } + case DECIMAL: return "decimal(" + std::to_string(precision) + "," + std::to_string(scale) + ")"; + case DATE: return "date"; + case VARCHAR: return "varchar(" + std::to_string(maximum_length) + ")"; + case CHAR: return "char(" + std::to_string(maximum_length) + ")"; + case GEOMETRY: return "geometry(" + crs + ")"; + case GEOGRAPHY: return "geography(" + crs + "," + algorithmToString(algorithm) + ")"; + } + return "unknown"; +} + +Type * Type::addStructField(const std::string & name, std::unique_ptr field) +{ + field_names.push_back(name); + return addChild(std::move(field)); +} + +Type * Type::addChild(std::unique_ptr child) +{ + subtypes.push_back(std::move(child)); + return subtypes.back().get(); +} + +UInt64 Type::assignIds(UInt64 start) +{ + column_id = start; + UInt64 next = start + 1; + for (auto & subtype : subtypes) + next = subtype->assignIds(next); + maximum_column_id = next - 1; + return next; +} + +void Type::toProto(std::vector & out) const +{ + size_t index = out.size(); + if (index != column_id) + throw Exception(ErrorCodes::LOGICAL_ERROR, "ORC type ids are not assigned in pre-order"); + out.emplace_back(); + { + auto & proto = out.back(); + proto.kind = kind; + for (const auto & subtype : subtypes) + proto.subtypes.push_back(subtype->getColumnId()); + proto.field_names = field_names; + if (kind == CHAR || kind == VARCHAR) + proto.maximum_length = static_cast(maximum_length); + if (kind == DECIMAL) + { + proto.precision = static_cast(precision); + proto.scale = static_cast(scale); + } + for (const auto & [key, value] : attributes) + proto.attributes.push_back({key, value}); + } + /// `out` may be reallocated by the recursive calls, so do not keep the reference. + for (const auto & subtype : subtypes) + subtype->toProto(out); +} + +std::unique_ptr Type::createPrimitiveType(TypeKind kind) +{ + return std::make_unique(kind); +} + +std::unique_ptr Type::createCharType(TypeKind kind, UInt64 maximum_length) +{ + auto result = std::make_unique(kind); + result->maximum_length = maximum_length; + return result; +} + +std::unique_ptr Type::createDecimalType(UInt64 precision, UInt64 scale) +{ + auto result = std::make_unique(DECIMAL); + result->precision = precision; + result->scale = scale; + return result; +} + +std::unique_ptr Type::createStructType() +{ + return std::make_unique(STRUCT); +} + +std::unique_ptr Type::createListType(std::unique_ptr element) +{ + auto result = std::make_unique(LIST); + result->addChild(std::move(element)); + return result; +} + +std::unique_ptr Type::createMapType(std::unique_ptr key, std::unique_ptr value) +{ + auto result = std::make_unique(MAP); + result->addChild(std::move(key)); + result->addChild(std::move(value)); + return result; +} + +std::unique_ptr Type::createUnionType() +{ + return std::make_unique(UNION); +} + +std::unique_ptr Type::fromProto(const std::vector & types) +{ + if (types.empty()) + throw Exception(ErrorCodes::INCORRECT_DATA, "ORC file footer is corrupt: it has no types"); + UInt64 next_id = 0; + auto root = convertType(types, 0, next_id, 0); + if (next_id != types.size()) + throw Exception(ErrorCodes::INCORRECT_DATA, + "ORC file footer is corrupt: {} of {} types are not reachable from the root type", types.size() - next_id, types.size()); + root->assignIds(0); + return root; +} + +std::unique_ptr Type::buildSelectedType(const Type & type, const std::vector & selected) +{ + if (!selected.at(type.column_id)) + return nullptr; + + auto result = std::make_unique(type.kind); + result->maximum_length = type.maximum_length; + result->precision = type.precision; + result->scale = type.scale; + result->crs = type.crs; + result->algorithm = type.algorithm; + result->attributes = type.attributes; + result->column_id = type.column_id; + result->maximum_column_id = type.maximum_column_id; + + for (size_t i = 0; i < type.subtypes.size(); ++i) + { + auto child = buildSelectedType(*type.subtypes[i], selected); + if (!child) + continue; + if (type.kind == STRUCT) + result->addStructField(type.field_names[i], std::move(child)); + else + result->addChild(std::move(child)); + } + return result; +} + +std::vector selectColumnsByTypeIds(const Type & root, const std::vector & type_ids) +{ + std::vector selected(root.getMaximumColumnId() + 1, false); + + /// Find the types by ids with a walk over the tree. + std::vector by_id(selected.size()); + std::vector stack{&root}; + while (!stack.empty()) + { + const Type * type = stack.back(); + stack.pop_back(); + by_id[type->getColumnId()] = type; + for (UInt64 i = 0; i < type->getSubtypeCount(); ++i) + stack.push_back(type->getSubtype(i)); + } + + for (UInt64 id : type_ids) + { + if (id >= selected.size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid ORC type id {} selected out of {}", id, selected.size()); + selectChildren(selected, *by_id[id]); + } + + selectParents(selected, root); + selected[0] = true; + return selected; +} + +} diff --git a/src/Processors/Formats/Impl/ORC/Type.h b/src/Processors/Formats/Impl/ORC/Type.h new file mode 100644 index 0000000000000..9a30f4455076c --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Type.h @@ -0,0 +1,114 @@ +#pragma once + +#include + +#include +#include +#include +#include + +namespace DB::ORC +{ + +enum TypeKind : UInt32 +{ + BOOLEAN = 0, + BYTE = 1, + SHORT = 2, + INT = 3, + LONG = 4, + FLOAT = 5, + DOUBLE = 6, + STRING = 7, + BINARY = 8, + TIMESTAMP = 9, + LIST = 10, + MAP = 11, + STRUCT = 12, + UNION = 13, + DECIMAL = 14, + DATE = 15, + VARCHAR = 16, + CHAR = 17, + TIMESTAMP_INSTANT = 18, + GEOMETRY = 19, + GEOGRAPHY = 20, +}; + +/// A node of the type tree of an ORC file. Every node is a column of the file; the column ids are +/// assigned in pre-order, so the subtree of a column `c` occupies the ids `[c, c.getMaximumColumnId()]`. +class Type +{ +public: + explicit Type(TypeKind kind_) : kind(kind_) {} + + TypeKind getKind() const { return kind; } + UInt64 getColumnId() const { return column_id; } + UInt64 getMaximumColumnId() const { return maximum_column_id; } + UInt64 getSubtypeCount() const { return subtypes.size(); } + const Type * getSubtype(UInt64 i) const { return subtypes.at(i).get(); } + /// Only for STRUCT. + const std::string & getFieldName(UInt64 i) const { return field_names.at(i); } + /// Only for CHAR and VARCHAR. + UInt64 getMaximumLength() const { return maximum_length; } + /// Only for DECIMAL. + UInt64 getPrecision() const { return precision; } + UInt64 getScale() const { return scale; } + + bool hasAttributeKey(const std::string & key) const { return attributes.contains(key); } + const std::string & getAttributeValue(const std::string & key) const { return attributes.at(key); } + void setAttribute(const std::string & key, const std::string & value) { attributes[key] = value; } + const std::map & getAttributes() const { return attributes; } + + /// The type name in the same format as in the reference implementation (Hive type names). + std::string toString() const; + + void setMaximumLength(UInt64 value) { maximum_length = value; } + void setPrecisionAndScale(UInt64 precision_, UInt64 scale_) { precision = precision_; scale = scale_; } + void setGeospatial(std::string crs_, UInt32 algorithm_) { crs = std::move(crs_); algorithm = algorithm_; } + + Type * addStructField(const std::string & name, std::unique_ptr field); + Type * addChild(std::unique_ptr child); + + /// Assigns the column ids in pre-order, starting from `start`. Returns the next free id. + UInt64 assignIds(UInt64 start); + + /// Appends the types of this subtree to `out` in pre-order. The ids must be assigned. + void toProto(std::vector & out) const; + + static std::unique_ptr createPrimitiveType(TypeKind kind); + static std::unique_ptr createCharType(TypeKind kind, UInt64 maximum_length); + static std::unique_ptr createDecimalType(UInt64 precision, UInt64 scale); + static std::unique_ptr createStructType(); + static std::unique_ptr createListType(std::unique_ptr element); + static std::unique_ptr createMapType(std::unique_ptr key, std::unique_ptr value); + static std::unique_ptr createUnionType(); + + /// Builds the type tree from the types in the file footer. Throws `INCORRECT_DATA` if they do + /// not form a tree with the ids in pre-order. + static std::unique_ptr fromProto(const std::vector & types); + + /// Builds a copy of the tree with only the selected columns (by id). The parent of every + /// selected column must also be selected. Returns nullptr if the root is not selected. + static std::unique_ptr buildSelectedType(const Type & type, const std::vector & selected); + +private: + TypeKind kind; + UInt64 column_id = 0; + UInt64 maximum_column_id = 0; + std::vector> subtypes; + std::vector field_names; + UInt64 maximum_length = 0; + UInt64 precision = 0; + UInt64 scale = 0; + std::string crs; + UInt32 algorithm = 0; + std::map attributes; +}; + +/// The column selection with the same semantics as in the reference implementation: +/// selecting a column selects its whole subtree, and then all parents of the selected columns +/// are selected. If only some branches of a UNION are selected, all its branches are selected. +std::vector selectColumnsByTypeIds(const Type & root, const std::vector & type_ids); + +} diff --git a/src/Processors/Formats/Impl/ORC/Vector.cpp b/src/Processors/Formats/Impl/ORC/Vector.cpp new file mode 100644 index 0000000000000..ce7b5cdac8de7 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Vector.cpp @@ -0,0 +1,143 @@ +#include + +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int NOT_IMPLEMENTED; + extern const int INCORRECT_DATA; +} +} + +namespace DB::ORC +{ + +ColumnPtr decodeDictionary(const ColumnString & dictionary, const Int64 * index, const char * not_null, size_t num_values) +{ + const size_t dictionary_size = dictionary.size(); + const auto & dictionary_offsets = dictionary.getOffsets(); + const auto & dictionary_chars = dictionary.getChars(); + + auto result = ColumnString::create(); + auto & chars = result->getChars(); + auto & offsets = result->getOffsets(); + offsets.resize(num_values); + + size_t total = 0; + for (size_t i = 0; i < num_values; ++i) + { + if (!not_null || not_null[i]) + { + Int64 entry = index[i]; + if (entry < 0 || static_cast(entry) >= dictionary_size) + throw Exception(ErrorCodes::INCORRECT_DATA, "ORC dictionary index {} at row {} is out of range [0, {})", entry, i, dictionary_size); + total += dictionary_offsets[entry] - dictionary_offsets[entry - 1]; + } + offsets[i] = total; + } + + chars.resize(total); + UInt8 * out = chars.data(); + for (size_t i = 0; i < num_values; ++i) + { + if (!not_null || not_null[i]) + { + size_t begin = dictionary_offsets[index[i] - 1]; + size_t size = dictionary_offsets[index[i]] - begin; + memcpy(out, dictionary_chars.data() + begin, size); + out += size; + } + } + return result; +} + +ColumnPtr getStringColumn(const StringVectorBatch & batch) +{ + if (!batch.isEncoded) + return batch.column; + + const auto & encoded = typeid_cast(batch); + return decodeDictionary( + assert_cast(*encoded.dictionary->column), + encoded.index.data(), + batch.hasNulls ? batch.notNull.data() : nullptr, + batch.numElements); +} + +std::unique_ptr createRowBatch(const Type & type, bool encoded) +{ + checkStackSize(); + + switch (type.getKind()) + { + case BOOLEAN: + return std::make_unique(); + case BYTE: + return std::make_unique(); + case SHORT: + return std::make_unique(); + case INT: + return std::make_unique(); + case LONG: + case DATE: + return std::make_unique(); + case FLOAT: + return std::make_unique(); + case DOUBLE: + return std::make_unique(); + case STRING: + case BINARY: + case CHAR: + case VARCHAR: + case GEOMETRY: + case GEOGRAPHY: + if (encoded) + return std::make_unique(); + return std::make_unique(); + case TIMESTAMP: + case TIMESTAMP_INSTANT: + return std::make_unique(); + case STRUCT: + { + auto result = std::make_unique(); + for (UInt64 i = 0; i < type.getSubtypeCount(); ++i) + result->addField(createRowBatch(*type.getSubtype(i), encoded)); + return result; + } + case LIST: + { + auto result = std::make_unique(); + if (type.getSubtypeCount()) + result->elements = createRowBatch(*type.getSubtype(0), encoded); + return result; + } + case MAP: + { + auto result = std::make_unique(); + if (type.getSubtypeCount() > 0) + result->keys = createRowBatch(*type.getSubtype(0), encoded); + if (type.getSubtypeCount() > 1) + result->elements = createRowBatch(*type.getSubtype(1), encoded); + return result; + } + case UNION: + { + auto result = std::make_unique(); + for (UInt64 i = 0; i < type.getSubtypeCount(); ++i) + result->addChild(createRowBatch(*type.getSubtype(i), encoded)); + return result; + } + case DECIMAL: + if (type.getPrecision() == 0 || type.getPrecision() > 18) + return std::make_unique(); + return std::make_unique(); + } + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "ORC type {} is not supported", type.toString()); +} + +} diff --git a/src/Processors/Formats/Impl/ORC/Vector.h b/src/Processors/Formats/Impl/ORC/Vector.h new file mode 100644 index 0000000000000..1e44a80a61d11 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Vector.h @@ -0,0 +1,174 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include + +/// Batches of column values filled by the reader. The values of the primitive types are decoded +/// directly into ClickHouse columns, which the conversion to the requested types takes without +/// copying. The structure follows the reference implementation: a batch has the null mask of its +/// values, and the batches of the compound types have the batches of their children. + +namespace DB::ORC +{ + +struct ColumnVectorBatch +{ + ColumnVectorBatch() = default; + virtual ~ColumnVectorBatch() = default; + + ColumnVectorBatch(const ColumnVectorBatch &) = delete; + ColumnVectorBatch & operator=(const ColumnVectorBatch &) = delete; + + /// Prepares the batch for `num_values` values. + void reset(size_t num_values) + { + numElements = num_values; + notNull.resize(num_values); + } + + size_t numElements = 0; + /// Non-zero for the non-null values, valid only if `hasNulls`. + PaddedPODArray notNull; + bool hasNulls = false; + /// For string batches read with lazy dictionary decoding. + bool isEncoded = false; +}; + +/// The values of a primitive type in a ClickHouse column (`ColumnVector` of `T`). The null values are zeros. +template +struct NumericVectorBatch : public ColumnVectorBatch +{ + using Container = PaddedPODArray; + + /// Creates a new column for the next values: the previous column can be still referenced. + Container & prepareColumn(size_t num_values) + { + auto new_column = ColumnVector::create(num_values); + auto & data = new_column->getData(); + column = std::move(new_column); + return data; + } + + const Container & getData() const { return assert_cast &>(*column).getData(); } + + ColumnPtr column; +}; + +/// BOOLEAN: 0 or 1. +using BoolVectorBatch = NumericVectorBatch; +using ByteVectorBatch = NumericVectorBatch; +using ShortVectorBatch = NumericVectorBatch; +using IntVectorBatch = NumericVectorBatch; +/// LONG and DATE (days since the epoch, not narrowed, so that the range can be checked). +using LongVectorBatch = NumericVectorBatch; +using FloatVectorBatch = NumericVectorBatch; +using DoubleVectorBatch = NumericVectorBatch; + +/// The values in a `ColumnString`. The null values are empty strings. +struct StringVectorBatch : public ColumnVectorBatch +{ + ColumnPtr column; +}; + +struct StringDictionary +{ + /// The entries of the dictionary. + ColumnPtr column; +}; + +/// A string batch with the dictionary indices, without decoding the values. `column` is not set. +struct EncodedStringVectorBatch : public StringVectorBatch +{ + std::shared_ptr dictionary; + PaddedPODArray index; +}; + +/// Builds a `ColumnString` from the dictionary entries by the indices (the null values are empty strings). +/// Throws `INCORRECT_DATA` if an index is out of range. +ColumnPtr decodeDictionary(const ColumnString & dictionary, const Int64 * index, const char * not_null, size_t num_values); + +/// Returns the values of a string batch as a `ColumnString`, decoding the dictionary for an encoded batch. +ColumnPtr getStringColumn(const StringVectorBatch & batch); + +struct StructVectorBatch : public ColumnVectorBatch +{ + void addField(std::unique_ptr field) + { + fields.push_back(field.get()); + owned_fields.push_back(std::move(field)); + } + + std::vector fields; + +private: + std::vector> owned_fields; +}; + +/// LIST and MAP: the end offsets of the values in the children, as in `ColumnArray`. +struct ListVectorBatch : public ColumnVectorBatch +{ + /// `ColumnArray::ColumnOffsets`. + ColumnPtr offsets; + std::unique_ptr elements; +}; + +struct MapVectorBatch : public ColumnVectorBatch +{ + /// `ColumnArray::ColumnOffsets`. + ColumnPtr offsets; + std::unique_ptr keys; + std::unique_ptr elements; +}; + +struct UnionVectorBatch : public ColumnVectorBatch +{ + void addChild(std::unique_ptr child) + { + children.push_back(child.get()); + owned_children.push_back(std::move(child)); + } + + /// The branch of each value. + PaddedPODArray tags; + /// The position of each value in its branch. + PaddedPODArray offsets; + std::vector children; + +private: + std::vector> owned_children; +}; + +/// For DECIMAL with the precision at most 18. +struct Decimal64VectorBatch : public ColumnVectorBatch +{ + Int32 precision = 0; + Int32 scale = 0; + PaddedPODArray values; +}; + +/// For DECIMAL with the precision above 18 or unspecified (zero). +struct Decimal128VectorBatch : public ColumnVectorBatch +{ + Int32 precision = 0; + Int32 scale = 0; + PaddedPODArray values; +}; + +struct TimestampVectorBatch : public ColumnVectorBatch +{ + /// Seconds since the epoch in UTC. + PaddedPODArray data; + PaddedPODArray nanoseconds; +}; + +/// Creates a batch tree matching the type. With `encoded`, the string columns get +/// `EncodedStringVectorBatch`, which the reader fills without decoding the dictionaries. +std::unique_ptr createRowBatch(const Type & type, bool encoded = false); + +} diff --git a/src/Processors/Formats/Impl/ORC/Writer.cpp b/src/Processors/Formats/Impl/ORC/Writer.cpp new file mode 100644 index 0000000000000..cfab65634955e --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Writer.cpp @@ -0,0 +1,1493 @@ +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include + + +namespace DB +{ +namespace ErrorCodes +{ + extern const int BAD_ARGUMENTS; + extern const int LOGICAL_ERROR; + extern const int NOT_IMPLEMENTED; + extern const int VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE; +} +} + +namespace DB::ORC +{ + +struct WriterContext +{ + Compressor * compressor = nullptr; + size_t block_size = 0; + bool enable_index = false; + double dictionary_key_size_threshold = 0; + std::optional timezone_offset; + bool timezone_is_utc = true; + /// 2015-01-01 00:00:00 in the time zone of the writer, as seconds since the epoch. + Int64 epoch_offset = 0; +}; + +namespace +{ + +struct StreamData +{ + Proto::StreamKind kind; + UInt32 column; + std::string data; +}; + +std::string compressWhole(const WriterContext & context, const std::string & data) +{ + OutputByteStream stream(context.compressor, context.block_size); + stream.write(data.data(), data.size()); + stream.finish(); + return stream.getOutput(); +} + +} + +class ColumnWriter +{ +public: + ColumnWriter(const Type & type, WriterContext & context_) + : column_id(type.getColumnId()) + , context(context_) + , present_stream(context.compressor, context.block_size) + , present_encoder(present_stream) + , index_statistics(type.getKind()) + , stripe_statistics(type.getKind()) + , file_statistics(type.getKind()) + { + } + + virtual ~ColumnWriter() = default; + + /// Must be called by the most derived constructor, after the streams are created. + void init() + { + if (context.enable_index) + recordPositions(); + } + + /// Adds the values `[offset, offset + num_values)` of `column`, which is not `Nullable`. + /// `null_map` (1 for a null) and `parent_null_map` (1 for a null in the parent struct, for which + /// no value is written at all) are for these values only, so they start at the first value. + virtual void add(const IColumn & /*column*/, const UInt8 * null_map, size_t /*offset*/, size_t num_values, const UInt8 * parent_null_map) + { + addPresent(null_map, num_values, parent_null_map); + } + + /// Writes the PRESENT stream for the values. + void addPresent(const UInt8 * null_map, size_t num_values, const UInt8 * parent_null_map) + { + for (size_t i = 0; i < num_values; ++i) + { + if (parent_null_map && parent_null_map[i]) + continue; + present_encoder.write(!null_map || !null_map[i]); + } + if (null_map) + { + for (size_t i = 0; i < num_values && !has_null_value; ++i) + if (null_map[i]) + has_null_value = true; + } + } + + virtual void createRowIndexEntry() + { + entries.push_back(Entry{std::move(current_present_positions), std::move(current_data_positions), index_statistics.toProto()}); + current_present_positions.clear(); + current_data_positions.clear(); + stripe_statistics.merge(index_statistics); + index_statistics.reset(); + recordPositions(); + } + + virtual void mergeRowGroupStatsIntoStripeStats() + { + stripe_statistics.merge(index_statistics); + index_statistics.reset(); + } + + /// Encodes the deferred data of the stripe. + virtual void finalizeStripe() {} + + virtual void writeIndex(std::vector & streams) + { + Proto::RowIndex index; + for (const auto & entry : entries) + { + auto & proto_entry = index.entry.emplace_back(); + /// The positions of the PRESENT stream are written only if the stream is written. + if (has_null_value) + proto_entry.positions = entry.present_positions; + proto_entry.positions.insert(proto_entry.positions.end(), entry.data_positions.begin(), entry.data_positions.end()); + proto_entry.statistics = entry.statistics; + } + streams.push_back({Proto::StreamKind::ROW_INDEX, static_cast(column_id), compressWhole(context, index.serialize())}); + } + + virtual void flush(std::vector & streams) + { + present_encoder.flush(); + present_stream.finish(); + if (has_null_value) + streams.push_back({Proto::StreamKind::PRESENT, static_cast(column_id), present_stream.getOutput()}); + } + + virtual void getColumnEncodings(std::vector & encodings) const + { + encodings.push_back({Proto::ColumnEncodingKind::DIRECT, 0, {}}); + } + + virtual void getStripeStatistics(std::vector & stats) const + { + stats.push_back(stripe_statistics.toProto()); + } + + virtual void getFileStatistics(std::vector & stats) const + { + stats.push_back(file_statistics.toProto()); + } + + virtual void mergeStripeStatsIntoFileStats() + { + file_statistics.merge(stripe_statistics); + stripe_statistics.reset(); + } + + virtual size_t estimateSize() const + { + return present_stream.estimateSize(); + } + + /// Starts a new stripe. + virtual void reset() + { + present_stream.reset(); + has_null_value = false; + entries.clear(); + current_present_positions.clear(); + current_data_positions.clear(); + resetStreams(); + if (context.enable_index) + recordPositions(); + } + +protected: + /// Appends the positions of the data streams, in the order the reader seeks them. + virtual void recordDataPositions(std::vector & positions) = 0; + /// Clears the data streams for a new stripe. + virtual void resetStreams() = 0; + + /// The number of the non-null values, also updates the statistics with the count. + size_t countValues(const UInt8 * null_map, size_t num_values) + { + size_t count = num_values; + if (null_map) + { + count = 0; + for (size_t i = 0; i < num_values; ++i) + count += !null_map[i]; + } + index_statistics.increase(count); + if (count < num_values) + index_statistics.setHasNull(); + return count; + } + + struct Entry + { + std::vector present_positions; + std::vector data_positions; + Proto::ColumnStatistics statistics; + }; + + const UInt64 column_id; + WriterContext & context; + OutputByteStream present_stream; + BooleanRleEncoder present_encoder; + bool has_null_value = false; + StatisticsBuilder index_statistics; + StatisticsBuilder stripe_statistics; + StatisticsBuilder file_statistics; + std::vector entries; + +private: + void recordPositions() + { + present_encoder.recordPosition(current_present_positions); + recordDataPositions(current_data_positions); + } + + std::vector current_present_positions; + std::vector current_data_positions; +}; + +namespace +{ + +std::unique_ptr buildColumnWriter(const Type & type, const DataTypePtr & data_type, WriterContext & context); + +/// Adds the values of a column that can be `Nullable`. The values that are null in the parent +/// (`outer_null_map`, for the fields of a struct) are null as well. +void addColumn( + ColumnWriter & writer, const IColumn & column, const UInt8 * outer_null_map, size_t offset, size_t num_values, const UInt8 * parent_null_map) +{ + if (const auto * nullable = typeid_cast(&column)) + { + const UInt8 * null_map = nullable->getNullMapData().data() + offset; + if (!outer_null_map) + { + writer.add(nullable->getNestedColumn(), null_map, offset, num_values, parent_null_map); + return; + } + PaddedPODArray merged(num_values); + for (size_t i = 0; i < num_values; ++i) + merged[i] = null_map[i] | outer_null_map[i]; + writer.add(nullable->getNestedColumn(), merged.data(), offset, num_values, parent_null_map); + return; + } + writer.add(column, outer_null_map, offset, num_values, parent_null_map); +} + +/// The raw values of a column with fixed-size values. +template +const T * getRawValues(const IColumn & column) +{ + auto raw = column.getRawData(); + if (column.sizeOfValueIfFixed() != sizeof(T)) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected column {} with values of size {} for ORC (expected {})", + column.getName(), column.sizeOfValueIfFixed(), sizeof(T)); + return reinterpret_cast(raw.data()); +} + +/// The value of the integer column (of any width up to 8 bytes) as a signed number of the same width, +/// as ORC does not have unsigned types. +Int64 readSignedValue(const char * data, size_t width, size_t row) +{ + switch (width) + { + case 1: return unalignedLoad(data + row); + case 2: return unalignedLoad(data + row * 2); + case 4: return unalignedLoad(data + row * 4); + default: return unalignedLoad(data + row * 8); + } +} + +/// The writers that have children call the same methods for them. +class CompoundColumnWriter : public ColumnWriter +{ +public: + using ColumnWriter::ColumnWriter; + + void createRowIndexEntry() override + { + ColumnWriter::createRowIndexEntry(); + for (auto & child : children) + child->createRowIndexEntry(); + } + + void mergeRowGroupStatsIntoStripeStats() override + { + ColumnWriter::mergeRowGroupStatsIntoStripeStats(); + for (auto & child : children) + child->mergeRowGroupStatsIntoStripeStats(); + } + + void finalizeStripe() override + { + for (auto & child : children) + child->finalizeStripe(); + } + + void writeIndex(std::vector & streams) override + { + ColumnWriter::writeIndex(streams); + for (auto & child : children) + child->writeIndex(streams); + } + + void flush(std::vector & streams) override + { + ColumnWriter::flush(streams); + flushOwnStreams(streams); + for (auto & child : children) + child->flush(streams); + } + + void getColumnEncodings(std::vector & encodings) const override + { + getOwnColumnEncoding(encodings); + for (const auto & child : children) + child->getColumnEncodings(encodings); + } + + void getStripeStatistics(std::vector & stats) const override + { + ColumnWriter::getStripeStatistics(stats); + for (const auto & child : children) + child->getStripeStatistics(stats); + } + + void getFileStatistics(std::vector & stats) const override + { + ColumnWriter::getFileStatistics(stats); + for (const auto & child : children) + child->getFileStatistics(stats); + } + + void mergeStripeStatsIntoFileStats() override + { + ColumnWriter::mergeStripeStatsIntoFileStats(); + for (auto & child : children) + child->mergeStripeStatsIntoFileStats(); + } + + size_t estimateSize() const override + { + size_t size = ColumnWriter::estimateSize() + estimateOwnSize(); + for (const auto & child : children) + size += child->estimateSize(); + return size; + } + + void reset() override + { + ColumnWriter::reset(); + for (auto & child : children) + (*child).reset(); + } + +protected: + virtual void flushOwnStreams(std::vector &) {} + virtual void getOwnColumnEncoding(std::vector & encodings) const + { + encodings.push_back({Proto::ColumnEncodingKind::DIRECT, 0, {}}); + } + virtual size_t estimateOwnSize() const { return 0; } + + std::vector> children; +}; + +class StructColumnWriter final : public CompoundColumnWriter +{ +public: + StructColumnWriter(const Type & type, const DataTypes & field_types, WriterContext & context_) + : CompoundColumnWriter(type, context_) + { + if (field_types.size() != type.getSubtypeCount()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "ORC struct {} has {} fields, the ClickHouse type has {}", + type.toString(), type.getSubtypeCount(), field_types.size()); + for (UInt64 i = 0; i < type.getSubtypeCount(); ++i) + children.push_back(buildColumnWriter(*type.getSubtype(i), field_types[i], context)); + init(); + } + + void add(const IColumn & column, const UInt8 * null_map, size_t offset, size_t num_values, const UInt8 * parent_null_map) override + { + const auto & tuple = assert_cast(column); + ColumnWriter::add(column, null_map, offset, num_values, parent_null_map); + /// The fields of the null values are null, and they are not written. + for (size_t i = 0; i < children.size(); ++i) + addColumn(*children[i], tuple.getColumn(i), null_map, offset, num_values, null_map); + countValues(null_map, num_values); + } + + /// Adds the rows of the root struct, which are the columns of the file. + void addRoot(const Columns & columns, size_t offset, size_t num_values) + { + if (columns.size() != children.size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected {} columns for ORC, got {}", children.size(), columns.size()); + addPresent(nullptr, num_values, nullptr); + for (size_t i = 0; i < children.size(); ++i) + addColumn(*children[i], *columns[i], nullptr, offset, num_values, nullptr); + countValues(nullptr, num_values); + } + +protected: + void recordDataPositions(std::vector &) override {} + void resetStreams() override {} +}; + +template +class CollectionColumnWriter final : public CompoundColumnWriter +{ +public: + CollectionColumnWriter(const Type & type, const DataTypes & child_types, WriterContext & context_) + : CompoundColumnWriter(type, context_) + , length_stream(context.compressor, context.block_size) + , length_encoder(length_stream, false) + { + if (child_types.size() != type.getSubtypeCount()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "ORC type {} does not match the ClickHouse type", type.toString()); + for (UInt64 i = 0; i < type.getSubtypeCount(); ++i) + children.push_back(buildColumnWriter(*type.getSubtype(i), child_types[i], context)); + init(); + } + + void add(const IColumn & column, const UInt8 * null_map, size_t offset, size_t num_values, const UInt8 * parent_null_map) override + { + const ColumnArray & array = is_map ? assert_cast(column).getNestedColumn() : assert_cast(column); + /// The offsets of `ColumnArray` are the end offsets, and the one before the first is zero. + const auto & offsets = array.getOffsets(); + + ColumnWriter::add(column, null_map, offset, num_values, parent_null_map); + + UInt64 element_offset = offsets[static_cast(offset) - 1]; + UInt64 total_elements = offsets[offset + num_values - 1] - element_offset; + + if constexpr (is_map) + { + const auto & key_value = assert_cast(array.getData()); + addColumn(*children[0], key_value.getColumn(0), nullptr, element_offset, total_elements, nullptr); + addColumn(*children[1], key_value.getColumn(1), nullptr, element_offset, total_elements, nullptr); + } + else + { + addColumn(*children[0], array.getData(), nullptr, element_offset, total_elements, nullptr); + } + + size_t count = 0; + for (size_t i = 0; i < num_values; ++i) + { + if (null_map && null_map[i]) + continue; + size_t row = offset + i; + UInt64 length = offsets[row] - offsets[static_cast(row) - 1]; + length_encoder.write(static_cast(length)); + index_statistics.updateCollection(length); + ++count; + } + index_statistics.increase(count); + if (count < num_values) + index_statistics.setHasNull(); + } + +protected: + void recordDataPositions(std::vector & positions) override + { + length_encoder.recordPosition(positions); + } + + void resetStreams() override + { + length_stream.reset(); + } + + void flushOwnStreams(std::vector & streams) override + { + length_encoder.flush(); + length_stream.finish(); + streams.push_back({Proto::StreamKind::LENGTH, static_cast(column_id), length_stream.getOutput()}); + } + + void getOwnColumnEncoding(std::vector & encodings) const override + { + encodings.push_back({Proto::ColumnEncodingKind::DIRECT_V2, 0, {}}); + } + + size_t estimateOwnSize() const override { return length_stream.estimateSize(); } + +private: + OutputByteStream length_stream; + IntRleEncoderV2 length_encoder; +}; + +class UnionColumnWriter final : public CompoundColumnWriter +{ +public: + /// The branches of the union are in the same order as the variants of the `Variant` type, + /// so the global discriminator of a value is its tag. + UnionColumnWriter(const Type & type, const DataTypes & variant_types, WriterContext & context_) + : CompoundColumnWriter(type, context_) + , tag_stream(context.compressor, context.block_size) + , tag_encoder(tag_stream) + { + if (variant_types.size() != type.getSubtypeCount()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "ORC union {} does not match the ClickHouse type", type.toString()); + for (UInt64 i = 0; i < type.getSubtypeCount(); ++i) + children.push_back(buildColumnWriter(*type.getSubtype(i), variant_types[i], context)); + init(); + } + + void add(const IColumn & column, const UInt8 * null_map, size_t offset, size_t num_values, const UInt8 * parent_null_map) override + { + const auto & variant = assert_cast(column); + + /// A `NULL` of `Variant` is a null value of the union. + PaddedPODArray union_null_map(num_values); + bool has_nulls = false; + for (size_t i = 0; i < num_values; ++i) + { + union_null_map[i] = (null_map && null_map[i]) || variant.globalDiscriminatorAt(offset + i) == ColumnVariant::NULL_DISCRIMINATOR; + has_nulls |= union_null_map[i]; + } + const UInt8 * union_nulls = has_nulls ? union_null_map.data() : nullptr; + + ColumnWriter::add(column, union_nulls, offset, num_values, parent_null_map); + + std::vector child_offset(children.size(), -1); + std::vector child_length(children.size(), 0); + size_t count = 0; + for (size_t i = 0; i < num_values; ++i) + { + if (union_nulls && union_nulls[i]) + continue; + size_t row = offset + i; + auto tag = variant.globalDiscriminatorAt(row); + if (tag >= children.size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Invalid union tag {} for union with {} children", static_cast(tag), children.size()); + if (child_offset[tag] == -1) + child_offset[tag] = static_cast(variant.offsetAt(row)); + ++child_length[tag]; + tag_encoder.write(static_cast(tag)); + ++count; + } + + /// The values of each branch are consecutive in its column. + for (size_t i = 0; i < children.size(); ++i) + if (child_length[i]) + addColumn(*children[i], variant.getVariantByGlobalDiscriminator(i), nullptr, static_cast(child_offset[i]), child_length[i], nullptr); + + index_statistics.increase(count); + if (count < num_values) + index_statistics.setHasNull(); + } + +protected: + void recordDataPositions(std::vector & positions) override + { + tag_encoder.recordPosition(positions); + } + + void resetStreams() override + { + tag_stream.reset(); + } + + void flushOwnStreams(std::vector & streams) override + { + tag_encoder.flush(); + tag_stream.finish(); + streams.push_back({Proto::StreamKind::DATA, static_cast(column_id), tag_stream.getOutput()}); + } + + size_t estimateOwnSize() const override { return tag_stream.estimateSize(); } + +private: + OutputByteStream tag_stream; + ByteRleEncoder tag_encoder; +}; + + +/// The writers of the primitive columns. +class PrimitiveColumnWriter : public ColumnWriter +{ +public: + using ColumnWriter::ColumnWriter; + + void getColumnEncodings(std::vector & encodings) const override + { + encodings.push_back({encoding_kind, dictionary_size, {}}); + } + +protected: + Proto::ColumnEncodingKind encoding_kind = Proto::ColumnEncodingKind::DIRECT; + UInt32 dictionary_size = 0; +}; + +class BooleanColumnWriter final : public PrimitiveColumnWriter +{ +public: + BooleanColumnWriter(const Type & type, WriterContext & context_) + : PrimitiveColumnWriter(type, context_) + , data_stream(context.compressor, context.block_size) + , encoder(data_stream) + { + init(); + } + + void add(const IColumn & column, const UInt8 * null_map, size_t offset, size_t num_values, const UInt8 * parent_null_map) override + { + ColumnWriter::add(column, null_map, offset, num_values, parent_null_map); + const UInt8 * data = getRawValues(column) + offset; + for (size_t i = 0; i < num_values; ++i) + { + if (null_map && null_map[i]) + continue; + encoder.write(data[i] != 0); + index_statistics.updateBoolean(data[i] != 0); + } + countValues(null_map, num_values); + } + + void flush(std::vector & streams) override + { + ColumnWriter::flush(streams); + encoder.flush(); + data_stream.finish(); + streams.push_back({Proto::StreamKind::DATA, static_cast(column_id), data_stream.getOutput()}); + } + + size_t estimateSize() const override { return ColumnWriter::estimateSize() + data_stream.estimateSize(); } + +protected: + void recordDataPositions(std::vector & positions) override { encoder.recordPosition(positions); } + void resetStreams() override { data_stream.reset(); } + +private: + OutputByteStream data_stream; + BooleanRleEncoder encoder; +}; + +class ByteColumnWriter final : public PrimitiveColumnWriter +{ +public: + ByteColumnWriter(const Type & type, WriterContext & context_) + : PrimitiveColumnWriter(type, context_) + , data_stream(context.compressor, context.block_size) + , encoder(data_stream) + { + init(); + } + + void add(const IColumn & column, const UInt8 * null_map, size_t offset, size_t num_values, const UInt8 * parent_null_map) override + { + ColumnWriter::add(column, null_map, offset, num_values, parent_null_map); + /// The unsigned values are written as signed ones of the same width. + const Int8 * data = getRawValues(column) + offset; + for (size_t i = 0; i < num_values; ++i) + { + if (null_map && null_map[i]) + continue; + encoder.write(static_cast(data[i])); + index_statistics.updateInteger(data[i]); + } + countValues(null_map, num_values); + } + + void flush(std::vector & streams) override + { + ColumnWriter::flush(streams); + encoder.flush(); + data_stream.finish(); + streams.push_back({Proto::StreamKind::DATA, static_cast(column_id), data_stream.getOutput()}); + } + + size_t estimateSize() const override { return ColumnWriter::estimateSize() + data_stream.estimateSize(); } + +protected: + void recordDataPositions(std::vector & positions) override { encoder.recordPosition(positions); } + void resetStreams() override { data_stream.reset(); } + +private: + OutputByteStream data_stream; + ByteRleEncoder encoder; +}; + +/// SHORT, INT, LONG and DATE. +class IntegerColumnWriter final : public PrimitiveColumnWriter +{ +public: + IntegerColumnWriter(const Type & type, WriterContext & context_) + : PrimitiveColumnWriter(type, context_) + , data_stream(context.compressor, context.block_size) + , encoder(data_stream, true) + , is_date(type.getKind() == DATE) + { + encoding_kind = Proto::ColumnEncodingKind::DIRECT_V2; + init(); + } + + void add(const IColumn & column, const UInt8 * null_map, size_t offset, size_t num_values, const UInt8 * parent_null_map) override + { + ColumnWriter::add(column, null_map, offset, num_values, parent_null_map); + /// The unsigned values are written as signed ones of the same width (except `Date`). + const size_t width = column.sizeOfValueIfFixed(); + if (width != 2 && width != 4 && width != 8) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected column {} for ORC integer", column.getName()); + const char * data = column.getRawData().data(); + for (size_t i = 0; i < num_values; ++i) + { + if (null_map && null_map[i]) + continue; + /// `Date` is the unsigned number of days (up to 2149), which must not wrap around + /// to the negative ones after 2059. `Date32` is signed, as the other integers. + Int64 value = (is_date && width == 2) + ? unalignedLoad(data + (offset + i) * sizeof(UInt16)) + : readSignedValue(data, width, offset + i); + encoder.write(value); + if (is_date) + index_statistics.updateDate(static_cast(value)); + else + index_statistics.updateInteger(value); + } + countValues(null_map, num_values); + } + + void flush(std::vector & streams) override + { + ColumnWriter::flush(streams); + encoder.flush(); + data_stream.finish(); + streams.push_back({Proto::StreamKind::DATA, static_cast(column_id), data_stream.getOutput()}); + } + + size_t estimateSize() const override { return ColumnWriter::estimateSize() + data_stream.estimateSize(); } + +protected: + void recordDataPositions(std::vector & positions) override { encoder.recordPosition(positions); } + void resetStreams() override { data_stream.reset(); } + +private: + OutputByteStream data_stream; + IntRleEncoderV2 encoder; + const bool is_date; +}; + +template +class FloatingColumnWriter final : public PrimitiveColumnWriter +{ +public: + FloatingColumnWriter(const Type & type, WriterContext & context_) + : PrimitiveColumnWriter(type, context_) + , data_stream(context.compressor, context.block_size) + { + init(); + } + + void add(const IColumn & column, const UInt8 * null_map, size_t offset, size_t num_values, const UInt8 * parent_null_map) override + { + ColumnWriter::add(column, null_map, offset, num_values, parent_null_map); + const ItemType * data = getRawValues(column) + offset; + if (!null_map && std::endian::native == std::endian::little) + { + data_stream.write(reinterpret_cast(data), num_values * sizeof(ItemType)); + for (size_t i = 0; i < num_values; ++i) + index_statistics.updateDouble(static_cast(data[i])); + } + else + { + for (size_t i = 0; i < num_values; ++i) + { + if (null_map && null_map[i]) + continue; + ItemType value = data[i]; + if constexpr (std::endian::native == std::endian::big) + value = std::bit_cast(std::byteswap(std::bit_cast>(value))); + data_stream.write(reinterpret_cast(&value), sizeof(value)); + index_statistics.updateDouble(static_cast(data[i])); + } + } + countValues(null_map, num_values); + } + + void flush(std::vector & streams) override + { + ColumnWriter::flush(streams); + data_stream.finish(); + streams.push_back({Proto::StreamKind::DATA, static_cast(column_id), data_stream.getOutput()}); + } + + size_t estimateSize() const override { return ColumnWriter::estimateSize() + data_stream.estimateSize(); } + +protected: + void recordDataPositions(std::vector & positions) override { data_stream.recordPosition(positions); } + void resetStreams() override { data_stream.reset(); } + +private: + OutputByteStream data_stream; +}; + +/// STRING, VARCHAR, CHAR and BINARY. The values of a stripe are kept in memory and encoded at the +/// end of the stripe, either directly or with a dictionary. +class StringColumnWriter final : public PrimitiveColumnWriter +{ +public: + StringColumnWriter(const Type & type, WriterContext & context_) + : PrimitiveColumnWriter(type, context_) + , is_binary(type.getKind() == BINARY) + , data_stream(context.compressor, context.block_size) + , length_stream(context.compressor, context.block_size) + , dictionary_data_stream(context.compressor, context.block_size) + , length_encoder(length_stream, false) + , index_encoder(data_stream, false) + { + /// The binary columns never use dictionaries, as in the reference implementation. + if (is_binary || context.dictionary_key_size_threshold <= 0) + use_dictionary = false; + encoding_kind = Proto::ColumnEncodingKind::DIRECT_V2; + init(); + } + + void add(const IColumn & column, const UInt8 * null_map, size_t offset, size_t num_values, const UInt8 * parent_null_map) override + { + ColumnWriter::add(column, null_map, offset, num_values, parent_null_map); + /// `String`, or the raw bytes of the values of fixed size (`FixedString`, `IPv6`, big integers). + auto add_value = [&](std::string_view value) + { + blob.append(value); + value_ends.push_back(blob.size()); + if (is_binary) + index_statistics.updateBinary(value.size()); + else + index_statistics.updateString(value); + }; + + if (const auto * strings = typeid_cast(&column)) + { + const auto & offsets = strings->getOffsets(); + const auto * chars = reinterpret_cast(strings->getChars().data()); + for (size_t i = 0; i < num_values; ++i) + { + if (null_map && null_map[i]) + continue; + size_t row = offset + i; + add_value(std::string_view(chars + offsets[row - 1], offsets[row] - offsets[row - 1])); + } + } + else + { + for (size_t i = 0; i < num_values; ++i) + { + if (null_map && null_map[i]) + continue; + add_value(column.getDataAt(offset + i)); + } + } + countValues(null_map, num_values); + } + + void createRowIndexEntry() override + { + if (!use_dictionary) + decideDictionary(); + ColumnWriter::createRowIndexEntry(); + } + + void finalizeStripe() override + { + if (!use_dictionary) + decideDictionary(); + + /// The start of the group without an entry is not needed. + group_starts.resize(entries.size()); + + if (*use_dictionary) + encodeWithDictionary(); + else + encodeDirect(); + } + + void flush(std::vector & streams) override + { + ColumnWriter::flush(streams); + if (*use_dictionary) + { + index_encoder.flush(); + data_stream.finish(); + length_encoder.flush(); + length_stream.finish(); + dictionary_data_stream.finish(); + streams.push_back({Proto::StreamKind::DATA, static_cast(column_id), data_stream.getOutput()}); + streams.push_back({Proto::StreamKind::DICTIONARY_DATA, static_cast(column_id), dictionary_data_stream.getOutput()}); + streams.push_back({Proto::StreamKind::LENGTH, static_cast(column_id), length_stream.getOutput()}); + } + else + { + data_stream.finish(); + length_encoder.flush(); + length_stream.finish(); + streams.push_back({Proto::StreamKind::DATA, static_cast(column_id), data_stream.getOutput()}); + streams.push_back({Proto::StreamKind::LENGTH, static_cast(column_id), length_stream.getOutput()}); + } + } + + size_t estimateSize() const override + { + return ColumnWriter::estimateSize() + blob.size() + value_ends.size() * sizeof(UInt64); + } + +protected: + void recordDataPositions(std::vector &) override + { + /// The positions are known only after the encoding, at the end of the stripe. + group_starts.push_back(value_ends.size()); + } + + void resetStreams() override + { + data_stream.reset(); + length_stream.reset(); + dictionary_data_stream.reset(); + blob.clear(); + value_ends.clear(); + group_starts.clear(); + dictionary_size = 0; + } + +private: + std::string_view getValue(size_t i) const + { + size_t begin = i ? value_ends[i - 1] : 0; + return std::string_view(blob.data() + begin, value_ends[i] - begin); + } + + /// The decision is made once for the file, by the values of the first row group (or the first + /// stripe without the row index), as in the reference implementation. + void decideDictionary() + { + std::unordered_set distinct; + for (size_t i = 0; i < value_ends.size(); ++i) + distinct.insert(getValue(i)); + /// Compare as doubles: the threshold is user-provided and the product may not fit in `size_t`. + use_dictionary = static_cast(distinct.size()) <= static_cast(value_ends.size()) * context.dictionary_key_size_threshold; + } + + void encodeDirect() + { + encoding_kind = Proto::ColumnEncodingKind::DIRECT_V2; + size_t group = 0; + for (size_t i = 0; i <= value_ends.size(); ++i) + { + while (group < group_starts.size() && group_starts[group] == i) + { + auto & positions = entries[group].data_positions; + data_stream.recordPosition(positions); + length_encoder.recordPosition(positions); + ++group; + } + if (i == value_ends.size()) + break; + auto value = getValue(i); + data_stream.write(value.data(), value.size()); + length_encoder.write(static_cast(value.size())); + } + } + + void encodeWithDictionary() + { + encoding_kind = Proto::ColumnEncodingKind::DICTIONARY_V2; + + /// The dictionary is sorted, as in the reference implementation. + std::unordered_map index_of; + std::vector keys; + for (size_t i = 0; i < value_ends.size(); ++i) + { + auto value = getValue(i); + if (index_of.emplace(value, 0).second) + keys.push_back(value); + } + std::sort(keys.begin(), keys.end()); + for (size_t i = 0; i < keys.size(); ++i) + index_of[keys[i]] = static_cast(i); + dictionary_size = static_cast(keys.size()); + + size_t group = 0; + for (size_t i = 0; i <= value_ends.size(); ++i) + { + while (group < group_starts.size() && group_starts[group] == i) + { + index_encoder.recordPosition(entries[group].data_positions); + ++group; + } + if (i == value_ends.size()) + break; + index_encoder.write(index_of[getValue(i)]); + } + + for (const auto & key : keys) + { + dictionary_data_stream.write(key.data(), key.size()); + length_encoder.write(static_cast(key.size())); + } + } + + const bool is_binary; + std::optional use_dictionary; + OutputByteStream data_stream; + OutputByteStream length_stream; + OutputByteStream dictionary_data_stream; + IntRleEncoderV2 length_encoder; + IntRleEncoderV2 index_encoder; + + std::string blob; + std::vector value_ends; + /// The index of the first value of each row group. + std::vector group_starts; +}; + +class TimestampColumnWriter final : public PrimitiveColumnWriter +{ +public: + /// `scale` is of `DateTime64`, or nullopt for `DateTime` (`UInt32` seconds). + TimestampColumnWriter(const Type & type, std::optional scale_, WriterContext & context_) + : PrimitiveColumnWriter(type, context_) + , scale(scale_) + , seconds_stream(context.compressor, context.block_size) + , nanos_stream(context.compressor, context.block_size) + , seconds_encoder(seconds_stream, true) + , nanos_encoder(nanos_stream, false) + , is_utc(type.getKind() == TIMESTAMP_INSTANT || context.timezone_is_utc) + , epoch_offset(type.getKind() == TIMESTAMP_INSTANT ? 1420070400 : context.epoch_offset) + { + encoding_kind = Proto::ColumnEncodingKind::DIRECT_V2; + init(); + } + + void add(const IColumn & column, const UInt8 * null_map, size_t offset, size_t num_values, const UInt8 * parent_null_map) override + { + ColumnWriter::add(column, null_map, offset, num_values, parent_null_map); + const char * data = column.getRawData().data(); + const Int64 multiplier = scale ? common::exp10_i64(static_cast(*scale)) : 1; + const Int64 nanos_multiplier = scale ? common::exp10_i64(9 - static_cast(*scale)) : 0; + for (size_t i = 0; i < num_values; ++i) + { + if (null_map && null_map[i]) + continue; + Int64 secs = 0; + Int64 nano = 0; + if (scale) + { + /// For the values before the epoch, the nanoseconds are negative (as in the Arrow ORC adapter). + Int64 value = unalignedLoad(data + (offset + i) * sizeof(Int64)); + secs = value / multiplier; + nano = (value % multiplier) * nanos_multiplier; + } + else + { + secs = unalignedLoad(data + (offset + i) * sizeof(UInt32)); + nano = 0; + } + + /// The statistics have the milliseconds rounded down and the rest of the nanoseconds, which is not negative. + Int64 nano_millis = nano / 1000000; + Int64 nano_rest = nano % 1000000; + if (nano_rest < 0) + { + nano_rest += 1000000; + --nano_millis; + } + Int64 utc_seconds = secs; + Int64 millis = 0; + Int64 encoded_seconds = 0; + if ((!is_utc && common::addOverflow(secs, (*context.timezone_offset)(secs), utc_seconds)) + || common::mulOverflow(utc_seconds, Int64{1000}, millis) + || common::addOverflow(millis, nano_millis, millis) + || common::subOverflow(secs + (secs < 0 && nano > 999999), epoch_offset, encoded_seconds)) + throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, + "Timestamp (seconds={}, nanoseconds={}) in ORC column {} is out of the range supported by the ORC format", secs, nano, column_id); + index_statistics.updateTimestamp(millis, static_cast(nano_rest)); + + seconds_encoder.write(encoded_seconds); + nanos_encoder.write(formatNanos(nano)); + } + countValues(null_map, num_values); + } + + void flush(std::vector & streams) override + { + ColumnWriter::flush(streams); + seconds_encoder.flush(); + seconds_stream.finish(); + nanos_encoder.flush(); + nanos_stream.finish(); + streams.push_back({Proto::StreamKind::DATA, static_cast(column_id), seconds_stream.getOutput()}); + streams.push_back({Proto::StreamKind::SECONDARY, static_cast(column_id), nanos_stream.getOutput()}); + } + + size_t estimateSize() const override + { + return ColumnWriter::estimateSize() + seconds_stream.estimateSize() + nanos_stream.estimateSize(); + } + +protected: + void recordDataPositions(std::vector & positions) override + { + seconds_encoder.recordPosition(positions); + nanos_encoder.recordPosition(positions); + } + + void resetStreams() override + { + seconds_stream.reset(); + nanos_stream.reset(); + } + +private: + /// The trailing zeros are removed and their number is kept in the lowest 3 bits. + static Int64 formatNanos(Int64 nanos) + { + if (nanos == 0) + return 0; + if (nanos % 100 != 0) + return nanos << 3; + nanos /= 100; + Int64 trailing_zeros = 1; + while (nanos % 10 == 0 && trailing_zeros < 7) + { + nanos /= 10; + ++trailing_zeros; + } + return (nanos << 3) | trailing_zeros; + } + + const std::optional scale; + OutputByteStream seconds_stream; + OutputByteStream nanos_stream; + IntRleEncoderV2 seconds_encoder; + IntRleEncoderV2 nanos_encoder; + const bool is_utc; + const Int64 epoch_offset; +}; + +class DecimalColumnWriter final : public PrimitiveColumnWriter +{ +public: + DecimalColumnWriter(const Type & type, WriterContext & context_) + : PrimitiveColumnWriter(type, context_) + , scale(static_cast(type.getScale())) + , value_stream(context.compressor, context.block_size) + , scale_stream(context.compressor, context.block_size) + , scale_encoder(scale_stream, true) + { + encoding_kind = Proto::ColumnEncodingKind::DIRECT_V2; + init(); + } + + void add(const IColumn & column, const UInt8 * null_map, size_t offset, size_t num_values, const UInt8 * parent_null_map) override + { + ColumnWriter::add(column, null_map, offset, num_values, parent_null_map); + /// `Decimal32`, `Decimal64` or `Decimal128`. + const size_t width = column.sizeOfValueIfFixed(); + const char * data = column.getRawData().data(); + for (size_t i = 0; i < num_values; ++i) + { + if (null_map && null_map[i]) + continue; + size_t row = offset + i; + Int128 value; + if (width == 16) + value = unalignedLoad(data + row * 16); + else if (width == 8) + value = unalignedLoad(data + row * 8); + else if (width == 4) + value = unalignedLoad(data + row * 4); + else + throw Exception(ErrorCodes::LOGICAL_ERROR, "Unexpected column {} for ORC decimal", column.getName()); + + /// Zigzag-encoded base 128 varint. + UInt128 encoded = (static_cast(value) << 1) ^ static_cast(value >> 127); + while (encoded >= 0x80) + { + char byte = static_cast(static_cast(encoded & 0x7F) | 0x80); + value_stream.write(&byte, 1); + encoded >>= 7; + } + char last = static_cast(static_cast(encoded)); + value_stream.write(&last, 1); + + scale_encoder.write(scale); + index_statistics.updateDecimal(value, scale); + } + countValues(null_map, num_values); + } + + void flush(std::vector & streams) override + { + ColumnWriter::flush(streams); + value_stream.finish(); + scale_encoder.flush(); + scale_stream.finish(); + streams.push_back({Proto::StreamKind::DATA, static_cast(column_id), value_stream.getOutput()}); + streams.push_back({Proto::StreamKind::SECONDARY, static_cast(column_id), scale_stream.getOutput()}); + } + + size_t estimateSize() const override + { + return ColumnWriter::estimateSize() + value_stream.estimateSize() + scale_stream.estimateSize(); + } + +protected: + void recordDataPositions(std::vector & positions) override + { + value_stream.recordPosition(positions); + scale_encoder.recordPosition(positions); + } + + void resetStreams() override + { + value_stream.reset(); + scale_stream.reset(); + } + +private: + const Int32 scale; + OutputByteStream value_stream; + OutputByteStream scale_stream; + IntRleEncoderV2 scale_encoder; +}; + +std::unique_ptr buildColumnWriter(const Type & type, const DataTypePtr & data_type_with_nulls, WriterContext & context) +{ + checkStackSize(); + + /// The nulls are handled by `addColumn`. + DataTypePtr data_type = removeNullable(data_type_with_nulls); + + switch (type.getKind()) + { + case BOOLEAN: + return std::make_unique(type, context); + case BYTE: + return std::make_unique(type, context); + case SHORT: + case INT: + case LONG: + case DATE: + return std::make_unique(type, context); + case FLOAT: + return std::make_unique>(type, context); + case DOUBLE: + return std::make_unique>(type, context); + case STRING: + case BINARY: + case CHAR: + case VARCHAR: + return std::make_unique(type, context); + case TIMESTAMP: + case TIMESTAMP_INSTANT: + { + std::optional scale; + if (const auto * datetime64 = typeid_cast(data_type.get())) + scale = datetime64->getScale(); + return std::make_unique(type, scale, context); + } + case DECIMAL: + return std::make_unique(type, context); + case STRUCT: + return std::make_unique(type, assert_cast(*data_type).getElements(), context); + case LIST: + return std::make_unique>( + type, DataTypes{assert_cast(*data_type).getNestedType()}, context); + case MAP: + { + const auto & map_type = assert_cast(*data_type); + return std::make_unique>(type, DataTypes{map_type.getKeyType(), map_type.getValueType()}, context); + } + case UNION: + return std::make_unique(type, assert_cast(*data_type).getVariants(), context); + case GEOMETRY: + case GEOGRAPHY: + break; + } + throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Writing ORC type {} is not supported", type.toString()); +} + +} + + +Writer::Writer(const Type & schema_, const DataTypes & column_types, OutputStream & out_, const WriterOptions & options_) + : out(out_), options(options_) +{ + if (options.compression_block_size == 0 || options.compression_block_size >= (1ULL << 23)) + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "The ORC compression block size must be positive and less than 8 MiB, got {}", options.compression_block_size); + + schema = Type::buildSelectedType(schema_, std::vector(schema_.getMaximumColumnId() + 1, true)); + schema->assignIds(0); + + compressor = createCompressor(options.compression); + + context = std::make_unique(); + context->compressor = compressor.get(); + context->block_size = options.compression_block_size; + context->enable_index = options.row_index_stride > 0; + context->dictionary_key_size_threshold = options.dictionary_key_size_threshold; + context->timezone_offset.emplace(DateLUT::instance(options.timezone_name)); + context->timezone_is_utc = options.timezone_name == "GMT" || options.timezone_name == "UTC"; + static constexpr Int64 UTC_2015_01_01 = 1420070400; + context->epoch_offset = UTC_2015_01_01 - (*context->timezone_offset)(UTC_2015_01_01); + + if (schema->getKind() != STRUCT) + throw Exception(ErrorCodes::LOGICAL_ERROR, "The root type of ORC file must be a struct"); + column_writer = std::make_unique(*schema, column_types, *context); + + write("ORC"); + footer.header_length = current_offset; + footer.row_index_stride = static_cast(options.row_index_stride); + /// The identifier of the C++ writer of the reference implementation, which this writer follows. + footer.writer = 1; + footer.software_version = "ClickHouse"; + schema->toProto(footer.types); +} + +Writer::~Writer() = default; + +void Writer::write(const std::string & data) +{ + out.write(data.data(), data.size()); + current_offset += data.size(); +} + +void Writer::add(const Columns & columns, size_t num_rows) +{ + auto & root = typeid_cast(*column_writer); + + if (context->enable_index) + { + size_t position = 0; + while (position < num_rows) + { + size_t chunk = std::min(num_rows - position, options.row_index_stride - index_rows); + root.addRoot(columns, position, chunk); + position += chunk; + index_rows += chunk; + stripe_rows += chunk; + if (index_rows >= options.row_index_stride) + { + column_writer->createRowIndexEntry(); + index_rows = 0; + } + } + } + else + { + stripe_rows += num_rows; + root.addRoot(columns, 0, num_rows); + } + + if (column_writer->estimateSize() >= options.stripe_size) + writeStripe(); +} + +void Writer::writeStripe() +{ + if (context->enable_index && index_rows != 0) + { + column_writer->createRowIndexEntry(); + index_rows = 0; + } + else + column_writer->mergeRowGroupStatsIntoStripeStats(); + + column_writer->finalizeStripe(); + + std::vector index_streams; + std::vector data_streams; + if (context->enable_index) + column_writer->writeIndex(index_streams); + column_writer->flush(data_streams); + + Proto::StripeFooter stripe_footer; + UInt64 index_length = 0; + UInt64 data_length = 0; + for (const auto & s : index_streams) + { + stripe_footer.streams.push_back({static_cast(s.kind), s.column, s.data.size()}); + index_length += s.data.size(); + } + for (const auto & s : data_streams) + { + stripe_footer.streams.push_back({static_cast(s.kind), s.column, s.data.size()}); + data_length += s.data.size(); + } + column_writer->getColumnEncodings(stripe_footer.columns); + stripe_footer.writer_timezone = options.timezone_name; + + auto & stripe_statistics = metadata.stripe_stats.emplace_back(); + column_writer->getStripeStatistics(stripe_statistics.col_stats); + column_writer->mergeStripeStatsIntoFileStats(); + + Proto::StripeInformation stripe; + stripe.offset = current_offset; + + for (const auto & s : index_streams) + write(s.data); + for (const auto & s : data_streams) + write(s.data); + + std::string footer_data = compressWhole(*context, stripe_footer.serialize()); + write(footer_data); + + stripe.index_length = index_length; + stripe.data_length = data_length; + stripe.footer_length = footer_data.size(); + stripe.number_of_rows = stripe_rows; + footer.stripes.push_back(stripe); + + total_rows += stripe_rows; + stripe_rows = 0; + index_rows = 0; + (*column_writer).reset(); +} + +void Writer::close() +{ + if (stripe_rows > 0) + writeStripe(); + + Proto::PostScript postscript; + + /// The content length covers only the stripes, not the metadata section with the stripe statistics. + footer.content_length = current_offset - footer.header_length; + + std::string metadata_data = compressWhole(*context, metadata.serialize()); + write(metadata_data); + postscript.metadata_length = metadata_data.size(); + + footer.number_of_rows = total_rows; + footer.statistics.clear(); + column_writer->getFileStatistics(footer.statistics); + std::string footer_data = compressWhole(*context, footer.serialize()); + write(footer_data); + + postscript.footer_length = footer_data.size(); + postscript.compression = options.compression; + postscript.compression_block_size = options.compression_block_size; + postscript.version = {0, 12}; + /// ORC-135: the timestamp statistics are in UTC. + postscript.writer_version = 6; + postscript.magic = "ORC"; + std::string postscript_data = postscript.serialize(); + if (postscript_data.size() > 255) + throw Exception(ErrorCodes::LOGICAL_ERROR, "ORC postscript is too large: {} bytes", postscript_data.size()); + write(postscript_data); + write(std::string(1, static_cast(postscript_data.size()))); +} + +} diff --git a/src/Processors/Formats/Impl/ORC/Writer.h b/src/Processors/Formats/Impl/ORC/Writer.h new file mode 100644 index 0000000000000..111c20ac0fab1 --- /dev/null +++ b/src/Processors/Formats/Impl/ORC/Writer.h @@ -0,0 +1,78 @@ +#pragma once + +#include +#include +#include +#include +#include + +#include +#include + +class DateLUTImpl; + +namespace DB::ORC +{ + +class OutputStream +{ +public: + virtual ~OutputStream() = default; + virtual void write(const void * buf, size_t length) = 0; +}; + +struct WriterOptions +{ + CompressionKind compression = CompressionKind::ZSTD; + UInt64 compression_block_size = 256 * 1024; + /// The number of rows between the row index entries; 0 means no row index. + UInt64 row_index_stride = 10000; + /// Use the dictionary encoding for a string column if the number of distinct values in the + /// first row group is at most this fraction of the number of values. 0 disables the dictionaries. + double dictionary_key_size_threshold = 0.0; + /// The time zone of the writer, which is written into the stripe footers. + std::string timezone_name = "GMT"; + /// A stripe is written when the estimated size of its data reaches this size. + UInt64 stripe_size = 64 * 1024 * 1024; +}; + +class ColumnWriter; +struct WriterContext; + +class Writer +{ +public: + /// `column_types` are the ClickHouse types (without `LowCardinality`) of the columns of the file, + /// which are the fields of `schema` (a struct). The values are converted as follows: + /// the unsigned integers are written as the signed ones of the same width (but `Date` as is); + /// `DateTime` and `DateTime64` as TIMESTAMP; `FixedString`, `IPv6` and the big integers + /// as their raw bytes; `Variant` as UNION with the branches in the order of its variants. + Writer(const Type & schema_, const DataTypes & column_types, OutputStream & out_, const WriterOptions & options_); + ~Writer(); + + /// Adds the rows of the columns (in the order of the fields of the schema). + void add(const Columns & columns, size_t num_rows); + + /// Writes the remaining data and the file footer. + void close(); + +private: + void writeStripe(); + void write(const std::string & data); + + std::unique_ptr schema; + OutputStream & out; + WriterOptions options; + std::unique_ptr compressor; + std::unique_ptr context; + std::unique_ptr column_writer; + + Proto::Footer footer; + Proto::Metadata metadata; + UInt64 current_offset = 0; + UInt64 stripe_rows = 0; + UInt64 index_rows = 0; + UInt64 total_rows = 0; +}; + +} diff --git a/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp index 7c2fa8963137d..896189c0a1fad 100644 --- a/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ORCBlockOutputFormat.cpp @@ -45,24 +45,24 @@ namespace ErrorCodes namespace { -orc::CompressionKind getORCCompression(FormatSettings::ORCCompression method) +ORC::CompressionKind getORCCompression(FormatSettings::ORCCompression method) { if (method == FormatSettings::ORCCompression::NONE) - return orc::CompressionKind::CompressionKind_NONE; + return ORC::CompressionKind::NONE; #if USE_SNAPPY if (method == FormatSettings::ORCCompression::SNAPPY) - return orc::CompressionKind::CompressionKind_SNAPPY; + return ORC::CompressionKind::SNAPPY; #endif if (method == FormatSettings::ORCCompression::ZSTD) - return orc::CompressionKind::CompressionKind_ZSTD; + return ORC::CompressionKind::ZSTD; if (method == FormatSettings::ORCCompression::LZ4) - return orc::CompressionKind::CompressionKind_LZ4; + return ORC::CompressionKind::LZ4; if (method == FormatSettings::ORCCompression::ZLIB) - return orc::CompressionKind::CompressionKind_ZLIB; + return ORC::CompressionKind::ZLIB; throw Exception(ErrorCodes::NOT_IMPLEMENTED, "Unsupported compression method"); } @@ -71,23 +71,23 @@ orc::CompressionKind getORCCompression(FormatSettings::ORCCompression method) /// them back as the same ClickHouse type: `BINARY` and `STRING` both read as `String`, and a nested /// `UNION` reads as a `Variant`, which sorts its branches, while ORC keeps them positional. /// A field name is length-prefixed so a name containing `:` or `,` cannot forge another key. -String orcTypeDedupKey(const orc::Type & type) +String orcTypeDedupKey(const ORC::Type & type) { const auto kind = type.getKind(); - if (kind == orc::TypeKind::BINARY || kind == orc::TypeKind::STRING) + if (kind == ORC::TypeKind::BINARY || kind == ORC::TypeKind::STRING) return "string"; - if (kind == orc::TypeKind::DECIMAL) + if (kind == ORC::TypeKind::DECIMAL) return fmt::format("decimal({},{})", type.getPrecision(), type.getScale()); std::vector children; for (size_t i = 0; i < type.getSubtypeCount(); ++i) { auto child = orcTypeDedupKey(*type.getSubtype(i)); - if (kind == orc::TypeKind::STRUCT) + if (kind == ORC::TypeKind::STRUCT) child = fmt::format("{}:{}:{}", type.getFieldName(i).size(), type.getFieldName(i), child); children.push_back(std::move(child)); } - if (kind == orc::TypeKind::UNION) + if (kind == ORC::TypeKind::UNION) std::sort(children.begin(), children.end()); /// Every remaining kind the writer emits is a parameterless primitive mapping to a distinct /// ClickHouse type, so for those the kind alone is the key and there are no children. @@ -98,17 +98,6 @@ String orcTypeDedupKey(const orc::Type & type) ORCOutputStream::ORCOutputStream(WriteBuffer & out_) : out(out_) {} -uint64_t ORCOutputStream::getLength() const -{ - return out.count(); -} - -uint64_t ORCOutputStream::getNaturalWriteSize() const -{ - out.nextIfAtEnd(); - return out.available(); -} - void ORCOutputStream::write(const void* buf, size_t length) { out.write(static_cast(buf), length); @@ -124,7 +113,7 @@ ORCBlockOutputFormat::ORCBlockOutputFormat(WriteBuffer & out_, SharedHeader head data_types.push_back(recursiveRemoveLowCardinality(type)); } -std::unique_ptr ORCBlockOutputFormat::getORCType(const DataTypePtr & type, const String & column_path) +std::unique_ptr ORCBlockOutputFormat::getORCType(const DataTypePtr & type, const String & column_path) { /// Nullable(T) maps to the same ORC node as T; only the Iceberg `required` flag differs. /// A complex container (list/map/struct) is never wrapped in Nullable, so its Iceberg @@ -135,63 +124,63 @@ std::unique_ptr ORCBlockOutputFormat::getORCType(const DataTypePtr & required = !column_mapper->isIcebergOptionalPath(column_path); const DataTypePtr unwrapped = removeNullable(type); - std::unique_ptr result; + std::unique_ptr result; switch (unwrapped->getTypeId()) { case TypeIndex::UInt8: { if (isBool(unwrapped)) - result = orc::createPrimitiveType(orc::TypeKind::BOOLEAN); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::BOOLEAN); else - result = orc::createPrimitiveType(orc::TypeKind::BYTE); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::BYTE); break; } case TypeIndex::Enum8: [[fallthrough]]; case TypeIndex::Int8: { - result = orc::createPrimitiveType(orc::TypeKind::BYTE); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::BYTE); break; } case TypeIndex::Enum16: [[fallthrough]]; case TypeIndex::UInt16: [[fallthrough]]; case TypeIndex::Int16: { - result = orc::createPrimitiveType(orc::TypeKind::SHORT); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::SHORT); break; } case TypeIndex::UInt32: [[fallthrough]]; case TypeIndex::IPv4: [[fallthrough]]; case TypeIndex::Int32: { - result = orc::createPrimitiveType(orc::TypeKind::INT); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::INT); break; } case TypeIndex::UInt64: [[fallthrough]]; case TypeIndex::Int64: { - result = orc::createPrimitiveType(orc::TypeKind::LONG); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::LONG); break; } case TypeIndex::Float32: { - result = orc::createPrimitiveType(orc::TypeKind::FLOAT); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::FLOAT); break; } case TypeIndex::Float64: { - result = orc::createPrimitiveType(orc::TypeKind::DOUBLE); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::DOUBLE); break; } case TypeIndex::Date32: [[fallthrough]]; case TypeIndex::Date: { - result = orc::createPrimitiveType(orc::TypeKind::DATE); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::DATE); break; } case TypeIndex::DateTime: [[fallthrough]]; case TypeIndex::DateTime64: { - result = orc::createPrimitiveType(orc::TypeKind::TIMESTAMP); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::TIMESTAMP); break; } case TypeIndex::Int128: [[fallthrough]]; @@ -199,7 +188,7 @@ std::unique_ptr ORCBlockOutputFormat::getORCType(const DataTypePtr & case TypeIndex::Int256: [[fallthrough]]; case TypeIndex::UInt256: [[fallthrough]]; case TypeIndex::Decimal256: - result = orc::createPrimitiveType(orc::TypeKind::BINARY); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::BINARY); break; case TypeIndex::String: { @@ -208,7 +197,7 @@ std::unique_ptr ORCBlockOutputFormat::getORCType(const DataTypePtr & bool force_string = format_settings.orc.output_string_as_string; if (column_mapper && column_mapper->hasIcebergStringInfo()) force_string = column_mapper->isIcebergStringPath(column_path); - result = orc::createPrimitiveType(force_string ? orc::TypeKind::STRING : orc::TypeKind::BINARY); + result = ORC::Type::createPrimitiveType(force_string ? ORC::TypeKind::STRING : ORC::TypeKind::BINARY); break; } case TypeIndex::FixedString: @@ -216,38 +205,38 @@ std::unique_ptr ORCBlockOutputFormat::getORCType(const DataTypePtr & /// Iceberg `fixed[L]` (FixedString) must stay ORC `binary`, so for Iceberg writes /// force `binary` regardless of the setting; keep the setting-driven choice otherwise. if (format_settings.orc.output_string_as_string && !column_mapper) - result = orc::createPrimitiveType(orc::TypeKind::STRING); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::STRING); else - result = orc::createPrimitiveType(orc::TypeKind::BINARY); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::BINARY); break; } case TypeIndex::IPv6: { - result = orc::createPrimitiveType(orc::TypeKind::BINARY); + result = ORC::Type::createPrimitiveType(ORC::TypeKind::BINARY); break; } case TypeIndex::Array: { const auto * array_type = assert_cast(unwrapped.get()); - result = orc::createListType(getORCType(array_type->getNestedType(), Nested::concatenateName(column_path, "element"))); + result = ORC::Type::createListType(getORCType(array_type->getNestedType(), Nested::concatenateName(column_path, "element"))); break; } case TypeIndex::Decimal32: { const auto * decimal_type = assert_cast *>(unwrapped.get()); - result = orc::createDecimalType(decimal_type->getPrecision(), decimal_type->getScale()); + result = ORC::Type::createDecimalType(decimal_type->getPrecision(), decimal_type->getScale()); break; } case TypeIndex::Decimal64: { const auto * decimal_type = assert_cast *>(unwrapped.get()); - result = orc::createDecimalType(decimal_type->getPrecision(), decimal_type->getScale()); + result = ORC::Type::createDecimalType(decimal_type->getPrecision(), decimal_type->getScale()); break; } case TypeIndex::Decimal128: { const auto * decimal_type = assert_cast *>(unwrapped.get()); - result = orc::createDecimalType(decimal_type->getPrecision(), decimal_type->getScale()); + result = ORC::Type::createDecimalType(decimal_type->getPrecision(), decimal_type->getScale()); break; } case TypeIndex::Tuple: @@ -255,7 +244,7 @@ std::unique_ptr ORCBlockOutputFormat::getORCType(const DataTypePtr & const auto * tuple_type = assert_cast(unwrapped.get()); const auto & nested_names = tuple_type->getElementNames(); const auto & nested_types = tuple_type->getElements(); - auto struct_type = orc::createStructType(); + auto struct_type = ORC::Type::createStructType(); for (size_t i = 0; i < nested_types.size(); ++i) struct_type->addStructField(nested_names[i], getORCType(nested_types[i], Nested::concatenateName(column_path, nested_names[i]))); result = std::move(struct_type); @@ -264,7 +253,7 @@ std::unique_ptr ORCBlockOutputFormat::getORCType(const DataTypePtr & case TypeIndex::Map: { const auto * map_type = assert_cast(unwrapped.get()); - result = orc::createMapType( + result = ORC::Type::createMapType( getORCType(map_type->getKeyType(), Nested::concatenateName(column_path, "key")), getORCType(map_type->getValueType(), Nested::concatenateName(column_path, "value")) ); @@ -273,7 +262,7 @@ std::unique_ptr ORCBlockOutputFormat::getORCType(const DataTypePtr & case TypeIndex::Variant: { const auto & variant_type = assert_cast(*unwrapped); - auto union_type = orc::createUnionType(); + auto union_type = ORC::Type::createUnionType(); /// ORC keeps one physical stream per union branch and identifies a branch only by its /// ORC type, so two variants that map to the same ORC type (e.g. `Int32` and `UInt32`, /// both ORC `int`) would produce a union with duplicate branches, which the reader @@ -293,7 +282,7 @@ std::unique_ptr ORCBlockOutputFormat::getORCType(const DataTypePtr & "Type {} is not supported for ORC output format: variant {} is written as ORC type '{}' and variant {} as " "'{}', which are read back as the same type, and ORC unions with duplicate branch types cannot be read back", unwrapped->getName(), it->second.first, it->second.second, nested_type->getName(), child_type->toString()); - union_type->addUnionChild(std::move(child_type)); + union_type->addChild(std::move(child_type)); } result = std::move(union_type); break; @@ -318,406 +307,18 @@ std::unique_ptr ORCBlockOutputFormat::getORCType(const DataTypePtr & return result; } -template -void ORCBlockOutputFormat::writeNumbers( - orc::ColumnVectorBatch & orc_column, - const IColumn & column, - const PaddedPODArray * /*null_bytemap*/, - ConvertFunc convert) -{ - NumberVectorBatch & number_orc_column = dynamic_cast(orc_column); - const auto & number_column = assert_cast &>(column); - - number_orc_column.data.resize(number_column.size()); - for (size_t i = 0; i != number_column.size(); ++i) - number_orc_column.data[i] = convert(number_column.getElement(i)); -} - -template -void ORCBlockOutputFormat::writeDecimals( - orc::ColumnVectorBatch & orc_column, - const IColumn & column, - DataTypePtr & type, - const PaddedPODArray * /*null_bytemap*/, - ConvertFunc convert) -{ - DecimalVectorBatch & decimal_orc_column = dynamic_cast(orc_column); - const auto & decimal_column = assert_cast &>(column); - const auto * decimal_type = assert_cast *>(type.get()); - decimal_orc_column.precision = decimal_type->getPrecision(); - decimal_orc_column.scale = decimal_type->getScale(); - decimal_orc_column.resize(decimal_column.size()); - - decimal_orc_column.values.resize(decimal_column.size()); - for (size_t i = 0; i != decimal_column.size(); ++i) - decimal_orc_column.values[i] = convert(decimal_column.getElement(i).value); -} - -template -void ORCBlockOutputFormat::writeStrings( - orc::ColumnVectorBatch & orc_column, - const IColumn & column, - const PaddedPODArray * null_bytemap) -{ - orc::StringVectorBatch & string_orc_column = dynamic_cast(orc_column); - const auto & string_column = assert_cast(column); - - string_orc_column.data.resize(string_column.size()); - string_orc_column.length.resize(string_column.size()); - for (size_t i = 0; i != string_column.size(); ++i) - { - const std::string_view string = string_column.getDataAt(i); - string_orc_column.data[i] = (null_bytemap && (*null_bytemap)[i]) ? nullptr : const_cast(string.data()); - string_orc_column.length[i] = string.size(); - } -} - -template -void ORCBlockOutputFormat::writeDateTimes( - orc::ColumnVectorBatch & orc_column, - const IColumn & column, - const PaddedPODArray * /*null_bytemap*/, - GetSecondsFunc get_seconds, - GetNanosecondsFunc get_nanoseconds) -{ - orc::TimestampVectorBatch & timestamp_orc_column = dynamic_cast(orc_column); - const auto & timestamp_column = assert_cast(column); - - timestamp_orc_column.data.resize(timestamp_column.size()); - timestamp_orc_column.nanoseconds.resize(timestamp_column.size()); - for (size_t i = 0; i != timestamp_column.size(); ++i) - { - timestamp_orc_column.data[i] = static_cast(get_seconds(timestamp_column.getElement(i))); - timestamp_orc_column.nanoseconds[i] = static_cast(get_nanoseconds(timestamp_column.getElement(i))); - } -} - -void ORCBlockOutputFormat::writeColumn( - orc::ColumnVectorBatch & orc_column, - const IColumn & column, - DataTypePtr & type, - const PaddedPODArray * null_bytemap) -{ - size_t rows = column.size(); - orc_column.resize(rows); - orc_column.numElements = rows; - - /// Calculate orc_column.hasNulls - if (null_bytemap) - orc_column.hasNulls = !memoryIsZero(null_bytemap->data(), 0, null_bytemap->size()); - else - orc_column.hasNulls = false; - - /// Fill orc_column.notNull - if (orc_column.hasNulls) - { - for (size_t i = 0; i < rows; ++i) - orc_column.notNull[i] = !(*null_bytemap)[i]; - } - else - { - for (size_t i = 0; i < rows; ++i) - orc_column.notNull[i] = 1; - } - - /// ORC doesn't have unsigned types, so cast everything to signed and sign-extend to Int64 to - /// make the ORC library calculate min and max correctly. - switch (type->getTypeId()) - { - case TypeIndex::Enum8: [[fallthrough]]; - case TypeIndex::Int8: - { - /// Note: Explicit cast to avoid clang-tidy error: 'signed char' to 'long' conversion; consider casting to 'unsigned char' first. - writeNumbers(orc_column, column, null_bytemap, [](const Int8 & value){ return Int64(Int8(value)); }); - break; - } - case TypeIndex::UInt8: - { - writeNumbers(orc_column, column, null_bytemap, [](const UInt8 & value){ return Int64(Int8(value)); }); - break; - } - case TypeIndex::Enum16: [[fallthrough]]; - case TypeIndex::Int16: - { - writeNumbers(orc_column, column, null_bytemap, [](const Int16 & value){ return value; }); - break; - } - case TypeIndex::Date: [[fallthrough]]; - case TypeIndex::UInt16: - { - writeNumbers(orc_column, column, null_bytemap, [](const UInt16 & value){ return Int64(Int16(value)); }); - break; - } - case TypeIndex::Date32: [[fallthrough]]; - case TypeIndex::Int32: - { - writeNumbers(orc_column, column, null_bytemap, [](const Int32 & value){ return value; }); - break; - } - case TypeIndex::UInt32: - { - writeNumbers(orc_column, column, null_bytemap, [](const UInt32 & value){ return Int64(Int32(value)); }); - break; - } - case TypeIndex::IPv4: - { - writeNumbers(orc_column, column, null_bytemap, [](const IPv4 & value){ return Int64(Int32(value.toUnderType())); }); - break; - } - case TypeIndex::Int64: - { - writeNumbers(orc_column, column, null_bytemap, [](const Int64 & value){ return value; }); - break; - } - case TypeIndex::UInt64: - { - writeNumbers(orc_column, column, null_bytemap, [](const UInt64 & value){ return value; }); - break; - } - case TypeIndex::Int128: - { - writeStrings(orc_column, column, null_bytemap); - break; - } - case TypeIndex::UInt128: - { - writeStrings(orc_column, column, null_bytemap); - break; - } - case TypeIndex::Int256: - { - writeStrings(orc_column, column, null_bytemap); - break; - } - case TypeIndex::UInt256: - { - writeStrings(orc_column, column, null_bytemap); - break; - } - case TypeIndex::Float32: - { - writeNumbers(orc_column, column, null_bytemap, [](const Float32 & value){ return static_cast(value); }); - break; - } - case TypeIndex::Float64: - { - writeNumbers(orc_column, column, null_bytemap, [](const Float64 & value){ return value; }); - break; - } - case TypeIndex::FixedString: - { - writeStrings(orc_column, column, null_bytemap); - break; - } - case TypeIndex::String: - { - writeStrings(orc_column, column, null_bytemap); - break; - } - case TypeIndex::IPv6: - { - writeStrings(orc_column, column, null_bytemap); - break; - } - case TypeIndex::DateTime: - { - writeDateTimes( - orc_column, - column, null_bytemap, - [](UInt32 value){ return value; }, - [](UInt32){ return 0; }); - break; - } - case TypeIndex::DateTime64: - { - const auto * timestamp_type = assert_cast(type.get()); - UInt32 scale = timestamp_type->getScale(); - writeDateTimes( - orc_column, - column, - null_bytemap, - [scale](Int64 value) { return value / Int64(std::pow(10, scale)); }, - [scale](Int64 value) { return (value % Int64(std::pow(10, scale))) * Int64(std::pow(10, 9 - scale)); }); - break; - } - case TypeIndex::Decimal32: { - writeDecimals( - orc_column, - column, - type, - null_bytemap, - [](Int32 value){ return value; }); - break; - } - case TypeIndex::Decimal64: - { - writeDecimals( - orc_column, - column, - type, - null_bytemap, - [](Int64 value){ return value; }); - break; - } - case TypeIndex::Decimal128: - { - writeDecimals( - orc_column, - column, - type, - null_bytemap, - [](Int128 value){ return orc::Int128(static_cast(value >> 64), static_cast((value << 64) >> 64)); }); - break; - } - case TypeIndex::Decimal256: - { - writeStrings>(orc_column, column, null_bytemap); - break; - } - case TypeIndex::Nullable: - { - chassert(!null_bytemap); - const auto & nullable_column = assert_cast(column); - const PaddedPODArray & new_null_bytemap = assert_cast &>(*nullable_column.getNullMapColumnPtr()).getData(); - auto nested_type = removeNullable(type); - writeColumn(orc_column, nullable_column.getNestedColumn(), nested_type, &new_null_bytemap); - break; - } - case TypeIndex::Array: - { - orc::ListVectorBatch & list_orc_column = dynamic_cast(orc_column); - const auto & list_column = assert_cast(column); - auto nested_type = assert_cast(*type).getNestedType(); - const ColumnArray::Offsets & offsets = list_column.getOffsets(); - - size_t column_size = list_column.size(); - list_orc_column.offsets.resize(column_size + 1); - - /// The length of list i in ListVectorBatch is offsets[i+1] - offsets[i]. - list_orc_column.offsets[0] = 0; - for (size_t i = 0; i != column_size; ++i) - list_orc_column.offsets[i + 1] = offsets[i]; - - orc::ColumnVectorBatch & nested_orc_column = *list_orc_column.elements; - writeColumn(nested_orc_column, list_column.getData(), nested_type, nullptr); - break; - } - case TypeIndex::Tuple: - { - orc::StructVectorBatch & struct_orc_column = dynamic_cast(orc_column); - const auto & tuple_column = assert_cast(column); - auto nested_types = assert_cast(type.get())->getElements(); - for (size_t i = 0; i != tuple_column.tupleSize(); ++i) - { - if (null_bytemap && nested_types[i]->isNullable()) - { - /// When both the struct and the element are nullable, we need to merge the two null bitmaps: - /// a child value is null if either the struct row is null OR the element itself is null. - const auto & nullable_col = assert_cast(tuple_column.getColumn(i)); - const auto & element_null_map = nullable_col.getNullMapData(); - PaddedPODArray merged_null_map(element_null_map.size()); - for (size_t j = 0; j < element_null_map.size(); ++j) - merged_null_map[j] = element_null_map[j] | (*null_bytemap)[j]; - - auto nested_type = removeNullable(nested_types[i]); - writeColumn(*struct_orc_column.fields[i], nullable_col.getNestedColumn(), nested_type, &merged_null_map); - } - else - { - /// Propagate the struct-level null_bytemap to children so the ORC library correctly handles - /// null struct rows (child values at null positions must also be marked null). - writeColumn(*struct_orc_column.fields[i], tuple_column.getColumn(i), nested_types[i], null_bytemap); - } - } - break; - } - case TypeIndex::Map: - { - orc::MapVectorBatch & map_orc_column = dynamic_cast(orc_column); - const auto & list_column = assert_cast(column).getNestedColumn(); - const auto & map_type = assert_cast(*type); - const ColumnArray::Offsets & offsets = list_column.getOffsets(); - - size_t column_size = list_column.size(); - - map_orc_column.offsets.resize(column_size + 1); - /// The length of list i in ListVectorBatch is offsets[i+1] - offsets[i]. - map_orc_column.offsets[0] = 0; - for (size_t i = 0; i != column_size; ++i) - map_orc_column.offsets[i + 1] = offsets[i]; - - const auto nested_columns = assert_cast(list_column.getDataPtr().get())->getColumns(); - - orc::ColumnVectorBatch & keys_orc_column = *map_orc_column.keys; - auto key_type = map_type.getKeyType(); - writeColumn(keys_orc_column, *nested_columns[0], key_type, nullptr); - - orc::ColumnVectorBatch & values_orc_column = *map_orc_column.elements; - auto value_type = map_type.getValueType(); - writeColumn(values_orc_column, *nested_columns[1], value_type, nullptr); - break; - } - case TypeIndex::Variant: - { - auto & union_orc_column = dynamic_cast(orc_column); - const auto & variant_column = assert_cast(column); - const auto & variant_types = assert_cast(*type).getVariants(); - - /// The ORC union branches are created in the same order as the Variant's variants (see - /// getORCType), so a global discriminator is exactly the ORC tag. A Variant NULL row (the - /// NULL discriminator) is written as a NULL union row. - bool has_nulls = orc_column.hasNulls; - for (size_t i = 0; i < rows; ++i) - { - auto global_discr = variant_column.globalDiscriminatorAt(i); - if (global_discr == ColumnVariant::NULL_DISCRIMINATOR || (null_bytemap && (*null_bytemap)[i])) - { - union_orc_column.notNull[i] = 0; - union_orc_column.tags[i] = 0; - union_orc_column.offsets[i] = 0; - has_nulls = true; - } - else - { - union_orc_column.tags[i] = static_cast(global_discr); - union_orc_column.offsets[i] = variant_column.offsetAt(i); - } - } - union_orc_column.hasNulls = has_nulls; - - /// Each branch column already contains exactly the rows routed to it (in offset order). - for (size_t i = 0; i < variant_types.size(); ++i) - { - auto nested_type = variant_types[i]; - writeColumn(*union_orc_column.children[i], variant_column.getVariantByGlobalDiscriminator(i), nested_type, nullptr); - } - break; - } - default: - throw Exception(ErrorCodes::ILLEGAL_COLUMN, "Type {} is not supported for ORC output format", type->getName()); - } -} - void ORCBlockOutputFormat::consume(Chunk chunk) { if (!writer) prepareWriter(); - size_t columns_num = chunk.getNumColumns(); size_t rows_num = chunk.getNumRows(); - - std::unique_ptr batch = writer->createRowBatch(chunk.getNumRows()); - orc::StructVectorBatch & root = dynamic_cast(*batch); - auto columns = chunk.detachColumns(); for (auto & column : columns) column = recursiveRemoveLowCardinality(column); - for (size_t i = 0; i != columns_num; ++i) - writeColumn(*root.fields[i], *columns[i], data_types[i], nullptr); - - root.numElements = rows_num; - writer->add(*batch); + /// The values are encoded directly from the columns. + writer->add(columns, rows_num); } void ORCBlockOutputFormat::finalizeImpl() @@ -736,20 +337,19 @@ void ORCBlockOutputFormat::resetFormatterImpl() void ORCBlockOutputFormat::prepareWriter() { const Block & header = getPort(PortKind::Main).getHeader(); - schema = orc::createStructType(); - options.setMemoryPool(&getORCMemoryPool()); - options.setCompression(getORCCompression(format_settings.orc.output_compression_method)); - options.setRowIndexStride(format_settings.orc.output_row_index_stride); - options.setDictionaryKeySizeThreshold(format_settings.orc.output_dictionary_key_size_threshold); - options.setCompressionBlockSize(format_settings.orc.output_compression_block_size); - options.setTimezoneName(format_settings.orc.writer_time_zone_name); + schema = ORC::Type::createStructType(); + options.compression = getORCCompression(format_settings.orc.output_compression_method); + options.row_index_stride = format_settings.orc.output_row_index_stride; + options.dictionary_key_size_threshold = format_settings.orc.output_dictionary_key_size_threshold; + options.compression_block_size = format_settings.orc.output_compression_block_size; + options.timezone_name = format_settings.orc.writer_time_zone_name; size_t columns_count = header.columns(); for (size_t i = 0; i != columns_count; ++i) { const auto & column_name = header.safeGetByPosition(i).name; schema->addStructField(column_name, getORCType(recursiveRemoveLowCardinality(data_types[i]), column_name)); } - writer = orc::createWriter(*schema, &output_stream, options); + writer = std::make_unique(*schema, data_types, output_stream, options); } void registerOutputFormatORC(FormatFactory & factory); diff --git a/src/Processors/Formats/Impl/ORCBlockOutputFormat.h b/src/Processors/Formats/Impl/ORCBlockOutputFormat.h index 10832fbbcbb6b..c7cac88d52cc8 100644 --- a/src/Processors/Formats/Impl/ORCBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/ORCBlockOutputFormat.h @@ -9,7 +9,7 @@ #include #include #include -#include +#include namespace DB @@ -21,22 +21,15 @@ using DataTypes = VectorWithMemoryTracking; class WriteBuffer; -/// orc::Writer writes only in orc::OutputStream -class ORCOutputStream : public orc::OutputStream +class ORCOutputStream : public ORC::OutputStream { public: explicit ORCOutputStream(WriteBuffer & out_); - uint64_t getLength() const override; - uint64_t getNaturalWriteSize() const override; void write(const void * buf, size_t length) override; - void close() override {} - const std::string& getName() const override { return name; } - private: WriteBuffer & out; - std::string name = "ORCOutputStream"; }; @@ -54,37 +47,16 @@ class ORCBlockOutputFormat final : public IOutputFormat /// For Iceberg writes, attaches the field-id `iceberg.id` type attribute so readers project /// by id. `column_path` is the dotted name (`t.x`, `arr.element`) used to look the id up. - std::unique_ptr getORCType(const DataTypePtr & type, const String & column_path); - - /// ConvertFunc is needed for type UInt8, because firstly UInt8 (char8_t) must be - /// converted to unsigned char (bugprone-signed-char-misuse in clang). - template - void writeNumbers(orc::ColumnVectorBatch & orc_column, const IColumn & column, const PaddedPODArray * null_bytemap, ConvertFunc convert); - - /// ConvertFunc is needed to convert ClickHouse Int128 to ORC Int128. - template - void writeDecimals(orc::ColumnVectorBatch & orc_column, const IColumn & column, DataTypePtr & type, - const PaddedPODArray * null_bytemap, ConvertFunc convert); - - template - void writeStrings(orc::ColumnVectorBatch & orc_column, const IColumn & column, const PaddedPODArray * null_bytemap); - - /// ORC column TimestampVectorBatch stores only seconds and nanoseconds, - /// GetSecondsFunc and GetNanosecondsFunc are needed to extract them from DateTime type. - template - void writeDateTimes(orc::ColumnVectorBatch & orc_column, const IColumn & column, const PaddedPODArray * null_bytemap, - GetSecondsFunc get_seconds, GetNanosecondsFunc get_nanoseconds); - - void writeColumn(orc::ColumnVectorBatch & orc_column, const IColumn & column, DataTypePtr & type, const PaddedPODArray * null_bytemap); + std::unique_ptr getORCType(const DataTypePtr & type, const String & column_path); void prepareWriter(); const FormatSettings format_settings; ORCOutputStream output_stream; DataTypes data_types; - std::unique_ptr writer; - std::unique_ptr schema; - orc::WriterOptions options; + std::unique_ptr writer; + std::unique_ptr schema; + ORC::WriterOptions options; /// Non-null only for Iceberg writes; maps dotted column name -> Iceberg field-id. ColumnMapperPtr column_mapper; }; diff --git a/src/Processors/Formats/Impl/Parquet/Decoding.cpp b/src/Processors/Formats/Impl/Parquet/Decoding.cpp index ca7b8f5057a35..87924b120ed70 100644 --- a/src/Processors/Formats/Impl/Parquet/Decoding.cpp +++ b/src/Processors/Formats/Impl/Parquet/Decoding.cpp @@ -1961,6 +1961,8 @@ Int96Converter::Int96Converter() void Int96Converter::convertColumn(std::span data, size_t num_values, IColumn & col) const { + const bool saturate = overflow_behavior == FormatSettings::DateTimeOverflowBehavior::Saturate; + std::span to_span = col.insertRawUninitialized(num_values); Int64 * to = reinterpret_cast(to_span.data()); for (size_t i = 0; i < num_values; ++i) @@ -1985,16 +1987,17 @@ void Int96Converter::convertColumn(std::span data, size_t num_values /// This interpretation is consistent with the arrow reader code (DecodeInt96Timestamp in /// arrow/cpp/src/parquet/types.h) and with a test file written by spark /// (tests/queries/0_stateless/02998_native_parquet_reader.sh). - bool overflow = false; - Int64 x = 0; - overflow |= common::subOverflow(julian_day, static_cast(2440588l), x); // unix day number - overflow |= common::mulOverflow(x, day_nanos, x); // unix nanoseconds - overflow |= common::addOverflow(x, nanos, x); - - if (overflow) - throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "INT96 timestamp out of range: julian day {}, time of day {} ns", julian_day, nanos); - to[i] = x; + /// julian_day comes from Int32, so neither the subtraction nor the product can overflow Int128. + const Int128 total = Int128(julian_day - 2440588l) * day_nanos + nanos; + if (total < std::numeric_limits::min() || total > std::numeric_limits::max()) + { + if (!saturate) + throw Exception(ErrorCodes::VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE, "INT96 timestamp out of range: julian day {}, time of day {} ns", julian_day, nanos); + to[i] = julian_day < 2440588l ? std::numeric_limits::min() : std::numeric_limits::max(); + } + else + to[i] = static_cast(total); } } diff --git a/src/Processors/Formats/Impl/Parquet/Decoding.h b/src/Processors/Formats/Impl/Parquet/Decoding.h index e0533fd295073..88f0cc76411d1 100644 --- a/src/Processors/Formats/Impl/Parquet/Decoding.h +++ b/src/Processors/Formats/Impl/Parquet/Decoding.h @@ -397,6 +397,9 @@ extern template struct BigEndianDecimalStringConverter; struct Int96Converter : public FixedSizeConverter { + /// A value that does not fit into Int64 nanoseconds is clamped to the `DateTime64(9)` range with Saturate, otherwise throws. + FormatSettings::DateTimeOverflowBehavior overflow_behavior = FormatSettings::DateTimeOverflowBehavior::Ignore; + Int96Converter(); void convertColumn(std::span data, size_t num_values, IColumn & col) const override; diff --git a/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp b/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp index bf5cd131f135e..d764b100b5c7e 100644 --- a/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp +++ b/src/Processors/Formats/Impl/Parquet/SchemaConverter.cpp @@ -1530,7 +1530,9 @@ void SchemaConverter::processPrimitiveColumn( case parq::Type::INT96: { out_inferred_type = std::make_shared(9, "UTC"); - out_decoder.fixed_size_converter = std::make_shared(); + auto converter = std::make_shared(); + converter->overflow_behavior = options.format.date_time_overflow_behavior; + out_decoder.fixed_size_converter = std::move(converter); /// (Leaving allow_stats == false because INT96 sort order is undefined.) return; } diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp index 28c3c0816d54a..b01ca4e03015a 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.cpp @@ -7,16 +7,12 @@ #include #include #include +#include #include #include +#include -namespace CurrentMetrics -{ - extern const Metric ParquetEncoderThreads; - extern const Metric ParquetEncoderThreadsActive; - extern const Metric ParquetEncoderThreadsScheduled; -} namespace DB { @@ -32,11 +28,10 @@ ParquetBlockOutputFormat::ParquetBlockOutputFormat(WriteBuffer & out_, SharedHea : IOutputFormat(header_, out_), format_settings{format_settings_}, format_filter_info(format_filter_info_) { if (format_settings.parquet.parallel_encoding && format_settings.max_threads > 1) - pool = std::make_unique( - CurrentMetrics::ParquetEncoderThreads, - CurrentMetrics::ParquetEncoderThreadsActive, - CurrentMetrics::ParquetEncoderThreadsScheduled, - format_settings.max_threads); + { + getParquetEncoderThreadPool().initializeWithDefaultSettingsIfNotInitialized(); + pool = &getParquetEncoderThreadPool().get(); + } using C = FormatSettings::ParquetCompression; switch (format_settings.parquet.output_compression_method) @@ -82,10 +77,14 @@ ParquetBlockOutputFormat::ParquetBlockOutputFormat(WriteBuffer & out_, SharedHea ParquetBlockOutputFormat::~ParquetBlockOutputFormat() { if (pool) - { - is_stopped = true; - pool->wait(); - } + stopThreads(); +} + +void ParquetBlockOutputFormat::stopThreads() +{ + is_stopped = true; + std::unique_lock lock(mutex); + condvar.wait(lock, [this] { return threads_running == 0; }); } void ParquetBlockOutputFormat::consume(Chunk chunk) @@ -263,8 +262,7 @@ void ParquetBlockOutputFormat::resetFormatterImpl() { if (pool) { - is_stopped = true; - pool->wait(); + stopThreads(); is_stopped = false; } @@ -413,17 +411,16 @@ void ParquetBlockOutputFormat::startMoreThreadsIfNeeded(const std::unique_lock switcher; + switcher.emplace(thread_group, ThreadName::PARQUET_ENCODER); + std::unique_lock lock(mutex); while (true) @@ -457,7 +457,12 @@ void ParquetBlockOutputFormat::threadFunction() { /// The check and the decrement need to be in the same critical section, to make sure /// we never get stuck with tasks but no threads. + /// Leave the query's thread group before stopThreads() can see the decrement. + switcher.reset(); + thread_group.reset(); --threads_running; + if (threads_running == 0) + condvar.notify_all(); return; } diff --git a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h index c38b1e5d0ce7f..cc7dd4b5128f4 100644 --- a/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h +++ b/src/Processors/Formats/Impl/ParquetBlockOutputFormat.h @@ -12,6 +12,9 @@ namespace DB { +class ThreadGroup; +using ThreadGroupPtr = std::shared_ptr; + class ParquetBlockOutputFormat final : public IOutputFormat { public: @@ -104,8 +107,9 @@ class ParquetBlockOutputFormat final : public IOutputFormat void writeRowGroupInOneThread(Chunk chunk); void writeRowGroupInParallel(std::vector chunks); - void threadFunction(); + void threadFunction(ThreadGroupPtr & thread_group); void startMoreThreadsIfNeeded(const std::unique_lock & lock); + void stopThreads(); /// Called in single-threaded fashion. Writes to the file. void reapCompletedRowGroups(std::unique_lock & lock); @@ -126,13 +130,15 @@ class ParquetBlockOutputFormat final : public IOutputFormat size_t base_offset = 0; // initial out.count(), just for assert std::mutex mutex; - std::condition_variable condvar; // wakes up consume() - std::unique_ptr pool; + std::condition_variable condvar; // wakes up consume() and stopThreads() + /// Shared by all Parquet writers of the process (`getParquetEncoderThreadPool`). + ThreadPool * pool = nullptr; std::atomic_bool is_stopped{false}; std::exception_ptr background_exception = nullptr; /// Invariant: if there's at least one task then there's at least one thread. + /// Counts the jobs scheduled on `pool` until they exit, queued ones included. size_t threads_running = 0; std::atomic bytes_in_flight{0}; diff --git a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp index 8d8cbabdd35f0..ad3bebffb872e 100644 --- a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.cpp @@ -64,9 +64,16 @@ ParquetV3BlockInputFormat::ParquetV3BlockInputFormat( format_filter_info = std::make_shared(); } -void ParquetV3BlockInputFormat::initializeIfNeeded() +void ParquetV3BlockInputFormat::prefetch() { - if (!reader) + if (need_only_count) + return; + readFileFooterIfNeeded(); +} + +void ParquetV3BlockInputFormat::readFileFooterIfNeeded() +{ + if (!reader && !reader_with_footer) { format_filter_info->initKeyConditionOnce(getPort().getHeader()); parser_shared_resources->initOnce([&] @@ -95,11 +102,24 @@ void ParquetV3BlockInputFormat::initializeIfNeeded() parser_shared_resources->opaque = ext; }); + auto new_reader = std::make_unique(); + new_reader->reader.prefetcher.init(in, read_options, parser_shared_resources); + new_reader->reader.file_metadata = getFileMetadata(new_reader->reader.prefetcher); + + std::lock_guard lock(reader_mutex); + reader_with_footer = std::move(new_reader); + } +} + +void ParquetV3BlockInputFormat::initializeIfNeeded() +{ + if (!reader) + { + readFileFooterIfNeeded(); + { std::lock_guard lock(reader_mutex); - reader.emplace(); - reader->reader.prefetcher.init(in, read_options, parser_shared_resources); - reader->reader.file_metadata = getFileMetadata(reader->reader.prefetcher); + reader = std::move(reader_with_footer); reader->reader.init(read_options, getPort().getHeader(), format_filter_info); reader->init(parser_shared_resources, buckets_to_read ? std::optional(buckets_to_read->row_group_ids) : std::nullopt); } @@ -203,6 +223,7 @@ void ParquetV3BlockInputFormat::resetReadBuffer() std::lock_guard lock(reader_mutex); if (reader) reader->shutdownTasks(); + reader_with_footer.reset(); } IInputFormat::resetReadBuffer(); } @@ -212,6 +233,7 @@ void ParquetV3BlockInputFormat::resetParser() { std::lock_guard lock(reader_mutex); reader.reset(); + reader_with_footer.reset(); } previous_block_missing_values.clear(); IInputFormat::resetParser(); diff --git a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.h b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.h index 0ee6d2226a554..b0767cd75886c 100644 --- a/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.h +++ b/src/Processors/Formats/Impl/ParquetV3BlockInputFormat.h @@ -68,6 +68,8 @@ class ParquetV3BlockInputFormat final : public IInputFormat private: Chunk read() override; + void prefetch() override; + void onCancel() noexcept override; const FormatSettings format_settings; @@ -77,17 +79,20 @@ class ParquetV3BlockInputFormat final : public IInputFormat ParquetMetadataCachePtr metadata_cache; const std::optional object_with_metadata; - /// (This mutex is not important. It protects `reader.emplace` in a weird case where onCancel() + /// (This mutex is not important. It protects setting `reader` in a weird case where onCancel() /// may be called in parallel with first read(). ReadManager itself is thread safe for that, - /// but initializing vs checking the std::optional would race without this mutex.) + /// but initializing vs checking the pointer would race without this mutex.) std::mutex reader_mutex; - std::optional reader; + std::unique_ptr reader; + /// Has the file footer, but is not initialized: initializing it starts reading and decoding the column chunks. + std::unique_ptr reader_with_footer; bool reported_count = false; // if need_only_count BlockMissingValues previous_block_missing_values; size_t previous_approx_bytes_read_for_chunk = 0; + void readFileFooterIfNeeded(); void initializeIfNeeded(); std::shared_ptr buckets_to_read; diff --git a/src/Processors/Formats/Impl/RawBLOBRowOutputFormat.cpp b/src/Processors/Formats/Impl/RawBLOBRowOutputFormat.cpp index 7d007ac8d451b..5bcbec3c45b6f 100644 --- a/src/Processors/Formats/Impl/RawBLOBRowOutputFormat.cpp +++ b/src/Processors/Formats/Impl/RawBLOBRowOutputFormat.cpp @@ -1,4 +1,6 @@ +#include #include +#include #include #include #include @@ -10,14 +12,24 @@ namespace DB RawBLOBRowOutputFormat::RawBLOBRowOutputFormat( WriteBuffer & out_, - SharedHeader header_) + SharedHeader header_, + const FormatSettings & format_settings_) : IRowOutputFormat(header_, out_) + , format_settings(format_settings_) { } -void RawBLOBRowOutputFormat::writeField(const IColumn & column, const ISerialization &, size_t row_num) +void RawBLOBRowOutputFormat::writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) { + /// `getDataAt` of `ColumnAggregateFunction` returns the bytes of the pointer to the state, not the state itself. + /// Write the serialized state, the same as its conversion to `String`. + if (typeid_cast(&column)) + { + serialization.serializeBinary(column, row_num, out, format_settings); + return; + } + if (!column.isNullAt(row_num)) { auto value = column.getDataAt(row_num); @@ -32,10 +44,10 @@ void registerOutputFormatRawBLOB(FormatFactory & factory) factory.registerOutputFormat("RawBLOB", []( WriteBuffer & buf, const Block & sample, - const FormatSettings &, + const FormatSettings & settings, FormatFilterInfoPtr /*format_filter_info*/) { - return std::make_shared(buf, std::make_shared(sample)); + return std::make_shared(buf, std::make_shared(sample), settings); }); /// The output is a verbatim copy of the column bytes, which are not guaranteed to be valid UTF-8 text. diff --git a/src/Processors/Formats/Impl/RawBLOBRowOutputFormat.h b/src/Processors/Formats/Impl/RawBLOBRowOutputFormat.h index e2afa209faaab..9418d3c33028c 100644 --- a/src/Processors/Formats/Impl/RawBLOBRowOutputFormat.h +++ b/src/Processors/Formats/Impl/RawBLOBRowOutputFormat.h @@ -1,5 +1,6 @@ #pragma once +#include #include @@ -28,12 +29,15 @@ class RawBLOBRowOutputFormat final : public IRowOutputFormat public: RawBLOBRowOutputFormat( WriteBuffer & out_, - SharedHeader header_); + SharedHeader header_, + const FormatSettings & format_settings_); String getName() const override { return "RawBLOBRowOutputFormat"; } private: - void writeField(const IColumn & column, const ISerialization &, size_t row_num) override; + void writeField(const IColumn & column, const ISerialization & serialization, size_t row_num) override; + + const FormatSettings format_settings; }; } diff --git a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp index ee68091b256df..c7cf243021544 100644 --- a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp +++ b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.cpp @@ -78,7 +78,7 @@ namespace Setting extern const SettingsBool query_plan_push_down_limit; extern const SettingsBool query_plan_push_limit_by_into_sort; extern const SettingsBool query_plan_top_k_through_join; - extern const SettingsBool query_plan_read_in_order_through_join; + extern const SettingsBoolForce query_plan_read_in_order_through_join; extern const SettingsBool optimize_aggregation_in_order_limit; extern const SettingsBool query_plan_read_in_order; extern const SettingsBool query_plan_remove_redundant_distinct; @@ -247,7 +247,8 @@ QueryPlanOptimizationSettings::QueryPlanOptimizationSettings( && from[Setting::query_plan_optimize_count_from_text_index] && from[Setting::optimize_trivial_count_query]; enable_full_text_index = from[Setting::enable_full_text_index]; - read_in_order_through_join = from[Setting::query_plan_read_in_order_through_join]; + const auto & through_join = from[Setting::query_plan_read_in_order_through_join]; + read_in_order_through_join = through_join.is_keyword ? 2 : through_join.base.value; /// In-memory buffer for correlated subqueries uses a non-serializable ChunkBuffer, /// incompatible with distributed execution. When make_distributed_plan is enabled, /// always use the materialization path (materializeQueryPlanReferences) instead. diff --git a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h index 7923e0cced7db..fafa82d0719b7 100644 --- a/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h +++ b/src/Processors/QueryPlan/Optimizations/QueryPlanOptimizationSettings.h @@ -106,7 +106,8 @@ struct QueryPlanOptimizationSettings bool optimize_projection; bool use_query_condition_cache; bool use_query_condition_cache_for_top_k; - bool read_in_order_through_join; + /// 0 - off, 1 - on, 2 - on even when the parts have no virtual rows (`force`). + UInt64 read_in_order_through_join; bool optimize_aggregation_in_order_limit; bool correlated_subqueries_use_in_memory_buffer; bool push_limit_by_into_sort; diff --git a/src/Processors/QueryPlan/Optimizations/SipHashingWriteBuffer.h b/src/Processors/QueryPlan/Optimizations/SipHashingWriteBuffer.h new file mode 100644 index 0000000000000..66fe4c0c15fac --- /dev/null +++ b/src/Processors/QueryPlan/Optimizations/SipHashingWriteBuffer.h @@ -0,0 +1,36 @@ +#pragma once + +#include +#include +#include + +namespace DB +{ + +/// Feeds everything written to it into a `SipHash` over a fixed-size window and discards the bytes, +/// so a stream that is only ever hash input never has to be held in memory to be hashed. +/// +/// `SipHash::update` carries state, including any sub-word remainder, across calls, so hashing a +/// stream in consecutive chunks gives the same 64-bit value as hashing one contiguous buffer holding +/// all of it: the window size is not observable in the result. +class SipHashingWriteBuffer final : public BufferWithOwnMemory +{ +public: + /// Any size is correct: `WriteBuffer` callers either loop over `available()` or check it before + /// writing into the buffer directly. + static constexpr size_t window_bytes = 32768; + + explicit SipHashingWriteBuffer(SipHash & hash_) : BufferWithOwnMemory(window_bytes), hash(hash_) { } + + /// This buffer may be destroyed with a stream still half-written: nothing reads the discarded + /// bytes, so abandoning one costs nothing and callers need not finalize. `~WriteBuffer` asserts + /// against being left neither finalized nor canceled, so cancel here. + ~SipHashingWriteBuffer() override { cancel(); } + +private: + void nextImpl() override { hash.update(working_buffer.begin(), offset()); } + + SipHash & hash; +}; + +} diff --git a/src/Processors/QueryPlan/Optimizations/calculateHashTableCacheKeys.cpp b/src/Processors/QueryPlan/Optimizations/calculateHashTableCacheKeys.cpp index 164c366fca56a..aaf7c22053231 100644 --- a/src/Processors/QueryPlan/Optimizations/calculateHashTableCacheKeys.cpp +++ b/src/Processors/QueryPlan/Optimizations/calculateHashTableCacheKeys.cpp @@ -11,6 +11,7 @@ #include #include #include +#include #include #include #include @@ -95,7 +96,11 @@ UInt64 calculateHashFromStep(const ITransformingStep & transform) && sameByteLayout(*transform.getOutputHeader(), *transform.getInputHeaders().front())) return 0; - WriteBufferFromOwnString wbuf; + /// This serialized form is only ever hash input - nothing reads the bytes back - so it is + /// hashed as it is produced rather than accumulated. A step can carry a whole constant-folded + /// literal here, which `serializeConstant` writes out in full. + SipHash hash; + SipHashingWriteBuffer wbuf(hash); SerializedSetsRegistry registry; registry.for_cache_key = true; /// The bytes are hashed in-process by the same binary that wrote them and never reach another @@ -110,8 +115,7 @@ UInt64 calculateHashFromStep(const ITransformingStep & transform) if (transform.isSerializable()) transform.serialize(ctx); - SipHash hash; - hash.update(wbuf.str()); + wbuf.finalize(); return hash.get64(); } diff --git a/src/Processors/QueryPlan/Optimizations/optimizeJoin.cpp b/src/Processors/QueryPlan/Optimizations/optimizeJoin.cpp index 6820bf9b3593d..2ca1a370b6f91 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeJoin.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeJoin.cpp @@ -17,6 +17,7 @@ #include #include +#include #include #include #include @@ -325,6 +326,18 @@ struct RuntimeHashStatisticsContext } }; +/// The sizes of the arrays are unknown, so an array join is assumed to produce this many rows per input row. +constexpr Float64 array_join_rows_multiplier = 100; + +static UInt64 toUInt64Saturating(Float64 estimate) +{ + if (!(estimate > 0)) + return 0; + if (estimate >= static_cast(std::numeric_limits::max())) + return std::numeric_limits::max(); + return static_cast(estimate); +} + static RelationStats estimateAggregatingStepStats(const AggregatingStep & aggregating_step, const RelationStats & input_stats) { const auto & aggregator_params = aggregating_step.getAggregatorParameters(); @@ -508,7 +521,32 @@ RelationStats estimateReadRowsCount(QueryPlan::Node & node, const ActionsDAG::No return estimated; } - if (const auto * expression_step = typeid_cast(step); expression_step && !expression_step->getExpression().hasArrayJoin()) + const auto * expression_step = typeid_cast(step); + const auto * array_join_step = typeid_cast(step); + if (array_join_step || (expression_step && expression_step->getExpression().hasArrayJoin())) + { + /// A filter above the array join can read its output, which the statistics of the input do not describe. + auto stats = estimateReadRowsCount(*node.children.front(), nullptr); + if (stats.estimated_rows && *stats.estimated_rows != 0) + { + /// Such a filter leaves the row count unknown unless the input is empty. + if (filter) + { + stats.estimated_rows.reset(); + stats.imprecise_estimate = true; + } + else + stats.estimated_rows = toUInt64Saturating(Float64(*stats.estimated_rows) * array_join_rows_multiplier); + } + if (expression_step) + remapColumnStats(stats.column_stats, expression_step->getExpression()); + else + for (const auto & column : array_join_step->getColumns()) + stats.column_stats.erase(column); + return stats; + } + + if (expression_step) { auto stats = estimateReadRowsCount(*node.children.front(), filter); remapColumnStats(stats.column_stats, expression_step->getExpression()); @@ -1124,6 +1162,10 @@ static QueryPlan::Node chooseJoinOrder(QueryGraphBuilder query_graph_builder, Qu QueryGraph query_graph; query_graph.relation_stats = std::move(query_graph_builder.relation_stats); query_graph.edges = std::move(query_graph_builder.join_edges); + BitSet null_supplying_relations; + for (const auto & [relation, restriction] : query_graph_builder.join_kinds) + if (restriction.second != JoinKind::Inner) + null_supplying_relations.set(relation); query_graph.join_kinds = std::move(query_graph_builder.join_kinds); query_graph.outer_join_conditions = std::move(query_graph_builder.outer_join_conditions); @@ -1318,11 +1360,13 @@ static QueryPlan::Node chooseJoinOrder(QueryGraphBuilder query_graph_builder, Qu /// input pos -> new input node std::unordered_map current_step_type_changes; - for (auto rel_id : {left_rels.getSingleBit(), right_rels.getSingleBit()}) + /// NULL padding applies where the relation is joined alone; other type changes only after all predicates on original columns. + const bool is_last_step = entry_idx + 1 == sequence.size(); + for (const auto & [relation, new_inputs] : query_graph_builder.type_changes) { - if (!rel_id.has_value()) + const bool joined_alone = left_rels.getSingleBit() == relation || right_rels.getSingleBit() == relation; + if (!(null_supplying_relations.test(relation) ? joined_alone : is_last_step)) continue; - const auto & new_inputs = query_graph_builder.type_changes[rel_id.value()]; for (const auto * new_input : new_inputs) { const auto * input_node = trackInputColumn(new_input); diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp index fab0836a8d946..c5e2f3bf05fb7 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp @@ -1151,7 +1151,7 @@ InputOrderInfoPtr buildInputOrderInfo( { FindReadingStepContext find_reading_ctx{ .allow_existing_order = false, - .read_in_order_through_join = optimization_settings.read_in_order_through_join, + .read_in_order_through_join = optimization_settings.read_in_order_through_join != 0, }; QueryPlan::Node * reading_node = findReadingStep(node, find_reading_ctx); if (!reading_node) @@ -1196,22 +1196,24 @@ InputOrderInfoPtr buildInputOrderInfo( virtual_row_reader = reading; } - if (!uses_virtual_row) - { - /// Virtual row is required for INNER JOIN to maintain read-in-order optimization. - /// Without it, the sorting step after join cannot correctly determine - /// which input stream to read from when most rows are filtered out, - /// potentially reading excessive amount of data. - for (const auto * join_step : find_reading_ctx.joins_to_keep_in_order) + /// Joins that can discard left rows need virtual rows from every selected part to avoid + /// excessive reading by the merge after the join. `LEFT ALL` and `LEFT ANY` preserve left rows. + /// `query_plan_read_in_order_through_join = 'force'` accepts the over-read. + const bool join_requires_virtual_rows = std::ranges::any_of( + find_reading_ctx.joins_to_keep_in_order, + [](const auto * join_step) { const auto & table_join = join_step->getJoin()->getTableJoin(); - auto strictness = table_join.strictness(); - if (table_join.kind() != JoinKind::Left || (strictness != JoinStrictness::All && strictness != JoinStrictness::Any)) - { - LOG_DEBUG(getLogger("optimizeReadInOrder"), "Skip using read in order for inner join without virtual row optimization"); - return nullptr; - } - } + const auto strictness = table_join.strictness(); + return table_join.kind() != JoinKind::Left + || (strictness != JoinStrictness::All && strictness != JoinStrictness::Any); + }); + if (join_requires_virtual_rows && optimization_settings.read_in_order_through_join != 2 + && (!uses_virtual_row || !reading->everyPartHasVirtualRowBound(order_info.input_order->direction < 0))) + { + LOG_DEBUG(getLogger("optimizeReadInOrder"), "Skip using read in order for inner join without virtual row optimization"); + reading->resetVirtualRowConversions(); + return nullptr; } bool can_read = reading->requestReadingInOrder( @@ -1221,7 +1223,11 @@ InputOrderInfoPtr buildInputOrderInfo( query_limit); if (!can_read) + { + /// E.g. a reverse `FINAL` read the engine does not support: the read stays as it was. + reading->resetVirtualRowConversions(); return nullptr; + } for (auto * join_step : find_reading_ctx.joins_to_keep_in_order) join_step->keepLeftPipelineInOrder(/* disable_squashing */ true); @@ -1276,7 +1282,7 @@ InputOrder buildInputOrderInfo(AggregatingStep & aggregating, QueryPlan::Node & { FindReadingStepContext find_reading_ctx { .allow_existing_order = false, - .read_in_order_through_join = optimization_settings.read_in_order_through_join, + .read_in_order_through_join = optimization_settings.read_in_order_through_join != 0, }; QueryPlan::Node * reading_node = findReadingStep(node, find_reading_ctx); if (!reading_node) @@ -1397,7 +1403,7 @@ InputOrder buildInputOrderInfo(DistinctStep & distinct, QueryPlan::Node & node, FindReadingStepContext find_reading_ctx { .allow_existing_order = true, - .read_in_order_through_join = optimization_settings.read_in_order_through_join, + .read_in_order_through_join = optimization_settings.read_in_order_through_join != 0, }; QueryPlan::Node * reading_node = findReadingStep(node, find_reading_ctx); if (!reading_node) @@ -1488,7 +1494,7 @@ InputOrder buildInputOrderInfo(LimitByStep & limit_by, QueryPlan::Node & node, c FindReadingStepContext find_reading_ctx{ .allow_existing_order = true, - .read_in_order_through_join = optimization_settings.read_in_order_through_join, + .read_in_order_through_join = optimization_settings.read_in_order_through_join != 0, }; QueryPlan::Node * reading_node = findReadingStep(node, find_reading_ctx); if (!reading_node) diff --git a/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp b/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp index 9556da9e95500..0d3d9a4cca666 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeTree.cpp @@ -92,7 +92,7 @@ void optimizeTreeFirstPass(const QueryPlanOptimizationSettings & optimization_se optimization_settings.max_limit_for_top_k_optimization, optimization_settings.use_skip_indexes_on_data_read, optimization_settings.read_in_order, - optimization_settings.read_in_order_through_join, + optimization_settings.read_in_order_through_join != 0, optimization_settings.join_swap_table, optimization_settings.enable_group_by_top_k_optimization, optimization_settings.top_k_optimization_observation_rows, @@ -223,7 +223,7 @@ void optimizeTreeSecondPass( optimization_settings.max_limit_for_top_k_optimization, optimization_settings.use_skip_indexes_on_data_read, optimization_settings.read_in_order, - optimization_settings.read_in_order_through_join, + optimization_settings.read_in_order_through_join != 0, optimization_settings.join_swap_table, optimization_settings.enable_group_by_top_k_optimization, optimization_settings.top_k_optimization_observation_rows, diff --git a/src/Processors/QueryPlan/PartsSplitter.cpp b/src/Processors/QueryPlan/PartsSplitter.cpp index 84629ef81b69c..cda59ee7e8427 100644 --- a/src/Processors/QueryPlan/PartsSplitter.cpp +++ b/src/Processors/QueryPlan/PartsSplitter.cpp @@ -1221,6 +1221,7 @@ SplitPartsWithRangesByPrimaryKeyResult splitPartsWithRangesByPrimaryKey( auto split_ranges = splitIntersectingPartsRangesIntoLayers( intersecting_parts_ranges, max_layers, primary_key.column_names.size(), in_reverse_order, logger); + result.borders = split_ranges.borders; result.merging_pipes = readByLayers(std::move(split_ranges), primary_key, create_merging_pipe, context); return result; } diff --git a/src/Processors/QueryPlan/PartsSplitter.h b/src/Processors/QueryPlan/PartsSplitter.h index ccab952da0a08..ba6a40d6ccd2a 100644 --- a/src/Processors/QueryPlan/PartsSplitter.h +++ b/src/Processors/QueryPlan/PartsSplitter.h @@ -35,6 +35,9 @@ struct SplitPartsWithRangesByPrimaryKeyResult { RangesInDataParts non_intersecting_parts_ranges; Pipes merging_pipes; + /// Borders between consecutive `merging_pipes` in the table's key order (see `SplitPartsByRanges::borders`); + /// empty when the intersecting ranges were not split into layers. + std::vector> borders; }; /** Splits parts ranges into: diff --git a/src/Processors/QueryPlan/ReadFromLoopStep.cpp b/src/Processors/QueryPlan/ReadFromLoopStep.cpp index 60e00d420ec99..4735796e0b744 100644 --- a/src/Processors/QueryPlan/ReadFromLoopStep.cpp +++ b/src/Processors/QueryPlan/ReadFromLoopStep.cpp @@ -72,7 +72,9 @@ namespace if (context->getSettingsRef()[Setting::allow_experimental_analyzer]) { - InterpreterSelectQueryAnalyzer interpreter(select_ast, context, options, column_names); + /// The analyzer substitutes the inserted block of a materialized view for its source table only when given it. + InterpreterSelectQueryAnalyzer interpreter( + select_ast, context, options, context->isViewInnerQuery() ? nullptr : context->getViewSource(), column_names); if (query_info.storage_limits) interpreter.addStorageLimits(*query_info.storage_limits); plan = std::move(interpreter).extractQueryPlan(); diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.cpp b/src/Processors/QueryPlan/ReadFromMergeTree.cpp index 38c99f94d19bc..7f8d7c27010d9 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.cpp +++ b/src/Processors/QueryPlan/ReadFromMergeTree.cpp @@ -652,8 +652,7 @@ Pipe ReadFromMergeTree::readFromPoolParallelReplicas( actions_settings, reader_settings, index_build_context, - lazy_materializing_rows, - &storage_snapshot->metadata->getColumns()); + lazy_materializing_rows); auto source = std::make_shared(std::move(processor), data.getLogName()); pipes.emplace_back(std::move(source)); @@ -770,8 +769,7 @@ Pipe ReadFromMergeTree::readFromPool( actions_settings, reader_settings, index_build_context, - lazy_materializing_rows, - &storage_snapshot->metadata->getColumns()); + lazy_materializing_rows); auto source = std::make_shared(std::move(processor), data.getLogName()); @@ -787,6 +785,95 @@ Pipe ReadFromMergeTree::readFromPool( return pipe; } +/// The index of `part` if it has the first `num_key_columns` PK columns, `nullptr` otherwise. +/// `optimizeIndexColumns` may drop trailing PK columns, and a reverse read needs the final mark. +static IMergeTreeDataPart::IndexPtr indexWithVirtualRowBound(const IMergeTreeDataPart & part, size_t num_key_columns, bool read_in_reverse) +{ + if (read_in_reverse && !part.index_granularity->hasFinalMark()) + return nullptr; + auto index = part.getIndex(); + if (index->size() < num_key_columns) + return nullptr; + return index; +} + +/// PK values at the first selected mark, or the end mark for a reverse read. +/// Returns `std::nullopt` if the index cannot provide them. +static std::optional getVirtualRowBound(const RangesInDataPart & part_with_ranges, const Block & pk_header, bool read_in_reverse) +{ + const auto index = indexWithVirtualRowBound(*part_with_ranges.data_part, pk_header.columns(), read_in_reverse); + if (!index) + return std::nullopt; + + const size_t mark_pos = read_in_reverse ? part_with_ranges.ranges.back().end : part_with_ranges.ranges.front().begin; + for (size_t j = 0; j < pk_header.columns(); ++j) + if ((*index)[j]->size() <= mark_pos) + return std::nullopt; + + /// Insert via `Field`: the index may have the type before an `ALTER MODIFY COLUMN` + /// (e.g. `String` vs `LowCardinality(String)`), because a mutation can keep the source part's index. + auto pk_columns = pk_header.cloneEmptyColumns(); + for (size_t j = 0; j < pk_columns.size(); ++j) + pk_columns[j]->insert((*(*index)[j])[mark_pos]); + return pk_header.cloneWithColumns(std::move(pk_columns)); +} + +/// Whether `lhs` comes before `rhs` in the table's order, reversed for a reverse read. +static bool virtualRowPrecedes(const Block & lhs, const Block & rhs, const std::vector & reverse_flags, bool read_in_reverse) +{ + for (size_t j = 0; j < lhs.columns(); ++j) + { + int cmp = lhs.getByPosition(j).column->compareAt(0, 0, *rhs.getByPosition(j).column, /*nan_direction_hint=*/ 1); + if (!reverse_flags.empty() && reverse_flags[j]) + cmp = -cmp; + if (cmp != 0) + return read_in_reverse ? cmp > 0 : cmp < 0; + } + return false; +} + +/// The virtual row of a `FINAL` merge over `parts`: the least of the parts' bounds (the greatest for a reverse read). +/// Returns `std::nullopt` if the index of some part does not have the bound. +static std::optional getFinalMergeVirtualRow( + const RangesInDataParts & parts, const Block & pk_header, const std::vector & reverse_flags, bool read_in_reverse) +{ + std::optional bound; + for (const auto & part : parts) + { + auto part_bound = getVirtualRowBound(part, pk_header, read_in_reverse); + if (!part_bound) + return std::nullopt; + if (!bound || virtualRowPrecedes(*part_bound, *bound, reverse_flags, read_in_reverse)) + bound = std::move(part_bound); + } + return bound; +} + +/// With `FINAL`, intersecting ranges are split into layers which are read in parallel. Borders between layers +/// are PK values which can be in the middle of a granule, so a border is a tighter bound than the index. +/// Use the border as the virtual row if it has all the needed PK columns. +static std::optional tightenFinalMergeVirtualRow( + std::optional virtual_row, + const std::vector & border, + const Block & pk_header, + const std::vector & reverse_flags, + bool read_in_reverse) +{ + /// A border has as many values as the parts' indexes share, which may be fewer than the virtual row needs. + if (border.size() < pk_header.columns()) + return virtual_row; + + auto columns = pk_header.cloneEmptyColumns(); + for (size_t j = 0; j < columns.size(); ++j) + columns[j]->insert(border[j]); + Block border_row = pk_header.cloneWithColumns(std::move(columns)); + + /// The border should always be at least as tight, compare just in case. + if (!virtual_row || virtualRowPrecedes(*virtual_row, border_row, reverse_flags, read_in_reverse)) + return border_row; + return virtual_row; +} + Pipe ReadFromMergeTree::readInOrder( RangesInDataParts parts_with_ranges, const MergeTreeIndexBuildContextPtr & index_build_context, @@ -801,7 +888,10 @@ Pipe ReadFromMergeTree::readInOrder( const bool has_hard_limit_below_one_block = read_type != ReadType::Default && read_limit && read_limit < block_size.max_block_size_rows; const bool has_soft_limit_below_one_block = read_type != ReadType::Default && query_task_size_limit && query_task_size_limit < block_size.max_block_size_rows; - const bool use_virtual_row = virtual_row_conversion && (read_type == ReadType::InOrder || read_type == ReadType::InReverseOrder); + /// The `FINAL` merges do not consume virtual rows: a `FINAL` read gets one per merge instead (see + /// `spreadMarkRangesAmongStreamsFinal`). + const bool use_virtual_row = virtual_row_conversion && !isQueryWithFinal() + && (read_type == ReadType::InOrder || read_type == ReadType::InReverseOrder); const bool use_virtual_row_per_block = use_virtual_row && context->getSettingsRef()[Setting::read_in_order_use_virtual_row_per_block]; if (use_virtual_row_per_block && read_type == ReadType::InReverseOrder) @@ -908,6 +998,8 @@ Pipe ReadFromMergeTree::readInOrder( const UInt64 in_order_limit = query_info.input_order_info ? query_info.input_order_info->limit : 0; const bool set_total_rows_approx = !is_parallel_reading_from_replicas || isParallelReplicasLocalPlanForInitiator(); + const Block pk_header = use_virtual_row ? makeVirtualRowHeader() : Block{}; + Pipes pipes; for (size_t i = 0; i < parts_with_ranges.size(); ++i) { @@ -953,28 +1045,12 @@ Pipe ReadFromMergeTree::readInOrder( actions_settings, reader_settings, index_build_context, - lazy_materializing_rows, - &storage_snapshot->metadata->getColumns()); + lazy_materializing_rows); processor->addPartLevelToChunk(isQueryWithFinal()); - Block pk_header; - if (use_virtual_row) - { - const auto & primary_key = storage_snapshot->metadata->primary_key; - size_t num_pk_columns_required = virtual_row_conversion->getRequiredColumnsWithTypes().size(); - - ColumnsWithTypeAndName pk_header_columns; - pk_header_columns.reserve(num_pk_columns_required); - for (size_t j = 0; j < num_pk_columns_required; ++j) - pk_header_columns.push_back( - {primary_key.data_types[j]->createColumn(), primary_key.data_types[j], primary_key.column_names[j]}); - - pk_header = Block(std::move(pk_header_columns)); - - if (use_virtual_row_per_block) - processor->setVirtualRowConversions(virtual_row_conversion, pk_header, read_type == ReadType::InReverseOrder); - } + if (use_virtual_row_per_block) + processor->setVirtualRowConversions(virtual_row_conversion, pk_header, read_type == ReadType::InReverseOrder); auto source = std::make_shared(std::move(processor), data.getLogName()); if (set_total_rows_approx) @@ -984,26 +1060,11 @@ Pipe ReadFromMergeTree::readInOrder( if (use_virtual_row) { - const auto & index = part_with_ranges.data_part->getIndex(); - - bool has_final_mark = part_with_ranges.data_part->index_granularity->hasFinalMark(); - bool read_in_direct_order = read_type == ReadType::InOrder; - size_t mark_range_pos = read_in_direct_order ? part_with_ranges.ranges.front().begin : part_with_ranges.ranges.back().end; - bool has_pk_value = (read_in_direct_order || has_final_mark) && std::ranges::all_of(*index, [&](const auto & col) { return col->size() > mark_range_pos; }); - - /// The index may have fewer columns than the primary key if suffix columns were - /// removed by optimizeIndexColumns (controlled by primary_key_ratio_of_unique_prefix_values_to_skip_suffix_columns). - /// In that case, we cannot apply virtual row optimization because we don't have all required columns. - auto pk_columns = pk_header.cloneEmptyColumns(); - if (index->size() >= pk_columns.size() && has_pk_value) + if (auto pk_block = getVirtualRowBound(part_with_ranges, pk_header, read_type == ReadType::InReverseOrder)) { - for (size_t j = 0; j < pk_columns.size(); ++j) - pk_columns[j]->insert((*(*index)[j])[mark_range_pos]); - - Block pk_block = pk_header.cloneWithColumns(std::move(pk_columns)); pipe.addSimpleTransform([&](const SharedHeader & header) { - return std::make_shared(header, pk_block, virtual_row_conversion); + return std::make_shared(header, *pk_block, virtual_row_conversion); }); } } @@ -2279,6 +2340,18 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( chassert(num_streams == requested_num_streams); num_streams = std::min(num_streams, settings[Setting::max_final_threads]); + /// Always false (`requestReadingInOrder` rejects reverse order with FINAL), kept as a value + /// so the virtual-row helpers take the real flag. + const bool read_in_reverse = reader_settings.read_in_order && query_info.input_order_info && query_info.input_order_info->direction < 0; + + /// A virtual row after each `FINAL` merge announces the key its layer starts from, so the merge downstream + /// defers the merge and the reading of its parts. Only when a step requested reading in order: the parts + /// of a `FINAL` read are in order for its merges anyway, but without the request nothing downstream + /// consumes virtual rows, and vertical `FINAL` may be on. + const bool use_virtual_row = virtual_row_conversion && reader_settings.read_in_order; + const Block virtual_row_header = use_virtual_row ? makeVirtualRowHeader() : Block{}; + const auto reverse_flags = storage_snapshot->metadata->getSortingKeyReverseFlags(); + /// If do_not_merge_across_partitions_select_final is true than we won't merge parts from different partitions. /// We have all parts in parts vector, where parts with same partition are nearby. /// So we will store iterators pointed to the beginning of each partition range (and parts.end()), @@ -2385,6 +2458,8 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( } Pipes pipes; + /// The virtual row of each of `pipes`, when its parts have one. + std::vector> pipes_virtual_rows; { RangesInDataParts new_parts; size_t current_ranges_marks = 0; @@ -2410,9 +2485,17 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( if (storage_snapshot->metadata->hasPrimaryKey()) { + /// `splitPartsWithRangesByPrimaryKey` calls the getter once per merging pipe, in their order. + std::vector> merging_pipes_virtual_rows; + // Let's split parts into non intersecting parts ranges and layers to ensure data parallelism of FINAL. - auto in_order_reading_step_getter = [this, &index_build_context, &column_names, &info](auto parts) + auto in_order_reading_step_getter + = [this, &index_build_context, &column_names, &info, read_in_reverse, + use_virtual_row, &virtual_row_header, &reverse_flags, &merging_pipes_virtual_rows](auto parts) { + if (use_virtual_row) + merging_pipes_virtual_rows.push_back(getFinalMergeVirtualRow(parts, virtual_row_header, reverse_flags, read_in_reverse)); + return this->read( std::move(parts), index_build_context, @@ -2498,10 +2581,29 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( /// reach `createProjection` or `addMergingFinal` below. Dropping it is safe here: /// unlike the join-by-shards path, the per-layer pipes are simply united, so their /// positions carry no meaning. - for (auto && merging_pipe : split_ranges_result.merging_pipes) + for (size_t i = 0; i < split_ranges_result.merging_pipes.size(); ++i) { - if (!merging_pipe.empty()) - pipes.push_back(std::move(merging_pipe)); + auto & merging_pipe = split_ranges_result.merging_pipes[i]; + if (merging_pipe.empty()) + continue; + + pipes.push_back(std::move(merging_pipe)); + if (use_virtual_row) + { + /// The parts' index bounds a layer loosely: a first granule of a sparse part spans many keys. + /// The border the layer starts from bounds it exactly; the layers run in the table's order, + /// so a forward read starts layer `i` after `borders[i - 1]`, a reverse one after `borders[i]`. + auto virtual_row = std::move(merging_pipes_virtual_rows.at(i)); + const auto & borders = split_ranges_result.borders; + const bool has_start_border = read_in_reverse ? (i < borders.size()) : (i > 0); + if (has_start_border) + { + const auto & border = borders[read_in_reverse ? i : (i - 1)]; + virtual_row = tightenFinalMergeVirtualRow( + std::move(virtual_row), border, virtual_row_header, reverse_flags, read_in_reverse); + } + pipes_virtual_rows.push_back(std::move(virtual_row)); + } } } else @@ -2517,6 +2619,9 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( pipes.back().addSimpleTransform([sorting_expr](const SharedHeader & header) { return std::make_shared(header, sorting_expr); }); + /// Without a primary key there is nothing to build a virtual row from. + if (use_virtual_row) + pipes_virtual_rows.emplace_back(); } /// Drop temporary columns, added by 'sorting_key_expr' @@ -2528,7 +2633,6 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( continue; Names sort_columns = storage_snapshot->metadata->getSortingKeyColumns(); - std::vector reverse_flags = storage_snapshot->metadata->getSortingKeyReverseFlags(); SortDescription sort_description; sort_description.compile_sort_description = settings[Setting::compile_sort_description]; sort_description.min_count_to_compile_sort_description = settings[Setting::min_count_to_compile_sort_description]; @@ -2544,15 +2648,25 @@ Pipe ReadFromMergeTree::spreadMarkRangesAmongStreamsFinal( sort_description.emplace_back(sort_columns[i], 1); } - for (auto & pipe : pipes) + for (size_t i = 0; i < pipes.size(); ++i) + { addMergingFinal( - pipe, + pipes[i], sort_description, data.merging_params, storage_snapshot->metadata, block_size.max_block_size_rows, enable_vertical_final); + if (use_virtual_row && pipes_virtual_rows[i]) + { + pipes[i].addSimpleTransform([&](const SharedHeader & header) + { + return std::make_shared(header, *pipes_virtual_rows[i], virtual_row_conversion); + }); + } + } + merging_pipes.emplace_back(Pipe::unitePipes(std::move(pipes))); } @@ -3810,14 +3924,82 @@ bool ReadFromMergeTree::requestReadingInOrder(size_t prefix_size, int direction, bool ReadFromMergeTree::setVirtualRowConversions(ActionsDAG virtual_row_conversion_) { - /// Disable virtual row for FINAL. - if (isQueryWithFinal() || !context->getSettingsRef()[Setting::read_in_order_use_virtual_row]) + if (!context->getSettingsRef()[Setting::read_in_order_use_virtual_row]) + return false; + + if (isQueryWithFinal() && !canUseVirtualRowWithFinal()) return false; virtual_row_conversion = std::make_shared(std::move(virtual_row_conversion_)); return true; } +bool ReadFromMergeTree::canUseVirtualRowWithFinal() const +{ + /// The virtual row of a `FINAL` merge announces the key its parts start from (see + /// `spreadMarkRangesAmongStreamsFinal`). That bounds the output only for the engines that never + /// emit a key below their inputs: `Graphite` rounds the time column down. + switch (data.merging_params.mode) + { + case MergeTreeData::MergingParams::Replacing: + case MergeTreeData::MergingParams::Collapsing: + case MergeTreeData::MergingParams::VersionedCollapsing: + case MergeTreeData::MergingParams::Summing: + case MergeTreeData::MergingParams::Aggregating: + case MergeTreeData::MergingParams::Coalescing: + break; + case MergeTreeData::MergingParams::Ordinary: + case MergeTreeData::MergingParams::Graphite: + return false; + } + + /// The lanes of a parallel-replicas `FINAL` are laid out by the coordinator (see `buildDistributedFinalPipe`); not covered yet. + if (is_parallel_reading_from_replicas) + return false; + + return true; +} + +RangesInDataParts ReadFromMergeTree::getPartsForPrewhere() const +{ + if (analyzed_result_ptr || !indexes) + return getParts(); + + /// Share all part filters with `selectRangesToRead`, including the snapshot boundary and statistics. + /// Keep this snapshot temporary: `PREWHERE` optimization can still change filters, + /// so execution must filter again with the final conditions. + IndexStats unused_stats; + return MergeTreeDataSelectExecutor::filterParts( + getParts(), *indexes, getStorageMetadata(), data, query_info, mutations_snapshot, getContext(), + max_block_numbers_to_read.get(), log, unused_stats); +} + +bool ReadFromMergeTree::everyPartHasVirtualRowBound(bool read_in_reverse) const +{ + /// Without part filters or an existing analysis, checking bounds would load indexes of unrelated parts. + if (!analyzed_result_ptr && !indexes) + return false; + + const size_t num_key_columns = virtual_row_conversion->getRequiredColumnsWithTypes().size(); + for (const auto & part : getPartsForPrewhere()) + if (!indexWithVirtualRowBound(*part.data_part, num_key_columns, read_in_reverse)) + return false; + return true; +} + +Block ReadFromMergeTree::makeVirtualRowHeader() const +{ + /// The leading primary key columns the conversions read: the merge downstream compares nothing else. + const auto & primary_key = storage_snapshot->metadata->primary_key; + const size_t num_columns = virtual_row_conversion->getRequiredColumnsWithTypes().size(); + + ColumnsWithTypeAndName columns; + columns.reserve(num_columns); + for (size_t j = 0; j < num_columns; ++j) + columns.push_back({primary_key.data_types[j]->createColumn(), primary_key.data_types[j], primary_key.column_names[j]}); + return Block(std::move(columns)); +} + bool ReadFromMergeTree::readsInOrder() const { diff --git a/src/Processors/QueryPlan/ReadFromMergeTree.h b/src/Processors/QueryPlan/ReadFromMergeTree.h index d7c306b5f9017..2c6ad2410aaea 100644 --- a/src/Processors/QueryPlan/ReadFromMergeTree.h +++ b/src/Processors/QueryPlan/ReadFromMergeTree.h @@ -360,6 +360,10 @@ class ReadFromMergeTree final : public SourceStepWithFilter bool requestReadingInOrder(size_t prefix_size, int direction, size_t read_limit, size_t query_limit = 0); bool setVirtualRowConversions(ActionsDAG virtual_row_conversion_); void resetVirtualRowConversions() { virtual_row_conversion = nullptr; } + + /// Whether every part surviving pruning has the value of its virtual row in the index. + /// Loads only those parts' primary indexes; returns false if part filters are unavailable. + bool everyPartHasVirtualRowBound(bool read_in_reverse) const; bool readsInOrder() const; const InputOrderInfoPtr & getInputOrder() const { return query_info.input_order_info; } const SortDescription & getSortDescription() const override { return result_sort_description; } @@ -435,6 +439,8 @@ class ReadFromMergeTree final : public SourceStepWithFilter void createReadTasksForTextIndex(const UsefulSkipIndexes & skip_indexes, const IndexReadColumns & added_columns, const Names & removed_columns, bool is_final); const std::optional & getIndexes() const { return indexes; } + /// A temporary part snapshot for PREWHERE costs; does not publish range analysis. + RangesInDataParts getPartsForPrewhere() const; ConditionSelectivityEstimatorPtr getConditionSelectivityEstimator(const Names & required_columns) const; /// Compose statistics over the part set of the given partition/PK analysis result /// instead of all prepared parts. Passing nullptr falls back to getParts(). @@ -617,6 +623,12 @@ class ReadFromMergeTree final : public SourceStepWithFilter /// a single pool reads the whole table (no splitting). std::optional split_index = std::nullopt); + /// Whether the `FINAL` merges of this read can be followed by a virtual row (see `spreadMarkRangesAmongStreamsFinal`). + bool canUseVirtualRowWithFinal() const; + + /// The columns of a virtual row: the leading primary key columns the conversions read. + Block makeVirtualRowHeader() const; + Pipe spreadMarkRanges( RangesInDataParts && parts_with_ranges, const MergeTreeIndexBuildContextPtr & index_build_context, diff --git a/src/Processors/QueryPlan/tests/gtest_siphashing_write_buffer_key_identity.cpp b/src/Processors/QueryPlan/tests/gtest_siphashing_write_buffer_key_identity.cpp new file mode 100644 index 0000000000000..cdb6c6d23698b --- /dev/null +++ b/src/Processors/QueryPlan/tests/gtest_siphashing_write_buffer_key_identity.cpp @@ -0,0 +1,98 @@ +#include + +#include +#include + +#include +#include +#include + +using namespace DB; + +namespace +{ + +/// The value `calculateHashFromStep` produced before it streamed: accumulate the whole stream, then +/// hash the contiguous result in one call. +UInt64 bufferedKey(const std::vector & writes) +{ + WriteBufferFromOwnString out; + for (const auto & w : writes) + out.write(w.data(), w.size()); + + SipHash hash; + hash.update(out.str()); + return hash.get64(); +} + +/// The value it produces now: hash each chunk as it is written, keeping no copy of the stream. +UInt64 streamedKey(const std::vector & writes) +{ + SipHash hash; + SipHashingWriteBuffer out(hash); + for (const auto & w : writes) + out.write(w.data(), w.size()); + out.finalize(); + return hash.get64(); +} + +/// Writes that between them cross the sink's flush boundary at every alignment: sub-word chunks +/// (which exercise `SipHash::update`'s carried remainder), chunks straddling one window, and a +/// single write many windows long. +std::vector> streamShapes() +{ + const size_t w = SipHashingWriteBuffer::window_bytes; + return { + {}, + {""}, + {"a"}, + {"a", "b", "c"}, + {std::string(7, 'x')}, + {std::string(8, 'x')}, + {std::string(9, 'x')}, + {std::string(1, 'a'), std::string(w - 1, 'b')}, + {std::string(w - 1, 'a'), std::string(1, 'b')}, + {std::string(w, 'a')}, + {std::string(w, 'a'), std::string(1, 'b')}, + {std::string(w + 1, 'a')}, + {std::string(3, 'a'), std::string(w, 'b'), std::string(5, 'c')}, + {std::string(4 * w + 3, 'a')}, + }; +} + +} + +/// The property the whole change rests on: streaming the key must not change its value. If this +/// fails, every cached hash-table-statistics entry and every parallel-replicas node match silently +/// stops agreeing with the plan that produced it. +TEST(SipHashingWriteBuffer, KeyIsIdenticalToBufferedHash) +{ + for (const auto & writes : streamShapes()) + ASSERT_EQ(streamedKey(writes), bufferedKey(writes)) << "stream shape with " << writes.size() << " write(s)"; +} + +/// Negative control for the test above. Perturbing the stream by a single byte in one shape must +/// change the key, otherwise `KeyIsIdenticalToBufferedHash` would pass for a sink that hashes +/// nothing at all. +TEST(SipHashingWriteBuffer, KeyDiffersWhenStreamIsPerturbed) +{ + const std::vector writes{std::string(SipHashingWriteBuffer::window_bytes + 1, 'a')}; + + auto perturbed = writes; + perturbed.front().back() = 'b'; + + ASSERT_NE(streamedKey(writes), streamedKey(perturbed)); + ASSERT_NE(streamedKey(writes), streamedKey({})); +} + +/// Destroying the sink with a stream still half-written must not trip `~WriteBuffer`'s "neither +/// finalized nor canceled" assertion, since callers are free to abandon a key half-computed. +TEST(SipHashingWriteBuffer, DestroyedWithoutFinalizeAfterPartialWrite) +{ + SipHash hash; + { + SipHashingWriteBuffer out(hash); + const std::string chunk(SipHashingWriteBuffer::window_bytes + 17, 'x'); + out.write(chunk.data(), chunk.size()); + } +} diff --git a/src/Processors/Transforms/AddingDefaultsTransform.cpp b/src/Processors/Transforms/AddingDefaultsTransform.cpp index 9e0fe0f5c260e..9650df7204bb4 100644 --- a/src/Processors/Transforms/AddingDefaultsTransform.cpp +++ b/src/Processors/Transforms/AddingDefaultsTransform.cpp @@ -157,7 +157,8 @@ void AddingDefaultsTransform::transform(Chunk & chunk) return; const auto * block_missing_values = input_format.getMissingValues(); - if (!block_missing_values) + /// `BlockMissingValues::empty` is also true when the mask holds no columns at all, where the lookups below would be out of range. + if (!block_missing_values || block_missing_values->empty()) return; const auto & header = getOutputPort().getHeader(); diff --git a/src/Processors/tests/gtest_ch_column_to_arrow_column_record_batch_split.cpp b/src/Processors/tests/gtest_ch_column_to_arrow_column_record_batch_split.cpp new file mode 100644 index 0000000000000..38367d69dc3e2 --- /dev/null +++ b/src/Processors/tests/gtest_ch_column_to_arrow_column_record_batch_split.cpp @@ -0,0 +1,127 @@ +#include "config.h" + +#if USE_ARROW || USE_PARQUET + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +/// Arrow offsets its `Utf8`, `Binary` and `List` buffers with 32-bit integers, so a chunk holding more +/// than that becomes several record batches, each covering a row range. A range not starting at row 0 has +/// to rebase those offsets against the row before it. Reaching the real limit needs more than 2 GiB of +/// data, so these tests lower it. +/// +/// `CHColumnToArrowColumn` is the Apache Arrow library writer, used by `ArrowFlight`, +/// `StorageArrowFlight` and Delta Lake writes rather than by the `Arrow` output format. + +namespace DB +{ + +namespace +{ + +constexpr size_t NUM_ROWS = 8; +constexpr size_t VALUE_SIZE = 16; + +std::string valueAt(size_t row) +{ + return std::string(VALUE_SIZE, static_cast('a' + row)); +} + +CHColumnToArrowColumn::Settings settingsWithLimit(UInt64 max_record_batch_bytes) +{ + CHColumnToArrowColumn::Settings settings; + settings.output_string_as_string = true; + settings.output_max_record_batch_bytes = max_record_batch_bytes; + return settings; +} + +std::shared_ptr convertToArrow(const DataTypePtr & type, MutableColumnPtr column, UInt64 max_record_batch_bytes) +{ + ColumnsWithTypeAndName header_columns{ColumnWithTypeAndName(type->createColumn(), type, "t")}; + + Columns columns; + columns.emplace_back(std::move(column)); + + std::vector chunks; + chunks.emplace_back(std::move(columns), NUM_ROWS); + + CHColumnToArrowColumn converter(header_columns, "Arrow", settingsWithLimit(max_record_batch_bytes)); + std::shared_ptr arrow_table; + converter.chChunkToArrowTable(arrow_table, chunks, 1); + return arrow_table; +} + +} + +TEST(CHColumnToArrowColumn, StringSplitAcrossRecordBatches) +{ + auto type = std::make_shared(); + auto column = type->createColumn(); + for (size_t row = 0; row < NUM_ROWS; ++row) + column->insert(valueAt(row)); + + /// Two values per batch, so most batches start past row 0. + auto table = convertToArrow(type, std::move(column), 2 * VALUE_SIZE); + + const auto & chunked = *table->column(0); + ASSERT_GT(chunked.num_chunks(), 1); + ASSERT_EQ(chunked.length(), static_cast(NUM_ROWS)); + + size_t row = 0; + for (int chunk_i = 0; chunk_i < chunked.num_chunks(); ++chunk_i) + { + const auto & array = assert_cast(*chunked.chunk(chunk_i)); + /// Every batch addresses its own values from zero. + ASSERT_EQ(array.value_offset(0), 0); + for (int64_t i = 0; i < array.length(); ++i, ++row) + ASSERT_EQ(array.GetString(i), valueAt(row)); + } + ASSERT_EQ(row, NUM_ROWS); +} + +TEST(CHColumnToArrowColumn, ArraySplitAcrossRecordBatches) +{ + auto type = std::make_shared(std::make_shared()); + auto column = type->createColumn(); + for (size_t row = 0; row < NUM_ROWS; ++row) + column->insert(Array{Field(valueAt(row)), Field(valueAt(row))}); + + auto table = convertToArrow(type, std::move(column), 2 * VALUE_SIZE); + + const auto & chunked = *table->column(0); + ASSERT_GT(chunked.num_chunks(), 1); + ASSERT_EQ(chunked.length(), static_cast(NUM_ROWS)); + + size_t row = 0; + for (int chunk_i = 0; chunk_i < chunked.num_chunks(); ++chunk_i) + { + const auto & list = assert_cast(*chunked.chunk(chunk_i)); + ASSERT_EQ(list.value_offset(0), 0); + for (int64_t i = 0; i < list.length(); ++i, ++row) + { + /// `value_slice` returns a new array by value: keep it alive while it is inspected. + const auto values_holder = list.value_slice(i); + const auto & values = assert_cast(*values_holder); + ASSERT_EQ(values.length(), 2); + ASSERT_EQ(values.GetString(0), valueAt(row)); + ASSERT_EQ(values.GetString(1), valueAt(row)); + } + } + ASSERT_EQ(row, NUM_ROWS); +} + +} + +#endif diff --git a/src/Processors/tests/gtest_pipeline_executor_late_cpu_slots.cpp b/src/Processors/tests/gtest_pipeline_executor_late_cpu_slots.cpp new file mode 100644 index 0000000000000..a50e9e78439d1 --- /dev/null +++ b/src/Processors/tests/gtest_pipeline_executor_late_cpu_slots.cpp @@ -0,0 +1,241 @@ +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include + +using namespace DB; + +namespace +{ + +constexpr size_t num_streams = 4; +constexpr auto timeout = std::chrono::seconds(60); + +struct LateSlotsState +{ + std::mutex mutex; + std::condition_variable cv; + + bool first_step_started = false; + bool slots_released = false; + + /// Threads that ran a step after the release. + std::set threads; + bool done = false; + bool timed_out = false; +}; + +/// The first step of the pipeline waits until the test releases the CPU slots held by another allocation. +/// Every later step waits until `num_streams` threads are in it. A thread is started for a granted slot only +/// at a task boundary of a running thread, so without one the only running thread waits here for the timeout. +class LateSlotsSource : public ISource +{ +public: + LateSlotsSource(SharedHeader header, LateSlotsState & state_) + : ISource(std::move(header)) + , state(state_) + { + } + + String getName() const override { return "LateSlotsSource"; } + +protected: + Chunk generate() override + { + std::unique_lock lock(state.mutex); + if (!state.first_step_started) + { + state.first_step_started = true; + state.cv.notify_all(); + state.cv.wait_for(lock, timeout, [&] { return state.slots_released; }); + return Chunk(Columns{ColumnUInt8::create(1, static_cast(1))}, 1); + } + + if (!state.done) + { + state.threads.insert(std::this_thread::get_id()); + if (state.threads.size() >= num_streams) + state.done = true; + else if (!state.cv.wait_for(lock, timeout, [&] { return state.done; })) + state.done = state.timed_out = true; + state.cv.notify_all(); + } + return {}; + } + +private: + LateSlotsState & state; +}; + +void checkThreadsForLateSlots(bool lazy_allocation) +{ + auto & concurrency_control = ConcurrencyControl::instance(); + const String prev_scheduler = concurrency_control.getScheduler(); + const bool prev_lazy_allocation = concurrency_control.getLazyAllocation(); + SCOPE_EXIT({ + concurrency_control.setMaxConcurrency(UnlimitedSlots); + concurrency_control.setScheduler(prev_scheduler); + concurrency_control.setLazyAllocation(prev_lazy_allocation); + }); + concurrency_control.setScheduler("fair_round_robin"); + concurrency_control.setLazyAllocation(lazy_allocation); + concurrency_control.setMaxConcurrency(num_streams); + + /// Another query holds all the competing slots, so the pipeline starts with its master thread only. + SlotAllocationPtr other_query = concurrency_control.allocate(0, num_streams); + + LateSlotsState state; + SharedHeader header = std::make_shared(Block{ColumnWithTypeAndName(ColumnUInt8::create(), std::make_shared(), "x")}); + auto processors = std::make_shared(); + for (size_t i = 0; i < num_streams; ++i) + { + auto source = std::make_shared(header, state); + auto sink = std::make_shared(source->getPort().getSharedHeader()); + connect(source->getPort(), sink->getPort()); + processors->emplace_back(std::move(source)); + processors->emplace_back(std::move(sink)); + } + + QueryStatusPtr element; + PipelineExecutor executor(processors, element); + std::thread query([&] { executor.execute(num_streams, /* concurrency_control */ true); }); + + { + std::unique_lock lock(state.mutex); + EXPECT_TRUE(state.cv.wait_for(lock, timeout, [&] { return state.first_step_started; })) << "The pipeline did not start"; + } + + /// The other query finishes: its slots are granted to the pipeline, which is in the middle of execution. + other_query.reset(); + { + std::lock_guard lock(state.mutex); + state.slots_released = true; + } + state.cv.notify_all(); + + query.join(); + + std::lock_guard lock(state.mutex); + EXPECT_FALSE(state.timed_out); + EXPECT_EQ(state.threads.size(), num_streams); +} + +} + +/// A CPU slot granted to a running pipeline gets a thread at the next task boundary, +/// even if no step pushes tasks to the shared queue. +TEST(PipelineExecutor, ThreadsForSlotsGrantedAfterStart) +{ + checkThreadsForLateSlots(/* lazy_allocation */ true); +} + +TEST(PipelineExecutor, ThreadsForSlotsGrantedAfterStartEagerAllocation) +{ + checkThreadsForLateSlots(/* lazy_allocation */ false); +} + +/// A thread requested in place of a downscaled one is kept until a slot is granted and the thread is started. +TEST(PipelineExecutor, ThreadRequestedAfterDownscale) +{ + ExecutorTasks tasks; + tasks.init(num_streams, 1, std::make_shared(num_streams), false, false, nullptr, nullptr); + tasks.upscale(0); + tasks.upscale(1); + EXPECT_FALSE(tasks.hasThreadsToSpawn()); + + tasks.preempt(1); + tasks.downscale(1); + EXPECT_FALSE(tasks.hasThreadsToSpawn()); + tasks.requestThreads(1); + EXPECT_TRUE(tasks.hasThreadsToSpawn()); + + tasks.upscale(1); + EXPECT_FALSE(tasks.hasThreadsToSpawn()); + tasks.finish(); +} + +/// A thread that finds no queued task to run drops the threads requested for queued tasks: +/// none are left for them. +TEST(PipelineExecutor, RequestedThreadsDroppedWhenThreadGoesIdle) +{ + ExecutorTasks tasks; + tasks.init(num_streams, 1, std::make_shared(num_streams), false, false, nullptr, nullptr); + tasks.upscale(0); + tasks.upscale(1); + tasks.requestThreads(2); + EXPECT_TRUE(tasks.hasThreadsToSpawn()); + + /// One of the two running threads waits for a task; the queues are empty. + std::thread idle([&] { tasks.tryGetTask(tasks.getThreadContext(1)); }); + const auto deadline = std::chrono::steady_clock::now() + timeout; + while (tasks.hasThreadsToSpawn() && std::chrono::steady_clock::now() < deadline) + std::this_thread::sleep_for(std::chrono::milliseconds(1)); + EXPECT_FALSE(tasks.hasThreadsToSpawn()); + + tasks.finish(); + idle.join(); +} + +#if defined(OS_LINUX) || defined(OS_DARWIN) +namespace +{ + +/// Waits for a pipe that is never written to. +class PendingAsyncProcessor : public IProcessor +{ +public: + explicit PendingAsyncProcessor(int fd_) : fd(fd_) {} + String getName() const override { return "PendingAsyncProcessor"; } + Status prepare() override { return Status::Async; } + int schedule() override { return fd; } + +private: + int fd; +}; + +} + +/// Requested threads are kept while asynchronous tasks are pending: they can run the tasks once these are ready. +TEST(PipelineExecutor, RequestedThreadsKeptWhileAsyncTasksPending) +{ + int fds[2]; + ASSERT_EQ(pipe(fds), 0); + SCOPE_EXIT({ + close(fds[0]); + close(fds[1]); + }); + Processors processors{std::make_shared(fds[0])}; + ExecutingGraph::Node node(processors.begin(), 0); + + ExecutorTasks tasks; + tasks.init(num_streams, 1, std::make_shared(num_streams), false, false, nullptr, nullptr); + tasks.upscale(0); + tasks.upscale(1); + + ExecutorTasks::Queue queue; + ExecutorTasks::Queue async_queue; + async_queue.push(&node); + tasks.pushTasks(queue, async_queue, tasks.getThreadContext(0)); + EXPECT_TRUE(tasks.hasThreadsToSpawn()); + + /// One of the two running threads waits for a task; only the asynchronous one is pending. + std::thread idle([&] { tasks.tryGetTask(tasks.getThreadContext(1)); }); + std::this_thread::sleep_for(std::chrono::milliseconds(200)); + EXPECT_TRUE(tasks.hasThreadsToSpawn()); + + tasks.finish(); + idle.join(); +} +#endif diff --git a/src/Processors/tests/gtest_write_orc_iceberg_required.cpp b/src/Processors/tests/gtest_write_orc_iceberg_required.cpp index c8917a248a924..2355d618525fd 100644 --- a/src/Processors/tests/gtest_write_orc_iceberg_required.cpp +++ b/src/Processors/tests/gtest_write_orc_iceberg_required.cpp @@ -17,7 +17,8 @@ # include # include -# include +# include +# include # include # include @@ -122,24 +123,24 @@ void writeOneEmptyBlock(const Block & header, ColumnMapperPtr mapper, const Stri /// Walk the ORC footer type tree, recording iceberg.required for every path (dotted convention: /// t.field, arr.element, m.key, m.value) that carries the attribute. -void collectRequired(const orc::Type & type, const String & path, std::map & out) +void collectRequired(const ORC::Type & type, const String & path, std::map & out) { if (type.hasAttributeKey("iceberg.required")) out[path] = type.getAttributeValue("iceberg.required"); switch (type.getKind()) { - case orc::TypeKind::STRUCT: + case ORC::TypeKind::STRUCT: for (uint64_t i = 0; i < type.getSubtypeCount(); ++i) { const String child = path.empty() ? type.getFieldName(i) : path + "." + type.getFieldName(i); collectRequired(*type.getSubtype(i), child, out); } break; - case orc::TypeKind::LIST: + case ORC::TypeKind::LIST: collectRequired(*type.getSubtype(0), path + ".element", out); break; - case orc::TypeKind::MAP: + case ORC::TypeKind::MAP: collectRequired(*type.getSubtype(0), path + ".key", out); collectRequired(*type.getSubtype(1), path + ".value", out); break; @@ -150,8 +151,9 @@ void collectRequired(const orc::Type & type, const String & path, std::map readIcebergRequired(const String & path) { - orc::ReaderOptions options; - auto reader = orc::createReader(orc::readLocalFile(path), options); + ReadBufferFromFile in(path); + std::atomic is_cancelled{0}; + auto reader = createORCReader(asORCInputStreamLoadIntoMemory(in, is_cancelled), ORC::ReaderOptions{}); std::map out; // The root is the file struct; descend into its top-level columns with an empty prefix. collectRequired(reader->getType(), "", out); diff --git a/src/Server/ArrowFlight/ArrowFlightServer.cpp b/src/Server/ArrowFlight/ArrowFlightServer.cpp index c560ba7838ed4..2548bb0884692 100644 --- a/src/Server/ArrowFlight/ArrowFlightServer.cpp +++ b/src/Server/ArrowFlight/ArrowFlightServer.cpp @@ -575,9 +575,7 @@ static arrow::Result, std::vector, std::vectorgetSettingsRef()[Setting::output_format_arrow_unsupported_types_as_binary]}); - - if (schema_modifier) { - auto status = schema_modifier(schema); - ARROW_RETURN_NOT_OK(status); - schema = status.ValueUnsafe(); - } + PullingPipelineExecutor executor{block_io.pipeline}; + schema = CHColumnToArrowColumn::calculateArrowSchema( + executor.getHeader().getColumnsWithTypeAndName(), + "Arrow", + nullptr, + {.output_string_as_string = true, .output_unsupported_types_as_binary = query_context->getSettingsRef()[Setting::output_format_arrow_unsupported_types_as_binary]}); + + if (schema_modifier) + { + auto status = schema_modifier(schema); + ARROW_RETURN_NOT_OK(status); + schema = status.ValueUnsafe(); + } - std::optional header; - std::vector chunks; - Block block; - while (executor.pull(block)) - { - if (!block.empty()) + std::optional header; + std::vector chunks; + Block block; + while (executor.pull(block)) { - if (block_modifier) - block_modifier(query_context, block); - if (!header) - header = getHeader(block.getColumnsWithTypeAndName()); - chunks.emplace_back(Chunk{block.getColumns(), block.rows()}); - if (!single_table) + if (!block.empty()) { - tables.emplace_back( - CHColumnToArrowColumn::calculateArrowTable( - *header, "Arrow", chunks, - {.output_string_as_string = true, .output_unsupported_types_as_binary = query_context->getSettingsRef()[Setting::output_format_arrow_unsupported_types_as_binary]}, - header->size(), schema)); - chunks.clear(); + if (block_modifier) + block_modifier(query_context, block); + if (!header) + header = getHeader(block.getColumnsWithTypeAndName()); + chunks.emplace_back(Chunk{block.getColumns(), block.rows()}); + if (!single_table) + { + tables.emplace_back( + CHColumnToArrowColumn::calculateArrowTable( + *header, "Arrow", chunks, + {.output_string_as_string = true, .output_unsupported_types_as_binary = query_context->getSettingsRef()[Setting::output_format_arrow_unsupported_types_as_binary]}, + header->size(), schema)); + chunks.clear(); + } } } - } - if (!header) - tables.emplace_back(getEmptyArrowTable(schema)); - else if (single_table) - tables.emplace_back( - CHColumnToArrowColumn::calculateArrowTable( - *header, "Arrow", chunks, - {.output_string_as_string = true, .output_unsupported_types_as_binary = query_context->getSettingsRef()[Setting::output_format_arrow_unsupported_types_as_binary]}, - header->size(), schema)); + if (!header) + tables.emplace_back(getEmptyArrowTable(schema)); + else if (single_table) + tables.emplace_back( + CHColumnToArrowColumn::calculateArrowTable( + *header, "Arrow", chunks, + {.output_string_as_string = true, .output_unsupported_types_as_binary = query_context->getSettingsRef()[Setting::output_format_arrow_unsupported_types_as_binary]}, + header->size(), schema)); + } + block_io.onFinish(); query_finished = true; } catch (...) @@ -790,9 +791,7 @@ arrow::Status ArrowFlightServer::GetSchema( bool query_finished = false; bool handling_exception = false; SCOPE_EXIT({ - if (query_finished) - block_io.onFinish(); - else if (!handling_exception) + if (!query_finished && !handling_exception) block_io.onCancelOrConnectionLoss(); }); @@ -801,18 +800,21 @@ arrow::Status ArrowFlightServer::GetSchema( ARROW_RETURN_NOT_OK(checkNoCustomFormat(ast)); ARROW_RETURN_NOT_OK(checkPipelineIsPulling(block_io.pipeline)); - PullingPipelineExecutor executor{block_io.pipeline}; - - schema = CHColumnToArrowColumn::calculateArrowSchema( - executor.getHeader().getColumnsWithTypeAndName(), "Arrow", nullptr, - {.output_string_as_string = true, .output_unsupported_types_as_binary = query_context->getSettingsRef()[Setting::output_format_arrow_unsupported_types_as_binary]}); - if (schema_modifier) { - auto status = schema_modifier(schema); - ARROW_RETURN_NOT_OK(status); - schema = status.ValueUnsafe(); + PullingPipelineExecutor executor{block_io.pipeline}; + + schema = CHColumnToArrowColumn::calculateArrowSchema( + executor.getHeader().getColumnsWithTypeAndName(), "Arrow", nullptr, + {.output_string_as_string = true, .output_unsupported_types_as_binary = query_context->getSettingsRef()[Setting::output_format_arrow_unsupported_types_as_binary]}); + if (schema_modifier) + { + auto status = schema_modifier(schema); + ARROW_RETURN_NOT_OK(status); + schema = status.ValueUnsafe(); + } } + block_io.onFinish(); query_finished = true; } catch (...) @@ -1087,9 +1089,7 @@ arrow::Status ArrowFlightServer::DoGet( bool query_finished = false; bool handling_exception = false; SCOPE_EXIT({ - if (query_finished) - block_io.onFinish(); - else if (!handling_exception) + if (!query_finished && !handling_exception) block_io.onCancelOrConnectionLoss(); }); @@ -1098,16 +1098,19 @@ arrow::Status ArrowFlightServer::DoGet( ARROW_RETURN_NOT_OK(checkNoCustomFormat(ast)); ARROW_RETURN_NOT_OK(checkPipelineIsPulling(block_io.pipeline)); - PullingPipelineExecutor executor{block_io.pipeline}; + { + PullingPipelineExecutor executor{block_io.pipeline}; - Block block; - while (executor.pull(block)) - chunks.emplace_back(Chunk(block.getColumns(), block.rows())); + Block block; + while (executor.pull(block)) + chunks.emplace_back(Chunk(block.getColumns(), block.rows())); - auto header = executor.getHeader(); - auto ch_to_arrow_converter = createCHToArrowConverter(header, query_context); - ch_to_arrow_converter->chChunkToArrowTable(table, chunks, header.columns()); + auto header = executor.getHeader(); + auto ch_to_arrow_converter = createCHToArrowConverter(header, query_context); + ch_to_arrow_converter->chChunkToArrowTable(table, chunks, header.columns()); + } + block_io.onFinish(); query_finished = true; } catch (...) @@ -1222,9 +1225,7 @@ arrow::Status ArrowFlightServer::DoPut( bool query_finished = false; bool handling_exception = false; SCOPE_EXIT({ - if (query_finished) - block_io.onFinish(); - else if (!handling_exception) + if (!query_finished && !handling_exception) block_io.onCancelOrConnectionLoss(); }); @@ -1252,6 +1253,7 @@ arrow::Status ArrowFlightServer::DoPut( executor.execute(); } + block_io.onFinish(); query_finished = true; } catch (...) diff --git a/src/Server/PostgreSQLHandler.cpp b/src/Server/PostgreSQLHandler.cpp index 653a311023ed4..87ada41e4a541 100644 --- a/src/Server/PostgreSQLHandler.cpp +++ b/src/Server/PostgreSQLHandler.cpp @@ -722,8 +722,8 @@ void PostgreSQLHandler::cancelRequest() inline std::unique_ptr PostgreSQLHandler::receiveStartupMessage(int payload_size) { /// The declared size is read from the wire before any authentication, and the message is read - /// into memory in full, so it has to be bounded. PostgreSQL uses the same limit. - static constexpr Int32 max_startup_message_size = 10000; + /// into memory in full, so it has to be bounded (`max_unauthenticated_message_size`). + static constexpr Int32 max_startup_message_size = PostgreSQLProtocol::Messaging::max_unauthenticated_message_size; std::unique_ptr message; try diff --git a/src/Storages/AlterCommands.cpp b/src/Storages/AlterCommands.cpp index 9a8c5912c7baf..6b37e1e372151 100644 --- a/src/Storages/AlterCommands.cpp +++ b/src/Storages/AlterCommands.cpp @@ -30,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -47,6 +48,7 @@ #include #include #include +#include #include #include #include @@ -1733,6 +1735,33 @@ bool AlterCommands::hasVectorSimilarityIndex(const StorageInMemoryMetadata & met return false; } +NameSet AlterCommands::getAddedIndexNames(const IndicesDescription & existing) const +{ + NameSet present; + for (const auto & index : existing) + present.insert(index.name); + + NameSet added; + for (const AlterCommand & command : *this) + { + if (command.ignore) + continue; + + if (command.type == AlterCommand::ADD_INDEX) + { + /// Otherwise `AlterCommand::apply` either skips it, with `IF NOT EXISTS`, or throws. + if (present.insert(command.index_name).second) + added.insert(command.index_name); + } + else if (command.type == AlterCommand::DROP_INDEX && !command.partition && !command.clear) + { + present.erase(command.index_name); + added.erase(command.index_name); + } + } + return added; +} + void AlterCommands::apply(StorageInMemoryMetadata & metadata, ContextPtr context, bool share_nested_offsets) const { if (!prepared) @@ -1792,6 +1821,16 @@ void AlterCommands::apply(StorageInMemoryMetadata & metadata, ContextPtr context } } + /// Only the declared ones, and only now: the rebuild above resolves them against the schema every + /// command produced, which is what gets stored. Existing indices stay unauthorised on purpose. + const NameSet declared_indices = getAddedIndexNames(metadata.secondary_indices); + + for (const auto & index : metadata_copy.secondary_indices) + { + if (declared_indices.contains(index.name)) + checkTextIndexPreAndPostProcessorPermissions(index, context); + } + /// Changes in columns may lead to changes in projections ProjectionsDescription new_projections; for (const auto & projection : metadata_copy.projections) @@ -2039,6 +2078,8 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const /// to its target using the target metadata and never evaluates its own column defaults, so a default /// over a virtual column is inert there and is left out of this set. NameSet insert_time_default_columns; + /// New `DEFAULT` and `MATERIALIZED` expressions of a `MergeTree` table, also computed in the background for existing parts. + ASTPtr background_default_expr_list = make_intrusive(); bool defaults_evaluated_at_insert_time = true; if (const auto * mv = dynamic_cast(table.get())) defaults_evaluated_at_insert_time = mv->hasInnerTable(); @@ -2408,6 +2449,16 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const /// Collect default expressions for MODIFY and ADD commands if (command.type == AlterCommand::MODIFY_COLUMN || command.type == AlterCommand::ADD_COLUMN) { + /// A column TTL resets the column to its existing default in merges. + if (command.ttl && !command.default_expression && all_columns.has(column_name)) + { + const auto & default_desc = all_columns.get(column_name).default_desc; + if (default_desc.expression + && (default_desc.kind == ColumnDefaultKind::Default || default_desc.kind == ColumnDefaultKind::Materialized)) + background_default_expr_list->children.emplace_back( + setAlias(default_desc.expression->clone(), column_name + "_tmp_ttl_default")); + } + if (command.default_expression) { DataTypePtr data_type_ptr; @@ -2429,6 +2480,9 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const if (defaults_evaluated_at_insert_time && (command.default_kind == ColumnDefaultKind::Default || command.default_kind == ColumnDefaultKind::Materialized)) insert_time_default_columns.insert(final_column_name); + + if (command.default_kind == ColumnDefaultKind::Default || command.default_kind == ColumnDefaultKind::Materialized) + background_default_expr_list->children.emplace_back(setAlias(command.default_expression->clone(), tmp_column_name)); } /// if we change data type for column with default else if (all_columns.has(column_name) && command.data_type) { @@ -2460,6 +2514,16 @@ void AlterCommands::validate(const StoragePtr & table, ContextPtr context) const throw Exception(ErrorCodes::BAD_ARGUMENTS, "Cannot DROP or CLEAR all columns"); validateColumnsDefaultsAndGetSampleBlock(default_expr_list, all_columns.getAll(), context, insert_time_default_columns); + + /// A replica replaying the `ALTER` of a Replicated database does not repeat the check. + auto txn = context->getZooKeeperMetadataTransaction(); + if (!background_default_expr_list->children.empty() && castStorage(table, DeferredTable::Load) + && (!txn || txn->isInitialQuery())) + { + auto background_context = Context::createCopy(context); + background_context->setAnalyzingBackgroundExpressions(true); + validateColumnsDefaultsAndGetSampleBlock(background_default_expr_list, all_columns.getAll(), background_context, {}); + } } bool AlterCommands::hasNonReplicatedAlterCommand() const diff --git a/src/Storages/AlterCommands.h b/src/Storages/AlterCommands.h index 04031c1ee8b59..546cabef6c02c 100644 --- a/src/Storages/AlterCommands.h +++ b/src/Storages/AlterCommands.h @@ -239,6 +239,9 @@ class AlterCommands : public std::vector /// stay consistent with prepare()/validate() for nested columns (see AlterCommand::apply). void apply(StorageInMemoryMetadata & metadata, ContextPtr context, bool share_nested_offsets = true) const; + /// Indices the commands actually add: `ADD INDEX IF NOT EXISTS` of an existing name adds nothing. + NameSet getAddedIndexNames(const IndicesDescription & existing) const; + /// At least one command modify settings or comments. bool hasNonReplicatedAlterCommand() const; diff --git a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp index 819576c2dabf7..8786820604c4d 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp +++ b/src/Storages/Distributed/DistributedAsyncInsertBatch.cpp @@ -21,6 +21,7 @@ #include #include +#include #include namespace CurrentMetrics @@ -234,6 +235,11 @@ bool DistributedAsyncInsertBatch::recoverBatch() } files.erase(files.begin(), first_existing_file); + /// The batch is sent with the `ClientInfo` of its first file. Older servers could put files of different + /// users (or originating hosts) into one batch, so a restored batch of mixed identities must not be resent + /// as a whole: it is dropped, and its files are batched again by identity like any other pending files. + std::optional> batch_identity; + /// A missing file inside the surviving suffix cannot be recovered safely. for (const auto & file : files) { @@ -252,6 +258,18 @@ bool DistributedAsyncInsertBatch::recoverBatch() total_rows += header.rows; total_bytes += header.bytes; } + + auto identity = getRemoteAuthenticationIdentity(header.client_info); + if (!batch_identity) + { + batch_identity = std::move(identity); + } + else if (identity != *batch_identity) + { + LOG_WARNING(parent.log, "File {} has a different user or originating host than the first file of the batch, " + "the files of the batch will be sent in new batches", file); + return false; + } } catch (Exception & e) { diff --git a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp index da636ba25c50c..469feb7731857 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp +++ b/src/Storages/Distributed/DistributedAsyncInsertDirectoryQueue.cpp @@ -525,8 +525,11 @@ struct DistributedAsyncInsertDirectoryQueue::BatchHeader bool operator==(const BatchHeader & other) const { + /// The whole batch is sent with the `ClientInfo` of its first file, so files that the shard would + /// authenticate differently must never be merged into one batch. return std::tie(settings, query, client_info.query_kind, client_info.client_agent, client_info.is_internal) == std::tie(other.settings, other.query, other.client_info.query_kind, other.client_info.client_agent, other.client_info.is_internal) && + getRemoteAuthenticationIdentity(client_info) == getRemoteAuthenticationIdentity(other.client_info) && blocksHaveEqualStructure(header, other.header); } @@ -536,6 +539,9 @@ struct DistributedAsyncInsertDirectoryQueue::BatchHeader { SipHash hash_state; hash_state.update(batch_header.query.data(), batch_header.query.size()); + const auto [initial_user, initial_host] = getRemoteAuthenticationIdentity(batch_header.client_info); + hash_state.update(initial_user.data(), initial_user.size()); + hash_state.update(initial_host.data(), initial_host.size()); hash_state.update(batch_header.client_info.client_agent.data(), batch_header.client_info.client_agent.size()); hash_state.update(batch_header.client_info.is_internal); batch_header.header.updateHash(hash_state); diff --git a/src/Storages/Distributed/DistributedAsyncInsertHelpers.cpp b/src/Storages/Distributed/DistributedAsyncInsertHelpers.cpp index 8cbb8461f5ea7..7890ecb8f89e0 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertHelpers.cpp +++ b/src/Storages/Distributed/DistributedAsyncInsertHelpers.cpp @@ -68,6 +68,14 @@ static void writeAndConvert(RemoteInserter & remote, const DistributedAsyncInser } } +std::pair getRemoteAuthenticationIdentity(const ClientInfo & client_info) +{ + String initial_host; + if (client_info.initial_address) + initial_host = client_info.initial_address->host().toString(); + return {client_info.initial_user, initial_host}; +} + void writeRemoteConvert( const DistributedAsyncInsertHeader & distributed_header, RemoteInserter & remote, diff --git a/src/Storages/Distributed/DistributedAsyncInsertHelpers.h b/src/Storages/Distributed/DistributedAsyncInsertHelpers.h index 202d9ff6fff8b..3cc335c169f98 100644 --- a/src/Storages/Distributed/DistributedAsyncInsertHelpers.h +++ b/src/Storages/Distributed/DistributedAsyncInsertHelpers.h @@ -1,12 +1,16 @@ #pragma once #include +#include + +#include namespace DB { struct DistributedAsyncInsertHeader; +class ClientInfo; class ReadBufferFromFile; class RemoteInserter; @@ -23,6 +27,11 @@ class RemoteInserter; /// and mark batch as broken. bool isDistributedSendBroken(int code, bool remote_error); +/// The identity the shard authenticates a queued insert as (with the interserver ``): `initial_user` +/// and the host of `initial_address`, which is checked against the allowed hosts of that user. +/// A batch is sent with the `ClientInfo` of its first file, so all files of a batch must have the same identity. +std::pair getRemoteAuthenticationIdentity(const ClientInfo & client_info); + void writeRemoteConvert( const DistributedAsyncInsertHeader & distributed_header, RemoteInserter & remote, diff --git a/src/Storages/Distributed/DistributedSink.cpp b/src/Storages/Distributed/DistributedSink.cpp index a94d739a18933..84f8f89df8657 100644 --- a/src/Storages/Distributed/DistributedSink.cpp +++ b/src/Storages/Distributed/DistributedSink.cpp @@ -966,17 +966,21 @@ void DistributedSink::writeToShard(const Cluster::ShardInfo & shard_info, const /// `client_agent` is intentionally excluded from the embedded `ClientInfo` here and written /// as a trailing header field below, so that older binaries draining these queue files read /// the embedded `ClientInfo` in the old, append-compatible layout. - if (OpenTelemetry::CurrentContext().isTraceEnabled()) { - // if the distributed tracing is enabled, use the trace context in current thread as parent of next span auto client_info = context->getClientInfo(); - client_info.client_trace_context = OpenTelemetry::CurrentContext(); + // if the distributed tracing is enabled, use the trace context in current thread as parent of next span + if (OpenTelemetry::CurrentContext().isTraceEnabled()) + client_info.client_trace_context = OpenTelemetry::CurrentContext(); + /// Background contexts (e.g. the ones of `Kafka` / `RabbitMQ` streaming into views) have + /// `query_kind = NO_QUERY`, for which `ClientInfo::write` stores nothing but the kind. The + /// replayed insert would then reach the shard with an empty `initial_user`, which a shard + /// accepting the interserver `` executes with full access, losing the identity + /// (e.g. the definer of a `SQL SECURITY DEFINER` view) that the query is bound to. + /// `RemoteInserter` sends the insert as a secondary query anyway, so persist it as such. + if (client_info.query_kind == ClientInfo::QueryKind::NO_QUERY) + client_info.query_kind = ClientInfo::QueryKind::SECONDARY_QUERY; client_info.write(header_buf, DBMS_TCP_PROTOCOL_VERSION, /*with_trailing_fields=*/ false); } - else - { - context->getClientInfo().write(header_buf, DBMS_TCP_PROTOCOL_VERSION, /*with_trailing_fields=*/ false); - } writeVarUInt(block.rows(), header_buf); writeVarUInt(block.bytes(), header_buf); diff --git a/src/Storages/Hive/HiveFile.cpp b/src/Storages/Hive/HiveFile.cpp index 378bb0840ef73..15d92e6ad1155 100644 --- a/src/Storages/Hive/HiveFile.cpp +++ b/src/Storages/Hive/HiveFile.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include @@ -34,23 +33,16 @@ namespace ErrorCodes extern const int BAD_ARGUMENTS; } -template -Range createRangeFromOrcStatistics(const StatisticsType * stats) +template +Range createRangeFromOrcStatistics(const std::optional & minimum, const std::optional & maximum) { /// Null values or NaN/Inf values of double type. - if (stats->hasMinimum() && stats->hasMaximum()) - { - return Range(FieldType(stats->getMinimum()), true, FieldType(stats->getMaximum()), true); - } - if (stats->hasMinimum()) - { - return Range::createLeftBounded(FieldType(stats->getMinimum()), true); - } - if (stats->hasMaximum()) - { - return Range::createRightBounded(FieldType(stats->getMaximum()), true); - } - + if (minimum && maximum) + return Range(FieldType(*minimum), true, FieldType(*maximum), true); + if (minimum) + return Range::createLeftBounded(FieldType(*minimum), true); + if (maximum) + return Range::createRightBounded(FieldType(*maximum), true); return Range::createWholeUniverseWithoutNull(); } @@ -109,48 +101,40 @@ void IHiveFile::loadSplitMinMaxIndexes() split_minmax_idxes_loaded = true; } -Range HiveORCFile::buildRange(const orc::ColumnStatistics * col_stats) +Range HiveORCFile::buildRange(const ORC::Proto::ColumnStatistics * col_stats) { - if (!col_stats || col_stats->hasNull()) + /// A missing `has_null` means that the column may contain nulls. + if (!col_stats || col_stats->has_null.value_or(true)) return Range::createWholeUniverseWithoutNull(); - if (const auto * int_stats = dynamic_cast(col_stats)) - { - return createRangeFromOrcStatistics(int_stats); - } - if (const auto * double_stats = dynamic_cast(col_stats)) - { - return createRangeFromOrcStatistics(double_stats); - } - if (const auto * string_stats = dynamic_cast(col_stats)) + if (const auto & int_stats = col_stats->int_statistics) + return createRangeFromOrcStatistics(int_stats->minimum, int_stats->maximum); + if (const auto & double_stats = col_stats->double_statistics) + return createRangeFromOrcStatistics(double_stats->minimum, double_stats->maximum); + if (const auto & string_stats = col_stats->string_statistics) + return createRangeFromOrcStatistics(string_stats->minimum, string_stats->maximum); + if (const auto & bool_stats = col_stats->bucket_statistics) { - return createRangeFromOrcStatistics(string_stats); - } - if (const auto * bool_stats = dynamic_cast(col_stats)) - { - auto false_cnt = bool_stats->getFalseCount(); - auto true_cnt = bool_stats->getTrueCount(); + if (bool_stats->count.empty()) + return Range::createWholeUniverseWithoutNull(); + UInt64 true_cnt = bool_stats->count[0]; + UInt64 false_cnt = col_stats->number_of_values.value_or(0) - true_cnt; if (false_cnt && true_cnt) - { return Range(UInt8(0), true, UInt8(1), true); - } if (false_cnt) - { return Range::createLeftBounded(UInt8(0), true); - } if (true_cnt) - { return Range::createRightBounded(UInt8(1), true); - } - } - else if (const auto * timestamp_stats = dynamic_cast(col_stats)) - { - return createRangeFromOrcStatistics(timestamp_stats); + return Range::createWholeUniverseWithoutNull(); } - else if (const auto * date_stats = dynamic_cast(col_stats)) + if (const auto & timestamp_stats = col_stats->timestamp_statistics) { - return createRangeFromOrcStatistics(date_stats); + /// The deprecated `minimum` and `maximum` are in the local time zone of the writer, which is + /// unknown for the statistics of the whole file, so only the UTC bounds are usable. + return createRangeFromOrcStatistics(timestamp_stats->minimum_utc, timestamp_stats->maximum_utc); } + if (const auto & date_stats = col_stats->date_statistics) + return createRangeFromOrcStatistics(date_stats->minimum, date_stats->maximum); return Range::createWholeUniverseWithoutNull(); } @@ -159,14 +143,12 @@ void HiveORCFile::prepareReader() in = std::make_unique(namenode_url, path, getContext()->getGlobalContext()->getConfigRef(), getContext()->getReadSettings()); auto format_settings = getFormatSettings(getContext()); std::atomic is_stopped{0}; - orc::ReaderOptions options; - options.setMemoryPool(getORCMemoryPool()); - reader = createORCReader(asORCInputStream(*in, format_settings, /*use_prefetch=*/false, is_stopped), options); + reader = createORCReader(asORCInputStream(*in, format_settings, /*use_prefetch=*/false, is_stopped), ORC::ReaderOptions{}); } void HiveORCFile::prepareColumnMapping() { - const orc::Type & type = reader->getType(); + const ORC::Type & type = reader->getType(); size_t count = type.getSubtypeCount(); for (size_t pos = 0; pos < count; pos++) { @@ -183,7 +165,7 @@ bool HiveORCFile::useFileMinMaxIndex() const } -std::unique_ptr HiveORCFile::buildMinMaxIndex(const orc::Statistics * statistics) +std::unique_ptr HiveORCFile::buildMinMaxIndex(const std::vector * statistics) { if (!statistics) return nullptr; @@ -206,7 +188,7 @@ std::unique_ptr HiveORCFile::buildMinMaxIndex(c { size_t pos = it->second; /// Attention: column statistics start from 1. 0 has special purpose. - const orc::ColumnStatistics * col_stats = statistics->getColumnStatistics(static_cast(pos + 1)); + const ORC::Proto::ColumnStatistics * col_stats = pos + 1 < statistics->size() ? &(*statistics)[pos + 1] : nullptr; idx->hyperrectangle[i] = buildRange(col_stats); } ++i; @@ -223,8 +205,7 @@ void HiveORCFile::loadFileMinMaxIndexImpl() prepareColumnMapping(); } - auto statistics = reader->getStatistics(); - file_minmax_idx = buildMinMaxIndex(statistics.get()); + file_minmax_idx = buildMinMaxIndex(&reader->getStatistics()); } bool HiveORCFile::useSplitMinMaxIndex() const @@ -243,7 +224,8 @@ void HiveORCFile::loadSplitMinMaxIndexesImpl() auto * raw_reader = reader.get(); auto stripe_num = raw_reader->getNumberOfStripes(); - auto stripe_stats_num = raw_reader->getNumberOfStripeStatistics(); + const auto & stripe_statistics = raw_reader->getMetadata().stripe_stats; + auto stripe_stats_num = stripe_statistics.size(); if (stripe_num != stripe_stats_num) throw Exception(ErrorCodes::BAD_ARGUMENTS, "orc file:{} has different strip num {} and strip statistics num {}", path, stripe_num, stripe_stats_num); @@ -251,8 +233,7 @@ void HiveORCFile::loadSplitMinMaxIndexesImpl() split_minmax_idxes.resize(stripe_num); for (size_t i = 0; i < stripe_num; ++i) { - auto stripe_stats = raw_reader->getStripeStatistics(i); - split_minmax_idxes[i] = buildMinMaxIndex(stripe_stats.get()); + split_minmax_idxes[i] = buildMinMaxIndex(&stripe_statistics[i].col_stats); } } diff --git a/src/Storages/Hive/HiveFile.h b/src/Storages/Hive/HiveFile.h index 61625c2c76c63..4b82ef293b6d8 100644 --- a/src/Storages/Hive/HiveFile.h +++ b/src/Storages/Hive/HiveFile.h @@ -8,7 +8,7 @@ #include #include -#include +#include #include #include @@ -225,18 +225,18 @@ class HiveORCFile : public IHiveFile bool useSplitMinMaxIndex() const override; private: - static Range buildRange(const orc::ColumnStatistics * col_stats); + static Range buildRange(const ORC::Proto::ColumnStatistics * col_stats); void loadFileMinMaxIndexImpl() override; void loadSplitMinMaxIndexesImpl() override; - std::unique_ptr buildMinMaxIndex(const orc::Statistics * statistics); + std::unique_ptr buildMinMaxIndex(const std::vector * statistics); void prepareReader(); void prepareColumnMapping(); std::optional getRowsImpl() override; std::unique_ptr in; - std::unique_ptr reader; + std::unique_ptr reader; std::map orc_column_positions; }; diff --git a/src/Storages/Hive/StorageHive.cpp b/src/Storages/Hive/StorageHive.cpp index acaba8fe71a25..7c71181264681 100644 --- a/src/Storages/Hive/StorageHive.cpp +++ b/src/Storages/Hive/StorageHive.cpp @@ -609,6 +609,10 @@ HiveFilesWithSkipSplits StorageHive::collectHiveFilesFromPartition( throw Exception(ErrorCodes::INVALID_PARTITION_VALUE, "Partition value size not match, expect {}, but got {}", partition_names.size(), partition.values.size()); + if (!partition_key_expr) + throw Exception(ErrorCodes::INVALID_PARTITION_VALUE, + "Hive metastore returned a partition for table {}.{}, but the table has no partition key", hive_database, hive_table); + /// Join partition values in CSV format WriteBufferFromOwnString wb; for (size_t i = 0; i < partition.values.size(); ++i) diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index 4307ed1df1c03..ee27ad3647b4e 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -198,6 +198,9 @@ class IStorage : public std::enable_shared_from_this, public TypePromo /// A storage that cannot serve synthesised subcolumns such as `.null`/`.size0` as standalone /// inputs may enable this while keeping supportsOptimizationToSubcolumns() false. virtual bool supportsOptimizationToTupleElementSubcolumns() const { return supportsOptimizationToSubcolumns(); } + /// Returns true if the storage writes `Map` columns with the bucketed serialization, where a single key + /// can be read without reading the whole column. Gates the `m['key']` -> `m.key_` rewrite. + virtual bool hasBucketedMapSerialization() const { return false; } /// Returns true if the storage supports transactions for SELECT, INSERT and ALTER queries. /// Storage may throw an exception later if some query kind is not fully supported. diff --git a/src/Storages/IStorageCluster.cpp b/src/Storages/IStorageCluster.cpp index 87c61dcae9cce..c6c040a629582 100644 --- a/src/Storages/IStorageCluster.cpp +++ b/src/Storages/IStorageCluster.cpp @@ -120,7 +120,7 @@ void IStorageCluster::read( query_to_send = interpreter.getQueryInfo().query->clone(); } - updateQueryToSendIfNeeded(query_to_send, storage_snapshot, context); + updateQueryToSendIfNeeded(query_to_send, storage_snapshot, context, /*add_structure_and_format=*/true); RestoreQualifiedNamesVisitor::Data data; data.distributed_table = DatabaseAndTableWithAlias(*getTableExpression(query_info.query->as(), 0)); diff --git a/src/Storages/IStorageCluster.h b/src/Storages/IStorageCluster.h index 6ecfdd22dc193..144fda69e3195 100644 --- a/src/Storages/IStorageCluster.h +++ b/src/Storages/IStorageCluster.h @@ -59,8 +59,15 @@ class IStorageCluster : public IStorage /// function (`url`, `s3`, ...) that `parallel_replicas_for_cluster_engines` converted into this cluster /// storage into its `*Cluster` variant with the cluster name argument, so that the nodes take their read /// tasks from the initiator instead of reading every file on their own. Called by `read` and by the - /// distributed `INSERT ... SELECT` in `InterpreterInsertQuery`, which forwards the query the same way. - virtual void updateQueryToSendIfNeeded(ASTPtr & /*query*/, const StorageSnapshotPtr & /*storage_snapshot*/, const ContextPtr & /*context*/) {} + /// distributed `INSERT ... SELECT` in `InterpreterInsertQuery`. + /// A `SELECT` needs the structure and format so all nodes produce the initiator's header. For a forwarded + /// `INSERT ... SELECT`, leave the arguments as written: an omitted or `'auto'` structure then takes the + /// destination table's columns and `DEFAULT` expressions on each node. + virtual void updateQueryToSendIfNeeded( + ASTPtr & /*query*/, + const StorageSnapshotPtr & /*storage_snapshot*/, + const ContextPtr & /*context*/, + bool /*add_structure_and_format*/) {} protected: virtual void updateBeforeRead(const ContextPtr &) {} diff --git a/src/Storages/IndicesDescription.cpp b/src/Storages/IndicesDescription.cpp index 8b4846aa8463f..e2780da4b1bba 100644 --- a/src/Storages/IndicesDescription.cpp +++ b/src/Storages/IndicesDescription.cpp @@ -1,12 +1,20 @@ +#include #include #include #include +#include #include +#include +#include +#include #include #include #include #include +#include +#include +#include #include #include #include @@ -28,27 +36,129 @@ namespace ErrorCodes namespace { -/// Only the `preprocessor` and `postprocessor` arguments contain column expressions. -void expandTextIndexTransformAliases(const ASTPtr & arguments, const ColumnsDescription & columns) +/// Calls `fn` with the value of each `preprocessor` / `postprocessor` argument of a text index. +template +void forEachTransformArgument(const ASTPtr & arguments, F && fn) { - using ReplaceAliasToExprVisitor = InDepthNodeVisitor; for (const auto & child : arguments->children) { - const auto * func = child->as(); + auto * func = child->as(); if (!func || func->name != "equals" || !func->arguments || func->arguments->children.size() != 2) continue; const auto * key = func->arguments->children[0]->as(); if (key && (key->name() == "preprocessor" || key->name() == "postprocessor")) + fn(func->arguments->children[1]); + } +} + +void expandTextIndexTransformAliases(const ASTPtr & arguments, const ColumnsDescription & columns) +{ + using ReplaceAliasToExprVisitor = InDepthNodeVisitor; + forEachTransformArgument(arguments, [&](ASTPtr & expression) + { + ReplaceAliasToExprVisitor::Data data{columns, {}, /*reject_lambda_capture=*/ true}; + ReplaceAliasToExprVisitor{data}.visit(expression); + }); +} + +/// Resolving a name runs the function's `create`, where `checkAccess` lives. Names only: building the +/// expression would report its type errors from here, not from the validator. +/// +/// A transform is analysed under the global context, so reading a table or a dictionary during analysis +/// would happen with full access. Such access depends on the arguments, not the function name, so +/// authorizing names cannot cover it, and these constructs are refused. Only here, where a user declares +/// the index: metadata stored by an older version must still load and stay alterable. +void checkFunctionsAsUser(const ASTPtr & ast, const ContextPtr & context) +{ + if (ast->as() || ast->as() || ast->as() || ast->as()) + throw Exception(ErrorCodes::INCORRECT_QUERY, "A text index transform expression must not contain subqueries or table references"); + + if (const auto * function = ast->as()) + { + if (functionIsInOrGlobalInOperator(function->name) && function->arguments && function->arguments->children.size() == 2) { - ReplaceAliasToExprVisitor::Data data{columns, {}, /*reject_lambda_capture=*/ true}; - ReplaceAliasToExprVisitor{data}.visit(func->arguments->children[1]); + const auto & set = function->arguments->children[1]; + if (set->as() || set->as()) + throw Exception(ErrorCodes::INCORRECT_QUERY, + "A text index transform expression must not use a table or a set as the right side of `{}`", function->name); } + + /// Only the name, which for `dictGet` and the like does not touch the dictionary; analysis would. + if (auto resolver = FunctionFactory::instance().tryGet(function->name, context); resolver && !resolver->isDeterministic()) + throw Exception(ErrorCodes::INCORRECT_QUERY, + "A text index transform expression must not contain non-deterministic functions. Got `{}`", function->name); } + + for (const auto & child : ast->children) + checkFunctionsAsUser(child, context); +} + +void checkTransformPermissions(const ASTPtr & arguments, const ContextPtr & context) +{ + forEachTransformArgument(arguments, [&](ASTPtr & argument) + { + auto expression = argument->clone(); + /// Inlines SQL UDF bodies, as `TreeRewriter` does, so a function used only inside one is seen. + if (!UserDefinedSQLFunctionFactory::instance().empty()) + UserDefinedSQLFunctionVisitor::visit(expression, context); + + checkFunctionsAsUser(expression, context); + }); } } +void checkTextIndexPreAndPostProcessorPermissions(const IndexDescription & index, const ContextPtr & context) +{ + if (index.type == TEXT_INDEX_NAME && index.arguments) + checkTransformPermissions(index.arguments, context); +} + +bool hasTextIndexPreOrPostProcessor(const IndexDescription & index) +{ + if (index.type != TEXT_INDEX_NAME || !index.arguments) + return false; + + bool found = false; + forEachTransformArgument(index.arguments, [&](ASTPtr &) { found = true; }); + return found; +} + +[[noreturn]] static void throwTransformNeedsInitiatorUser() +{ + throw Exception( + ErrorCodes::INCORRECT_QUERY, + "A text index preprocessor or postprocessor is authorized on the host the query is submitted to, and the " + "other hosts resolve it against their own schema without that user. Let every host run the query as the " + "initiating user instead: enable the server setting distributed_ddl_use_initial_user_and_roles on every " + "host, and set distributed_ddl_entry_format_version to at least {}", + DDLLogEntry::INITIATOR_USER_VERSION); +} + +void rejectTextIndexTransformWithoutInitiatorUser(const ASTIndexDeclaration & declaration) +{ + if (hasTextIndexPreOrPostProcessor(declaration)) + throwTransformNeedsInitiatorUser(); +} + +void rejectTextIndexTransformWithoutInitiatorUser(const IndexDescription & index) +{ + if (hasTextIndexPreOrPostProcessor(index)) + throwTransformNeedsInitiatorUser(); +} + +bool hasTextIndexPreOrPostProcessor(const ASTIndexDeclaration & declaration) +{ + const auto * type = declaration.getType() ? declaration.getType()->as() : nullptr; + if (!type || Poco::toLower(type->name) != TEXT_INDEX_NAME || !type->arguments) + return false; + + bool found = false; + forEachTransformArgument(type->arguments, [&](ASTPtr &) { found = true; }); + return found; +} + IndexDescription::IndexDescription(const IndexDescription & other) : definition_ast(other.definition_ast ? other.definition_ast->clone() : nullptr) , expression_list_ast(other.expression_list_ast ? other.expression_list_ast->clone() : nullptr) diff --git a/src/Storages/IndicesDescription.h b/src/Storages/IndicesDescription.h index 5c3cf33b3191e..f53855ae21717 100644 --- a/src/Storages/IndicesDescription.h +++ b/src/Storages/IndicesDescription.h @@ -16,6 +16,7 @@ namespace DB class ExpressionActions; using ExpressionActionsPtr = std::shared_ptr; +class ASTIndexDeclaration; /// Description of non-primary index for Storage struct IndexDescription @@ -86,6 +87,20 @@ struct IndexDescription bool isSimpleSingleColumnIndex() const; }; +/// Checks that the functions referenced by the text index preprocessor and postprocessor expressions +/// may run with the permissions of `context`, and that they read no table, set or dictionary. Call only +/// where a user declares an index, on the node they submitted it to; elsewhere these resolve under the +/// global context, and stored metadata is not re-checked. +void checkTextIndexPreAndPostProcessorPermissions(const IndexDescription & index, const ContextPtr & context); + +/// Refuses a transform when the other hosts of an `ON CLUSTER` query do not run it as the initiator. +void rejectTextIndexTransformWithoutInitiatorUser(const ASTIndexDeclaration & declaration); +void rejectTextIndexTransformWithoutInitiatorUser(const IndexDescription & index); + +/// Whether the declaration is a text index with a `preprocessor` or a `postprocessor` argument. +bool hasTextIndexPreOrPostProcessor(const ASTIndexDeclaration & declaration); +bool hasTextIndexPreOrPostProcessor(const IndexDescription & index); + /// All secondary indices in storage struct IndicesDescription : public std::vector, IHints<> { diff --git a/src/Storages/MaterializedView/RefreshSchedule.cpp b/src/Storages/MaterializedView/RefreshSchedule.cpp index 91c164133bd8c..f6a54a31068de 100644 --- a/src/Storages/MaterializedView/RefreshSchedule.cpp +++ b/src/Storages/MaterializedView/RefreshSchedule.cpp @@ -2,6 +2,8 @@ #include +#include + namespace DB { @@ -70,7 +72,10 @@ std::chrono::sys_seconds RefreshSchedule::advance(std::chrono::sys_seconds last_ std::chrono::system_clock::time_point RefreshSchedule::addRandomSpread(std::chrono::system_clock::time_point when, Int64 randomness) const { - return when + std::chrono::milliseconds(Int64(static_cast(spread.minSeconds()) * 1e3 / 2 * static_cast(randomness) / 1e9)); + /// RANDOMIZE FOR is unbounded, so compute in Int128 and cap the offset at about 95 years. + constexpr Int64 max_offset_ms = 3'000'000'000'000; + const Int128 offset_ms = Int128(spread.minSeconds()) * randomness / 2'000'000; + return when + std::chrono::milliseconds(Int64(std::clamp(offset_ms, -max_offset_ms, max_offset_ms))); } } diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp index 37a9ad04aa93f..0d0896b98830e 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDiskBase.cpp @@ -252,6 +252,7 @@ void DataPartStorageOnDiskBase::setRelativePath(const std::string & path) std::lock_guard lock(skip_indices_packed_mutex); skip_indices_packed_probed = false; skip_indices_packed_reader.reset(); + skip_indices_packed_unsupported = false; } } @@ -1218,7 +1219,10 @@ std::shared_ptr DataPartStorageOnDiskBase::getSkipIndic Expect404ResponseScope scope; try { - skip_indices_packed_reader = std::make_shared(disk, packed_path, ReadSettings{}); + auto buf = disk->readFile(packed_path, ReadSettings{}.adjustBufferSize(4096)); + auto index = readSkipIndicesPackedIndex(*buf, packed_path); + skip_indices_packed_unsupported = !index; + skip_indices_packed_reader = std::make_shared(index ? std::move(*index) : PackedFilesIO::Index{}); } catch (const Exception &) { @@ -1232,6 +1236,23 @@ std::shared_ptr DataPartStorageOnDiskBase::getSkipIndic return skip_indices_packed_reader; } +std::optional DataPartStorageOnDiskBase::readSkipIndicesPackedIndex(ReadBuffer & in, const String & archive_path) +{ + /// An empty buffer is left to readIndex, which reports the truncated archive. + if (!in.eof()) + { + const auto version = static_cast(*in.position()); + if (version > PackedFilesIO::MAX_VERSION) + { + LOG_DEBUG(getLogger("DataPartStorageOnDiskBase"), + "Skip indices archive {} has format version {}, newer than the supported {}. Skip indices stored in it will not be used", + archive_path, static_cast(version), static_cast(PackedFilesIO::MAX_VERSION)); + return std::nullopt; + } + } + return PackedFilesReader::readIndex(in); +} + void DataPartStorageOnDiskBase::seedSkipIndicesPackedReader(const PackedFilesIO::Index & index) const { std::lock_guard lock(skip_indices_packed_mutex); @@ -1265,6 +1286,14 @@ bool DataPartStorageOnDiskBase::hasSkipIndicesPackedArchive() const return getSkipIndicesPackedReader() != nullptr; } +bool DataPartStorageOnDiskBase::hasUnsupportedSkipIndicesPackedArchive() const +{ + if (!getSkipIndicesPackedReader()) + return false; + std::lock_guard lock(skip_indices_packed_mutex); + return skip_indices_packed_unsupported; +} + void DataPartStorageOnDiskBase::copyArchiveEntryTo( const PackedFilesReader & source_archive, const String & file_name, diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskBase.h b/src/Storages/MergeTree/DataPartStorageOnDiskBase.h index bceb343c342db..627292c787fb4 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDiskBase.h +++ b/src/Storages/MergeTree/DataPartStorageOnDiskBase.h @@ -79,6 +79,10 @@ class DataPartStorageOnDiskBase : public IDataPartStorage /// to know whether the archive must be rebuilt/excluded from hardlinks. bool hasSkipIndicesPackedArchive() const; + /// True iff skp_idx.packed has a newer format version than this server reads. Its skip indices + /// are not used and cannot be kept up to date, so a mutated part must not inherit the archive. + bool hasUnsupportedSkipIndicesPackedArchive() const; + /// Copy the named virtual files from this storage's skp_idx.packed archive into @target so /// the writer can ship a complete archive after also writing fresh recalc'd entries. Used by /// mutations to preserve surviving in-archive indices that aren't being recomputed when the @@ -197,6 +201,10 @@ class DataPartStorageOnDiskBase : public IDataPartStorage /// so the disk-probe default always misses. virtual std::shared_ptr getSkipIndicesPackedReader() const; + /// Reads the index of a skp_idx.packed archive. Returns nullopt for an archive written in a newer + /// format version: its skip indices are not used, while the part itself stays readable. + static std::optional readSkipIndicesPackedIndex(ReadBuffer & in, const String & archive_path); + /// Cheap pre-filtered lookup for the file-read overlay: returns the archive reader only when /// @name is a "skp_idx_..." substream that the archive actually contains, else nullptr. The /// prefix gate keeps unrelated files (checksums.txt, count.txt, columns.txt, ...) from loading @@ -245,6 +253,8 @@ class DataPartStorageOnDiskBase : public IDataPartStorage mutable std::mutex skip_indices_packed_mutex; mutable bool skip_indices_packed_probed TSA_GUARDED_BY(skip_indices_packed_mutex) = false; mutable std::shared_ptr skip_indices_packed_reader TSA_GUARDED_BY(skip_indices_packed_mutex); + /// The archive exists but has an unsupported format version; the reader then has an empty index. + mutable bool skip_indices_packed_unsupported TSA_GUARDED_BY(skip_indices_packed_mutex) = false; template void executeWriteOperation(Op && op) diff --git a/src/Storages/MergeTree/DataPartStorageOnDiskPacked.cpp b/src/Storages/MergeTree/DataPartStorageOnDiskPacked.cpp index d6f8debe5bf2d..d5a182f2c6d0f 100644 --- a/src/Storages/MergeTree/DataPartStorageOnDiskPacked.cpp +++ b/src/Storages/MergeTree/DataPartStorageOnDiskPacked.cpp @@ -696,8 +696,12 @@ std::shared_ptr DataPartStorageOnDiskPacked::getSkipInd { auto inner_archive_buf = reader->readFile( volume->getDisk(), data_path, String(SKIP_INDICES_PACKED_FILENAME), getReadSettings(), std::nullopt); - auto inner_index = PackedFilesReader::readIndex(*inner_archive_buf); - seedSkipIndicesPackedReader(inner_index); + auto index = readSkipIndicesPackedIndex(*inner_archive_buf, data_path + "/" + String(SKIP_INDICES_PACKED_FILENAME)); + { + std::lock_guard lock(skip_indices_packed_mutex); + skip_indices_packed_unsupported = !index; + } + seedSkipIndicesPackedReader(index ? *index : PackedFilesIO::Index{}); } catch (const Exception &) { diff --git a/src/Storages/MergeTree/KeyCondition.cpp b/src/Storages/MergeTree/KeyCondition.cpp index b45e96877aa3a..cc1cfddae41a3 100644 --- a/src/Storages/MergeTree/KeyCondition.cpp +++ b/src/Storages/MergeTree/KeyCondition.cpp @@ -5210,6 +5210,14 @@ std::optional KeyCondition::applyMonotonicFunctionsChainToRange( DataTypePtr current_type, bool single_point) { + if (functions.empty()) + return key_range; + + /// A bound where `isNull()` holds stands in for NULL or for an infinity, and is left untransformed below, + /// so no transformed value would remain to compare against an atom in the function's result domain. + if (single_point && (key_range.left.isNull() || key_range.right.isNull())) + return {}; + /// The chain was built against a recursively `LowCardinality`-stripped key type, so seed it with the /// stripped type here rather than in each caller: several of them pass the key column's raw type. current_type = recursiveRemoveLowCardinality(current_type); @@ -5241,17 +5249,21 @@ std::optional KeyCondition::applyMonotonicFunctionsChainToRange( /// If we apply function to open interval, we can get empty intervals in result. /// E.g. for ('2020-01-03', '2020-01-20') after applying 'toYYYYMM' we will get ('202001', '202001'). /// To avoid this we make range left and right included. - /// Any function that treats NULL specially is not monotonic. - /// Thus we can safely use isNull() as an -Inf/+Inf indicator here. + /// A NULL bound is an -inf/+inf stand-in and is left as is. A function that returns NULL for a non-NULL + /// argument leaves a bound with no position in the `Range` order, so the range cannot be analyzed. if (!key_range.left.isNull()) { key_range.left = applyFunction(func, current_type, key_range.left); + if (key_range.left.isNull()) + return {}; key_range.left_included = true; } if (!key_range.right.isNull()) { key_range.right = applyFunction(func, current_type, key_range.right); + if (key_range.right.isNull()) + return {}; key_range.right_included = true; } } @@ -6252,52 +6264,23 @@ BoolMask KeyCondition::checkInHyperrectangle( } else { - Range key_range = sparse_hyperrectangle[sparse_pos]; + /// The column may be wrapped in a chain of possibly monotonic functions; for an empty chain + /// the helper returns the range unchanged. + std::optional new_range = applyMonotonicFunctionsChainToRange( + sparse_hyperrectangle[sparse_pos], + element.monotonic_functions_chain, + sparse_data_types[sparse_pos], + single_point); - /// The case when the column is wrapped in a chain of possibly monotonic functions. - if (!element.monotonic_functions_chain.empty()) + if (!new_range) { - std::optional new_range = applyMonotonicFunctionsChainToRange( - key_range, - element.monotonic_functions_chain, - sparse_data_types[sparse_pos], - single_point); - - if (!new_range) - { - /// Cannot determine monotonicity on this range – unknown. - rpn_stack.emplace_back(true, true); - } - else - { - key_range = *new_range; - - bool intersects = element.range.intersectsRange(key_range); - bool contains = element.range.containsRange(key_range); - - /// NaN doesn't satisfy any comparison condition in SQL (e.g., NaN > 0 is false/NULL). - /// In ClickHouse sort order, NaN has a defined position (after +inf), so Range-based - /// analysis may incorrectly include NaN values. - /// - If left bound is NaN: all values in the range are NaN (NaN sorts last), - /// so no comparison condition can be true. - /// - If only right bound is NaN: the range extends into NaN territory, - /// so it cannot be fully contained (NaN values don't satisfy the condition). - if (unlikely(key_range.left.isNaN())) - { - intersects = false; - contains = false; - } - else if (unlikely(key_range.right.isNaN())) - { - contains = false; - } - - rpn_stack.emplace_back(intersects, !contains); - /// we don't create bloom_filter_data if monotonic_functions_chain is present - } + /// Cannot determine monotonicity on this range – unknown. + rpn_stack.emplace_back(true, true); } else { + const Range & key_range = *new_range; + bool intersects = element.range.intersectsRange(key_range); bool contains = element.range.containsRange(key_range); @@ -6319,7 +6302,6 @@ BoolMask KeyCondition::checkInHyperrectangle( } rpn_stack.emplace_back(intersects, !contains); - } } diff --git a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp index 14679f3bca78e..88f6194694df7 100644 --- a/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp +++ b/src/Storages/MergeTree/MergeTreeBlockReadUtils.cpp @@ -558,8 +558,7 @@ MergeTreeReadTaskColumns getReadTaskColumns( index_read_tasks, actions_settings, reader_settings.enable_multiple_prewhere_read_steps, - reader_settings.force_short_circuit_execution, - &storage_snapshot->metadata->getColumns()); + reader_settings.force_short_circuit_execution); for (const auto & step : prewhere_actions.steps) add_step(*step); diff --git a/src/Storages/MergeTree/MergeTreeData.cpp b/src/Storages/MergeTree/MergeTreeData.cpp index d56e8d105b718..d382d77c1fa4a 100644 --- a/src/Storages/MergeTree/MergeTreeData.cpp +++ b/src/Storages/MergeTree/MergeTreeData.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -81,6 +82,7 @@ #include #include #include +#include #include #include #include @@ -361,6 +363,7 @@ namespace MergeTreeSetting extern const MergeTreeSettingsMergeTreeStringSerializationVersion string_serialization_version; extern const MergeTreeSettingsMergeTreeNullableSerializationVersion nullable_serialization_version; extern const MergeTreeSettingsMergeTreeMapSerializationVersion map_serialization_version; + extern const MergeTreeSettingsMergeTreeMapSerializationVersion map_serialization_version_for_zero_level_parts; extern const MergeTreeSettingsUInt32 min_level_for_wide_part; extern const MergeTreeSettingsBool propagate_types_serialization_versions_to_nested_types; } @@ -980,6 +983,14 @@ bool MergeTreeData::supportsFinal() const || merging_params.mode == MergingParams::VersionedCollapsing; } +bool MergeTreeData::hasBucketedMapSerialization() const +{ + /// Inserted parts use the zero-level setting, merged parts the other one. + auto settings = getSettings(); + return (*settings)[MergeTreeSetting::map_serialization_version] == MergeTreeMapSerializationVersion::WITH_BUCKETS + || (*settings)[MergeTreeSetting::map_serialization_version_for_zero_level_parts] == MergeTreeMapSerializationVersion::WITH_BUCKETS; +} + static void checkKeyExpression(const ExpressionActions & expr, const Block & sample_block, const String & key_name, bool allow_nullable_key) { if (expr.hasArrayJoin()) @@ -1172,6 +1183,19 @@ void MergeTreeData::checkProperties( if (!attach && !allow_minmax_index_for_json) checkMinMaxIndexForJSON(index); MergeTreeIndexFactory::instance().validate(index, attach, *getSettings()); + + if (!attach) + { + /// The create path and the projection recursion pass the same metadata object as both arguments, + /// so nothing can be inherited there. + const IndexDescription * old_index = nullptr; + if (&old_metadata != &new_metadata) + for (const auto & candidate : old_metadata.secondary_indices) + if (candidate.name == index.name) + old_index = &candidate; + + bloomFilterIndexTextCheckTotalSize(index, old_index); + } } catch (Exception & e) { @@ -5104,6 +5128,47 @@ Names expressionSourceColumns(const ASTPtr & ast, const ColumnsDescription & col } +void MergeTreeData::checkStoredExpressions(const StorageInMemoryMetadata & metadata, const ContextPtr & local_context) +{ + auto context = Context::createCopy(local_context); + context->setAnalyzingBackgroundExpressions(true); + + const auto & columns = metadata.columns; + const auto & virtuals = metadata.virtuals; + + for (const auto * key : {&metadata.partition_key, &metadata.sorting_key, &metadata.sampling_key, &metadata.unique_key}) + { + if (key->definition_ast) + KeyDescription::getKeyFromAST(key->definition_ast, columns, virtuals, context); + } + if (metadata.primary_key.definition_ast) + KeyDescription::getPrimaryKeyFromAST(metadata.primary_key.definition_ast, metadata.sorting_key, columns, virtuals, context); + + for (const auto & index : metadata.secondary_indices) + { + if (!index.isImplicitlyCreated()) + IndexDescription::getIndexFromAST(index.definition_ast, columns, false, index.escape_filenames, context); + } + + for (const auto & projection : metadata.projections) + ProjectionDescription::getProjectionFromAST(projection.definition_ast, columns, &metadata.partition_key, context); + + /// A column TTL resets the column to its `DEFAULT` or `MATERIALIZED` expression in merges. + ASTPtr ttl_default_expr_list = make_intrusive(); + for (const auto & [name, ttl_ast] : columns.getColumnTTLs()) + { + TTLDescription::getTTLFromAST(ttl_ast, columns, context, metadata.primary_key, TTLValidationMode::SkipValidation); + const auto & default_desc = columns.get(name).default_desc; + if (default_desc.expression && (default_desc.kind == ColumnDefaultKind::Default || default_desc.kind == ColumnDefaultKind::Materialized)) + ttl_default_expr_list->children.emplace_back(setAlias(default_desc.expression->clone(), name + "_tmp_ttl_default")); + } + if (!ttl_default_expr_list->children.empty()) + validateColumnsDefaultsAndGetSampleBlock(ttl_default_expr_list, columns.getAll(), context); + if (metadata.table_ttl.definition_ast) + TTLTableDescription::getTTLForTableFromAST( + metadata.table_ttl.definition_ast, columns, context, metadata.primary_key, TTLValidationMode::SkipValidation); +} + void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, ContextPtr local_context) const { /// Reject schema-changing ALTER while a streaming query holds a subscription on this storage. @@ -5383,7 +5448,12 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context } removeImplicitStatistics(new_metadata.columns); - commands.apply(new_metadata, local_context, share_nested_offsets); + /// Keys, TTL and projections, and the indices the commands add, are built with this context. They are executed in the + /// background, so AI functions reject it. A replica replaying an ALTER the initiator already committed must not fail on it. + auto apply_context = Context::createCopy(local_context); + if (auto txn = local_context->getZooKeeperMetadataTransaction(); !txn || txn->isInitialQuery()) + apply_context->setAnalyzingBackgroundExpressions(true); + commands.apply(new_metadata, apply_context, share_nested_offsets); /// The `Quantize(...)` codec is immutable via ALTER: it cannot be added, removed, or changed, and the TYPE of a /// Quantize-coded column cannot be changed either. Both reach `ColumnsDescription::modify` as metadata-only changes @@ -6116,7 +6186,11 @@ void MergeTreeData::checkAlterIsPossible(const AlterCommands & commands, Context if (setting_name == "storage_policy") { - checkStoragePolicy(local_context->getStoragePolicy(new_value.safeGet())); + const auto new_storage_policy = local_context->getStoragePolicy(new_value.safeGet()); + /// Only a changed policy is checked, so a table that already has it can still be altered otherwise. + if (new_storage_policy != getStoragePolicy()) + DiskFromAST::ensureStoragePolicyIsAllowed(new_storage_policy->getName(), local_context); + checkStoragePolicy(new_storage_policy); found_storage_policy_setting = true; } else if (setting_name == "disk") @@ -12263,7 +12337,7 @@ MergeTreeData::LightweightUpdateResult MergeTreeData::updateLightweightImpl(cons if (it != commands.end()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Got unexpected command with type {} in lightweight update", it->type); - LOG_DEBUG(log, "Executing lightweight update with commands: {}", commands.toString(false)); + LOG_DEBUG(log, "Executing lightweight update with commands: {}", commands.ast(false)->formatForLogging()); NamesAndTypesList system_columns; MutationCommands commands_to_run; @@ -12868,6 +12942,16 @@ StorageSnapshotPtr MergeTreeData::getStorageSnapshot(const StorageMetadataPtr & return cache->emplace(this, createStorageSnapshot(metadata_snapshot, query_context, false)).first->second; } +void MergeTreeData::discardSharedStorageSnapshot(const ContextPtr & query_context) const +{ + auto query_metadata_cache = query_context->getQueryMetadataCache(); + if (!query_metadata_cache) + return; + + auto [cache, lock] = query_metadata_cache->getStorageSnapshotCache(); + cache->erase(this); +} + StorageSnapshotPtr MergeTreeData::getStorageSnapshotWithoutData(const StorageMetadataPtr & metadata_snapshot, ContextPtr query_context) const { diff --git a/src/Storages/MergeTree/MergeTreeData.h b/src/Storages/MergeTree/MergeTreeData.h index 1e48e4e85f6a7..aa97f33a44ebd 100644 --- a/src/Storages/MergeTree/MergeTreeData.h +++ b/src/Storages/MergeTree/MergeTreeData.h @@ -575,6 +575,8 @@ class MergeTreeData : public WithMutableContext, public IStorage, public IBackgr bool supportsSubcolumns() const override { return true; } + bool hasBucketedMapSerialization() const override; + bool supportsTTL() const override { return true; } bool supportsStatistics() const override { return true; } @@ -702,6 +704,11 @@ class MergeTreeData : public WithMutableContext, public IStorage, public IBackgr /// The same as above but does not hold vector of data parts. StorageSnapshotPtr getStorageSnapshotWithoutData(const StorageMetadataPtr & metadata_snapshot, ContextPtr query_context) const override; + /// Removes the snapshot of this table from the query-scoped snapshot cache of + /// `enable_shared_storage_snapshot_in_query`, so that the next `getStorageSnapshot` in the query + /// takes a fresh one. Holders of the removed snapshot keep it. + void discardSharedStorageSnapshot(const ContextPtr & query_context) const; + /// Load the set of data parts from disk. Call once - immediately after the object is created. void loadDataParts(bool skip_sanity_checks, std::optional> expected_parts); @@ -1047,6 +1054,11 @@ class MergeTreeData : public WithMutableContext, public IStorage, public IBackgr /// If something is wrong, throws an exception. void checkAlterIsPossible(const AlterCommands & commands, ContextPtr context) const override; + /// The expressions stored in the metadata (keys, TTL, skip indices, projections) are built with the global context, + /// which has full access, and are executed in the background. Builds them again with a copy of the query context, + /// which checks the user's access to their functions and rejects functions that are not executed in the background. + static void checkStoredExpressions(const StorageInMemoryMetadata & metadata, const ContextPtr & local_context); + /// Throw exception if command is some kind of DROP command (drop column, drop index, etc) or rename command /// and we have unfinished mutation which need this column to finish. void checkDropOrRenameCommandDoesntAffectInProgressMutations( diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp index ff799d6aeabdc..f7b12e9f78026 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.cpp @@ -691,6 +691,24 @@ std::optional> MergeTreeDataSelectExecutor::filterPar return result; } +RangesInDataParts MergeTreeDataSelectExecutor::filterParts( + const RangesInDataParts & parts, + const ReadFromMergeTree::Indexes & indexes, + const StorageMetadataPtr & metadata_snapshot, + const MergeTreeData & data, + const SelectQueryInfo & query_info, + const MergeTreeData::MutationsSnapshotPtr & mutations_snapshot, + const ContextPtr & context, + const PartitionIdToMaxBlock * max_block_numbers_to_read, + LoggerPtr log, + ReadFromMergeTree::IndexStats & index_stats) +{ + auto res = filterPartsByPartition( + parts, indexes.partition_pruner, indexes.minmax_idx_condition, indexes.part_values, + metadata_snapshot, data, context, max_block_numbers_to_read, log, index_stats); + return filterPartsByStatistics(res, metadata_snapshot, query_info, mutations_snapshot, context, log, index_stats); +} + RangesInDataParts MergeTreeDataSelectExecutor::filterPartsByPartition( const RangesInDataParts & parts, const std::optional & partition_pruner, diff --git a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h index 5c169c583900f..c925b34dbdaaf 100644 --- a/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h +++ b/src/Storages/MergeTree/MergeTreeDataSelectExecutor.h @@ -202,6 +202,19 @@ class MergeTreeDataSelectExecutor const ActionsDAG::Node * predicate, ContextPtr context); + /// Apply snapshot, virtual-column, min-max, partition and statistics filters before range analysis. + static RangesInDataParts filterParts( + const RangesInDataParts & parts, + const ReadFromMergeTree::Indexes & indexes, + const StorageMetadataPtr & metadata_snapshot, + const MergeTreeData & data, + const SelectQueryInfo & query_info, + const MergeTreeData::MutationsSnapshotPtr & mutations_snapshot, + const ContextPtr & context, + const PartitionIdToMaxBlock * max_block_numbers_to_read, + LoggerPtr log, + ReadFromMergeTree::IndexStats & index_stats); + /// Filter parts using minmax index and partition key. static RangesInDataParts filterPartsByPartition( const RangesInDataParts & parts, diff --git a/src/Storages/MergeTree/MergeTreeDataWriter.cpp b/src/Storages/MergeTree/MergeTreeDataWriter.cpp index 0df5d547c2a03..f008dd050d143 100644 --- a/src/Storages/MergeTree/MergeTreeDataWriter.cpp +++ b/src/Storages/MergeTree/MergeTreeDataWriter.cpp @@ -592,6 +592,11 @@ Block MergeTreeDataWriter::mergeBlock( span.addAttribute("clickhouse.rows", block_size); span.addAttribute("clickhouse.columns", header->columns()); + /// The block can lose all its rows before this point, e.g. when insert deduplication removes + /// duplicate rows of a materialized view block. Merging algorithms do not expect an empty input. + if (block_size == 0) + return *header; + auto get_merging_algorithm = [&]() -> std::shared_ptr { switch (merging_params.mode) diff --git a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp index 19d5a3516a5b7..7d21b561a7575 100644 --- a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp @@ -29,6 +29,7 @@ #include #include #include +#include namespace DB @@ -42,6 +43,7 @@ namespace Setting namespace ErrorCodes { extern const int BAD_ARGUMENTS; + extern const int CORRUPTED_DATA; extern const int ILLEGAL_COLUMN; extern const int INCORRECT_QUERY; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; @@ -101,9 +103,22 @@ void MergeTreeIndexGranuleBloomFilter::deserializeBinary(ReadBuffer & istr, Merg throw Exception(ErrorCodes::LOGICAL_ERROR, "Unknown index version {}.", version); readVarUInt(total_rows, istr); + if (total_rows == 0) + throw Exception(ErrorCodes::CORRUPTED_DATA, "Bloom filter index granule has zero rows"); - static size_t atom_size = 8; - size_t bytes_size = (bits_per_row * total_rows + atom_size - 1) / atom_size; + static constexpr size_t atom_size = 8; + size_t bits_size = 0; + if (common::mulOverflow(bits_per_row, total_rows, bits_size) + || common::addOverflow(bits_size, atom_size - 1, bits_size)) + { + throw Exception( + ErrorCodes::CORRUPTED_DATA, + "Bloom filter index granule size overflows for {} rows and {} bits per row", + total_rows, + bits_per_row); + } + + size_t bytes_size = bits_size / atom_size; size_t read_size = bytes_size; for (auto & filter : bloom_filters) { diff --git a/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp b/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp index 0ef8f588cc09b..8c8360b3c75e5 100644 --- a/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexBloomFilterText.cpp @@ -1013,4 +1013,34 @@ void bloomFilterIndexTextValidator(const IndexDescription & index, bool /*attach args[first_bf_param_idx+2].safeGet()); } +static size_t bloomFilterIndexTextTotalSize(const IndexDescription & index) +{ + if (index.type != NgramsTokenizer::getName() + && index.type != SplitByNonAlphaTokenizer::getName() + && index.type != SparseGramsTokenizer::getBloomFilterIndexName()) + return 0; + + /// The size of bloom filter is the first of the last three arguments, whatever the tokenizer. + /// The arguments are already checked by `bloomFilterIndexTextValidator`, and the size is at most `MAX_BLOOM_FILTER_SIZE`, + /// so the multiplication cannot overflow. + FieldVector args = getFieldsFromIndexArgumentsAST(index.arguments); + return args.at(args.size() - 3).safeGet() * index.column_names.size(); +} + +void bloomFilterIndexTextCheckTotalSize(const IndexDescription & index, const IndexDescription * old_index) +{ + const size_t total_size = bloomFilterIndexTextTotalSize(index); + if (total_size <= MAX_BLOOM_FILTER_SIZE) + return; + + /// An index inherited from the old metadata may have been created before this check, so it is accepted as long as it does not grow. + if (old_index && bloomFilterIndexTextTotalSize(*old_index) >= total_size) + return; + + throw Exception(ErrorCodes::BAD_ARGUMENTS, + "The total size of bloom filters in a granule of index {} cannot be more than {}: " + "the index has {} columns and the total size of their bloom filters is {}", + backQuote(index.name), MAX_BLOOM_FILTER_SIZE, index.column_names.size(), total_size); +} + } diff --git a/src/Storages/MergeTree/MergeTreeIndices.h b/src/Storages/MergeTree/MergeTreeIndices.h index f749aaf3302ec..6c85669609e34 100644 --- a/src/Storages/MergeTree/MergeTreeIndices.h +++ b/src/Storages/MergeTree/MergeTreeIndices.h @@ -408,6 +408,11 @@ void setIndexValidator(const IndexDescription & index, bool attach, const MergeT MergeTreeIndexPtr bloomFilterIndexTextCreator(StorageMetadataPtr metadata_snapshot, const IndexDescription & index, const MergeTreeSettings & settings); void bloomFilterIndexTextValidator(const IndexDescription & index, bool attach, const MergeTreeSettings & settings); +/// Every granule of a text bloom filter index eagerly allocates one filter per indexed column, so the total size of the filters +/// (not only the size of a single filter) is limited. The check is done only for indexes being declared or changed, not on `ATTACH` +/// and not for indexes inherited unchanged from `old_index`, so that tables created before the check still load and can be altered. +/// Does nothing for other index types. +void bloomFilterIndexTextCheckTotalSize(const IndexDescription & index, const IndexDescription * old_index); MergeTreeIndexPtr bloomFilterIndexCreator(StorageMetadataPtr metadata_snapshot, const IndexDescription & index, const MergeTreeSettings & settings); void bloomFilterIndexValidator(const IndexDescription & index, bool attach, const MergeTreeSettings & settings); diff --git a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp index 66fae3c1dbdb9..982696184e25c 100644 --- a/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderCompactSingleBuffer.cpp @@ -93,7 +93,7 @@ try } catch (...) { - if (!isRetryableException(std::current_exception())) + if (!isRetryableException(std::current_exception()) && !isQueryCancellation(std::current_exception())) data_part_info_for_read->reportBroken(); /// Better diagnostics. @@ -126,7 +126,7 @@ try } catch (...) { - if (!isRetryableException(std::current_exception())) + if (!isRetryableException(std::current_exception()) && !isQueryCancellation(std::current_exception())) data_part_info_for_read->reportBroken(); throw; } diff --git a/src/Storages/MergeTree/MergeTreeReaderWide.cpp b/src/Storages/MergeTree/MergeTreeReaderWide.cpp index 6f8b49239eba8..6db282c14290d 100644 --- a/src/Storages/MergeTree/MergeTreeReaderWide.cpp +++ b/src/Storages/MergeTree/MergeTreeReaderWide.cpp @@ -80,7 +80,7 @@ MergeTreeReaderWide::MergeTreeReaderWide( } catch (...) { - if (!isRetryableException(std::current_exception())) + if (!isRetryableException(std::current_exception()) && !isQueryCancellation(std::current_exception())) data_part_info_for_read->reportBroken(); throw; } @@ -106,7 +106,7 @@ void MergeTreeReaderWide::prefetchBeginOfRange(Priority priority) } catch (...) { - if (!isRetryableException(std::current_exception())) + if (!isRetryableException(std::current_exception()) && !isQueryCancellation(std::current_exception())) data_part_info_for_read->reportBroken(); throw; } @@ -232,7 +232,7 @@ size_t MergeTreeReaderWide::readRows( } catch (...) { - if (!isRetryableException(std::current_exception())) + if (!isRetryableException(std::current_exception()) && !isQueryCancellation(std::current_exception())) data_part_info_for_read->reportBroken(); /// Better diagnostics. diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp index e360ea2ff1a78..feb6446456375 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.cpp @@ -163,8 +163,7 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor( const ExpressionActionsSettings & actions_settings_, const MergeTreeReaderSettings & reader_settings_, MergeTreeIndexBuildContextPtr merge_tree_index_build_context_, - LazyMaterializingRowsPtr lazy_materializing_rows_, - const ColumnsDescription * columns_) + LazyMaterializingRowsPtr lazy_materializing_rows_) : pool(std::move(pool_)) , algorithm(std::move(algorithm_)) , row_level_filter(row_level_filter_) @@ -176,8 +175,7 @@ MergeTreeSelectProcessor::MergeTreeSelectProcessor( index_read_tasks_, actions_settings, reader_settings_.enable_multiple_prewhere_read_steps, - reader_settings_.force_short_circuit_execution, - columns_)) + reader_settings_.force_short_circuit_execution)) , reader_settings(reader_settings_) , result_header(transformHeader(pool->getHeader(), row_level_filter, prewhere_info)) , merge_tree_index_build_context(std::move(merge_tree_index_build_context_)) @@ -205,8 +203,7 @@ PrewhereExprInfo MergeTreeSelectProcessor::getPrewhereActions( const IndexReadTasks & index_read_tasks, const ExpressionActionsSettings & actions_settings, bool enable_multiple_prewhere_read_steps, - bool force_short_circuit_execution, - const ColumnsDescription * columns) + bool force_short_circuit_execution) { PrewhereExprInfo prewhere_actions; @@ -239,7 +236,7 @@ PrewhereExprInfo MergeTreeSelectProcessor::getPrewhereActions( } if (prewhere_info && - (!enable_multiple_prewhere_read_steps || !tryBuildPrewhereSteps(prewhere_info, actions_settings, prewhere_actions, force_short_circuit_execution, columns))) + (!enable_multiple_prewhere_read_steps || !tryBuildPrewhereSteps(prewhere_info, actions_settings, prewhere_actions, force_short_circuit_execution))) { PrewhereExprStep prewhere_step { diff --git a/src/Storages/MergeTree/MergeTreeSelectProcessor.h b/src/Storages/MergeTree/MergeTreeSelectProcessor.h index 9c04ebde42663..3e48c004ce1b3 100644 --- a/src/Storages/MergeTree/MergeTreeSelectProcessor.h +++ b/src/Storages/MergeTree/MergeTreeSelectProcessor.h @@ -119,8 +119,7 @@ class MergeTreeSelectProcessor : private boost::noncopyable const ExpressionActionsSettings & actions_settings_, const MergeTreeReaderSettings & reader_settings_, MergeTreeIndexBuildContextPtr merge_tree_index_build_context_ = {}, - LazyMaterializingRowsPtr lazy_materializing_rows_ = {}, - const ColumnsDescription * columns_ = nullptr); + LazyMaterializingRowsPtr lazy_materializing_rows_ = {}); String getName() const; @@ -149,8 +148,7 @@ class MergeTreeSelectProcessor : private boost::noncopyable const IndexReadTasks & index_read_tasks, const ExpressionActionsSettings & actions_settings, bool enable_multiple_prewhere_read_steps, - bool force_short_circuit_execution, - const ColumnsDescription * columns = nullptr); + bool force_short_circuit_execution); void addPartLevelToChunk(bool add_part_level_) { add_part_level = add_part_level_; } diff --git a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp index 91ee5dfa67d6c..4db98cb754f5c 100644 --- a/src/Storages/MergeTree/MergeTreeSequentialSource.cpp +++ b/src/Storages/MergeTree/MergeTreeSequentialSource.cpp @@ -294,7 +294,7 @@ try catch (...) { /// Suspicion of the broken part. A part is added to the queue for verification. - if (!isRetryableException(std::current_exception())) + if (!isRetryableException(std::current_exception()) && !isQueryCancellation(std::current_exception())) read_task_info->data_part_info->reportBroken(); throw; } diff --git a/src/Storages/MergeTree/MergeTreeSettings.cpp b/src/Storages/MergeTree/MergeTreeSettings.cpp index e2ee832cac906..4d9115458d8de 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.cpp +++ b/src/Storages/MergeTree/MergeTreeSettings.cpp @@ -1293,7 +1293,7 @@ Deprecated alias of `deduplication_hashes_cache_update_wait_ms`, kept for one re compatibility. It is honored only when `deduplication_hashes_cache_update_wait_ms` is left at its default; this setting will be removed in a future release. )", 0) \ - DECLARE(UInt64, max_replicated_logs_to_keep, 1000, R"( + DECLARE(NonZeroUInt64, max_replicated_logs_to_keep, 1000, R"( How many records may be in the ClickHouse Keeper log if there is inactive replica. An inactive replica becomes lost when when this number exceed. @@ -2482,7 +2482,7 @@ DECLARE_SETTINGS_TRAITS(MergeTreeSettingsTraits, LIST_OF_MERGE_TREE_SETTINGS, ME struct MergeTreeSettingsImpl : public BaseSettings { /// NOTE: will rewrite the AST to add immutable settings. - void loadFromQuery(ASTStorage & storage_def, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database); + void loadFromQuery(ASTStorage & storage_def, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database, bool stored_definition); /// Check that the values are sane taking also query-level settings into account. void sanityCheck(size_t background_pool_tasks, bool background_pool_auto_lowered) const; @@ -2520,7 +2520,8 @@ static void validateTableDisk(const DiskPtr & disk) IMPLEMENT_SETTINGS_TRAITS_CUSTOM_IMPL(MergeTreeSettingsTraits, LIST_OF_MERGE_TREE_SETTINGS, MergeTreeSettings, MergeTreeSetting) -void MergeTreeSettingsImpl::loadFromQuery(ASTStorage & storage_def, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database) +void MergeTreeSettingsImpl::loadFromQuery( + ASTStorage & storage_def, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database, bool stored_definition) { if (storage_def.settings) { @@ -2532,7 +2533,7 @@ void MergeTreeSettingsImpl::loadFromQuery(ASTStorage & storage_def, ContextPtr c DiskPtr disk; auto changes = storage_def.settings->changes; - MergeTreeSettings::resolveDiskSetting(changes, context, is_loading_from_existing_metadata, for_system_database); + MergeTreeSettings::resolveDiskSetting(changes, context, is_loading_from_existing_metadata, for_system_database, stored_definition); for (const auto & [name, value, _] : changes) { @@ -2874,13 +2875,15 @@ void MergeTreeSettings::applyChange(const SettingChange & change, ContextPtr con impl->applyChange(resolved_change); } -void MergeTreeSettings::resolveDiskSetting(SettingsChanges & changes, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database) +void MergeTreeSettings::resolveDiskSetting( + SettingsChanges & changes, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database, bool stored_definition) { for (auto & change : changes) - resolveDiskSetting(change, context, is_loading_from_existing_metadata, for_system_database); + resolveDiskSetting(change, context, is_loading_from_existing_metadata, for_system_database, stored_definition); } -void MergeTreeSettings::resolveDiskSetting(SettingChange & change, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database) +void MergeTreeSettings::resolveDiskSetting( + SettingChange & change, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database, bool stored_definition) { if (change.name != "disk") return; @@ -2892,7 +2895,8 @@ void MergeTreeSettings::resolveDiskSetting(SettingChange & change, ContextPtr co if (value_as_custom_ast && isDiskFunction(value_as_custom_ast)) { - auto disk_name = DiskFromAST::createCustomDisk(value_as_custom_ast, context, is_loading_from_existing_metadata, for_system_database); + auto disk_name = DiskFromAST::createCustomDisk( + value_as_custom_ast, context, is_loading_from_existing_metadata, for_system_database, stored_definition); LOG_DEBUG(getLogger("MergeTreeSettings"), "Created custom disk {}", disk_name); change.value = disk_name; } @@ -2985,9 +2989,10 @@ SettingsTierType MergeTreeSettings::getTier(std::string_view name) const return impl->getTier(name); } -void MergeTreeSettings::loadFromQuery(ASTStorage & storage_def, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database) +void MergeTreeSettings::loadFromQuery( + ASTStorage & storage_def, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database, bool stored_definition) { - impl->loadFromQuery(storage_def, context, is_loading_from_existing_metadata, for_system_database); + impl->loadFromQuery(storage_def, context, is_loading_from_existing_metadata, for_system_database, stored_definition); } void MergeTreeSettings::loadFromConfig(const String & config_elem, const Poco::Util::AbstractConfiguration & config) diff --git a/src/Storages/MergeTree/MergeTreeSettings.h b/src/Storages/MergeTree/MergeTreeSettings.h index 1a7910394b21f..609589144aee1 100644 --- a/src/Storages/MergeTree/MergeTreeSettings.h +++ b/src/Storages/MergeTree/MergeTreeSettings.h @@ -110,7 +110,9 @@ struct MergeTreeSettings void applyCompatibilitySetting(const String & compatibility_value); /// NOTE: will rewrite the AST to add immutable settings. - void loadFromQuery(ASTStorage & storage_def, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database = false); + void loadFromQuery( + ASTStorage & storage_def, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database = false, + bool stored_definition = false); void loadFromConfig(const String & config_elem, const Poco::Util::AbstractConfiguration & config); bool needSyncPart(size_t input_rows, size_t input_bytes) const; @@ -132,8 +134,12 @@ struct MergeTreeSettings static bool isPartFormatSetting(const String & name); static bool isDiskSettingChanged(const SettingsChanges & old_changes, const SettingsChanges & new_changes); - static void resolveDiskSetting(SettingsChanges & changes, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database = false); - static void resolveDiskSetting(SettingChange & change, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database = false); + static void resolveDiskSetting( + SettingsChanges & changes, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database = false, + bool stored_definition = false); + static void resolveDiskSetting( + SettingChange & change, ContextPtr context, bool is_loading_from_existing_metadata, bool for_system_database = false, + bool stored_definition = false); /// Cloud only static bool isSMTReadonlySetting(const String & name); diff --git a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp index cbeefa44a7ba9..0611d0d28676f 100644 --- a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp +++ b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.cpp @@ -1,7 +1,6 @@ #include #include -#include namespace DB @@ -22,29 +21,10 @@ namespace struct NodeInfo { NameSet required_columns; - /// Column names resolved to their physical storage names (subcolumn suffix stripped). - /// Used for grouping: conditions on subcolumns of the same storage column are placed into one step. - NameSet required_storage_columns; }; -/// Resolves a column name to its storage (physical) name. -/// For subcolumns like `map.key_k0`, returns `map`. -/// For regular columns, returns the name unchanged. -String resolveStorageColumnName(const String & column_name, const ColumnsDescription * columns) -{ - if (columns) - { - if (auto col = columns->tryGetColumnOrSubcolumn(GetColumnsOptions::AllPhysical, column_name)) - return col->getNameInStorage(); - } - return column_name; -} - /// Fills the list of required columns for a node in the DAG. -void fillRequiredColumns( - const ActionsDAG::Node * node, - std::unordered_map & nodes_info, - const ColumnsDescription * columns) +void fillRequiredColumns(const ActionsDAG::Node * node, std::unordered_map & nodes_info) { if (nodes_info.contains(node)) return; @@ -54,16 +34,14 @@ void fillRequiredColumns( if (node->type == ActionsDAG::ActionType::INPUT) { node_info.required_columns.insert(node->result_name); - node_info.required_storage_columns.insert(resolveStorageColumnName(node->result_name, columns)); return; } for (const auto & child : node->children) { - fillRequiredColumns(child, nodes_info, columns); + fillRequiredColumns(child, nodes_info); const auto & child_info = nodes_info[child]; node_info.required_columns.insert(child_info.required_columns.begin(), child_info.required_columns.end()); - node_info.required_storage_columns.insert(child_info.required_storage_columns.begin(), child_info.required_storage_columns.end()); } } @@ -225,8 +203,7 @@ bool tryBuildPrewhereSteps( PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, PrewhereExprInfo & prewhere, - bool force_short_circuit_execution, - const ColumnsDescription * columns) + bool force_short_circuit_execution) { if (!prewhere_info) return true; @@ -242,37 +219,21 @@ bool tryBuildPrewhereSteps( std::unordered_map nodes_info; for (const auto & node : condition_nodes) { - fillRequiredColumns(node, nodes_info, columns); + fillRequiredColumns(node, nodes_info); } /// 3. Sort condition nodes by the number of columns used in them and the overall size of those columns /// TODO: not sorting for now because the conditions are already sorted by Where Optimizer - /// 4. Group adjacent conditions that read from the same set of physical storage columns into a single step. - /// Conditions on subcolumns of the same column (e.g. `map.key_k0` and `map.key_k1`) are placed into one group - /// when they appear next to each other in the condition list. - /// - /// Only adjacent conditions are merged to preserve the user's explicit PREWHERE evaluation order. - /// Non-adjacent conditions on the same storage column are kept in separate steps even though this - /// may cause redundant reads, because the user may have intentionally interleaved a guard predicate - /// (e.g. `PREWHERE tags['safe'] != '' AND value > 0 AND toUInt64(tags['unsafe']) > 0` — the - /// `value > 0` step must filter rows before evaluating the potentially-throwing conversion). - /// - /// For the WHERE-to-PREWHERE path this is not a problem: `MergeTreeWhereOptimizer` already groups - /// conditions by their physical storage columns, so subcolumns of the same column arrive here adjacent. + /// 4. Group conditions with the same set of columns into a single read/compute step std::vector> condition_groups; for (const auto & node : condition_nodes) { const auto & node_info = nodes_info[node]; - if (!condition_groups.empty() - && nodes_info[condition_groups.back().front()].required_storage_columns == node_info.required_storage_columns) - { - condition_groups.back().push_back(node); - } + if (!condition_groups.empty() && nodes_info[condition_groups.back().back()].required_columns == node_info.required_columns) + condition_groups.back().push_back(node); /// Add to the last group else - { - condition_groups.push_back({node}); - } + condition_groups.push_back({node}); /// Start new group } /// 5. Build DAGs for each step diff --git a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.h b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.h index 93f40b1af8c33..40d5427b2f46e 100644 --- a/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.h +++ b/src/Storages/MergeTree/MergeTreeSplitPrewhereIntoReadSteps.h @@ -11,13 +11,10 @@ namespace DB { -class ColumnsDescription; - bool tryBuildPrewhereSteps( PrewhereInfoPtr prewhere_info, const ExpressionActionsSettings & actions_settings, PrewhereExprInfo & prewhere, - bool force_short_circuit_execution, - const ColumnsDescription * columns = nullptr); + bool force_short_circuit_execution); } diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index 7f5460817a718..174bf4e179e83 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -386,13 +386,10 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTree { RPNBuilderTreeNode node; NameSet columns; - /// Column names resolved to their physical storage names (subcolumn suffix stripped). - /// Used for grouping: conditions on subcolumns of the same storage column - /// (e.g. `map.key_k0` and `map.key_k1`) are grouped and moved to PREWHERE together. - NameSet storage_columns; bool has_invalid_column = false; bool may_use_primary_index = true; bool viable = false; + bool expensive = false; }; std::vector infos; @@ -400,20 +397,9 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTree for (const auto & conjunct : conjuncts) { - ConjunctInfo info{conjunct, {}, {}, false, true, false}; + ConjunctInfo info{conjunct, {}, false, true, false, false}; collectColumns(conjunct, nullptr, table_columns, info.columns, info.has_invalid_column, info.may_use_primary_index); - /// Resolve each column to its physical storage name so that subcolumns - /// of the same column (e.g. `map.key_k0`, `map.key_k1`) share one group. - const auto & storage_columns_description = storage_metadata->getColumns(); - for (const auto & col : info.columns) - { - if (auto resolved = storage_columns_description.tryGetColumnOrSubcolumn(GetColumnsOptions::AllPhysical, col)) - info.storage_columns.insert(resolved->getNameInStorage()); - else - info.storage_columns.insert(col); - } - /// Is it possible to use primary index for this condition? For conditions like `lower(country) = 'xx'`, may_use_primary_index will be false. info.viable = !info.has_invalid_column @@ -428,20 +414,20 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTree /// Do not move conditions involving all queried columns. && info.columns.size() < queried_columns.size(); + info.expensive = isExpensiveExpression(conjunct); + infos.push_back(std::move(info)); } - /// Group viable conjuncts by their required storage column set so same-column conditions + /// Group viable conjuncts by their required column set so same-column conditions /// are treated as a single unit and their combined selectivity can be estimated. - /// Grouping by storage columns (rather than the exact column set) keeps subcolumns of the - /// same column (e.g. `map.key_k0` and `map.key_k1`) in one group, so they are moved to - /// PREWHERE together and arrive adjacent for the prewhere-splitting step. /// Non-viable conjuncts stay as individual Conditions. + /// 'expensive' conjuncts each get their own group. /// /// We use a simple linear search to find groups (WHERE clauses are short). struct Group { - NameSet storage_columns; + NameSet columns; std::vector indices; }; std::vector groups; @@ -449,20 +435,29 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTree auto find_group_idx = [&](const NameSet & cols) -> std::optional { for (size_t g = 0; g < groups.size(); ++g) - if (groups[g].storage_columns == cols) + if (groups[g].columns == cols && !infos[groups[g].indices.front()].expensive) return g; return std::nullopt; }; + /// Index of the group of each viable conjunct. + std::vector group_of(infos.size()); + for (size_t i = 0; i < infos.size(); ++i) { if (!infos[i].viable) continue; - auto g = find_group_idx(infos[i].storage_columns); + auto g = infos[i].expensive ? std::nullopt : find_group_idx(infos[i].columns); if (!g.has_value()) - groups.push_back({infos[i].storage_columns, {i}}); + { + group_of[i] = groups.size(); + groups.push_back({infos[i].columns, {i}}); + } else + { + group_of[i] = *g; groups[*g].indices.push_back(i); + } } /// Emit Conditions in the original AND-chain order: @@ -491,8 +486,7 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTree else { /// Emit the whole column-set group at the position of the first conjunct. - auto g_idx = find_group_idx(info.storage_columns); - const auto & group = groups[*g_idx]; + const auto & group = groups[group_of[i]]; for (size_t idx : group.indices) emitted[idx] = true; @@ -516,6 +510,7 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTree cond.columns_size = getColumnsSize(group_columns); cond.viable = true; cond.good = group_good; + cond.expensive = info.expensive; if (where_optimizer_context.use_statistics) { @@ -687,6 +682,10 @@ std::optional MergeTreeWhereOptimizer:: if (!it->viable) break; + /// If any non-expensive conditions stay in WHERE then don't move expensive conditions to PREWHERE + if (it->expensive && std::ranges::any_of(where_conditions, [](const Condition & cond) { return !cond.expensive; })) + break; + if (!where_optimizer_context.move_all_conditions_to_prewhere) { bool moved_enough = false; @@ -878,4 +877,23 @@ NameSet MergeTreeWhereOptimizer::determineArrayJoinedNames(const ASTSelectQuery return array_joined_names; } +bool MergeTreeWhereOptimizer::isExpensiveExpression(const RPNBuilderTreeNode & node) +{ + if (!node.isFunction()) + return false; + + auto function_node = node.toFunctionNode(); + if (auto function_base = function_node.getFunctionBase(); function_base && function_base->isExpensive()) + return true; + + size_t arguments_size = function_node.getArgumentsSize(); + for (size_t i = 0; i < arguments_size; ++i) + { + if (isExpensiveExpression(function_node.getArgumentAt(i))) + return true; + } + + return false; +} + } diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.h b/src/Storages/MergeTree/MergeTreeWhereOptimizer.h index a5c115872b2a1..8089e072eb111 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.h +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.h @@ -76,6 +76,10 @@ class MergeTreeWhereOptimizer : private boost::noncopyable /// Can condition be moved to prewhere? bool viable = false; + /// Does the condition contain a function that reports `isExpensive`? + /// Such a condition is moved to PREWHERE after all other conditions. + bool expensive = false; + /// Does the condition presumably have good selectivity? bool good = false; @@ -103,10 +107,11 @@ class MergeTreeWhereOptimizer : private boost::noncopyable names += n.getColumnName(); } return fmt::format( - "Condition(exp:{} viable: {}, good: {}, min_position_in_primary_key: {}, estimated_row_count: {}, " + "Condition(exp:{} viable: {}, expensive: {}, good: {}, min_position_in_primary_key: {}, estimated_row_count: {}, " "columns_size: {}, bytes_per_rejected_row: {}, table_columns.size: {})", names, viable, + expensive, good, min_position_in_primary_key, estimated_row_count, @@ -117,7 +122,7 @@ class MergeTreeWhereOptimizer : private boost::noncopyable auto tuple() const { - return std::make_tuple(!viable, !good, -min_position_in_primary_key, bytes_per_rejected_row, table_columns.size()); + return std::make_tuple(!viable, expensive, !good, -min_position_in_primary_key, bytes_per_rejected_row, table_columns.size()); } /// Is condition a better candidate for moving to PREWHERE? @@ -183,6 +188,9 @@ class MergeTreeWhereOptimizer : private boost::noncopyable static NameSet determineArrayJoinedNames(const ASTSelectQuery & select); + /// Whether the expression contains a function that reports `isExpensive`. + static bool isExpensiveExpression(const RPNBuilderTreeNode & node); + ConditionSelectivityEstimatorPtr estimator; const NameSet table_columns; diff --git a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp index 10c3dfaf0ca49..fd361e332e5b5 100644 --- a/src/Storages/MergeTree/MutateFromLogEntryTask.cpp +++ b/src/Storages/MergeTree/MutateFromLogEntryTask.cpp @@ -180,7 +180,7 @@ ReplicatedMergeMutateTaskBase::PrepareResult MutateFromLogEntryTask::prepare() Strings mutation_ids; commands = std::make_shared(storage.queue.getMutationCommands(source_part, new_part_info.mutation, mutation_ids)); LOG_TRACE(log, "Mutating part {} with mutation commands from {} mutations ({}): {}", - entry.new_part_name, commands->size(), fmt::join(mutation_ids, ", "), commands->toString(true)); + entry.new_part_name, commands->size(), fmt::join(mutation_ids, ", "), commands->ast(true)->formatForLogging()); /// mutation_ids can be empty here. mutation_ids_for_log = mutation_ids; diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index cead60e122d59..80ed02c63cdf0 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -4014,8 +4014,10 @@ void updateIndicesToRecalculateAndDrop(std::shared_ptr & ctx) (*ctx->data->getSettings())[MergeTreeSetting::packed_skip_index_max_bytes] > 0 && !ctx->indices_to_recalc.empty(); + /// An unsupported archive cannot be updated, so it would keep granules of the old data. bool archive_dirty = !ctx->dropped_skip_index_archive_file_names.empty() - || (source_has_archive && writer_can_open_archive); + || (source_has_archive && writer_can_open_archive) + || source_disk_storage->hasUnsupportedSkipIndicesPackedArchive(); if (!archive_dirty) for (const auto & idx : ctx->indices_to_recalc) if (source_part->isSkipIndexInPackedArchive(*idx)) { archive_dirty = true; break; } diff --git a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp index f3e273840ced9..82d904c7670cc 100644 --- a/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp +++ b/src/Storages/MergeTree/ReplicatedMergeTreeCleanupThread.cpp @@ -12,7 +12,7 @@ namespace MergeTreeSetting { extern const MergeTreeSettingsUInt64 finished_mutations_to_keep; extern const MergeTreeSettingsSeconds lock_acquire_timeout_for_background_operations; - extern const MergeTreeSettingsUInt64 max_replicated_logs_to_keep; + extern const MergeTreeSettingsNonZeroUInt64 max_replicated_logs_to_keep; extern const MergeTreeSettingsUInt64 min_replicated_logs_to_keep; extern const MergeTreeSettingsUInt64 replicated_deduplication_window; extern const MergeTreeSettingsUInt64 replicated_deduplication_window_for_async_inserts; diff --git a/src/Storages/MergeTree/checkDataPart.cpp b/src/Storages/MergeTree/checkDataPart.cpp index 87ec70710e29c..7ed6784f38d95 100644 --- a/src/Storages/MergeTree/checkDataPart.cpp +++ b/src/Storages/MergeTree/checkDataPart.cpp @@ -56,6 +56,9 @@ namespace ErrorCodes extern const int ABORTED; extern const int CANNOT_WRITE_TO_OSTREAM; extern const int CACHE_CANNOT_WRITE_TO_CACHE_DISK; + extern const int QUERY_WAS_CANCELLED; + extern const int QUERY_WAS_CANCELLED_BY_CLIENT; + extern const int TIMEOUT_EXCEEDED; } @@ -154,6 +157,30 @@ bool isRetryableException(std::exception_ptr exception_ptr) } } +/// A query that was killed or hit `max_execution_time` can throw from inside a part read, e.g. from +/// `FileSegment::wait`, from the S3 client or from the marks loader. Such an exception says nothing +/// about the data on disk, so a reader must not report the part as broken because of it. +/// This is deliberately not part of `isRetryableException`: inside `checkDataPart` a retryable +/// exception yields empty checksums, and a cancelled `CHECK TABLE` would then pass an unchecked part. +bool isQueryCancellation(std::exception_ptr exception_ptr) +{ + try + { + rethrow_exception(exception_ptr); + } + catch (const Exception & e) + { + return e.code() == ErrorCodes::QUERY_WAS_CANCELLED + || e.code() == ErrorCodes::QUERY_WAS_CANCELLED_BY_CLIENT + || e.code() == ErrorCodes::TIMEOUT_EXCEEDED; + } + catch (...) + { + /// Ok: only a `DB::Exception` can carry a cancellation code. + return false; + } +} + static IMergeTreeDataPart::Checksums checkDataPart( MergeTreeData::DataPartPtr data_part, const IDataPartStorage & data_part_storage, diff --git a/src/Storages/MergeTree/checkDataPart.h b/src/Storages/MergeTree/checkDataPart.h index 2f9894bc70f0f..d7049f7161a7b 100644 --- a/src/Storages/MergeTree/checkDataPart.h +++ b/src/Storages/MergeTree/checkDataPart.h @@ -16,5 +16,8 @@ IMergeTreeDataPart::Checksums checkDataPart( bool isNotEnoughMemoryErrorCode(int code); bool isRetryableException(std::exception_ptr exception_ptr); +/// True for the cancellation of the current query (`KILL QUERY`, a client disconnect or `max_execution_time`), +/// which can surface from inside a part read and says nothing about the part. +bool isQueryCancellation(std::exception_ptr exception_ptr); } diff --git a/src/Storages/MergeTree/registerStorageMergeTree.cpp b/src/Storages/MergeTree/registerStorageMergeTree.cpp index e07d37416dc3d..b80c017d5e9e3 100644 --- a/src/Storages/MergeTree/registerStorageMergeTree.cpp +++ b/src/Storages/MergeTree/registerStorageMergeTree.cpp @@ -889,7 +889,7 @@ static StoragePtr create(const StorageFactory::Arguments & args) /// `LoadingStrictnessLevel::ATTACH` and must still be subject to these checks. storage_settings->loadFromQuery( *args.storage_def, args.getLocalContext(), isLoadingFromExistingMetadata(args.mode), - args.table_id.database_name == DatabaseCatalog::SYSTEM_DATABASE); + args.table_id.database_name == DatabaseCatalog::SYSTEM_DATABASE, /* stored_definition */ args.query.attach_short_syntax); /// What this query changes from the settings the server has in effect, which already include the /// `merge_tree` config section and `compatibility`: those are not changes made by the query. A @@ -898,6 +898,10 @@ static StoragePtr create(const StorageFactory::Arguments & args) args.getLocalContext()->checkMergeTreeSettingsConstraints( initial_storage_settings, storage_settings->changesFrom(initial_storage_settings)); + /// After the constraints, so a forbidden `storage_policy` is reported as such and not as an unknown policy. + if ((*storage_settings)[MergeTreeSetting::storage_policy].changed) + args.getStoragePolicy((*storage_settings)[MergeTreeSetting::storage_policy]); + /// `MergeTreeData` runs `sanityCheck` on the settings it is given when the mode says `CREATE`, so a /// full-definition `ATTACH` is the fresh definition left to check. Without this it can state a value /// such as `index_granularity = 0` that the same `CREATE` refuses, and the table is created broken. @@ -1130,6 +1134,14 @@ static StoragePtr create(const StorageFactory::Arguments & args) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Table TTL is not allowed for MergeTree in old syntax"); } + /// Only a fresh definition written now, not one loaded from stored metadata or replayed from another replica's DDL. + if (is_fresh_definition && !args.getLocalContext()->isRecoveryFromStoredMetadata()) + { + const auto txn = args.getLocalContext()->getZooKeeperMetadataTransaction(); + if (!txn || txn->isInitialQuery()) + MergeTreeData::checkStoredExpressions(metadata, args.getLocalContext()); + } + DataTypes data_types = metadata.partition_key.data_types; if (args.mode <= LoadingStrictnessLevel::CREATE && !(*storage_settings)[MergeTreeSetting::allow_floating_point_partition_key]) { @@ -2198,6 +2210,7 @@ Storage policies configuration markup: - `disk` — a disk within a volume. - `max_data_part_size_bytes` — the maximum size of a part that can be stored on any of the volume's disks. If the a size of a merged part estimated to be bigger than `max_data_part_size_bytes` then this part will be written to a next volume. Basically this feature allows to keep new/small parts on a hot (SSD) volume and move them to a cold (HDD) volume when they reach large size. Do not use this setting if your policy has only one volume. - `move_factor` — when the amount of available space gets lower than this factor, data automatically starts to move on the next volume if any (by default, 0.1). ClickHouse sorts existing parts by size from largest to smallest (in descending order) and selects parts with the total size that is sufficient to meet the `move_factor` condition. If the total size of all parts is insufficient, all parts will be moved. +- `backup_metadata_only` — the data of tables on this policy is provided by the storage itself, e.g. a shared read-only dataset (by default, `false`). All volumes of the policy must be read-only. `BACKUP` stores only the definitions of such tables, and `RESTORE` recreates them without restoring data and keeps their UUIDs from the backup, so that they find their data on the storage again. `BACKUP` and `RESTORE` of specific partitions of such a table fail. Only the plain `MergeTree` family is supported: a replicated or shared `MergeTree` table, or a materialized view or a `TimeSeries` table with an inner table on such a policy, is backed up and restored as usual, with a warning. - `perform_ttl_move_on_insert` — Disables TTL move on data part INSERT. By default (if enabled) if we insert a data part that already expired by the TTL move rule it immediately goes to a volume/disk declared in move rule. This can significantly slowdown insert in case if destination volume/disk is slow (e.g. S3). If disabled then already expired data part is written into a default volume and then right after moved to TTL volume. - `load_balancing` - Policy for disk balancing, `round_robin` or `least_used`. - `least_used_ttl_ms` - Configure timeout (in milliseconds) for the updating available space on all disks (`0` - update always, `-1` - never update, default is `60000`). Note, if the disk can be used by ClickHouse only and is not subject to a online filesystem resize/shrink you can use `-1`, in all other cases it is not recommended, since eventually it will lead to incorrect space distribution. diff --git a/src/Storages/MergeTree/tests/gtest_merge_tree_index_bloom_filter.cpp b/src/Storages/MergeTree/tests/gtest_merge_tree_index_bloom_filter.cpp new file mode 100644 index 0000000000000..007d155521ef5 --- /dev/null +++ b/src/Storages/MergeTree/tests/gtest_merge_tree_index_bloom_filter.cpp @@ -0,0 +1,74 @@ +#include + +#include +#include +#include +#include +#include + +#include + +using namespace DB; + +namespace DB::ErrorCodes +{ + extern const int CORRUPTED_DATA; +} + +namespace +{ + +String encodeRowCount(size_t row_count) +{ + WriteBufferFromOwnString out; + writeVarUInt(row_count, out); + return out.str(); +} + +void expectCorruptedRowCount(size_t bits_per_row, size_t row_count) +{ + MergeTreeIndexGranuleBloomFilter granule(bits_per_row, 3, 1); + String encoded = encodeRowCount(row_count); + ReadBufferFromString in(encoded); + + try + { + granule.deserializeBinary(in, 1); + FAIL() << "Expected a corrupted Bloom-filter granule to be rejected"; + } + catch (const Exception & exception) + { + EXPECT_EQ(exception.code(), ErrorCodes::CORRUPTED_DATA); + } +} + +} + +TEST(MergeTreeIndexBloomFilterTest, DeserializesValidGranule) +{ + String encoded = encodeRowCount(1); + encoded.append(sizeof(BloomFilter::UnderType), '\0'); + + MergeTreeIndexGranuleBloomFilter granule(12, 3, 1); + ReadBufferFromString in(encoded); + + EXPECT_NO_THROW(granule.deserializeBinary(in, 1)); + EXPECT_FALSE(granule.empty()); +} + +TEST(MergeTreeIndexBloomFilterTest, RejectsZeroRowGranule) +{ + expectCorruptedRowCount(12, 0); +} + +TEST(MergeTreeIndexBloomFilterTest, RejectsFilterSizeOverflow) +{ + constexpr size_t bits_per_row = 12; + constexpr size_t overflowing_rows = std::numeric_limits::max() / bits_per_row + 1; + expectCorruptedRowCount(bits_per_row, overflowing_rows); +} + +TEST(MergeTreeIndexBloomFilterTest, RejectsRoundedFilterSizeOverflow) +{ + expectCorruptedRowCount(1, std::numeric_limits::max()); +} diff --git a/src/Storages/ObjectStorage/Azure/Configuration.cpp b/src/Storages/ObjectStorage/Azure/Configuration.cpp index 4ca0c16780bd3..ac661969cb7af 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.cpp +++ b/src/Storages/ObjectStorage/Azure/Configuration.cpp @@ -44,7 +44,6 @@ namespace ErrorCodes { extern const int BAD_ARGUMENTS; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; - extern const int LOGICAL_ERROR; } const std::unordered_set required_configuration_keys = { @@ -69,7 +68,7 @@ const std::unordered_set optional_configuration_keys = { void StorageAzureConfiguration::check(ContextPtr context) { - auto url = Poco::URI(connection_params.getConnectionURL()); + auto url = Poco::URI(connection_params.getConnectionURLWithoutQuery()); context->getGlobalContext()->getRemoteHostFilter().checkURL(url); StorageObjectStorageConfiguration::check(context); } @@ -166,9 +165,9 @@ void AzureStorageParsedArguments::fillBlobsFromURLCommon(String & connection_url if (pos_container != std::string::npos) { - String container_blob_path = connection_url.substr(pos_container+5); - connection_url = connection_url.substr(0,pos_container+4); - container_name = connection_url.substr(pos_container+4); + size_t host_end = pos_container + suffix.size(); + String container_blob_path = host_end < connection_url.size() ? connection_url.substr(host_end + 1) : ""; + connection_url.resize(host_end); auto pos_blob_path = container_blob_path.find('/'); if (pos_blob_path != std::string::npos) @@ -188,9 +187,9 @@ void AzureStorageParsedArguments::fillBlobsFromURLCommon(String & connection_url auto pos_net = connection_url.find(suffix); if (pos_slash == std::string::npos || pos_at == std::string::npos|| pos_dot == std::string::npos || pos_net == std::string::npos - || pos_at-pos_slash-3 <= 0 || pos_dot-pos_at-1 <= 0) + || pos_at <= pos_slash + 3 || pos_dot <= pos_at + 1) { - throw Exception(ErrorCodes::LOGICAL_ERROR, "Incorrect url format for a abfss url {}", connection_url); + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Incorrect url format for a abfss url {}", connection_url); } auto container_name_abfss = connection_url.substr(pos_slash+3, pos_at-pos_slash-3); auto name = connection_url.substr(pos_at+1, pos_dot-pos_at-1); @@ -281,6 +280,21 @@ static ASTPtr extractExtraCredentials(ASTs & args) return nullptr; } +/// Returns the number of arguments before the trailing `identifier = value` arguments. +static size_t countPositionalArguments(const ASTs & args) +{ + size_t count = args.size(); + while (count > 0) + { + const auto * function = args[count - 1]->as(); + if (!function || function->name != "equals" || !function->arguments || function->arguments->children.size() != 2 + || !function->arguments->children[0]->as()) + break; + --count; + } + return count; +} + bool AzureStorageParsedArguments::collectCredentials( ASTPtr maybe_credentials, std::optional & client_id, std::optional & tenant_id, ContextPtr local_context) { @@ -374,7 +388,31 @@ void AzureStorageParsedArguments::fromAST(ASTs & engine_args, ContextPtr context { auto extra_credentials = extractExtraCredentials(engine_args); - if (engine_args.empty() || engine_args.size() > AzureStorageParsedArguments::getMaxNumberOfArguments(with_structure)) + const size_t count = countPositionalArguments(engine_args); + /// Like the positional arguments, the key-value arguments are persisted with evaluated values. + for (size_t i = count; i < engine_args.size(); ++i) + { + const auto & key_value = engine_args[i]->as().arguments->children; + engine_args[i] = makeASTOperator("equals", key_value[0], evaluateConstantExpressionOrIdentifierAsLiteral(key_value[1], context)); + } + const auto key_value_args = parseKeyValueArguments(ASTs(engine_args.begin() + count, engine_args.end()), context); + for (const auto & [key, _] : key_value_args) + { + if (key != "partition_strategy" && key != "partition_columns_in_data_file") + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Unexpected key-value argument `{}`: only `partition_strategy` and `partition_columns_in_data_file` " + "can be passed as key-value arguments", + key); + } + + if (!key_value_args.empty() && count < 3) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "Key-value arguments require at least three positional arguments: " + "connection string or storage account URL, container name and blob path"); + + if (count == 0 || count > AzureStorageParsedArguments::getMaxNumberOfArguments(with_structure)) { throw Exception( ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, @@ -383,12 +421,12 @@ void AzureStorageParsedArguments::fromAST(ASTs & engine_args, ContextPtr context AzureStorageParsedArguments::getSignatures(with_structure)); } - for (auto & engine_arg : engine_args) - engine_arg = evaluateConstantExpressionOrIdentifierAsLiteral(engine_arg, context); + for (size_t i = 0; i < count; ++i) + engine_args[i] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[i], context); /// This is only for lightweight loading of tables, so does not contain credentials /// for listing tables of Unity Catalog - if (engine_args.size() == 1) + if (count == 1) { connection_params.endpoint.storage_account_url = checkAndGetLiteralArgument(engine_args[0], "connection_string/storage_account_url"); @@ -396,7 +434,7 @@ void AzureStorageParsedArguments::fromAST(ASTs & engine_args, ContextPtr context return; } - if (engine_args.size() == 2) + if (count == 2) { String connection_url = checkAndGetLiteralArgument(engine_args[0], "connection_string/storage_account_url"); String sas_token = checkAndGetLiteralArgument(engine_args[1], "sas_token"); @@ -425,7 +463,7 @@ void AzureStorageParsedArguments::fromAST(ASTs & engine_args, ContextPtr context return s == "auto" || FormatFactory::instance().getAllFormats().contains(Poco::toLower(s)); }; - if (engine_args.size() == 4) + if (count == 4) { auto fourth_arg = checkAndGetLiteralArgument(engine_args[3], "format/account_name"); if (is_format_arg(fourth_arg)) @@ -442,7 +480,7 @@ void AzureStorageParsedArguments::fromAST(ASTs & engine_args, ContextPtr context "Unknown format or account name specified without account key: {}", fourth_arg); } } - else if (engine_args.size() == 5) + else if (count == 5) { auto fourth_arg = checkAndGetLiteralArgument(engine_args[3], "format/account_name"); if (is_format_arg(fourth_arg)) @@ -456,7 +494,7 @@ void AzureStorageParsedArguments::fromAST(ASTs & engine_args, ContextPtr context account_key = checkAndGetLiteralArgument(engine_args[4], "account_key"); } } - else if (engine_args.size() == 6) + else if (count == 6) { auto fourth_arg = checkAndGetLiteralArgument(engine_args[3], "format/account_name"); if (is_format_arg(fourth_arg)) @@ -499,7 +537,7 @@ void AzureStorageParsedArguments::fromAST(ASTs & engine_args, ContextPtr context } } } - else if (engine_args.size() == 7) + else if (count == 7) { const auto fourth_arg = checkAndGetLiteralArgument(engine_args[3], "format/account_name"); @@ -552,7 +590,7 @@ void AzureStorageParsedArguments::fromAST(ASTs & engine_args, ContextPtr context compression_method = checkAndGetLiteralArgument(engine_args[6], "compression"); } } - else if (engine_args.size() == 8) + else if (count == 8) { auto fourth_arg = checkAndGetLiteralArgument(engine_args[3], "format/account_name"); @@ -608,7 +646,7 @@ void AzureStorageParsedArguments::fromAST(ASTs & engine_args, ContextPtr context } } } - else if (engine_args.size() == 9) + else if (count == 9) { auto fourth_arg = checkAndGetLiteralArgument(engine_args[3], "format/account_name"); account_name = fourth_arg; @@ -646,7 +684,7 @@ void AzureStorageParsedArguments::fromAST(ASTs & engine_args, ContextPtr context partition_columns_in_data_file_was_set = true; } } - else if (engine_args.size() == 10 && with_structure) + else if (count == 10 && with_structure) { auto fourth_arg = checkAndGetLiteralArgument(engine_args[3], "format/account_name"); account_name = fourth_arg; @@ -671,6 +709,36 @@ void AzureStorageParsedArguments::fromAST(ASTs & engine_args, ContextPtr context structure = checkAndGetLiteralArgument(engine_args[9], "structure"); } + if (auto it = key_value_args.find("partition_strategy"); it != key_value_args.end()) + { + if (partition_strategy_was_set) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, "`partition_strategy` is specified both as a positional and as a key-value argument"); + + const auto & partition_strategy_name = it->second.safeGet(); + const auto partition_strategy_type_opt = magic_enum::enum_cast(partition_strategy_name, magic_enum::case_insensitive); + if (!partition_strategy_type_opt) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "Unknown partition strategy {}", partition_strategy_name); + + partition_strategy_type = partition_strategy_type_opt.value(); + partition_strategy_was_set = true; + } + + if (auto it = key_value_args.find("partition_columns_in_data_file"); it != key_value_args.end()) + { + if (partition_columns_in_data_file_was_set) + throw Exception( + ErrorCodes::BAD_ARGUMENTS, + "`partition_columns_in_data_file` is specified both as a positional and as a key-value argument"); + + partition_columns_in_data_file = it->second.safeGet(); + partition_columns_in_data_file_was_set = true; + } + + /// A key-value strategy gets the same default as the named-collection form. + if (key_value_args.contains("partition_strategy") && !partition_columns_in_data_file_was_set) + partition_columns_in_data_file = partition_strategy_type != PartitionStrategyFactory::StrategyType::HIVE; + connection_params = getAzureConnectionParams(connection_url, container_name, account_name, account_key, client_id, tenant_id, context); } @@ -700,6 +768,14 @@ static void addStructureAndFormatToArgsIfNeededAzure( } else { + /// extra_credentials is not a constant expression; take it out while rewriting. + auto extra_credentials = extractExtraCredentials(args); + + /// The trailing key-value arguments are appended back after the positional ones. + const size_t count = countPositionalArguments(args); + ASTs key_value_args(args.begin() + count, args.end()); + args.resize(count); + if (args.size() < 3 || args.size() > AzureStorageParsedArguments::getMaxNumberOfArguments()) throw Exception( ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, @@ -843,6 +919,10 @@ static void addStructureAndFormatToArgsIfNeededAzure( if (with_structure && checkAndGetLiteralArgument(args[7], "structure") == "auto") args[7] = structure_literal; } + + args.insert(args.end(), key_value_args.begin(), key_value_args.end()); + if (extra_credentials) + args.push_back(extra_credentials); } } diff --git a/src/Storages/ObjectStorage/Azure/Configuration.h b/src/Storages/ObjectStorage/Azure/Configuration.h index c9d45d0bdb8a5..0c6d57af8e14c 100644 --- a/src/Storages/ObjectStorage/Azure/Configuration.h +++ b/src/Storages/ObjectStorage/Azure/Configuration.h @@ -106,7 +106,7 @@ class StorageAzureConfiguration : public StorageObjectStorageConfiguration void setPaths(const Paths & paths) override { blobs_paths = paths; } String getNamespace() const override { return connection_params.getContainer(); } - String getDataSourceDescription() const override { return std::filesystem::path(connection_params.getConnectionURL()) / connection_params.getContainer(); } + String getDataSourceDescription() const override { return std::filesystem::path(connection_params.getConnectionURLWithoutQuery()) / connection_params.getContainer(); } StorageObjectStorageQuerySettings getQuerySettings(const ContextPtr &) const override; String getContainer() const { return connection_params.getContainer(); } diff --git a/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp b/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp index f383336c95964..48835c0af5569 100644 --- a/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.cpp @@ -54,6 +54,7 @@ try } metadata = manifest_file_reader->metadata(); + bytes_read = buffer->count(); parsed_column = std::move(columns[0]); parsed_column_data_type = std::dynamic_pointer_cast(data_type); parsed_manifest_file_entries.resize(parsed_column->size()); diff --git a/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.h b/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.h index ea2a54706b60f..84e867e9076eb 100644 --- a/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.h +++ b/src/Storages/ObjectStorage/DataLakes/Common/AvroForIcebergDeserializer.h @@ -45,6 +45,8 @@ class AvroForIcebergDeserializer std::map> metadata; + size_t bytes_read = 0; + /// Shared mutex to protect mutable cache members for thread safety mutable SharedMutex cache_mutex; @@ -71,6 +73,9 @@ class AvroForIcebergDeserializer size_t rows() const; + /// The constructor consumes the whole file, so this is its size on storage. + size_t bytesRead() const { return bytes_read; } + /// Allow to access avro paths like "a.b.c" bool hasPath(const std::string & path) const; DB::TypeIndex getTypeForPath(const std::string & path) const; diff --git a/src/Storages/ObjectStorage/DataLakes/DeltaLake/PartitionPruner.cpp b/src/Storages/ObjectStorage/DataLakes/DeltaLake/PartitionPruner.cpp index 182015d014d7d..afd0e08b3de94 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeltaLake/PartitionPruner.cpp +++ b/src/Storages/ObjectStorage/DataLakes/DeltaLake/PartitionPruner.cpp @@ -46,7 +46,8 @@ namespace DB::ColumnsDescription getPartitionColumnsDescription( const DB::Names & partition_columns, - const DB::NamesAndTypesList & table_schema) + const DB::NamesAndTypesList & table_schema, + bool keep_nullable) { DB::NamesAndTypesList names_and_types; for (const auto & column_name : partition_columns) @@ -54,7 +55,7 @@ namespace auto column = table_schema.tryGetByName(column_name); if (!column.has_value()) throw DB::Exception(DB::ErrorCodes::LOGICAL_ERROR, "Not found partition column in schema: {}", column_name); - names_and_types.emplace_back(column_name, removeNullable(column->type)); + names_and_types.emplace_back(column_name, keep_nullable ? column->type : removeNullable(column->type)); } return DB::ColumnsDescription(names_and_types); } @@ -70,7 +71,8 @@ PartitionPruner::PartitionPruner( { if (!partition_columns_.empty()) { - const auto partition_columns_description = getPartitionColumnsDescription(partition_columns_, table_schema_); + const auto partition_columns_description + = getPartitionColumnsDescription(partition_columns_, table_schema_, /* keep_nullable */ false); const auto partition_key_ast = createPartitionKeyAST(partition_columns_); partition_key = DB::KeyDescription::getKeyFromAST( @@ -78,10 +80,17 @@ PartitionPruner::PartitionPruner( partition_columns_description, {}, context); + nullable_partition_key = DB::KeyDescription::getKeyFromAST( + partition_key_ast, + getPartitionColumnsDescription(partition_columns_, table_schema_, /* keep_nullable */ true), + {}, + context); DB::ActionsDAGWithInversionPushDown inverted_dag(filter_dag.getOutputs().front(), context, /* boolean_context */ true); key_condition.emplace( inverted_dag, context, partition_key.column_names, partition_key.expression, true /* single_point */); + nullable_key_condition.emplace( + inverted_dag, context, nullable_partition_key.column_names, nullable_partition_key.expression, true /* single_point */); } if (!physical_names_map_.empty()) { @@ -110,20 +119,28 @@ bool PartitionPruner::canBePruned(const DB::ObjectInfo & object_info) const DB::Row partition_key_values; partition_key_values.reserve(partition_values.size()); + bool holds_null = false; for (auto && value : partition_values) { if (value.isNull()) + { + holds_null = true; partition_key_values.push_back(DB::POSITIVE_INFINITY); /// NULL_LAST + } else partition_key_values.push_back(std::move(value)); } + /// The key without Nullable drops the NULL elements of a set, so it is exact only for a tuple without NULL. + const auto & key = holds_null ? nullable_partition_key : partition_key; + const auto & condition = holds_null ? *nullable_key_condition : *key_condition; + std::vector partition_key_values_ref(partition_key_values.begin(), partition_key_values.end()); - bool can_be_true = key_condition->mayBeTrueInRange( + bool can_be_true = condition.mayBeTrueInRange( partition_key_values_ref.size(), partition_key_values_ref.data(), partition_key_values_ref.data(), - partition_key.data_types); + key.data_types); return !can_be_true; } diff --git a/src/Storages/ObjectStorage/DataLakes/DeltaLake/PartitionPruner.h b/src/Storages/ObjectStorage/DataLakes/DeltaLake/PartitionPruner.h index 1ff9e2e5559bf..ef89f33852bc6 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeltaLake/PartitionPruner.h +++ b/src/Storages/ObjectStorage/DataLakes/DeltaLake/PartitionPruner.h @@ -32,6 +32,9 @@ class PartitionPruner private: std::optional key_condition; DB::KeyDescription partition_key; + /// The same key over the schema's Nullable types, for a partition tuple that holds NULL. + std::optional nullable_key_condition; + DB::KeyDescription nullable_partition_key; DB::Names physical_partition_columns; }; diff --git a/src/Storages/ObjectStorage/DataLakes/DeltaLake/TableSnapshot.cpp b/src/Storages/ObjectStorage/DataLakes/DeltaLake/TableSnapshot.cpp index 5c14fcf47aec9..302c600c34f9e 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeltaLake/TableSnapshot.cpp +++ b/src/Storages/ObjectStorage/DataLakes/DeltaLake/TableSnapshot.cpp @@ -66,6 +66,30 @@ namespace DB::FailPoints namespace DeltaLake { +namespace +{ + +/// A data file with the size its `add` action records. +struct DataObjectInfo : public DB::ObjectInfo +{ + DataObjectInfo(std::string path_, int64_t size_) + : DB::ObjectInfo(DB::RelativePathWithMetadata(std::move(path_))) + , size(size_) + { + } + + std::optional getFileSizeHint() const override + { + if (size > 0) + return static_cast(size); + return std::nullopt; + } + + const int64_t size; +}; + +} + class TableSnapshot::Iterator final : public DB::IObjectIterator, private DB::WithContext { private: @@ -96,7 +120,6 @@ class TableSnapshot::Iterator final : public DB::IObjectIterator, private DB::Wi const TableSchema & table_schema_, const DB::NameToNameMap & physical_names_map_, const DB::Names & partition_columns_, - DB::ObjectStoragePtr object_storage_, const DB::ActionsDAG * filter_, DB::IDataLakeMetadata::FileProgressCallback callback_, size_t list_batch_size_, @@ -113,7 +136,6 @@ class TableSnapshot::Iterator final : public DB::IObjectIterator, private DB::Wi , read_schema(read_schema_) , expression_schema(table_schema_) , partition_columns(partition_columns_) - , object_storage(object_storage_) , callback(callback_) , list_batch_size(list_batch_size_) , log(log_) @@ -415,13 +437,9 @@ class TableSnapshot::Iterator final : public DB::IObjectIterator, private DB::Wi } parseDVHandle(*scan_item, object); - object->setObjectMetadata(object_storage->getObjectMetadata(object->getPath(), /*with_tags=*/ false)); if (callback) - { - chassert(object->getObjectMetadata()); - callback(DB::FileProgress(0, object->getObjectMetadata()->size_bytes)); - } + callback(DB::FileProgress(0, object->getFileSizeHint().value_or(0))); return object; } } @@ -595,7 +613,7 @@ class TableSnapshot::Iterator final : public DB::IObjectIterator, private DB::Wi std::string full_path = DB::resolvePathInsideTable( context->getDataPath(), DB::unescapeForFileName(KernelUtils::fromDeltaString(path))); - auto object = std::make_shared(DB::RelativePathWithMetadata(std::move(full_path))); + auto object = std::make_shared(std::move(full_path), size); object->data_lake_metadata.emplace(); LOG_TEST( @@ -639,7 +657,6 @@ class TableSnapshot::Iterator final : public DB::IObjectIterator, private DB::Wi DB::NamesAndTypesList read_schema; DB::NamesAndTypesList expression_schema; DB::Names partition_columns; - const DB::ObjectStoragePtr object_storage; const DB::IDataLakeMetadata::FileProgressCallback callback; const size_t list_batch_size; const LoggerPtr log; @@ -994,7 +1011,6 @@ DB::ObjectIterator TableSnapshot::iterate( schema->table_schema, schema->physical_names_map, schema->partition_columns, - object_storage, filter_dag, callback, list_batch_size, diff --git a/src/Storages/ObjectStorage/DataLakes/DeltaLake/WriteTransaction.cpp b/src/Storages/ObjectStorage/DataLakes/DeltaLake/WriteTransaction.cpp index 8dcb11bd7e722..d51fe061cd40b 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeltaLake/WriteTransaction.cpp +++ b/src/Storages/ObjectStorage/DataLakes/DeltaLake/WriteTransaction.cpp @@ -49,17 +49,13 @@ UInt64 getCurrentTime() return std::chrono::duration_cast(std::chrono::system_clock::now().time_since_epoch()).count(); } -void exportTable( - std::shared_ptr table, +void exportRecordBatch( + const arrow::RecordBatch & batch, ffi::FFI_ArrowArray & array, ffi::FFI_ArrowSchema & schema) { - auto batch = table->CombineChunksToBatch(); - if (!batch.ok()) - DB::throwFromArrowStatus(batch.status(), DB::ErrorCodes::UNKNOWN_EXCEPTION, "Failed to create chunks batch"); - arrow::Status status = arrow::ExportRecordBatch( - **batch, + batch, reinterpret_cast(&array), reinterpret_cast(&schema)); @@ -250,30 +246,60 @@ void WriteTransaction::commit(const std::vector & files) LOG_TEST(log, "Will commit {} files", files.size()); auto write_metadata = getWriteMetadata(files, log); - ffi::FFI_ArrowArray array{}; - ffi::FFI_ArrowSchema schema{}; - SCOPE_EXIT({ - if (schema.release) - schema.release(&schema); - }); - exportTable(write_metadata, array, schema); - - KernelEngineData engine_data; - try + /// Metadata too large for one Arrow record batch comes back as several table chunks. Recombining them + /// would hit the same 32-bit offset limit, so each record batch is added to the transaction on its own. + /// `TableBatchReader` yields nothing for an empty table, which still has to reach the kernel as before. + arrow::RecordBatchVector batches; + if (write_metadata->num_rows() == 0) + { + auto empty_batch = arrow::RecordBatch::MakeEmpty(write_metadata->schema()); + if (!empty_batch.ok()) + DB::throwFromArrowStatus(empty_batch.status(), DB::ErrorCodes::UNKNOWN_EXCEPTION, "Failed to create an empty record batch"); + batches.push_back(*std::move(empty_batch)); + } + else { - /// Takes ownership of `array` (but not`schema`) if successfully called. - engine_data = DeltaLake::KernelUtils::unwrapResult( - ffi::get_engine_data(array, &schema, &KernelUtils::allocateError), - "get_engine_data"); + arrow::TableBatchReader batch_reader(*write_metadata); + while (true) + { + std::shared_ptr batch; + arrow::Status read_status = batch_reader.ReadNext(&batch); + if (!read_status.ok()) + DB::throwFromArrowStatus(read_status, DB::ErrorCodes::UNKNOWN_EXCEPTION, "Failed to read a record batch"); + if (!batch) + break; + batches.push_back(std::move(batch)); + } } - catch (...) + + for (const auto & batch : batches) { - if (array.release) - array.release(&array); - throw; + ffi::FFI_ArrowArray array{}; + ffi::FFI_ArrowSchema schema{}; + SCOPE_EXIT({ + if (schema.release) + schema.release(&schema); + }); + exportRecordBatch(*batch, array, schema); + + KernelEngineData engine_data; + try + { + /// Takes ownership of `array` (but not`schema`) if successfully called. + engine_data = DeltaLake::KernelUtils::unwrapResult( + ffi::get_engine_data(array, &schema, &KernelUtils::allocateError), + "get_engine_data"); + } + catch (...) + { + if (array.release) + array.release(&array); + throw; + } + + ffi::add_files(transaction.get(), engine_data.release()); } - ffi::add_files(transaction.get(), engine_data.release()); using KernelCommittedTransaction = DeltaLake::KernelPointerWrapper; KernelCommittedTransaction committed(DeltaLake::KernelUtils::unwrapResult( ffi::commit(transaction.release(), engine.get()), diff --git a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.cpp b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.cpp index 1bc42de2adecd..7c16dda7b8bef 100644 --- a/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.cpp +++ b/src/Storages/ObjectStorage/DataLakes/DeltaLakeMetadataDeltaKernel.cpp @@ -602,10 +602,17 @@ ReadFromFormatInfo DeltaLakeMetadataDeltaKernel::prepareReadingFromFormat( info.format_header.insert(ColumnWithTypeAndName{result_name_and_type.type, result_name_and_type.name}); } - for (const auto & name_and_type : info.columns_description) - info.columns_description.rename( - name_and_type.name, - DeltaLake::getPhysicalName(name_and_type.name, physical_names_map)); + /// Build a new description rather than rename in place: the physical name of a column may be the logical name of + /// another column that is not renamed yet (`k` renamed to `k_old` under column mapping keeps the physical name `k`), + /// and `ColumnsDescription::rename` to an existing name erases the column, under the iteration over the description. + ColumnsDescription physical_columns_description; + for (const auto & column : info.columns_description) + { + ColumnDescription physical_column = column; + physical_column.name = DeltaLake::getPhysicalName(column.name, physical_names_map); + physical_columns_description.add(std::move(physical_column)); + } + info.columns_description = std::move(physical_columns_description); LOG_TEST(log, "Format header: {}", info.format_header.dumpStructure()); LOG_TEST(log, "Source header: {}", info.source_header.dumpStructure()); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp index 099f445fa0fab..5f76f328c9abe 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergIterator.cpp @@ -389,8 +389,7 @@ Iceberg::ManifestIteratorPtr IcebergIterator::createManifestIterator(const Manif persistent_components, local_context, logger, - manifest_list_entry.manifest_file_path, - manifest_list_entry.manifest_file_byte_size); + manifest_list_entry.manifest_file_path); return Iceberg::ManifestFileIterator::create( manifest_file_cacheable_part.deserializer, diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp index 67ecc542c68cf..1cb962cb4b2e1 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadata.cpp @@ -82,6 +82,7 @@ #include #include +#include #include #include @@ -123,6 +124,11 @@ extern const int SUPPORT_IS_DISABLED; extern const int FILE_ALREADY_EXISTS; } +namespace FailPoints +{ +extern const char iceberg_metadata_prefetch_pause[]; +} + namespace Setting { extern const SettingsNonZeroUInt64 max_block_size; @@ -278,6 +284,9 @@ void IcebergMetadata::backgroundMetadataPrefetcherThread() try { + /// Lets a test hold the refresher before it reads the latest metadata + FailPointInjection::pauseFailPoint(FailPoints::iceberg_metadata_prefetch_pause); + if (!Context::getGlobalContextInstance()) { /// Should never happen, but if seen, this is clear indicator that the task should be started/stopped via startup/shutdown mechanism (check TODOs above) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadataFilesCache.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadataFilesCache.h index 32fcc97d435dc..ceb715eeedbe9 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadataFilesCache.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/IcebergMetadataFilesCache.h @@ -75,7 +75,7 @@ struct IcebergMetadataFilesCacheCell : private boost::noncopyable /// - manifest list consists of cache keys which will retrieve the manifest file from cache [file_path --> ManifestFileCacheKeys] /// - manifest file [file_path --> Iceberg::ManifestFileCacheableInfo] std::variant cached_element; - Int64 memory_bytes; + size_t memory_bytes; explicit IcebergMetadataFilesCacheCell(String && metadata_json_str) : cached_element(std::move(metadata_json_str)) diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp index 812a6c62ef038..375d7699b5b89 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/MetadataGenerator.cpp @@ -124,6 +124,16 @@ bool checkValidSchemaEvolution(Poco::Dynamic::Var old_type, Poco::Dynamic::Var n return false; } +/// `last-column-id` written by some writers does not count nested field ids, so the ids of all schemas are taken into account too. +Int32 getLastAssignedFieldId(const Poco::JSON::Object::Ptr & metadata_object) +{ + Int64 result = metadata_object->getValue(Iceberg::f_last_column_id); + auto schemas = metadata_object->getArray(Iceberg::f_schemas); + for (UInt32 i = 0; i < schemas->size(); ++i) + result = std::max(result, Iceberg::IcebergSchemaProcessor::getMaxFieldId(schemas->getObject(i)->getArray(Iceberg::f_fields))); + return static_cast(result); +} + } MetadataGenerator::MetadataGenerator(Poco::JSON::Object::Ptr metadata_object_) @@ -475,12 +485,11 @@ void MetadataGenerator::generateAddColumnMetadata(const String & column_name, Da throw Exception(ErrorCodes::BAD_ARGUMENTS, "Column {} already exists", column_name); } - auto last_column_id = metadata_object->getValue(Iceberg::f_last_column_id); /// The new top-level field takes the next id; nested tuple/array/map /// children (when `type` is complex) take the ids after it via the shared /// `iter`. After getIcebergType `iter` is the max assigned id, which is what /// last-column-id must record so every added child gets a globally unique id. - Int32 new_field_id = last_column_id + 1; + Int32 new_field_id = getLastAssignedFieldId(metadata_object) + 1; Int32 iter = new_field_id; auto new_type = Iceberg::getIcebergType(type, iter); metadata_object->set(Iceberg::f_last_column_id, iter); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/MultipleFileWriter.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/MultipleFileWriter.cpp index 1935421c3701f..a9045b1423847 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/MultipleFileWriter.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/MultipleFileWriter.cpp @@ -99,6 +99,9 @@ void MultipleFileWriter::finalize() completed_file_stats.push_back(std::move(current_file_stats)); data_file_byte_counts.push_back(file_bytes); data_file_row_counts.push_back(current_file_num_rows.value_or(0)); + + /// A finished format can still hold resources: a Parquet one keeps its idle encoder threads. + release(); } void MultipleFileWriter::release() diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp index 6ffc50d493b88..a77a367fa008e 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/Mutations.cpp @@ -277,6 +277,8 @@ static std::optional writeDataFiles( path_resolver.resolve(delete_data_result[partition_key].path), /*with_tags=*/ false).size_bytes; delete_data_result[partition_key].total_bytes = static_cast(delete_bytes); } + delete_data_writers[partition_key].reset(); + delete_data_write_buffers[partition_key].reset(); } } @@ -351,6 +353,8 @@ static std::optional writeDataFiles( path_resolver.resolve(update_data_result[partition_key].path), /*with_tags=*/ false).size_bytes; update_data_result[partition_key].total_bytes = static_cast(update_bytes); } + update_data_writers[partition_key].reset(); + update_data_write_buffers[partition_key].reset(); } } diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp index 5978dc1a41851..ba9f49a03d8fd 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.cpp @@ -230,6 +230,42 @@ String canonicalizeTypeSpacing(const String & s) return result; } +/// Calls `callback(id, type)` for `field_id` and for the ids of the fields, list elements and map keys/values nested in it. +/// `type` is a JSON string (primitive leaf) or object (complex type) describing `field_id`. +template +void forEachFieldId(const Poco::Dynamic::Var & type, Int64 field_id, Callback && callback) +{ + checkStackSize(); + callback(field_id, type); + if (type.isString()) + return; + + const auto & type_object = type.extract(); + auto type_str = type_object->getValue(f_type); + if (type_str == "map") + { + if (type_object->has(f_key)) + forEachFieldId(type_object->get(f_key), type_object->getValue(f_key_id), callback); + if (type_object->has(f_value)) + forEachFieldId(type_object->get(f_value), type_object->getValue(f_value_id), callback); + } + else if (type_str == "list") + { + if (type_object->has(f_element)) + forEachFieldId(type_object->get(f_element), type_object->getValue(f_element_id), callback); + } + else if (type_str == "struct") + { + auto fields = type_object->getArray(f_fields); + for (UInt32 i = 0; i < fields->size(); ++i) + { + auto field = fields->getObject(i); + if (field->has(f_type)) + forEachFieldId(field->get(f_type), field->getValue(f_id), callback); + } + } +} + /// Compare two Iceberg type descriptors for the same field. A type is either a primitive /// string ("long", "decimal(20, 0)", ...) or a nested object ("struct" with a fields array, /// or "list" / "map" wrappers whose element/key/value members are themselves types). @@ -964,6 +1000,20 @@ std::unordered_map IcebergSchemaProcessor::traverseSchema(Poco::J return result; } +Int64 IcebergSchemaProcessor::getMaxFieldId(Poco::JSON::Array::Ptr schema) +{ + Int64 result = 0; + for (UInt32 i = 0; i < schema->size(); ++i) + { + auto current_object = schema->getObject(i); + if (current_object->has(Iceberg::f_type)) + forEachFieldId( + current_object->get(Iceberg::f_type), current_object->getValue(Iceberg::f_id), + [&](Int64 field_id, const Poco::Dynamic::Var &) { result = std::max(result, field_id); }); + } + return result; +} + std::unordered_set IcebergSchemaProcessor::collectIcebergStringPaths(Poco::JSON::Array::Ptr schema) { std::unordered_set result; diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h index e2353b4a6e357..5866405401f18 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/SchemaProcessor.h @@ -119,6 +119,9 @@ class IcebergSchemaProcessor static std::unordered_map traverseSchema(Poco::JSON::Array::Ptr schema); + /// The largest field id in `schema`, nested ids included. + static Int64 getMaxFieldId(Poco::JSON::Array::Ptr schema); + /// Paths whose Iceberg logical type is `string` (not `binary`); both read as DataTypeString. static std::unordered_set collectIcebergStringPaths(Poco::JSON::Array::Ptr schema); diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.cpp b/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.cpp index 9b3a648938b96..57645b512f92b 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.cpp +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.cpp @@ -75,8 +75,7 @@ Iceberg::ManifestFileCacheableInfo getManifestFile( const PersistentTableComponents & persistent_table_components, ContextPtr local_context, LoggerPtr log, - const IcebergPathFromMetadata & filename, - size_t bytes_size) + const IcebergPathFromMetadata & filename) { auto log_level = local_context->getSettingsRef()[Setting::iceberg_metadata_log_level].value; @@ -102,7 +101,8 @@ Iceberg::ManifestFileCacheableInfo getManifestFile( auto manifest_file_deserializer = std::make_unique( std::move(buffer), filename, getFormatSettings(local_context)); - return Iceberg::ManifestFileCacheableInfo{std::move(manifest_file_deserializer), bytes_size}; + const size_t manifest_file_bytes = manifest_file_deserializer->bytesRead(); + return Iceberg::ManifestFileCacheableInfo{std::move(manifest_file_deserializer), manifest_file_bytes}; }; if (use_iceberg_metadata_cache && persistent_table_components.table_uuid.has_value()) @@ -127,8 +127,7 @@ Iceberg::ManifestFileIterator::ManifestFileEntriesHandle getManifestFileEntriesH persistent_table_components, local_context, log, - cache_key.manifest_file_path, - static_cast(cache_key.manifest_file_byte_size)); + cache_key.manifest_file_path); auto iterator = Iceberg::ManifestFileIterator::create( cacheable_info.deserializer, diff --git a/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.h b/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.h index 2bb2edfdff13c..f78d7eb4e4749 100644 --- a/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.h +++ b/src/Storages/ObjectStorage/DataLakes/Iceberg/StatelessMetadataFileGetter.h @@ -25,8 +25,7 @@ Iceberg::ManifestFileCacheableInfo getManifestFile( const PersistentTableComponents & persistent_table_components, ContextPtr local_context, LoggerPtr log, - const IcebergPathFromMetadata & filename, - size_t bytes_size); + const IcebergPathFromMetadata & filename); /// Creates a fully initialized ManifestFileIterator from a cache key. /// All entries are drained so that aggregate methods (e.g. getRowsCountInAllFilesExcludingDeleted) diff --git a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp index 8e04a3f576f20..a42ceb4e9c310 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageCluster.cpp @@ -275,7 +275,8 @@ std::optional StorageObjectStorageCluster::totalBytes(ContextPtr query_c void StorageObjectStorageCluster::updateQueryToSendIfNeeded( ASTPtr & query, const DB::StorageSnapshotPtr & storage_snapshot, - const ContextPtr & context) + const ContextPtr & context, + bool add_structure_and_format) { auto * table_function = extractTableFunctionFromSelectQuery(query); if (!table_function) @@ -313,7 +314,8 @@ void StorageObjectStorageCluster::updateQueryToSendIfNeeded( if (!endsWith(table_function->name, "Cluster")) { - configuration->addStructureAndFormatToArgsIfNeeded(args, structure, configuration->format, context, /*with_structure=*/true); + if (add_structure_and_format) + configuration->addStructureAndFormatToArgsIfNeeded(args, structure, configuration->format, context, /*with_structure=*/true); /// When a non-cluster table function (e.g. `s3`) was auto-converted to cluster mode /// by the `parallel_replicas_for_cluster_engines` setting, rename it to the Cluster variant @@ -334,7 +336,8 @@ void StorageObjectStorageCluster::updateQueryToSendIfNeeded( { ASTPtr cluster_name_arg = args.front(); args.erase(args.begin()); - configuration->addStructureAndFormatToArgsIfNeeded(args, structure, configuration->format, context, /*with_structure=*/true); + if (add_structure_and_format) + configuration->addStructureAndFormatToArgsIfNeeded(args, structure, configuration->format, context, /*with_structure=*/true); args.insert(args.begin(), cluster_name_arg); } if (settings_temporary_storage) diff --git a/src/Storages/ObjectStorage/StorageObjectStorageCluster.h b/src/Storages/ObjectStorage/StorageObjectStorageCluster.h index 9815b14f6a173..2d7f64ea87467 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageCluster.h +++ b/src/Storages/ObjectStorage/StorageObjectStorageCluster.h @@ -77,7 +77,8 @@ class StorageObjectStorageCluster : public IStorageCluster void updateQueryToSendIfNeeded( ASTPtr & query, const StorageSnapshotPtr & storage_snapshot, - const ContextPtr & context) override; + const ContextPtr & context, + bool add_structure_and_format) override; const String engine_name; const StorageObjectStorageConfigurationPtr configuration; diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp index 5fc14e1df78f5..d96efabe196dd 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.cpp @@ -170,10 +170,16 @@ namespace return path.substr(0, position); } + String getObjectStorageHash(const IObjectStorage & object_storage) + { + const String description = object_storage.getDescription(); + return sipHash128String(description.substr(0, description.find('?')) + '\0' + object_storage.getObjectsNamespace()); + } + String getPageCachePathForObjectStorage(const RelativePathWithMetadata & object_info, const ObjectStoragePtr & object_storage) { if (object_storage->getType() != ObjectStorageType::Web) - return "s3:" + object_info.getPath(); + return fmt::format("s3:{}:{}", getObjectStorageHash(*object_storage), object_info.getPath()); if (!object_info.read_source_index) return "web:" + object_info.getPath(); @@ -929,7 +935,7 @@ void StorageObjectStorageSource::addNumRowsToCache(const ObjectInfo & object_inf schema_cache.addNumRows(cache_key, num_rows); } -StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReader() +StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReader(bool prefetch_input_format) { return createReader( 0, @@ -945,7 +951,8 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade max_block_size, parser_shared_resources, format_filter_info, - need_only_count); + need_only_count, + prefetch_input_format); } StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReader( @@ -962,7 +969,8 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade size_t max_block_size, FormatParserSharedResourcesPtr parser_shared_resources, FormatFilterInfoPtr format_filter_info, - bool need_only_count) + bool need_only_count, + bool prefetch_input_format) { ObjectInfoPtr object_info; auto query_settings = configuration->getQuerySettings(context_); @@ -1372,6 +1380,9 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade if (need_only_count) input_format->needOnlyCount(); + if (prefetch_input_format) + input_format->prefetch(); + builder.init(Pipe(input_format)); if (!identity_partition_columns.empty()) @@ -1535,7 +1546,7 @@ StorageObjectStorageSource::ReaderHolder StorageObjectStorageSource::createReade std::future StorageObjectStorageSource::createReaderAsync() { - return create_reader_scheduler([=, this] { return createReader(); }, Priority{}); + return create_reader_scheduler([=, this] { return createReader(/* prefetch_input_format */ true); }, Priority{}); } std::unique_ptr createReadBuffer( @@ -1689,6 +1700,7 @@ std::unique_ptr createReadBuffer( else { SipHash hash; + hash.update(getObjectStorageHash(*object_storage)); hash.update(object_info.getPath()); hash.update(object_info.metadata->etag); diff --git a/src/Storages/ObjectStorage/StorageObjectStorageSource.h b/src/Storages/ObjectStorage/StorageObjectStorageSource.h index ec9b155d08608..7d3654b5ee646 100644 --- a/src/Storages/ObjectStorage/StorageObjectStorageSource.h +++ b/src/Storages/ObjectStorage/StorageObjectStorageSource.h @@ -154,9 +154,10 @@ class StorageObjectStorageSource final : public ISource size_t max_block_size, FormatParserSharedResourcesPtr parser_shared_resources, FormatFilterInfoPtr format_filter_info, - bool need_only_count); + bool need_only_count, + bool prefetch_input_format = false); - ReaderHolder createReader(); + ReaderHolder createReader(bool prefetch_input_format = false); std::future createReaderAsync(); diff --git a/src/Storages/ObjectStorageQueue/AzureQueue_fwd.h b/src/Storages/ObjectStorageQueue/AzureQueue_fwd.h index db7c6314d7214..a338ae7dead0d 100644 --- a/src/Storages/ObjectStorageQueue/AzureQueue_fwd.h +++ b/src/Storages/ObjectStorageQueue/AzureQueue_fwd.h @@ -20,8 +20,7 @@ static inline std::unordered_map SETTINGS_TO_HIDE = std::string masked_value; if (!value.tryGet(masked_value)) return {}; - DB::maskConnectionStringKey(masked_value, "AccountKey="); - DB::maskConnectionStringKey(masked_value, "SharedAccessSignature="); + DB::maskAzureConnectionStringSecrets(masked_value); return fmt::format("'{}'", masked_value); }}, }; diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMoveDestinationAccess.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMoveDestinationAccess.cpp new file mode 100644 index 0000000000000..e845fa34a6c0d --- /dev/null +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMoveDestinationAccess.cpp @@ -0,0 +1,119 @@ +#include + +#include "config.h" + +#include +#include +#include +#include + +#include + +#include +#include +#include + + +namespace DB +{ + +namespace +{ + +enum class MoveDestinationKind : uint8_t +{ + S3, + Azure, +}; + +std::optional getMoveDestinationKind(std::string_view name) +{ + /// The prefix is accepted for every queue setting, see `normalizeSetting` in `StorageObjectStorageQueue.cpp`. + static constexpr std::string_view legacy_prefix = "s3queue_"; + if (name.starts_with(legacy_prefix)) + name.remove_prefix(legacy_prefix.size()); + + if (name == "after_processing_move_uri") + return MoveDestinationKind::S3; + if (name == "after_processing_move_connection_string" || name == "after_processing_move_container") + return MoveDestinationKind::Azure; + return std::nullopt; +} + +AccessTypeObjects::Source getAccessSource(MoveDestinationKind kind) +{ + switch (kind) + { + case MoveDestinationKind::S3: + return AccessTypeObjects::Source::S3; + case MoveDestinationKind::Azure: + return AccessTypeObjects::Source::AZURE; + } +} + +/// A value that is not a `String` is never empty here: it is checked like a destination that cannot be parsed. +bool isEmptyValue(const Field & value) +{ + return value.getType() == Field::Types::String && value.safeGet().empty(); +} + +/// A table moves files only to the destination of its own storage type, see `ObjectStorageQueuePostProcessor::process`. +AccessRightsElements getMoveDestinationAccess(const SettingsChanges & changes, MoveDestinationKind table_kind) +{ + const bool sets_destination = std::ranges::any_of( + changes, + [&](const SettingChange & change) { return getMoveDestinationKind(change.name) == table_kind && !isEmptyValue(change.value); }); + if (!sets_destination) + return {}; + + /// Unfiltered on purpose: a move has no single destination path to match a filter against. + /// For S3, the move writes keys built from `after_processing_move_prefix` and the source keys, and uses only + /// the bucket of the URI, not its key. For Azure, the `azureBlobStorage` function matches source filters against + /// the blob path (`StorageAzureConfiguration::getRawURI` returns `blob_path.path`). + const auto source = getAccessSource(table_kind); + AccessRightsElements res; + res.emplace_back(AccessType::WRITE, AccessTypeObjects::toStringSource(source)); + return res; +} + +} + +AccessRightsElements getS3QueueMoveDestinationAccess(const SettingsChanges & changes) +{ + return getMoveDestinationAccess(changes, MoveDestinationKind::S3); +} + +AccessRightsElements getAzureQueueMoveDestinationAccess(const SettingsChanges & changes) +{ + return getMoveDestinationAccess(changes, MoveDestinationKind::Azure); +} + +void checkObjectStorageQueueS3MoveDestinationHost( + [[maybe_unused]] const String & move_uri, + [[maybe_unused]] const RemoteHostFilter & filter) +{ +#if USE_AWS_S3 + if (!move_uri.empty()) + S3::URI(move_uri).checkRemoteHostFilter(filter); +#endif +} + +void checkObjectStorageQueueAzureMoveDestinationHost( + [[maybe_unused]] const String & move_connection_string, + [[maybe_unused]] const String & move_container, + [[maybe_unused]] const RemoteHostFilter & filter) +{ +#if USE_AZURE_BLOB_STORAGE + if (!move_connection_string.empty()) + { + AzureBlobStorage::ConnectionParams connection_params; + AzureBlobStorage::processURL(move_connection_string, move_container, connection_params.endpoint, connection_params.auth_method); + /// Without the query: a connection string can carry a SAS, which the message of `checkURL` must not show. + String url = connection_params.getConnectionURL(); + url = url.substr(0, url.find_first_of("?#")); + filter.checkURL(Poco::URI(url)); + } +#endif +} + +} diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueMoveDestinationAccess.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMoveDestinationAccess.h new file mode 100644 index 0000000000000..40601b4b11458 --- /dev/null +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueMoveDestinationAccess.h @@ -0,0 +1,37 @@ +#pragma once + +#include +#include + +namespace DB +{ + +class RemoteHostFilter; + +/// The move destination of an `S3Queue` or `AzureQueue` table is the same data flow as +/// `INSERT INTO FUNCTION s3('') SELECT * FROM
`: the table reads every processed file +/// with its own source credentials and writes it to the destination. So setting the destination needs +/// `WRITE` on the `S3` (or `AZURE`) source without a URL filter: the move writes keys built from +/// `after_processing_move_prefix` and the source keys, not from the URI, so a filtered grant cannot authorize them. +/// For `ALTER` it also needs `SELECT` on the table, because the creator of the table, not the user of the `ALTER`, +/// supplied the source and its credentials. + +/// The `StorageFactory::GetExportAccessFn` of `S3Queue` and `AzureQueue`: unfiltered `WRITE` if a setting moves files outside +/// the source bucket or container. Only these settings choose such a destination: `after_processing_move_uri` for S3, +/// `after_processing_move_connection_string` and `after_processing_move_container` for Azure, with or without the legacy +/// `s3queue_` prefix. Changing only the move keys or prefix stays inside a destination an authorized user set. +/// An empty value disables the move destination, so it is not a destination and needs no grant. +AccessRightsElements getS3QueueMoveDestinationAccess(const SettingsChanges & changes); +AccessRightsElements getAzureQueueMoveDestinationAccess(const SettingsChanges & changes); + +/// Throws `UNACCEPTABLE_URL` if `remote_url_allow_hosts` does not allow the destination of a move to another S3 bucket. +/// The move builds its client directly, so the destination never passes `StorageS3Configuration::check`. +/// Does nothing when `move_uri` is empty. +void checkObjectStorageQueueS3MoveDestinationHost(const String & move_uri, const RemoteHostFilter & filter); + +/// The same for a move to another Azure container (see `StorageAzureConfiguration::check`). +/// Does nothing when `move_connection_string` is empty. +void checkObjectStorageQueueAzureMoveDestinationHost( + const String & move_connection_string, const String & move_container, const RemoteHostFilter & filter); + +} diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueuePostProcessor.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueuePostProcessor.cpp index c520d97da2bb0..6f351f28f8841 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueuePostProcessor.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueuePostProcessor.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include @@ -51,6 +52,12 @@ namespace S3AuthSetting extern const S3AuthSettingsString google_adc_client_id; extern const S3AuthSettingsString google_adc_client_secret; extern const S3AuthSettingsString google_adc_refresh_token; + extern const S3AuthSettingsS3UriStyle uri_style; +} + +namespace S3RequestSetting +{ + extern const S3RequestSettingsBool allow_native_copy; } #endif @@ -307,6 +314,10 @@ void ObjectStorageQueuePostProcessor::moveS3Objects(const StoredObjects & object throw Exception(ErrorCodes::BAD_ARGUMENTS, "Not enough settings to move S3 objects"); } + /// The settings are checked at `CREATE` and `ALTER` too. This is defense in depth for a table that was created before + /// those checks existed, or before the list of allowed hosts was tightened. If it throws, nothing is copied and nothing is removed. + checkObjectStorageQueueS3MoveDestinationHost(move_uri, getContext()->getRemoteHostFilter()); + if (auto * s3_storage = dynamic_cast(object_storage.get()); s3_storage != nullptr) { auto src_client = s3_storage->getS3StorageClient(); @@ -337,6 +348,10 @@ void ObjectStorageQueuePostProcessor::moveS3Objects(const StoredObjects & object /// headers and SSE-C/SSE-KMS keys) merged from the server `` config: otherwise the server's /// headers or encryption keys would be sent to the user-supplied move destination. s3_settings->auth_settings.clearServerManagedRequestAuth(); + /// The host check and `dst_uri` below parse `move_uri` with automatic style detection. The client must use the same + /// parse, so the URI style of the server `` config or of the `s3_uri_style` query setting must not apply here: + /// otherwise the requests can go to a host that `checkObjectStorageQueueS3MoveDestinationHost` did not check. + s3_settings->auth_settings[S3AuthSetting::uri_style] = S3UriStyle::AUTO; std::shared_ptr dst_client = getClient( move_uri, *s3_settings, @@ -344,6 +359,10 @@ void ObjectStorageQueuePostProcessor::moveS3Objects(const StoredObjects & object /* for_disk_s3 */ false ); auto dst_uri = S3::URI(move_uri); + /// Native copy sends `CopyObject` through the source client, so it writes to the source endpoint. That is the checked + /// destination only if both clients use the same endpoint, otherwise use the buffered copy, which uploads with `dst_client`. + if (src_client->getInitialEndpoint() != dst_client->getInitialEndpoint()) + s3_settings->request_settings[S3RequestSetting::allow_native_copy] = false; auto read_settings = getReadSettings(); const auto read_settings_to_use = s3_storage->patchSettings(read_settings); auto scheduler = threadPoolCallbackRunnerUnsafe( @@ -430,6 +449,9 @@ void ObjectStorageQueuePostProcessor::moveAzureBlobs(const StoredObjects & objec throw Exception(ErrorCodes::BAD_ARGUMENTS, "Not enough settings to move Azure blobs"); } + /// See the comment in `moveS3Objects`: if it throws, nothing is copied and nothing is removed. + checkObjectStorageQueueAzureMoveDestinationHost(move_connection_string, move_container, getContext()->getRemoteHostFilter()); + if (auto * azure_storage = dynamic_cast(object_storage.get()); azure_storage != nullptr) { auto contextPtr = getContext(); diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp index 2678a4a03f540..c54b74cb82c01 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.cpp @@ -5,7 +5,9 @@ #include #include #include +#include #include +#include #include #include #include @@ -59,15 +61,15 @@ namespace ErrorCodes DECLARE(String, partition_component, "", "Name of the capture group from partition_regex to use as partition key. Required when using partitioning_mode='regex'. Example: 'hostname'", 0) \ DECLARE(ObjectStorageQueueBucketingMode, bucketing_mode, ObjectStorageQueueBucketingMode::PATH, "Bucketing strategy for Ordered mode parallel processing: PATH (hash full file path, default), PARTITION (hash partition key, requires partitioning_mode != NONE)", 0) \ DECLARE(UInt32, after_processing_retries, 10, "Number of retries for the after_processing action before giving up", 0) \ - DECLARE(String, after_processing_move_uri, "", "S3 bucket URL to move processed files to", 0) \ + DECLARE(String, after_processing_move_uri, "", "S3 bucket URL to move processed files to. Setting it requires the `WRITE` grant on the `S3` source (not limited to a URL filter), and `ALTER` also requires `SELECT` on the table. The URL must be allowed by `remote_url_allow_hosts`", 0) \ DECLARE(String, after_processing_move_prefix, "", "Path prefix to move processed files to", 0) \ DECLARE(Bool, after_processing_move_preserve_path, false, "If true, preserve the full source path under after_processing_move_prefix instead of using only the file name", 0) \ DECLARE(String, after_processing_tag_key, "", "Tag key to tag processed files in the storage", 0) \ DECLARE(String, after_processing_tag_value, "", "Tag value to tag processed files in the storage", 0) \ DECLARE(String, after_processing_move_access_key_id, "", "S3 Access Key ID accompanying after_processing_move_uri", 0) \ DECLARE(String, after_processing_move_secret_access_key, "", "S3 Secret Access Key accompanying after_processing_move_uri", 0) \ - DECLARE(String, after_processing_move_connection_string, "", "Azure connection string to move processed files to", 0) \ - DECLARE(String, after_processing_move_container, "", "Azure container accompanying after_processing_move_connection_string", 0) \ + DECLARE(String, after_processing_move_connection_string, "", "Azure connection string to move processed files to. Setting it requires the `WRITE` grant on the `AZURE` source (not limited to a URL filter), and `ALTER` also requires `SELECT` on the table. The endpoint must be allowed by `remote_url_allow_hosts`", 0) \ + DECLARE(String, after_processing_move_container, "", "Azure container accompanying after_processing_move_connection_string. Setting it requires the `WRITE` grant on the `AZURE` source (not limited to a URL filter), and `ALTER` also requires `SELECT` on the table", 0) \ DECLARE(Bool, use_hive_partitioning, false, "DEPRECATED: Use partitioning_mode='hive' instead. Whether path contains hive partitioning and this engine should process such files with a separate processed files tracking per partition.", 0) \ #define LIST_OF_OBJECT_STORAGE_QUEUE_SETTINGS(M, ALIAS) \ @@ -92,7 +94,8 @@ void ObjectStorageQueueSettings::dumpToSystemEngineSettingsColumns( MutableColumnsAndConstraints & params, const std::string & table_name, const std::string & database_name, - const StorageObjectStorageQueue & storage) const + const StorageObjectStorageQueue & storage, + bool display_secrets) const { MutableColumns & res_columns = params.res_columns; auto metadata_snapshot = storage.getInMemoryMetadataPtr(CurrentThread::tryGetQueryContext(), false); @@ -113,6 +116,24 @@ void ObjectStorageQueueSettings::dumpToSystemEngineSettingsColumns( { return StorageObjectStorageQueue::isSettingChangeable(setting_name, (*this)[ObjectStorageQueueSetting::mode]); }; + /// Hides what `SHOW CREATE TABLE` hides. A secret is hidden whole, as its masker renders SQL. + auto render_value = [&](const std::string & setting_name, const Field & value) -> String + { + static constexpr std::string_view hidden_secret = "[HIDDEN]"; + if (!display_secrets) + { + /// Both queue engines take the same settings, so the secrets of both apply. + for (const auto * settings_to_hide : {&S3Queue::SETTINGS_TO_HIDE, &AzureQueue::SETTINGS_TO_HIDE}) + { + auto it = settings_to_hide->find(setting_name); + if (it == settings_to_hide->end()) + continue; + if (auto masked = it->second(value); masked && masked->contains(hidden_secret)) + return String(hidden_secret); + } + } + return convertFieldToString(value); + }; for (const auto & change : impl->all()) { @@ -121,7 +142,7 @@ void ObjectStorageQueueSettings::dumpToSystemEngineSettingsColumns( res_columns[i++]->insert(database_name); res_columns[i++]->insert(table_name); res_columns[i++]->insert(name); - res_columns[i++]->insert(convertFieldToString(change.getValue())); + res_columns[i++]->insert(render_value(name, change.getValue())); res_columns[i++]->insert(change.getTypeName()); res_columns[i++]->insert(is_changed(name)); res_columns[i++]->insert(change.getDescription()); diff --git a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.h b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.h index 4455fadff3bf2..6dbd51326fbb6 100644 --- a/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.h +++ b/src/Storages/ObjectStorageQueue/ObjectStorageQueueSettings.h @@ -63,7 +63,8 @@ struct ObjectStorageQueueSettings MutableColumnsAndConstraints & params, const std::string & table_name, const std::string & database_name, - const StorageObjectStorageQueue & storage) const; + const StorageObjectStorageQueue & storage, + bool display_secrets) const; void loadFromQuery(ASTStorage & storage_def, bool is_attach, const StorageID & storage_id); diff --git a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp index 093276d481ac5..6692ac9ccc268 100644 --- a/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp +++ b/src/Storages/ObjectStorageQueue/StorageObjectStorageQueue.cpp @@ -1,3 +1,5 @@ +#include +#include #include #include @@ -28,6 +30,7 @@ #include #include #include +#include #include #include #include @@ -40,6 +43,7 @@ #include #include #include +#include #include #include #include @@ -171,9 +175,26 @@ namespace ErrorCodes namespace { + /// The move uses only the destination of the storage type of the table, so only that one is checked. + void checkMoveDestinationHost( + ObjectStorageType type, + const String & move_uri, + const String & move_connection_string, + const String & move_container, + const RemoteHostFilter & filter) + { + if (type == ObjectStorageType::S3) + checkObjectStorageQueueS3MoveDestinationHost(move_uri, filter); + else if (type == ObjectStorageType::Azure) + checkObjectStorageQueueAzureMoveDestinationHost(move_connection_string, move_container, filter); + } + void validateSettings( ObjectStorageQueueSettings & queue_settings, - bool is_attach) + ObjectStorageType type, + const RemoteHostFilter & remote_host_filter, + bool is_attach, + bool is_loading_from_existing_metadata) { if (!is_attach && !queue_settings[ObjectStorageQueueSetting::mode].changed) { @@ -232,6 +253,21 @@ namespace "Move of processed objects requires specifying target bucket/container or prefix"); } } + + /// The host filter of `configuration` covers the source only, so the move destination is checked here. + /// It is checked whenever it is set, not only for `after_processing = 'move'`: otherwise a table created with + /// `after_processing = 'keep'` and a destination that is not allowed could be switched to `move` by an `ALTER` + /// that does not change the destination. A table loaded from existing metadata must never fail to load. + if (!is_loading_from_existing_metadata) + { + checkMoveDestinationHost( + type, + queue_settings[ObjectStorageQueueSetting::after_processing_move_uri].value, + queue_settings[ObjectStorageQueueSetting::after_processing_move_connection_string].value, + queue_settings[ObjectStorageQueueSetting::after_processing_move_container].value, + remote_host_filter); + } + if (queue_settings[ObjectStorageQueueSetting::after_processing] == ObjectStorageQueueAction::TAG) { if (queue_settings[ObjectStorageQueueSetting::after_processing_tag_key].toString().empty()) @@ -340,7 +376,7 @@ StorageObjectStorageQueue::StorageObjectStorageQueue( } const bool is_attach = mode > LoadingStrictnessLevel::CREATE; - validateSettings(*queue_settings_, is_attach); + validateSettings(*queue_settings_, type, context_->getRemoteHostFilter(), is_attach, isLoadingFromExistingMetadata(mode)); /// The object storage S3 client is built once here and reused by background threads, so the effective /// per-session credential restriction must be captured now from the CREATE query (its @@ -1567,6 +1603,32 @@ void StorageObjectStorageQueue::checkAlterIsPossible(const AlterCommands & comma } } } + + /// The host filter of `configuration` covers the source only, so a new move destination is checked here. + /// `checkAlterIsPossible` runs on every host and the check depends only on the server config, so it is correct everywhere. + static constexpr std::array move_destination_settings{ + "after_processing_move_uri", "after_processing_move_connection_string", "after_processing_move_container"}; + + auto get_move_destination_value = [](const SettingsChanges * settings, std::string_view name) -> String + { + const Field * value = settings ? settings->tryGet(name) : nullptr; + return value ? convertFieldToString(*value) : String{}; + }; + + const bool move_destination_changed = std::ranges::any_of( + move_destination_settings, + [&](std::string_view name) + { return get_move_destination_value(old_settings, name) != get_move_destination_value(&new_settings, name); }); + + if (move_destination_changed) + { + checkMoveDestinationHost( + type, + get_move_destination_value(&new_settings, "after_processing_move_uri"), + get_move_destination_value(&new_settings, "after_processing_move_connection_string"), + get_move_destination_value(&new_settings, "after_processing_move_container"), + local_context->getRemoteHostFilter()); + } } void StorageObjectStorageQueue::alter( diff --git a/src/Storages/ObjectStorageQueue/registerQueueStorage.cpp b/src/Storages/ObjectStorageQueue/registerQueueStorage.cpp index 0ec97a2f1402b..6de88a0c3f836 100644 --- a/src/Storages/ObjectStorageQueue/registerQueueStorage.cpp +++ b/src/Storages/ObjectStorageQueue/registerQueueStorage.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -168,6 +169,7 @@ void registerStorageS3Queue(StorageFactory & factory) .supports_schema_inference = true, .source_access_type = AccessTypeObjects::Source::S3, .has_builtin_setting_fn = ObjectStorageQueueSettings::hasBuiltin, + .get_export_access_fn = getS3QueueMoveDestinationAccess, }, Documentation{ .description = R"DOCS_MD( @@ -358,6 +360,8 @@ Default value: empty string. URI of S3 bucket to move successfully processed files to, if the destination is another S3 bucket. +Setting it is the same data flow as `INSERT INTO FUNCTION s3('') SELECT * FROM
`, so it requires the `WRITE` grant on the `S3` source. A grant limited by a URL filter is not enough, because the move writes keys that are built from `after_processing_move_prefix` and the source keys, not from the URI. `ALTER TABLE ... MODIFY SETTING` also requires `SELECT` on the table. The destination must be allowed by `remote_url_allow_hosts`. The move always parses the URI with automatic style detection, as the `s3` table function does with the default `s3_uri_style`, so the `uri_style` server setting and the `s3_uri_style` query setting do not apply to it. + Possible values: - String. @@ -758,6 +762,7 @@ void registerStorageAzureQueue(StorageFactory & factory) .supports_schema_inference = true, .source_access_type = AccessTypeObjects::Source::AZURE, .has_builtin_setting_fn = ObjectStorageQueueSettings::hasBuiltin, + .get_export_access_fn = getAzureQueueMoveDestinationAccess, }, Documentation{ .description = R"DOCS_MD( @@ -804,6 +809,8 @@ Below are the settings only compatible with AzureQueue and not applicable for S3 Connection string for Azure Blob Storage to move successfully processed files to, if the destination is another Azure container. +Setting it requires the `WRITE` grant on the `AZURE` source. A grant limited by a URL filter is not enough, because a move has no single blob path and writes keys that are built from `after_processing_move_prefix` and the source keys. `ALTER TABLE ... MODIFY SETTING` also requires `SELECT` on the table. The destination must be allowed by `remote_url_allow_hosts`. + Possible values: - String. @@ -814,6 +821,8 @@ Default value: empty string. Container name to move successfully processed files to, if the destination is another Azure container. +Changing it redirects the move to another container of the same account, so it needs the same grants as `after_processing_move_connection_string`: the `WRITE` grant on the `AZURE` source without a URL filter, and `SELECT` on the table for `ALTER TABLE ... MODIFY SETTING`. + Possible values: - String. diff --git a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp index f49ecdbd0f847..7de0d39d40044 100644 --- a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp +++ b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.cpp @@ -346,6 +346,14 @@ bool StorageMaterializedPostgreSQL::supportsOptimizationToSubcolumns() const } +bool StorageMaterializedPostgreSQL::hasBucketedMapSerialization() const +{ + if (auto nested = tryGetNested()) + return nested->hasBucketedMapSerialization(); + return false; +} + + IStorage::ColumnSizeByName StorageMaterializedPostgreSQL::getColumnSizes() const { if (auto nested = tryGetNested()) diff --git a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.h b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.h index 0a41ea7e74ed1..8b8e5f6a8a028 100644 --- a/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.h +++ b/src/Storages/PostgreSQL/StorageMaterializedPostgreSQL.h @@ -105,6 +105,7 @@ class StorageMaterializedPostgreSQL final : public IStorage, WithContext bool supportedPrewhereColumnsIncludeSubcolumns() const override; bool supportsSubcolumns() const override; bool supportsOptimizationToSubcolumns() const override; + bool hasBucketedMapSerialization() const override; ColumnSizeByName getColumnSizes() const override; ColumnSizeByName getColumnSizes(const Names & columns, bool calculate_subcolumn_sizes) const override; diff --git a/src/Storages/StorageAlias.cpp b/src/Storages/StorageAlias.cpp index eff18fef021cf..dff2af5530826 100644 --- a/src/Storages/StorageAlias.cpp +++ b/src/Storages/StorageAlias.cpp @@ -1,4 +1,5 @@ #include +#include #include #include #include @@ -19,6 +20,7 @@ #include #include #include +#include namespace DB @@ -250,6 +252,26 @@ SinkToStoragePtr StorageAlias::write( return std::make_shared(*this, local_context, target_metadata, async_insert); } +void StorageAlias::checkAlterIsPossible(const AlterCommands & commands, ContextPtr local_context) const +{ + /// `InterpreterAlterQuery` checks the export grants (`SELECT` on the table and its row policy) on the name in the query, + /// which is the alias, not the target. The refusal does not need the user, so it also holds on the hosts of `ON CLUSTER`. + for (const auto & command : commands) + { + if (command.type == AlterCommand::MODIFY_SETTING && !StorageFactory::instance().getExportAccessOfAnyEngine(command.settings_changes).empty()) + { + throw Exception( + ErrorCodes::NOT_IMPLEMENTED, + "Changing the settings that send the data of a table elsewhere through an alias is not supported. " + "Execute the ALTER directly on the target table: {}.{}", + backQuoteIfNeed(target_database), + backQuoteIfNeed(target_table)); + } + } + + getTargetTable()->checkAlterIsPossible(commands, local_context); +} + void StorageAlias::alter( const AlterCommands & params, ContextPtr local_context, diff --git a/src/Storages/StorageAlias.h b/src/Storages/StorageAlias.h index 3fa188702e069..7c60ab2006db1 100644 --- a/src/Storages/StorageAlias.h +++ b/src/Storages/StorageAlias.h @@ -68,7 +68,7 @@ class StorageAlias final : public IStorage, WithContext /// Distributed write to target table std::optional distributedWrite(const ASTInsertQuery & query, ContextPtr local_context) override; - void checkAlterIsPossible(const AlterCommands & commands, ContextPtr local_context) const override { getTargetTable()->checkAlterIsPossible(commands, local_context); } + void checkAlterIsPossible(const AlterCommands & commands, ContextPtr local_context) const override; /// Alter target table void alter( @@ -162,6 +162,11 @@ class StorageAlias final : public IStorage, WithContext auto target = tryGetTargetTable(); return target && target->supportsOptimizationToTupleElementSubcolumns(); } + bool hasBucketedMapSerialization() const override + { + auto target = tryGetTargetTable(); + return target && target->hasBucketedMapSerialization(); + } bool supportsParallelInsert() const override { auto target = tryGetTargetTable(); diff --git a/src/Storages/StorageBuffer.cpp b/src/Storages/StorageBuffer.cpp index a949586df39b8..7e546320fd0d4 100644 --- a/src/Storages/StorageBuffer.cpp +++ b/src/Storages/StorageBuffer.cpp @@ -1052,6 +1052,13 @@ bool StorageBuffer::supportsOptimizationToTupleElementSubcolumns() const return false; } +bool StorageBuffer::hasBucketedMapSerialization() const +{ + if (auto destination = getDestinationTable()) + return destination->hasBucketedMapSerialization(); + return false; +} + bool StorageBuffer::checkThresholds(const Buffer & buffer, bool direct, time_t current_time, size_t additional_rows, size_t additional_bytes) const { time_t time_passed = 0; diff --git a/src/Storages/StorageBuffer.h b/src/Storages/StorageBuffer.h index 1b69b005805e7..c060bcab7133d 100644 --- a/src/Storages/StorageBuffer.h +++ b/src/Storages/StorageBuffer.h @@ -133,6 +133,7 @@ friend class BufferSink; /// Distributed). Fails closed like supportsPrewhere(): no destination means no rewrite. bool supportsOptimizationToSubcolumns() const override; bool supportsOptimizationToTupleElementSubcolumns() const override; + bool hasBucketedMapSerialization() const override; bool supportsFinal() const override { return true; } void checkAlterIsPossible(const AlterCommands & commands, ContextPtr context) const override; diff --git a/src/Storages/StorageDistributed.cpp b/src/Storages/StorageDistributed.cpp index 645ab311957da..7d6d2c37e6e11 100644 --- a/src/Storages/StorageDistributed.cpp +++ b/src/Storages/StorageDistributed.cpp @@ -2,6 +2,9 @@ #include #include +#include +#include +#include #include @@ -183,6 +186,7 @@ namespace Setting extern const SettingsBool prefer_global_in_and_join; extern const SettingsBool skip_unavailable_shards; extern const SettingsBool enable_global_with_statement; + extern const SettingsMap additional_table_filters; } namespace DistributedSetting @@ -1517,6 +1521,7 @@ std::optional StorageDistributed::distributedWrite(const ASTInser auto & select = query.select->as(); StoragePtr src_storage; + bool src_is_table_function = false; /// Distributed write only works in the most trivial case INSERT ... SELECT /// without any unions or joins on the right side @@ -1533,6 +1538,7 @@ std::optional StorageDistributed::distributedWrite(const ASTInser if (joined_tables.tablesCount() == 1) { src_storage = joined_tables.getLeftTableStorage(); + src_is_table_function = joined_tables.isLeftTableFunction(); } } } @@ -1540,8 +1546,22 @@ std::optional StorageDistributed::distributedWrite(const ASTInser if (!src_storage) return {}; + /// This is the only place source SELECT access is enforced in this case + /// (table functions have been already checked in ITableFunction::execute) + const auto src_id = src_storage->getStorageID(); + if (!src_is_table_function + && !local_context->getAccess()->isGranted(AccessType::SELECT, src_id.database_name, src_id.table_name)) + return {}; + if (auto row_policy_filter = local_context->getRowPolicyFilter(src_id.database_name, src_id.table_name, RowPolicyFilterType::SELECT_FILTER); + row_policy_filter && !row_policy_filter->isAlwaysTrue()) + return {}; + /// It is not easy to match this table only, analyzer also do not do it + if (!settings[Setting::additional_table_filters].value.empty()) + return {}; + if (auto src_distributed = std::dynamic_pointer_cast(src_storage)) { + src_distributed->checkLocalShardAccess(AccessType::SELECT, local_context); return distributedWriteBetweenDistributedTables(*src_distributed, query, local_context); } if (auto src_storage_cluster = std::dynamic_pointer_cast(src_storage)) @@ -2286,7 +2306,7 @@ void registerStorageDistributed(StorageFactory & factory) if (engine_args.size() >= 5) { engine_args[4] = evaluateConstantExpressionOrIdentifierAsLiteral(engine_args[4], local_context); - storage_policy = checkAndGetLiteralArgument(engine_args[4], "storage_policy"); + storage_policy = args.getStoragePolicy(checkAndGetLiteralArgument(engine_args[4], "storage_policy"))->getName(); } /// TODO: move some arguments from the arguments to the SETTINGS. diff --git a/src/Storages/StorageExecutable.cpp b/src/Storages/StorageExecutable.cpp index cd6eaf90454b2..c8b438f4430b0 100644 --- a/src/Storages/StorageExecutable.cpp +++ b/src/Storages/StorageExecutable.cpp @@ -202,7 +202,11 @@ void StorageExecutable::readImpl( { QueryPipelineBuilder builder; if (context->getSettingsRef()[Setting::allow_experimental_analyzer]) - builder = InterpreterSelectQueryAnalyzer(input_query, context, {}).buildQueryPipeline(); + { + /// The analyzer substitutes the inserted block of a materialized view for its source table only when given it. + builder = InterpreterSelectQueryAnalyzer( + input_query, context, {}, context->isViewInnerQuery() ? nullptr : context->getViewSource()).buildQueryPipeline(); + } else builder = InterpreterSelectWithUnionQuery(input_query, context, {}).buildQueryPipeline(); inputs.emplace_back(QueryPipelineBuilder::getPipe(std::move(builder), resources)); diff --git a/src/Storages/StorageFactory.cpp b/src/Storages/StorageFactory.cpp index ad3d71376f151..921a6495154b7 100644 --- a/src/Storages/StorageFactory.cpp +++ b/src/Storages/StorageFactory.cpp @@ -1,9 +1,13 @@ #include +#include #include +#include #include #include #include +#include #include +#include #include #include #include @@ -53,6 +57,22 @@ ContextMutablePtr StorageFactory::Arguments::getLocalContext() const return ptr; } +DiskPtr StorageFactory::Arguments::getDisk(const String & name) const +{ + auto disk = getContext()->getDisk(name); + if (!isLoadingFromExistingMetadata(mode)) + DiskFromAST::ensureDiskIsAllowed(disk, getLocalContext()); + return disk; +} + +StoragePolicyPtr StorageFactory::Arguments::getStoragePolicy(const String & name) const +{ + auto storage_policy = getContext()->getStoragePolicy(name); + if (!isLoadingFromExistingMetadata(mode)) + DiskFromAST::ensureStoragePolicyIsAllowed(name, getLocalContext()); + return storage_policy; +} + void StorageFactory::registerStorage(const std::string & name, CreatorFn creator_fn, StorageFeatures features, Documentation documentation) { @@ -277,4 +297,37 @@ const StorageFactory::StorageFeatures & StorageFactory::getStorageFeatures(const return it->second.features; } +AccessRightsElements StorageFactory::getExportAccess(const ASTCreateQuery & create) const +{ + AccessRightsElements res; + auto add = [&](const ASTStorage & storage) + { + if (!storage.engine || !storage.settings) + return; + auto it = storages.find(storage.engine->name); + if (it != storages.end() && it->second.features.get_export_access_fn) + res.append_range(it->second.features.get_export_access_fn(storage.settings->changes)); + }; + + if (create.storage) + add(*create.storage); + if (create.targets) + { + for (const ASTStorage * inner_engine : create.targets->getInnerEngines()) + add(*inner_engine); + } + return res; +} + +AccessRightsElements StorageFactory::getExportAccessOfAnyEngine(const SettingsChanges & changes) const +{ + AccessRightsElements res; + for (const auto & [_, creator] : storages) + { + if (creator.features.get_export_access_fn) + res.append_range(creator.features.get_export_access_fn(changes)); + } + return res; +} + } diff --git a/src/Storages/StorageFactory.h b/src/Storages/StorageFactory.h index f5085d18b827a..bf1784b0a6988 100644 --- a/src/Storages/StorageFactory.h +++ b/src/Storages/StorageFactory.h @@ -17,6 +17,12 @@ namespace DB class Context; class ASTCreateQuery; class ASTStorage; +class IDisk; +using DiskPtr = std::shared_ptr; +class IStoragePolicy; +using StoragePolicyPtr = std::shared_ptr; +class AccessRightsElements; +class SettingsChanges; struct StorageID; struct ConstraintsDescription; @@ -34,6 +40,10 @@ class StorageFactory : private boost::noncopyable, public IHints<> /// Used to validate if table settings belong to the engine or the query before the start of the query interpretation using HasBuiltinSettingFn = bool(std::string_view); + /// The grants needed by settings that make a table send its data to another place, e.g. the move destination of a queue. + /// Returns nothing if `changes` set no such place. It reads only the changes, so the table need not exist on this host. + using GetExportAccessFn = AccessRightsElements(const SettingsChanges & changes); + struct Arguments { const String & engine_name; @@ -55,6 +65,11 @@ class StorageFactory : private boost::noncopyable, public IHints<> ContextMutablePtr getContext() const; ContextMutablePtr getLocalContext() const; + + /// A disk or a storage policy the table definition names. Refused, unless loading from existing metadata, when the + /// query may not use it (see `DiskFromAST::ensureDiskIsAllowed`). + DiskPtr getDisk(const String & name) const; + StoragePolicyPtr getStoragePolicy(const String & name) const; }; /// Analog of the IStorage::supports*() helpers @@ -82,6 +97,7 @@ class StorageFactory : private boost::noncopyable, public IHints<> std::optional source_access_type = std::nullopt; HasBuiltinSettingFn * has_builtin_setting_fn = nullptr; + GetExportAccessFn * get_export_access_fn = nullptr; }; using CreatorFn = std::function; @@ -120,6 +136,7 @@ class StorageFactory : private boost::noncopyable, public IHints<> .supports_sql_security = false, .source_access_type = std::nullopt, .has_builtin_setting_fn = nullptr, + .get_export_access_fn = nullptr, }, Documentation documentation = {}); const Storages & getAllStorages() const @@ -149,6 +166,13 @@ class StorageFactory : private boost::noncopyable, public IHints<> const StorageFeatures & getStorageFeatures(const String & storage_name) const; + /// See `GetExportAccessFn`. For the engine of the table and the inner engines of a materialized view or a `TimeSeries` + /// table. An unknown engine needs nothing here: creating it fails later. + AccessRightsElements getExportAccess(const ASTCreateQuery & create) const; + /// The grants of every engine for which `changes` send the data elsewhere. It does not depend on the engine of the table, + /// so `ALTER ... ON CLUSTER` can be authorized on an initiator that does not have the table or has another engine. + AccessRightsElements getExportAccessOfAnyEngine(const SettingsChanges & changes) const; + private: Storages storages; }; diff --git a/src/Storages/StorageFileCluster.cpp b/src/Storages/StorageFileCluster.cpp index d8b6a06f28413..8b41dc6c6fdd1 100644 --- a/src/Storages/StorageFileCluster.cpp +++ b/src/Storages/StorageFileCluster.cpp @@ -105,18 +105,20 @@ void StorageFileCluster::updateBeforeRead(const ContextPtr & context) checkWriteAccessIfFilesAreRenamed(context); } -void StorageFileCluster::updateQueryToSendIfNeeded(DB::ASTPtr & query, const StorageSnapshotPtr & storage_snapshot, const DB::ContextPtr & context) +void StorageFileCluster::updateQueryToSendIfNeeded( + DB::ASTPtr & query, const StorageSnapshotPtr & storage_snapshot, const DB::ContextPtr & context, bool add_structure_and_format) { auto * table_function = extractTableFunctionFromSelectQuery(query); if (!table_function) throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected SELECT query from table function fileCluster, got '{}'", query->formatForErrorMessage()); - TableFunctionFileCluster::updateStructureAndFormatArgumentsIfNeeded( - table_function, - storage_snapshot->metadata->getColumns().getAll().toNamesAndTypesDescription(), - format_name, - context - ); + if (add_structure_and_format) + TableFunctionFileCluster::updateStructureAndFormatArgumentsIfNeeded( + table_function, + storage_snapshot->metadata->getColumns().getAll().toNamesAndTypesDescription(), + format_name, + context + ); } RemoteQueryExecutor::Extension StorageFileCluster::getTaskIteratorExtension( diff --git a/src/Storages/StorageFileCluster.h b/src/Storages/StorageFileCluster.h index fab05dd2f6459..b656f7fbbabbe 100644 --- a/src/Storages/StorageFileCluster.h +++ b/src/Storages/StorageFileCluster.h @@ -36,7 +36,8 @@ class StorageFileCluster : public IStorageCluster StorageMetadataPtr) const override; private: - void updateQueryToSendIfNeeded(ASTPtr & query, const StorageSnapshotPtr & storage_snapshot, const ContextPtr & context) override; + void updateQueryToSendIfNeeded( + ASTPtr & query, const StorageSnapshotPtr & storage_snapshot, const ContextPtr & context, bool add_structure_and_format) override; void updateBeforeRead(const ContextPtr & context) override; Strings paths; diff --git a/src/Storages/StorageInMemoryMetadata.cpp b/src/Storages/StorageInMemoryMetadata.cpp index 79df13dafb692..3f52ddca92d9e 100644 --- a/src/Storages/StorageInMemoryMetadata.cpp +++ b/src/Storages/StorageInMemoryMetadata.cpp @@ -229,6 +229,43 @@ ContextMutablePtr StorageInMemoryMetadata::getSQLSecurityOverriddenContext(Conte new_context->setUser(getDefinerID(context)); + /// `Context::setUser` only sets the access-control identity (`user_id`); it does not touch the + /// user names in `ClientInfo`. That is harmless when there is an invoker whose names were copied + /// above, but a scheduled refresh of a refreshable materialized view builds its context from the + /// global context, where `current_user`, `initial_user` and `authenticated_user` are empty. + /// `RemoteQueryExecutor` sends the names as-is, and a shard that accepts the interserver + /// `` treats an empty `initial_user` as "interserver mode": it skips authentication + /// (see `TCPHandler::receiveQuery`) and executes the query with full access + /// (see `Context::getAccess`, `full_access = !user_id`). Stamp the definer into the empty fields + /// so the remote leg of the query is bounded by the same identity as the local one. Non-empty + /// fields are left alone so `currentUser` inside an ordinary `DEFINER` view still returns the + /// invoker. `AsynchronousInsertQueue` and `InterpreterExecuteAsQuery` do the same. + { + const auto & new_client_info = new_context->getClientInfo(); + const bool current_user_empty = new_client_info.current_user.empty(); + const bool initial_user_empty = new_client_info.initial_user.empty(); + const bool authenticated_user_empty = new_client_info.authenticated_user.empty(); + + if (current_user_empty || initial_user_empty || authenticated_user_empty) + { + const auto definer_name = new_context->getUserName(); + if (current_user_empty) + new_context->setCurrentUserName(definer_name); + if (initial_user_empty) + { + new_context->setInitialUserName(definer_name); + /// The shard authenticates the definer by `initial_user` and checks `initial_address` + /// against the allowed hosts of that user. Without an invoker, the address is the + /// unspecified `0.0.0.0`, which only matches users with `HOST ANY`. The query is + /// produced by this server itself, so present it as coming from the local host. + if (!new_client_info.initial_address || new_client_info.initial_address->host().isWildcard()) + new_context->setInitialAddress(Poco::Net::SocketAddress(Poco::Net::IPAddress("127.0.0.1"), 0)); + } + if (authenticated_user_empty) + new_context->setAuthenticatedUserName(definer_name); + } + } + new_context->clampToSettingsConstraints(changed_settings, SettingSource::QUERY); new_context->applySettingsChanges(changed_settings); new_context->setSetting("allow_ddl", 1); diff --git a/src/Storages/StorageJoin.cpp b/src/Storages/StorageJoin.cpp index cdc7a5c14c2f6..501c1b60d0a68 100644 --- a/src/Storages/StorageJoin.cpp +++ b/src/Storages/StorageJoin.cpp @@ -454,7 +454,7 @@ void registerStorageJoin(StorageFactory & factory) } } - DiskPtr disk = args.getContext()->getDisk(disk_name); + DiskPtr disk = args.getDisk(disk_name); if (engine_args.size() < 3) throw Exception(ErrorCodes::NUMBER_OF_ARGUMENTS_DOESNT_MATCH, diff --git a/src/Storages/StorageLog.cpp b/src/Storages/StorageLog.cpp index 9fc213141e70e..a46099190cd6f 100644 --- a/src/Storages/StorageLog.cpp +++ b/src/Storages/StorageLog.cpp @@ -1415,7 +1415,7 @@ void registerStorageLog(StorageFactory & factory) args.engine_name, args.engine_args.size()); String disk_name = getDiskName(*args.storage_def, args.getContext()); - DiskPtr disk = args.getContext()->getDisk(disk_name); + DiskPtr disk = args.getDisk(disk_name); return std::make_shared( args.engine_name, diff --git a/src/Storages/StorageMaterializedView.cpp b/src/Storages/StorageMaterializedView.cpp index cb772f006fc75..5f4a0b9ffb35c 100644 --- a/src/Storages/StorageMaterializedView.cpp +++ b/src/Storages/StorageMaterializedView.cpp @@ -639,11 +639,19 @@ ContextMutablePtr StorageMaterializedView::createRefreshContext(const String & l { ContextPtr table_context = getContext(); ClientInfo client_info = table_context->getClientInfo(); - client_info.interface = ClientInfo::Interface::BACKGROUND; - client_info.client_name = "refreshable materialized view"; auto view_metadata = getInMemoryMetadataPtr(getContext(), false); auto refresh_context = view_metadata->getSQLSecurityOverriddenContext(table_context, &client_info); - refresh_context->setClientInfo(client_info); + /// For `SQL SECURITY DEFINER`, `getSQLSecurityOverriddenContext` fills the empty user names in + /// the context's `ClientInfo` with the definer. Do not assign the local `client_info` copy to + /// the context: `setClientInfo` replaces the whole struct and would wipe that identity out. + /// Re-read it instead, so that a view without SQL security (which gets a plain copy) also gets + /// these fields. + { + ClientInfo refresh_client_info = refresh_context->getClientInfo(); + refresh_client_info.interface = ClientInfo::Interface::BACKGROUND; + refresh_client_info.client_name = "refreshable materialized view"; + refresh_context->setClientInfo(refresh_client_info); + } refresh_context->setSetting("database_replicated_allow_replicated_engine_arguments", 3); refresh_context->setSetting("log_comment", log_comment); refresh_context->setQueryKind(ClientInfo::QueryKind::INITIAL_QUERY); diff --git a/src/Storages/StorageMaterializedView.h b/src/Storages/StorageMaterializedView.h index d6c72eae7c332..c4fda9236afec 100644 --- a/src/Storages/StorageMaterializedView.h +++ b/src/Storages/StorageMaterializedView.h @@ -46,6 +46,7 @@ class StorageMaterializedView final : public StorageWithCommonVirtualColumns, Wi /// initiator must not rewrite functions to subcolumns when the target opts out (e.g. Distributed). bool supportsOptimizationToSubcolumns() const override { return getTargetTable()->supportsOptimizationToSubcolumns(); } bool supportsOptimizationToTupleElementSubcolumns() const override { return getTargetTable()->supportsOptimizationToTupleElementSubcolumns(); } + bool hasBucketedMapSerialization() const override { return getTargetTable()->hasBucketedMapSerialization(); } bool supportsColumnsWithDynamicStructure() const override; bool supportsTransactions() const override { return getTargetTable()->supportsTransactions(); } diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index b46619e7d2450..69c5fce2f15ba 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -382,6 +382,12 @@ bool StorageMerge::supportsOptimizationToTupleElementSubcolumns() const return traverseTablesUntil([](const auto & table) { return !table->supportsOptimizationToTupleElementSubcolumns(); }) == nullptr; } +bool StorageMerge::hasBucketedMapSerialization() const +{ + /// The rewritten query is sent to every underlying table, so all of them must benefit from it. + return traverseTablesUntil([](const auto & table) { return !table->hasBucketedMapSerialization(); }) == nullptr; +} + bool StorageMerge::canMoveConditionsToPrewhere() const { /// NOTE: This check and the above check are used during query analysis as condition for applying @@ -1641,9 +1647,18 @@ ReadFromMerge::ChildPlan ReadFromMerge::createPlanForTable( /// Converting query to AST because types might be different in the source table. /// Need to resolve types again. auto ast = modified_query_info.query_tree->toAST(); + /// The analyzer substitutes the inserted block of a materialized view for its source table only when given it. + /// A child that is the source table itself reads the table, as it does at `FetchColumns`. + auto view_source = modified_context->isViewInnerQuery() ? nullptr : modified_context->getViewSource(); + if (view_source && view_source->getStorageID().getFullNameNotQuoted() == storage->getStorageID().getFullNameNotQuoted()) + { + modified_context->clearViewSource(); + view_source = nullptr; + } InterpreterSelectQueryAnalyzer interpreter(ast, modified_context, - child_select_query_options); + child_select_query_options, + view_source); auto & planner = interpreter.getPlanner(); planner.buildQueryPlanIfNeeded(); diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index 1ad5d636bd395..e5fdaecaf70e9 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -56,6 +56,7 @@ class StorageMerge final : public IStorage, WithContext /// initiator rewrite functions to subcolumns, or a skip index on the shard would be missed. bool supportsOptimizationToSubcolumns() const override; bool supportsOptimizationToTupleElementSubcolumns() const override; + bool hasBucketedMapSerialization() const override; bool supportsColumnsWithDynamicStructure() const override { return true; } bool supportsPrewhere() const override; std::optional supportedPrewhereColumns() const override; diff --git a/src/Storages/StorageMergeTree.cpp b/src/Storages/StorageMergeTree.cpp index 4b05914c42094..39ed4ac29ddb1 100644 --- a/src/Storages/StorageMergeTree.cpp +++ b/src/Storages/StorageMergeTree.cpp @@ -1249,6 +1249,11 @@ QueryPipeline StorageMergeTree::updateLightweight(const MutationCommands & comma /// Updates currently don't work with parallel replicas. context_copy->setSetting("max_parallel_replicas", Field(1)); + /// A snapshot cached by `enable_shared_storage_snapshot_in_query` before the wait for committing + /// inserts above (e.g. by the validation of `IN` operands in `InterpreterUpdateQuery`) would hide + /// the parts committed during the wait from the update. See `StorageReplicatedMergeTree::updateLightweight`. + discardSharedStorageSnapshot(context_copy); + auto [pipeline, patch_metadata] = updateLightweightImpl(commands, context_copy); auto sink = std::make_shared( diff --git a/src/Storages/StorageMongoDB.cpp b/src/Storages/StorageMongoDB.cpp index b1a68b4c56ea1..e902d3ca65d68 100644 --- a/src/Storages/StorageMongoDB.cpp +++ b/src/Storages/StorageMongoDB.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -38,6 +39,7 @@ #include #include +#include #include using bsoncxx::builder::basic::document; @@ -57,6 +59,7 @@ MongoDBInstanceHolder & MongoDBInstanceHolder::instance() namespace ErrorCodes { + extern const int ACCESS_DENIED; extern const int BAD_ARGUMENTS; extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int NOT_IMPLEMENTED; @@ -86,6 +89,58 @@ void MongoDBConfiguration::checkCollection() const throw Exception(ErrorCodes::BAD_ARGUMENTS, "MongoDB collection name must be non-empty and must not contain NUL characters"); } +namespace +{ + +void throwIfServerCredentials(const mongocxx::uri & uri, std::string_view remedy) +{ + /// Only these take the credentials from the connection string; `MONGODB-OIDC`, `MONGODB-AWS`, `GSSAPI` and + /// `MONGODB-X509` use the server's. Without a mechanism the driver uses SCRAM or no authentication. Names are case-insensitive. + static constexpr std::string_view allowed_mechanisms[] = {"SCRAM-SHA-1", "SCRAM-SHA-256", "PLAIN"}; + + const String mechanism = uri.auth_mechanism(); + const bool is_allowed_mechanism = mechanism.empty() + || std::ranges::any_of(allowed_mechanisms, [&](std::string_view allowed) { return equalsCaseInsensitive(mechanism, allowed); }); + + if (!is_allowed_mechanism) + throw Exception( + ErrorCodes::ACCESS_DENIED, + "MongoDB authentication mechanism '{}' is not allowed, because it authenticates with " + "credentials of the server process. Use SCRAM-SHA-1, SCRAM-SHA-256 or PLAIN with a username and a password " + "in the connection string{}.", + mechanism, remedy); + + /// The driver stores the deprecated `sslClientCertificateKeyFile` under the canonical name, so this covers both. + if (uri.tls_certificate_key_file()) + throw Exception( + ErrorCodes::ACCESS_DENIED, + "MongoDB connection option `tlsCertificateKeyFile` is not allowed, because it makes the server present a " + "client certificate from its own filesystem. Use SCRAM-SHA-1, SCRAM-SHA-256 or PLAIN with a username and a " + "password in the connection string{}.", + remedy); +} + +} + +void MongoDBConfiguration::checkServerCredentialsForTable(const ContextPtr & context) const +{ + if (!context->shouldRestrictUserQueryS3Credentials()) + return; + + throwIfServerCredentials(*uri, +#if CLICKHOUSE_CLOUD + "" +#else + ", or enable the setting `s3_allow_server_credentials_in_user_queries`" +#endif + ); +} + +void MongoDBConfiguration::checkServerCredentialsForDictionary() const +{ + throwIfServerCredentials(*uri, ", or define the dictionary in the server configuration files"); +} + StorageMongoDB::StorageMongoDB( const StorageID & table_id_, MongoDBConfiguration configuration_, @@ -121,6 +176,9 @@ Pipe StorageMongoDB::read( size_t max_block_size, size_t /*num_streams*/) { + /// Checked per read, with the query's settings: a table created with `mongodb(...)` is built with the global context. + configuration.checkServerCredentialsForTable(context); + storage_snapshot->check(column_names); Block sample_block; diff --git a/src/Storages/StorageMongoDB.h b/src/Storages/StorageMongoDB.h index 65c6e5a508d7d..de1aa96ab48f5 100644 --- a/src/Storages/StorageMongoDB.h +++ b/src/Storages/StorageMongoDB.h @@ -71,6 +71,9 @@ struct MongoDBConfiguration void checkHosts(const ContextPtr & context) const; void checkCollection() const; + void checkServerCredentialsForTable(const ContextPtr & context) const; + void checkServerCredentialsForDictionary() const; + bool isOidColumn(const std::string & name) const { return oid_fields.contains(name); diff --git a/src/Storages/StorageMySQL.cpp b/src/Storages/StorageMySQL.cpp index dfe38fcda5f94..46f9e5c462d14 100644 --- a/src/Storages/StorageMySQL.cpp +++ b/src/Storages/StorageMySQL.cpp @@ -682,7 +682,7 @@ The table structure can differ from the original MySQL table structure: **Engine Parameters** -- `host:port` — MySQL server address. +- `host:port` — MySQL server address. The host must be a plain host name or IP address and is reached over TCP at the given port, also for `localhost`: Unix sockets are not supported. Other host syntax of the MySQL client library, such as connection strings or comma-separated host lists, is rejected. - `database` — Remote database name. - `table` — Remote table name, or a query passed to MySQL as is (see [Passing a query instead of a table name](#passing-a-query)). - `user` — MySQL user. diff --git a/src/Storages/StorageProxy.h b/src/Storages/StorageProxy.h index 2dc249080a2cf..b91f6d387a3d8 100644 --- a/src/Storages/StorageProxy.h +++ b/src/Storages/StorageProxy.h @@ -45,6 +45,7 @@ class StorageProxy : public IStorage /// storage that opts out of the rewrite (e.g. Distributed) does not re-advertise true. bool supportsOptimizationToSubcolumns() const override { return getNested()->supportsOptimizationToSubcolumns(); } bool supportsOptimizationToTupleElementSubcolumns() const override { return getNested()->supportsOptimizationToTupleElementSubcolumns(); } + bool hasBucketedMapSerialization() const override { return getNested()->hasBucketedMapSerialization(); } bool supportsColumnsWithDynamicStructure() const override { return getNested()->supportsColumnsWithDynamicStructure(); } /// `ReadFromMerge::getSelectedTables` prunes children by name based on this flag; a lazy /// `StorageTableProxy` around a delegating storage (`Distributed`, `Merge`, `Buffer`, `Alias`) diff --git a/src/Storages/StorageReplicatedMergeTree.cpp b/src/Storages/StorageReplicatedMergeTree.cpp index 561a9eb4d24e6..fcdb29f957217 100644 --- a/src/Storages/StorageReplicatedMergeTree.cpp +++ b/src/Storages/StorageReplicatedMergeTree.cpp @@ -269,6 +269,7 @@ namespace FailPoints extern const char rmt_delay_execute_drop_range[]; extern const char replicated_table_remove_zk_before_get_children[]; extern const char replicated_table_remove_zk_before_final_multi[]; + extern const char rmt_restore_replica_pause_before_metadata_version_raise[]; extern const char check_table_inject_retryable_zk_error[]; } @@ -7357,6 +7358,7 @@ void StorageReplicatedMergeTree::restoreMetadataInZooKeeper( createReplica(metadata_snapshot, zookeeper_retries_info); createNewZooKeeperNodes(zookeeper_retries_info); + FailPointInjection::pauseFailPoint(FailPoints::rmt_restore_replica_pause_before_metadata_version_raise); /// Bump the /metadata ZNode version to match the max parts metadata version. /// The table's in-memory metadata_version is derived from the /metadata ZNode stat.version @@ -7364,12 +7366,41 @@ void StorageReplicatedMergeTree::restoreMetadataInZooKeeper( if (max_parts_metadata_version > 0) { auto zookeeper = getZooKeeper(); - String metadata_str = zookeeper->get(zookeeper_path + "/metadata"); - for (int32_t i = 0; i < max_parts_metadata_version; ++i) - zookeeper->set(zookeeper_path + "/metadata", metadata_str); - zookeeper->set(replica_path + "/metadata_version", std::to_string(max_parts_metadata_version)); + /// Each batch logs an ALTER_METADATA entry of the version it reaches, so a replica that cloned an earlier one catches up. + Coordination::Stat metadata_stat; + Coordination::Stat columns_stat; + String metadata_str = zookeeper->get(zookeeper_path + "/metadata", &metadata_stat); + String columns_str = zookeeper->get(zookeeper_path + "/columns", &columns_stat); + while (metadata_stat.version < max_parts_metadata_version) + { + const auto target = static_cast(std::min( + max_parts_metadata_version, Int64{metadata_stat.version} + static_cast(zkutil::MULTI_BATCH_SIZE))); + + Coordination::Requests ops; + ops.emplace_back(zkutil::makeCheckRequest(zookeeper_path + "/columns", columns_stat.version)); + for (int32_t version = metadata_stat.version; version < target; ++version) + ops.emplace_back(zkutil::makeSetRequest(zookeeper_path + "/metadata", version + 1 == target ? metadata_str : "", version)); + + ReplicatedMergeTreeLogEntryData entry; + entry.type = LogEntry::ALTER_METADATA; + entry.source_replica = replica_name; + entry.metadata_str = metadata_str; + entry.columns_str = columns_str; + entry.alter_version = target; + entry.create_time = time(nullptr); + entry.have_mutation = false; + ops.emplace_back(zkutil::makeCreateRequest(zookeeper_path + "/log/log-", entry.toString(), zkutil::CreateMode::PersistentSequential)); - LOG_INFO(log, "Bumped metadata version to {} to match parts", max_parts_metadata_version); + Coordination::Responses responses; + const Coordination::Error code = zookeeper->tryMulti(ops, responses); + if (code != Coordination::Error::ZOK && code != Coordination::Error::ZBADVERSION) + zkutil::KeeperMultiException::check(code, ops, responses); + + metadata_str = zookeeper->get(zookeeper_path + "/metadata", &metadata_stat); + columns_str = zookeeper->get(zookeeper_path + "/columns", &columns_stat); + } + + LOG_INFO(log, "Raised metadata version to {} to match parts", metadata_stat.version); } LOG_INFO(log, "Created ZK nodes for table"); @@ -8755,6 +8786,14 @@ QueryPipeline StorageReplicatedMergeTree::updateLightweight(const MutationComman waitForCommittingOpsToFinish(zookeeper, context_copy->getPartitionIdToMaxBlock(getStorageID().uuid), ops, backoff_ms, sync_timeout); } + /// With `enable_shared_storage_snapshot_in_query` the update reads with the snapshot of this table + /// cached for the whole query. Anything analyzed before this point may have cached one: the pruning + /// analysis in `allocateBlockNumbersInAffectedPartitions`, or the validation of `IN` operands in + /// `InterpreterUpdateQuery`. Such a snapshot predates the wait for committing inserts above, so the + /// parts that arrived during the wait would be invisible to the update and their rows would stay + /// unchanged. Discard it, so that the update and its subqueries share a snapshot taken now. + discardSharedStorageSnapshot(context_copy); + auto [pipeline, patch_metadata] = updateLightweightImpl(commands, context_copy); auto sink = std::make_shared( diff --git a/src/Storages/StorageSet.cpp b/src/Storages/StorageSet.cpp index 525941a5a0710..51626384a86f0 100644 --- a/src/Storages/StorageSet.cpp +++ b/src/Storages/StorageSet.cpp @@ -360,7 +360,7 @@ void registerStorageSet(StorageFactory & factory) if (has_settings) set_settings.loadFromQuery(*args.storage_def); - DiskPtr disk = args.getContext()->getDisk(set_settings[SetSetting::disk]); + DiskPtr disk = args.getDisk(set_settings[SetSetting::disk]); return std::make_shared( disk, args.relative_data_path, args.table_id, args.columns, args.constraints, args.comment, set_settings[SetSetting::persistent]); }, StorageFactory::StorageFeatures{ .supports_settings = true, .has_builtin_setting_fn = SetSettings::hasBuiltin, }, diff --git a/src/Storages/StorageStripeLog.cpp b/src/Storages/StorageStripeLog.cpp index 81211ad319eed..61c049df78f88 100644 --- a/src/Storages/StorageStripeLog.cpp +++ b/src/Storages/StorageStripeLog.cpp @@ -765,7 +765,7 @@ void registerStorageStripeLog(StorageFactory & factory) args.engine_name, args.engine_args.size()); String disk_name = getDiskName(*args.storage_def, args.getContext()); - DiskPtr disk = args.getContext()->getDisk(disk_name); + DiskPtr disk = args.getDisk(disk_name); return std::make_shared( disk, diff --git a/src/Storages/StorageURLCluster.cpp b/src/Storages/StorageURLCluster.cpp index e6378b222182a..3d89d3d8f7cfc 100644 --- a/src/Storages/StorageURLCluster.cpp +++ b/src/Storages/StorageURLCluster.cpp @@ -109,7 +109,8 @@ StorageURLCluster::StorageURLCluster( setInMemoryMetadata(storage_metadata); } -void StorageURLCluster::updateQueryToSendIfNeeded(ASTPtr & query, const StorageSnapshotPtr & storage_snapshot, const ContextPtr & context) +void StorageURLCluster::updateQueryToSendIfNeeded( + ASTPtr & query, const StorageSnapshotPtr & storage_snapshot, const ContextPtr & context, bool add_structure_and_format) { auto * table_function = extractTableFunctionFromSelectQuery(query); if (!table_function) @@ -119,12 +120,13 @@ void StorageURLCluster::updateQueryToSendIfNeeded(ASTPtr & query, const StorageS if (!expression_list) throw Exception(ErrorCodes::LOGICAL_ERROR, "Expected SELECT query from table function urlCluster, got '{}'", query->formatForErrorMessage()); - TableFunctionURLCluster::updateStructureAndFormatArgumentsIfNeeded( - table_function, - storage_snapshot->metadata->getColumns().getAll().toNamesAndTypesDescription(), - format_name, - context - ); + if (add_structure_and_format) + TableFunctionURLCluster::updateStructureAndFormatArgumentsIfNeeded( + table_function, + storage_snapshot->metadata->getColumns().getAll().toNamesAndTypesDescription(), + format_name, + context + ); /// When a non-cluster table function (`url`) was auto-converted to cluster mode /// by the `parallel_replicas_for_cluster_engines` setting, rename it to the Cluster variant diff --git a/src/Storages/StorageURLCluster.h b/src/Storages/StorageURLCluster.h index bee90c4ba7b0d..9c267b5e1bc6a 100644 --- a/src/Storages/StorageURLCluster.h +++ b/src/Storages/StorageURLCluster.h @@ -38,7 +38,8 @@ class StorageURLCluster : public IStorageCluster StorageMetadataPtr) const override; private: - void updateQueryToSendIfNeeded(ASTPtr & query, const StorageSnapshotPtr & storage_snapshot, const ContextPtr & context) override; + void updateQueryToSendIfNeeded( + ASTPtr & query, const StorageSnapshotPtr & storage_snapshot, const ContextPtr & context, bool add_structure_and_format) override; String uri; String format_name; diff --git a/src/Storages/StorageView.cpp b/src/Storages/StorageView.cpp index 1d27a13b7d6e3..1f0fb18f96c1f 100644 --- a/src/Storages/StorageView.cpp +++ b/src/Storages/StorageView.cpp @@ -250,7 +250,7 @@ StoragePtr tryGetTrivialViewUnderlyingStorage(const ASTPtr & inner_query, Contex || select->limitLength() || select->limitOffset() || select->limitBy() || select->limit_by_all || select->limitByLength() || select->limitByOffset() - || select->distinct || select->arrayJoinExpressionList().first + || select->distinct || select->settings()) { return nullptr; @@ -284,6 +284,7 @@ StoragePtr tryGetTrivialViewUnderlyingStorage(const ASTPtr & inner_query, Contex } const auto * tables = select->tables().get(); + /// Every JOIN and ARRAY JOIN clause is a separate element of this list. if (!tables || tables->children.size() != 1) { return nullptr; diff --git a/src/Storages/System/StorageSystemDashboards.cpp b/src/Storages/System/StorageSystemDashboards.cpp index c6ae173c713bf..db530c6c31c76 100644 --- a/src/Storages/System/StorageSystemDashboards.cpp +++ b/src/Storages/System/StorageSystemDashboards.cpp @@ -1262,6 +1262,9 @@ GROUP BY t ORDER BY t WITH FILL STEP {rounding:UInt32} SETTINGS skip_unavailable_shards = 1 )EOQ") } }, + /// With `asynchronous_metrics_key_values_mode = 'both'` every network interface is published twice: + /// as `NetworkReceiveBytes` with a key and as the legacy `NetworkReceiveBytes_eth0`, and summing the + /// LIKE pattern counts every byte twice. Prefer the key-value form, fall back to the legacy names. { { "dashboard", "Cloud overview" }, { "title", "Network receive bytes/sec" }, @@ -1270,7 +1273,7 @@ WITH toDateTimeOrDefault({from:String}, '', now() - {seconds:UInt32}) AS from, toDateTimeOrDefault({to:String}, '', now()) AS to SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(value) FROM ( - SELECT event_time, sum(value) AS value + SELECT event_time, if(countIf(metric = 'NetworkReceiveBytes') != 0, sumIf(value, metric = 'NetworkReceiveBytes'), sum(value)) AS value FROM clusterAllReplicas(default, merge('system', '^asynchronous_metric_log')) WHERE event_date BETWEEN toDate(from) AND toDate(to) AND event_time BETWEEN from AND to AND metric LIKE 'NetworkReceiveBytes%' @@ -1288,7 +1291,7 @@ WITH toDateTimeOrDefault({from:String}, '', now() - {seconds:UInt32}) AS from, toDateTimeOrDefault({to:String}, '', now()) AS to SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, avg(value) FROM ( - SELECT event_time, sum(value) AS value + SELECT event_time, if(countIf(metric = 'NetworkSendBytes') != 0, sumIf(value, metric = 'NetworkSendBytes'), sum(value)) AS value FROM clusterAllReplicas(default, merge('system', '^asynchronous_metric_log')) WHERE event_date BETWEEN toDate(from) AND toDate(to) AND event_time BETWEEN from AND to AND metric LIKE 'NetworkSendBytes%' @@ -2054,7 +2057,7 @@ WITH toDateTimeOrDefault({from:String}, '', now() - {seconds:UInt32}) AS from, toDateTimeOrDefault({to:String}, '', now()) AS to SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, hostname, avg(value) FROM ( - SELECT event_time, hostname, sum(value) AS value + SELECT event_time, hostname, if(countIf(metric = 'NetworkReceiveBytes') != 0, sumIf(value, metric = 'NetworkReceiveBytes'), sum(value)) AS value FROM clusterAllReplicas(default, merge('system', '^asynchronous_metric_log')) WHERE event_date BETWEEN toDate(from) AND toDate(to) AND event_time BETWEEN from AND to AND metric LIKE 'NetworkReceiveBytes%' @@ -2072,7 +2075,7 @@ WITH toDateTimeOrDefault({from:String}, '', now() - {seconds:UInt32}) AS from, toDateTimeOrDefault({to:String}, '', now()) AS to SELECT toStartOfInterval(event_time, INTERVAL {rounding:UInt32} SECOND)::INT AS t, hostname, avg(value) FROM ( - SELECT event_time, hostname, sum(value) AS value + SELECT event_time, hostname, if(countIf(metric = 'NetworkSendBytes') != 0, sumIf(value, metric = 'NetworkSendBytes'), sum(value)) AS value FROM clusterAllReplicas(default, merge('system', '^asynchronous_metric_log')) WHERE event_date BETWEEN toDate(from) AND toDate(to) AND event_time BETWEEN from AND to AND metric LIKE 'NetworkSendBytes%' diff --git a/src/Storages/System/StorageSystemMutations.cpp b/src/Storages/System/StorageSystemMutations.cpp index 095673f0f43c0..8e31750dce96d 100644 --- a/src/Storages/System/StorageSystemMutations.cpp +++ b/src/Storages/System/StorageSystemMutations.cpp @@ -11,14 +11,67 @@ #include #include #include +#include #include #include #include +#include +#include +#include namespace DB { +namespace Setting +{ + extern const SettingsUInt64 max_parser_depth; + extern const SettingsUInt64 max_parser_backtracks; +} + +namespace ErrorCodes +{ + extern const int SYNTAX_ERROR; + extern const int TOO_DEEP_RECURSION; + extern const int TOO_SLOW_PARSING; +} + +/// The mutation entry keeps the real text of the command, because the mutation is executed from it +/// (on every replica, for replicated tables). Hide the secrets on the way out instead, the way +/// `system.distributed_ddl_queue` does. The command is formatted the same way as the stored text, +/// so only the secrets differ from it. +static String hideSecretsInCommand(const ContextPtr & context, const String & command) +{ + /// Nothing to parse, see `MutationCommand::ast`. + if (command.empty()) + return command; + + const auto & settings = context->getSettingsRef(); + ParserAlterCommandList parser; + ASTPtr ast; + try + { + ast = parseQuery(parser, command, 0, settings[Setting::max_parser_depth], settings[Setting::max_parser_backtracks]); + } + catch (const Exception & e) + { + /// The text may be unparseable on another server version, so hide the whole command instead of failing the table. + if (e.code() != ErrorCodes::SYNTAX_ERROR + && e.code() != ErrorCodes::TOO_DEEP_RECURSION + && e.code() != ErrorCodes::TOO_SLOW_PARSING) + throw; + return "[HIDDEN]"; + } + + return ast->formatWithPossiblyHidingSensitiveData( + /*max_length=*/0, + /*one_line=*/true, + /*show_secrets=*/false, + /*print_pretty_type_names=*/false, + /*identifier_quoting_rule=*/IdentifierQuotingRule::WhenNecessary, + /*identifier_quoting_style=*/IdentifierQuotingStyle::Backticks); +} + ColumnsDescription StorageSystemMutations::getColumnsDescription() { @@ -72,10 +125,11 @@ Block StorageSystemMutations::getFilterSampleBlock() const }; } -void StorageSystemMutations::fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node * predicate, std::vector) const +void StorageSystemMutations::fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node * predicate, std::vector columns_mask) const { const auto access = context->getAccess(); const bool check_access_for_databases = !access->isGranted(AccessType::SHOW_TABLES); + const bool show_secrets = canDisplaySecrets(context); /// Collect a set of *MergeTree tables. std::map> merge_tree_tables; @@ -176,26 +230,45 @@ void StorageSystemMutations::fillData(MutableColumns & res_columns, ContextPtr c parts_postpone_reasons_map.emplace_back(std::move(key_value)); } - size_t col_num = 0; - res_columns[col_num++]->insert(database); - res_columns[col_num++]->insert(table); - - res_columns[col_num++]->insert(status.id); - res_columns[col_num++]->insert(status.command); - res_columns[col_num++]->insert(UInt64(status.create_time)); - res_columns[col_num++]->insert(UInt64(status.finish_time)); - res_columns[col_num++]->insert(block_partition_ids); - res_columns[col_num++]->insert(block_numbers); - res_columns[col_num++]->insert(parts_in_progress_names); - res_columns[col_num++]->insert(parts_to_do_names); - res_columns[col_num++]->insert(parts_to_do_names.size()); - res_columns[col_num++]->insert(parts_postpone_reasons_map); - res_columns[col_num++]->insert(status.is_done); - res_columns[col_num++]->insert(status.is_killed); - res_columns[col_num++]->insert(status.latest_failed_part); - res_columns[col_num++]->insert(UInt64(status.latest_fail_time)); - res_columns[col_num++]->insert(status.latest_fail_reason); - res_columns[col_num++]->insert(status.latest_fail_error_code_name); + size_t src_index = 0; + size_t res_index = 0; + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(database); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(table); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(status.id); + /// Parsing the command costs much more than everything else here, so it is done only when the column is queried. + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(show_secrets ? status.command : hideSecretsInCommand(context, status.command)); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(UInt64(status.create_time)); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(UInt64(status.finish_time)); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(block_partition_ids); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(block_numbers); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(parts_in_progress_names); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(parts_to_do_names); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(parts_to_do_names.size()); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(parts_postpone_reasons_map); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(status.is_done); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(status.is_killed); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(status.latest_failed_part); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(UInt64(status.latest_fail_time)); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(status.latest_fail_reason); + if (columns_mask[src_index++]) + res_columns[res_index++]->insert(status.latest_fail_error_code_name); } } } diff --git a/src/Storages/System/StorageSystemMutations.h b/src/Storages/System/StorageSystemMutations.h index 5341838a65e3c..c3f897491fc76 100644 --- a/src/Storages/System/StorageSystemMutations.h +++ b/src/Storages/System/StorageSystemMutations.h @@ -21,8 +21,9 @@ class StorageSystemMutations final : public IStorageSystemOneBlock protected: using IStorageSystemOneBlock::IStorageSystemOneBlock; - void fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node * predicate, std::vector) const override; + void fillData(MutableColumns & res_columns, ContextPtr context, const ActionsDAG::Node * predicate, std::vector columns_mask) const override; Block getFilterSampleBlock() const override; + bool supportsColumnsMask() const override { return true; } }; } diff --git a/src/Storages/System/StorageSystemObjectStorageQueueSettings.cpp b/src/Storages/System/StorageSystemObjectStorageQueueSettings.cpp index ec3f099ba390c..d0208fe4c1300 100644 --- a/src/Storages/System/StorageSystemObjectStorageQueueSettings.cpp +++ b/src/Storages/System/StorageSystemObjectStorageQueueSettings.cpp @@ -7,6 +7,7 @@ #include #include #include +#include #include #include #include @@ -45,6 +46,7 @@ void StorageSystemObjectStorageQueueSettings::fillData( const ActionsDAG::Node *, std::vector) const { + const bool display_secrets = canDisplaySecrets(context); auto add_table = [&]( const DatabaseTablesIteratorPtr & it, StorageObjectStorageQueue & storage) { @@ -54,7 +56,7 @@ void StorageSystemObjectStorageQueueSettings::fillData( auto constraints_and_current_profiles = context->getSettingsConstraintsAndCurrentProfiles(); const auto & constraints = constraints_and_current_profiles->constraints; MutableColumnsAndConstraints params(res_columns, constraints); - storage.getSettings().dumpToSystemEngineSettingsColumns(params, it->name(), it->databaseName(), storage); + storage.getSettings().dumpToSystemEngineSettingsColumns(params, it->name(), it->databaseName(), storage, display_secrets); }; const auto access = context->getAccess(); diff --git a/src/Storages/tests/gtest_azure_configuration.cpp b/src/Storages/tests/gtest_azure_configuration.cpp index 89ed7547bf72c..d793dc8ec9fc2 100644 --- a/src/Storages/tests/gtest_azure_configuration.cpp +++ b/src/Storages/tests/gtest_azure_configuration.cpp @@ -296,6 +296,40 @@ TEST(StorageAzureConfiguration, AddStructureAndFormatTooManyArgs) "Expected 3 to"); } +/// Distributed query preparation rewrites the args; extra_credentials must be skipped and kept. +TEST(StorageAzureConfiguration, AddStructureAndFormatKeepsExtraCredentials) +{ + std::string query = "DESCRIBE TABLE azureBlobStorage('https://azurite1:10000/devstoreaccount1', 'test_container', 'test_blob.csv', extra_credentials(client_id='test_client_id', tenant_id='test_tenant_id'))"; + + ASTs engine_args = getEngineArgs(query); + StorageAzureConfigurationFriend conf; + conf.addStructureAndFormatToArgsIfNeeded(engine_args, "a Int32", "Parquet", Context::getGlobalContextInstance(), /*with_structure=*/true); + + /// (url, container, blobpath, format, compression, structure, extra_credentials) + ASSERT_EQ(engine_args.size(), 7u); + EXPECT_EQ(engine_args[3]->as()->value.safeGet(), "Parquet"); + EXPECT_EQ(engine_args[5]->as()->value.safeGet(), "a Int32"); + const auto * credentials = engine_args.back()->as(); + ASSERT_TRUE(credentials); + EXPECT_EQ(credentials->name, "extra_credentials"); +} + +TEST(StorageAzureConfiguration, AddStructureAndFormatKeepsKeyValueArgumentsAndExtraCredentials) +{ + std::string query = "DESCRIBE TABLE azureBlobStorage('https://azurite1:10000/devstoreaccount1', 'test_container', 'test_blob.csv', extra_credentials(client_id='test_client_id', tenant_id='test_tenant_id'), partition_strategy = 'none')"; + + ASTs engine_args = getEngineArgs(query); + StorageAzureConfigurationFriend conf; + conf.addStructureAndFormatToArgsIfNeeded(engine_args, "a Int32", "Parquet", Context::getGlobalContextInstance(), /*with_structure=*/true); + + /// (url, container, blobpath, format, compression, structure, partition_strategy = 'none', extra_credentials) + ASSERT_EQ(engine_args.size(), 8u); + EXPECT_EQ(engine_args[5]->as()->value.safeGet(), "a Int32"); + EXPECT_EQ(engine_args[6]->formatWithSecretsOneLine(), "partition_strategy = 'none'"); + const auto * credentials = engine_args[7]->as(); + ASSERT_TRUE(credentials); + EXPECT_EQ(credentials->name, "extra_credentials"); +} } #endif diff --git a/src/TableFunctions/ITableFunction.cpp b/src/TableFunctions/ITableFunction.cpp index f55ab64d7ce64..a1009901d26a0 100644 --- a/src/TableFunctions/ITableFunction.cpp +++ b/src/TableFunctions/ITableFunction.cpp @@ -45,6 +45,13 @@ void ITableFunction::checkSourceAccess(ContextPtr context, bool is_insert_query) } } +void ITableFunction::checkCreateTemporaryTableAccess(ContextPtr context, bool is_insert_query) const +{ + auto table_function_properties = TableFunctionFactory::instance().tryGetProperties(getName()); + if (is_insert_query || !(table_function_properties && table_function_properties->allow_readonly)) + context->checkAccess(AccessType::CREATE_TEMPORARY_TABLE); +} + void ITableFunction::checkEngineAccess(ContextPtr context) const { if (!requiresTableEngineGrant()) @@ -60,6 +67,10 @@ ColumnsDescription ITableFunction::getActualTableStructureWithAccess(ContextPtr /// Resolving table structure is always a read operation. checkSourceAccess(context, /*is_insert_query*/ false); checkEngineAccess(context); + /// Resolving the structure may access the source (e.g. schema inference fetches a URL), so it must be + /// gated in the same way as `execute`: otherwise `DESCRIBE TABLE url(...)` would let a user who cannot + /// run the table function (e.g. under `readonly = 1`) make the server request an endpoint. + checkCreateTemporaryTableAccess(context, /*is_insert_query*/ false); return getActualTableStructure(context, is_insert_query); } @@ -74,9 +85,8 @@ StoragePtr ITableFunction::execute(const ASTPtr & ast_function, ContextPtr conte /// Deliberately not under `check_source_access`: no caller takes over the engine check. checkEngineAccess(context); - auto table_function_properties = TableFunctionFactory::instance().tryGetProperties(getName()); - if (check_create_temporary_table && (is_insert_query || !(table_function_properties && table_function_properties->allow_readonly))) - context->checkAccess(AccessType::CREATE_TEMPORARY_TABLE); + if (check_create_temporary_table) + checkCreateTemporaryTableAccess(context, is_insert_query); auto context_to_use = use_global_context ? context->getGlobalContext() : context; diff --git a/src/TableFunctions/ITableFunction.h b/src/TableFunctions/ITableFunction.h index f34337773e668..5cbfb803c94f9 100644 --- a/src/TableFunctions/ITableFunction.h +++ b/src/TableFunctions/ITableFunction.h @@ -107,13 +107,18 @@ class ITableFunction : public std::enable_shared_from_this StoragePtr execute(const ASTPtr & ast_function, ContextPtr context, const std::string & table_name, ColumnsDescription cached_columns_ = {}, bool use_global_context = false, bool is_insert_query = false, bool check_create_temporary_table = true, bool check_source_access = true) const; - /// Returns actual table structure after enforcing source access checks. - /// Use this instead of getActualTableStructure() from outside execute(). + /// Returns actual table structure after enforcing the same access checks as `execute`: source, engine, + /// and `CREATE TEMPORARY TABLE` (for table functions not allowed in readonly mode). + /// Use this instead of `getActualTableStructure` from outside `execute`. ColumnsDescription getActualTableStructureWithAccess(ContextPtr context, bool is_insert_query) const; /// Check that the user has the required source access (e.g. READ ON MYSQL, WRITE ON S3). void checkSourceAccess(ContextPtr context, bool is_insert_query) const; + /// Check the `CREATE TEMPORARY TABLE` grant, required to use a table function unless it is allowed + /// in readonly mode and the query is not an insert. + void checkCreateTemporaryTableAccess(ContextPtr context, bool is_insert_query) const; + /// The URI of the function for permission checking. Can be an empty string if not applicable. /// For example, for url('https://foo.bar') the URI would be 'https://foo.bar'. virtual const String & getFunctionURI() const diff --git a/src/TableFunctions/TableFunctionMySQL.cpp b/src/TableFunctions/TableFunctionMySQL.cpp index d454a9b0c4f11..681e541f01edf 100644 --- a/src/TableFunctions/TableFunctionMySQL.cpp +++ b/src/TableFunctions/TableFunctionMySQL.cpp @@ -187,7 +187,7 @@ mysql({host:port, database, table, user, password[, replace_query, on_duplicate_ | Argument | Description | |---------------------|-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| -| `host:port` | MySQL server address. | +| `host:port` | MySQL server address. The host must be a plain host name or IP address and is reached over TCP at the given port, also for `localhost`: Unix sockets are not supported. Other host syntax of the MySQL client library, such as connection strings or comma-separated host lists, is rejected. | | `database` | Remote database name. | | `table` | Remote table name, or a query passed to MySQL as is (see [Passing a query instead of a table name](#passing-a-query)). | | `user` | MySQL user. | diff --git a/src/TableFunctions/TableFunctionViewIfPermitted.cpp b/src/TableFunctions/TableFunctionViewIfPermitted.cpp index a30e3ac004695..610bb6f63f094 100644 --- a/src/TableFunctions/TableFunctionViewIfPermitted.cpp +++ b/src/TableFunctions/TableFunctionViewIfPermitted.cpp @@ -91,6 +91,9 @@ void TableFunctionViewIfPermitted::parseArguments(const ASTPtr & ast_function, C ColumnsDescription TableFunctionViewIfPermitted::getActualTableStructure(ContextPtr context, bool is_insert_query) const { + /// Resolving the structure of the `ELSE` table function may access its source (e.g. schema inference + /// fetches a URL), so it requires the same `CREATE TEMPORARY TABLE` grant as executing it. + else_table_function->checkCreateTemporaryTableAccess(context, /*is_insert_query=*/ false); return else_table_function->getActualTableStructure(context, is_insert_query); } diff --git a/src/configure_config.cmake b/src/configure_config.cmake index fd24de2f1ba87..5cbe9efaaf1b2 100644 --- a/src/configure_config.cmake +++ b/src/configure_config.cmake @@ -160,7 +160,8 @@ if (TARGET ch_contrib::parquet) set(USE_PARQUET 1) set(USE_ARROW 1) endif() -if (TARGET _orc) +# The ORC reader and writer are implemented in ClickHouse and need only the compression libraries. +if (TARGET ch_contrib::snappy) set(USE_ORC 1) endif() if (TARGET ch_contrib::liburing) diff --git a/tests/integration/conftest.py b/tests/integration/conftest.py index 52d86d70fa6e2..b5c6beca7e631 100644 --- a/tests/integration/conftest.py +++ b/tests/integration/conftest.py @@ -68,8 +68,38 @@ def tune_local_port_range(): ) +def is_xdist_replacement_worker(): + """True when this worker was spawned to replace one that died mid-run. + + `pytest-xdist` numbers a replacement past the size of the original pool, so `gw8` in a pool of + eight (`gw0`..`gw7`) is a replacement. A replacement starts a fresh pytest *session*, which + re-runs every session-scoped fixture - and `cleanup_environment` kills every container on the + host, including the ones the workers that are still running own. One worker dying, from an OOM + kill for instance, would otherwise take the containers of the whole job with it. + """ + worker = os.environ.get("PYTEST_XDIST_WORKER", "") + count = os.environ.get("PYTEST_XDIST_WORKER_COUNT", "") + if not worker.startswith("gw") or not count.isdigit(): + return False + + index = worker.removeprefix("gw") + if not index.isdigit(): + return False + + return int(index) >= int(count) + + @pytest.fixture(autouse=True, scope="session") def cleanup_environment(): + if is_xdist_replacement_worker(): + logging.info( + "Worker %s replaces a worker that died, so the containers of the workers which are " + "still running are not ours to clean up. Skipping the environment cleanup.", + os.environ["PYTEST_XDIST_WORKER"], + ) + yield + return + try: result = run_and_check(["docker ps | wc -l"], shell=True) if int(result) > 1: diff --git a/tests/integration/helpers/keeper_snapshot_utils.py b/tests/integration/helpers/keeper_snapshot_utils.py index 30610a82d9568..2231a44823308 100644 --- a/tests/integration/helpers/keeper_snapshot_utils.py +++ b/tests/integration/helpers/keeper_snapshot_utils.py @@ -108,11 +108,38 @@ def stop_zk(zk): pass +# A Keeper these tests have partitioned with `iptables ... DROP` can lose a response, and kazoo's +# own pings keep the socket readable, so `KazooClient.get` and `exists` wait on an `AsyncResult` +# that never completes. Unbounded, one lost response burns the whole 900s `pytest-timeout` budget +# and takes the rest of the module with it, reported as a bare `Timeout (>900.0s) from +# pytest-timeout` that names neither the request nor the node. Bound every wait instead. +ZK_REQUEST_TIMEOUT_SEC = 60 + + +def zk_get(zk, path, timeout=ZK_REQUEST_TIMEOUT_SEC): + """`zk.get(path)`, giving up after `timeout` instead of waiting forever.""" + return zk.retry(lambda: zk.get_async(path).get(timeout=timeout)) + + +def zk_exists(zk, path, timeout=ZK_REQUEST_TIMEOUT_SEC): + """`zk.exists(path)`, giving up after `timeout` instead of waiting forever.""" + return zk.retry(lambda: zk.exists_async(path).get(timeout=timeout)) + + def fill_test_tree(zk, base, count=300): import os as _os + from kazoo.exceptions import NodeExistsError zk.ensure_path(base) for i in range(count): - zk.create(f"{base}/{i}", _os.urandom(1024)) # random to resist ZSTD compression + data = _os.urandom(1024) # random to resist ZSTD compression + try: + zk.create(f"{base}/{i}", data) + except NodeExistsError: + # `KazooClientWithImplicitRetries.create` retries, and `create` is not idempotent: a + # create whose response was lost has already taken effect, so the retry fails on a node + # it wrote itself. Write the data the caller asked for and carry on - the postcondition + # is that the node exists with that data, however many attempts it took. + zk.set(f"{base}/{i}", data) for i in range(0, count, 10): zk.delete(f"{base}/{i}") @@ -135,9 +162,9 @@ def verify_test_tree(leader_zk, lagging_zk, base, count=300): lagging_zk.sync(base) for i in range(count): if i % 10 != 0: - assert lagging_zk.get(f"{base}/{i}")[0] == leader_zk.get(f"{base}/{i}")[0] + assert zk_get(lagging_zk, f"{base}/{i}")[0] == zk_get(leader_zk, f"{base}/{i}")[0] else: - assert lagging_zk.exists(f"{base}/{i}") is None + assert zk_exists(lagging_zk, f"{base}/{i}") is None def get_kill_timestamp(node): diff --git a/tests/integration/helpers/s3_mocks/broken_s3.py b/tests/integration/helpers/s3_mocks/broken_s3.py index d0adeeb5df79d..a3fd6ab5852dc 100644 --- a/tests/integration/helpers/s3_mocks/broken_s3.py +++ b/tests/integration/helpers/s3_mocks/broken_s3.py @@ -229,6 +229,36 @@ def inject_error(self, request_handler): ) request_handler.write_error(500, data) + class NoSuchUploadAction: + # Answers directly instead of redirecting, so the upload is never completed upstream and + # whatever object the key already holds stays in place. + def __init__(self, early=None): + self.early = early == "1" + + def inject_error(self, request_handler): + data = ( + '' + "" + "NoSuchUpload" + "The specified upload does not exist. The upload ID may be invalid, " + "or the upload may have been aborted or completed." + "txfbd566d03042474888193-00608d7538" + "" + ) + if self.early: + request_handler.send_response(404) + request_handler.send_header("Content-Type", "text/xml") + request_handler.send_header("Content-Length", str(len(data))) + request_handler.send_header("Connection", "close") + request_handler.end_headers() + request_handler.wfile.write(bytes(data, "UTF-8")) + # like MinIO: stop sending, wait, then close with the body unread, which resets the connection + request_handler.connection.shutdown(socket.SHUT_WR) + time.sleep(0.5) + request_handler.connection.close() + else: + request_handler.write_error(404, data) + class SlowDownAction: def inject_error(self, request_handler): data = ( @@ -396,6 +426,10 @@ def __init__( ) elif self.action == "timeout": self.error_handler = _ServerRuntime.TimeoutAction() + elif self.action == "no_such_upload": + self.error_handler = _ServerRuntime.NoSuchUploadAction( + *self.action_args + ) else: self.error_handler = _ServerRuntime.Expected500ErrorAction() diff --git a/tests/integration/test_ai_functions/mock_ai_server.py b/tests/integration/test_ai_functions/mock_ai_server.py index 41368618f5b7e..859bc38e84ff4 100644 --- a/tests/integration/test_ai_functions/mock_ai_server.py +++ b/tests/integration/test_ai_functions/mock_ai_server.py @@ -8,6 +8,19 @@ `aiTranslate`'s `instructions` argument is forwarded in the prompt, or that the `Authorization` header is omitted when the named collection has no `api_key`). Header names are lower-cased for case-insensitive lookup. + GET /concurrency — returns JSON `{"requests": N, "max_concurrency": M}` describing + the requests the slow endpoints below have served since the last `/reset-concurrency`, where + `max_concurrency` is the highest number that were ever being served at the same moment. Used + to assert that `ai_function_max_concurrent_requests_per_thread` controls how many requests are in flight. + GET /reset-concurrency — zeroes the counters above + POST /v1/chat/slow — like `/v1/chat/completions`, but sleeps SLOW_RESPONSE_SECONDS + before answering, so overlapping requests are observable in `/concurrency`. + POST /v1/embeddings_slow — like `/v1/embeddings`, but slow in the same way. + POST /v1/chat/hang — like `/v1/chat/slow`, but sleeps HANG_RESPONSE_SECONDS, so a + request is still in flight when the server shuts down. + POST /v1/chat/jitter — like `/v1/chat/completions` without a schema (echoes the user + message), but sleeps a random 0 to JITTER_MAX_SECONDS first, so concurrent requests complete out + of order. GET /set-flaky?count=N — arm the flaky endpoints below to fail their next N requests with a simulated transient network error (used to exercise retries). `count=0` disarms. POST /v1/chat/flaky — like `/v1/chat/completions`, but drops the connection without @@ -49,6 +62,7 @@ the opposite of the `max_tokens` case. POST /v1/anthropic/tool_use — Anthropic-shaped HTTP 200 with `stop_reason="tool_use"`, a successful structured-output (forced tool call) response that must NOT be rejected. + POST /v1/rate_limited — always returns HTTP 429, a provider rate limit. POST /v1/error — always returns HTTP 500, a transient/server-side error that the url table function (and so the AI functions) retries. POST /v1/bad_request — always returns HTTP 400, a deterministic client error that @@ -58,7 +72,9 @@ import http.server import json +import random import threading +import time from urllib.parse import urlparse, parse_qs MOCK_PORT = 18123 @@ -74,6 +90,20 @@ # Set via `GET /set-flaky?count=N`. Used to exercise the network-error retry path. FLAKY = {"fails_remaining": 0} +# How long the slow endpoints take to answer. Long enough that concurrent requests overlap +# observably, short enough not to slow the test down. +SLOW_RESPONSE_SECONDS = 0.5 + +# How long `/v1/chat/hang` takes to answer. +HANG_RESPONSE_SECONDS = 5 + +# Upper bound of the random delay of `/v1/chat/jitter`. +JITTER_MAX_SECONDS = 0.3 + +# Requests the slow endpoints have served, and the high-water mark of how many they were serving +# simultaneously, since the last `/reset-concurrency`. Guarded by `_LOCK`. +CONCURRENCY = {"requests": 0, "in_flight": 0, "max_concurrency": 0} + def extract_user_message(body): data = json.loads(body) @@ -237,6 +267,24 @@ def do_GET(self): self._send_json(200, snapshot) return + if parsed.path == "/concurrency": + with _LOCK: + snapshot = { + "requests": CONCURRENCY["requests"], + "max_concurrency": CONCURRENCY["max_concurrency"], + } + self._send_json(200, snapshot) + return + + if parsed.path == "/reset-concurrency": + with _LOCK: + CONCURRENCY.update(requests=0, in_flight=0, max_concurrency=0) + self.send_response(200) + self.send_header("Content-Type", "text/plain") + self.end_headers() + self.wfile.write(b"OK") + return + if parsed.path == "/set-flaky": qs = parse_qs(parsed.query) with _LOCK: @@ -260,6 +308,23 @@ def do_POST(self): LAST_REQUEST["body"] = body LAST_REQUEST["headers"] = {k.lower(): v for k, v in self.headers.items()} + if parsed.path == "/v1/chat/slow": + self._serve_slowly(lambda: make_success_response(extract_user_message(body))) + return + + if parsed.path == "/v1/embeddings_slow": + self._serve_slowly(lambda: make_embeddings_response(body)) + return + + if parsed.path == "/v1/chat/hang": + self._serve_slowly(lambda: make_success_response(extract_user_message(body)), HANG_RESPONSE_SECONDS) + return + + if parsed.path == "/v1/chat/jitter": + time.sleep(random.uniform(0, JITTER_MAX_SECONDS)) + self._send_json(200, make_success_response(extract_user_message(body))) + return + if parsed.path in ("/v1/chat/flaky", "/v1/embeddings_flaky"): with _LOCK: should_fail = FLAKY["fails_remaining"] > 0 @@ -368,6 +433,10 @@ def do_POST(self): self._send_json(200, make_anthropic_tool_use_response(body)) return + if parsed.path == "/v1/rate_limited": + self._send_json(429, make_error_response("rate limit exceeded", error_type="rate_limit_error")) + return + if parsed.path == "/v1/error": self._send_json(500, make_error_response("permanent failure")) return @@ -414,6 +483,25 @@ def do_POST(self): self.send_response(404) self.end_headers() + def _serve_slowly(self, make_response, seconds=SLOW_RESPONSE_SECONDS): + """Sleep, then answer, counting the request as in flight for the duration of the sleep. + The window closes before the response is written, so a handler that has already answered + can never overlap in `max_concurrency` with the request that answer unblocks.""" + with _LOCK: + CONCURRENCY["requests"] += 1 + CONCURRENCY["in_flight"] += 1 + CONCURRENCY["max_concurrency"] = max( + CONCURRENCY["max_concurrency"], CONCURRENCY["in_flight"] + ) + try: + time.sleep(seconds) + response = make_response() + finally: + with _LOCK: + CONCURRENCY["in_flight"] -= 1 + + self._send_json(200, response) + def _send_json(self, status, obj): body = json.dumps(obj).encode("utf-8") self.send_response(status) diff --git a/tests/integration/test_ai_functions/test.py b/tests/integration/test_ai_functions/test.py index 1c7e063b7f5d1..ea4bf564c7711 100644 --- a/tests/integration/test_ai_functions/test.py +++ b/tests/integration/test_ai_functions/test.py @@ -19,7 +19,7 @@ SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) cluster = ClickHouseCluster(__file__) -instance = cluster.add_instance("node") +instance = cluster.add_instance("node", stay_alive=True) def run_mock_server(): @@ -98,6 +98,34 @@ def started_cluster() -> typing.Generator[ClickHouseCluster, None, None]: f"model = 'test-model', " f"api_key = 'test-key'" ) + # Slow endpoints, used to observe how many requests an AI function has in flight at once. + instance.query( + f"CREATE NAMED COLLECTION ai_slow AS " + f"provider = 'openai', " + f"endpoint = 'http://localhost:{MOCK_PORT}/v1/chat/slow', " + f"model = 'test-model', " + f"api_key = 'test-key'" + ) + instance.query( + f"CREATE NAMED COLLECTION ai_hang AS " + f"provider = 'openai', " + f"endpoint = 'http://localhost:{MOCK_PORT}/v1/chat/hang', " + f"model = 'test-model', " + f"api_key = 'test-key'" + ) + instance.query( + f"CREATE NAMED COLLECTION ai_jitter AS " + f"provider = 'openai', " + f"endpoint = 'http://localhost:{MOCK_PORT}/v1/chat/jitter', " + f"model = 'test-model', " + f"api_key = 'test-key'" + ) + instance.query( + f"CREATE NAMED COLLECTION ai_embed_slow AS " + f"provider = 'openai', " + f"endpoint = 'http://localhost:{MOCK_PORT}/v1/embeddings_slow', " + f"api_key = 'test-key'" + ) instance.query( f"CREATE NAMED COLLECTION ai_error AS " f"provider = 'openai', " @@ -236,6 +264,14 @@ def started_cluster() -> typing.Generator[ClickHouseCluster, None, None]: f"endpoint = 'http://localhost:{MOCK_PORT}/v1/embeddings_wrong_count', " f"api_key = 'test-key'" ) + # Endpoint that always replies HTTP 429, as a rate-limited provider would. + instance.query( + f"CREATE NAMED COLLECTION ai_rate_limited AS " + f"provider = 'openai', " + f"endpoint = 'http://localhost:{MOCK_PORT}/v1/rate_limited', " + f"model = 'test-model', " + f"api_key = 'test-key'" + ) # Endpoints that drop the connection for the first N requests (armed via /set-flaky), # used to test that transient network failures are retried like the url table function. instance.query( @@ -409,6 +445,8 @@ def test_generate_truncated_response_counts_tokens(started_cluster): result = instance.query( "SELECT aiGenerate(x, map('credentials', 'ai_truncated')) FROM test_input", settings={ + # One request at a time, so the token quota is seen before the next request. + "ai_function_max_concurrent_requests_per_thread": 1, "ai_function_throw_on_error": 0, "ai_function_max_output_tokens_per_query": 5, "ai_function_throw_on_quota_exceeded": 0, @@ -769,6 +807,54 @@ def test_filter_where(started_cluster): assert lines == ["also good", "great product"] +PREWHERE_SETTINGS = { + "enable_analyzer": 1, + "optimize_move_to_prewhere": 1, + "move_all_conditions_to_prewhere": 1, + "allow_reorder_prewhere_conditions": 1, +} + + +def count_filter_calls(prefix, where): + """Runs `SELECT count()` over 16 rows, half of them with `flag = 1`, and returns the number of + rows and of AI requests.""" + instance.query( + "CREATE TABLE IF NOT EXISTS prewhere_ai (flag UInt8, text String) ENGINE = MergeTree ORDER BY tuple()" + ) + instance.query("TRUNCATE TABLE prewhere_ai") + instance.query( + "INSERT INTO prewhere_ai SELECT number % 2, 'row ' || toString(number) FROM numbers(16)" + ) + qid = unique_query_id(prefix) + rows = instance.query( + f"SELECT count() FROM prewhere_ai WHERE {where}", + query_id=qid, + settings=PREWHERE_SETTINGS, + ) + return int(rows), int(get_profile_events(qid)["api_calls"]) + + +def test_filter_prewhere_after_cheap_condition(started_cluster): + """The AI condition is moved to PREWHERE after `flag = 1`, so only the rows passing it are sent.""" + rows, api_calls = count_filter_calls( + "filter_prewhere_after_cheap", + "aiFilter(text, 'matches', map('credentials', 'ai_mock')) AND flag = 1", + ) + assert rows == 8 + assert api_calls == 8 + + +def test_filter_prewhere_behind_condition_left_in_where(started_cluster): + """`length(text) = flag` uses every queried column, so it stays in WHERE. The AI condition + stays there too and is evaluated only for the rows passing it, which are none.""" + rows, api_calls = count_filter_calls( + "filter_prewhere_behind_where", + "length(text) = flag AND aiFilter(text, 'matches', map('credentials', 'ai_mock'))", + ) + assert rows == 0 + assert api_calls == 0 + + def test_filter_truncated_response_throw(started_cluster): """aiFilter shares the FunctionBaseAI rejection path. A provider-signalled incomplete reply (here `finish_reason="length"`) is an error under the default `ai_function_throw_on_error=1`, @@ -1361,6 +1447,8 @@ def test_embed_quota_input_tokens_exceeded(started_cluster): result = instance.query( "SELECT aiEmbed(x, 'test-embed-model', map('credentials', 'ai_embed')) FROM test_input", settings={ + # One request at a time, so the token quota is seen before the next request. + "ai_function_max_concurrent_requests_per_thread": 1, "ai_function_embedding_max_batch_size": 1, "ai_function_max_input_tokens_per_query": 5, "ai_function_throw_on_quota_exceeded": 0, @@ -1411,6 +1499,62 @@ def test_generate_retries_on_network_error(started_cluster): assert int(events["rows_processed"]) == 1 +def _request_outcome_events(query_id): + instance.query("SYSTEM FLUSH LOGS") + return [ + int(v) + for v in instance.query( + f"SELECT ProfileEvents['AIAPICallsRetried'], ProfileEvents['AIAPICallsFailed'], " + f"ProfileEvents['AIAPICallsThrottled'] FROM system.query_log " + f"WHERE query_id = '{query_id}' AND type = 'QueryFinish'" + ).split() + ] + + +def _server_event(name): + return int(instance.query(f"SELECT sum(value) FROM system.events WHERE event = '{name}'").strip() or 0) + + +def test_request_outcome_profile_events(started_cluster): + """`AIAPICallsRetried`, `AIAPICallsFailed` and `AIAPICallsThrottled` count request outcomes per query + and for the whole server, including failures swallowed by `ai_function_throw_on_error = 0`.""" + settings = {"ai_function_throw_on_error": 0, "ai_function_retry_initial_delay_ms": 1} + + # Two dropped connections, then success: two retries, no failure. + set_flaky(2) + qid = unique_query_id("outcome_recovered") + instance.query( + "SELECT aiGenerate('x', map('credentials', 'ai_flaky')) FORMAT Null", + settings={**settings, "ai_function_max_retries": 5}, + query_id=qid, + ) + assert _request_outcome_events(qid) == [2, 0, 0] + + # Every attempt dropped: one retry, then the request fails and the row gets a default value. + set_flaky(10) + try: + qid = unique_query_id("outcome_failed") + instance.query( + "SELECT aiGenerate('x', map('credentials', 'ai_flaky')) FORMAT Null", + settings={**settings, "ai_function_max_retries": 1}, + query_id=qid, + ) + finally: + set_flaky(0) + assert _request_outcome_events(qid) == [1, 1, 0] + + # HTTP 429 on both attempts: both are throttled, one is retried, the request fails. + throttled_before = _server_event("AIAPICallsThrottled") + qid = unique_query_id("outcome_throttled") + instance.query( + "SELECT aiGenerate('x', map('credentials', 'ai_rate_limited')) FORMAT Null", + settings={**settings, "ai_function_max_retries": 1}, + query_id=qid, + ) + assert _request_outcome_events(qid) == [1, 1, 2] + assert _server_event("AIAPICallsThrottled") - throttled_before == 2 + + def test_generate_network_error_not_retried_when_disabled(started_cluster): """With `ai_function_max_retries = 0`, a network failure is surfaced rather than retried.""" set_flaky(10) @@ -1561,6 +1705,61 @@ def test_function_name_header(started_cluster): assert last_request()["headers"].get("x-clickhouse-ai-function") == name +def test_query_id_header(started_cluster): + """The OpenAI provider sends the `query_id` of the query that makes the request as the + `X-ClickHouse-Query-Id` header and its `initial_query_id` as the `X-ClickHouse-Initial-Query-Id` header: + on a remote shard the first is the shard's own query id, the second is the initiator's one. + Covers the chat and the embedding paths.""" + local_cases = [ + "SELECT aiGenerate('hi', map('credentials', 'ai_mock'))", + "SELECT aiEmbed('hi', 'test-embed-model', map('credentials', 'ai_embed'))", + ] + for i, query in enumerate(local_cases): + qid = unique_query_id(f"query_id_header_{i}") + instance.query(query, query_id=qid) + headers = last_request()["headers"] + assert headers.get("x-clickhouse-query-id") == qid + assert headers.get("x-clickhouse-initial-query-id") == qid + + remote_cases = [ + "SELECT aiGenerate(toString(dummy), map('credentials', 'ai_mock')) FROM remote('127.0.0.2', system.one)", + "SELECT aiEmbed(toString(dummy), 'test-embed-model', map('credentials', 'ai_embed')) FROM remote('127.0.0.2', system.one)", + ] + for i, query in enumerate(remote_cases): + qid = unique_query_id(f"query_id_header_remote_{i}") + instance.query(query, query_id=qid) + headers = last_request()["headers"] + sent_query_id = headers.get("x-clickhouse-query-id") + assert sent_query_id and sent_query_id != qid + assert headers.get("x-clickhouse-initial-query-id") == qid + instance.query("SYSTEM FLUSH LOGS query_log") + assert ( + instance.query( + f"SELECT initial_query_id FROM system.query_log WHERE query_id = '{sent_query_id}' AND NOT is_initial_query AND type = 'QueryFinish'" + ).strip() + == qid + ) + + +def test_query_id_header_control_characters(started_cluster): + """A query id from the native protocol can contain control characters. They are replaced with spaces + in the `X-ClickHouse-Query-Id` and `X-ClickHouse-Initial-Query-Id` headers, so no extra header is injected.""" + suffix = uuid.uuid4().hex[:8] + qid = f"query_id_crlf_{suffix}\r\nX-Injected: 1" + instance.query("SELECT aiGenerate('hi', map('credentials', 'ai_mock'))", query_id=qid) + instance.query("SYSTEM FLUSH LOGS query_log") + assert ( + instance.query( + f"SELECT count() FROM system.query_log WHERE query_id = 'query_id_crlf_{suffix}\\r\\nX-Injected: 1' AND type = 'QueryFinish'" + ).strip() + == "1" + ) + headers = last_request()["headers"] + assert headers.get("x-clickhouse-query-id") == f"query_id_crlf_{suffix} X-Injected: 1" + assert headers.get("x-clickhouse-initial-query-id") == f"query_id_crlf_{suffix} X-Injected: 1" + assert "x-injected" not in headers + + def test_embed_retry_respects_api_call_quota(started_cluster): """The embedding path enforces the same per-attempt API-call quota: a retriable HTTP 500 is not retried past `ai_function_max_api_calls_per_query`.""" @@ -2074,6 +2273,8 @@ def test_input_token_quota_is_per_query(started_cluster): instance.query( f"SELECT {CHAT_CALL} FROM quota_tokens FORMAT Null", settings={ + # One request at a time, so the token quota is seen before the next request. + "ai_function_max_concurrent_requests_per_thread": 1, **_QUOTA_SCOPE_SETTINGS, "ai_function_max_input_tokens_per_query": limit, "ai_function_throw_on_quota_exceeded": 0, @@ -2177,3 +2378,305 @@ def test_api_call_quota_ignores_subquery_settings(started_cluster): assert subquery_only == 64, ( f"expected all 64 rows to run (a quota set only in the subquery is ignored), got {subquery_only}" ) + + +SLOW_CHAT_CALL = "aiClassify(toString(number), ['positive','negative'], map('credentials', 'ai_slow'))" +SLOW_EMBED_CALL = "aiEmbed(toString(number), 'test-model', map('credentials', 'ai_embed_slow'))" + + +def _concurrency_stats(): + return json.loads( + instance.exec_in_container( + ["curl", "-s", f"http://localhost:{MOCK_PORT}/concurrency"] + ) + ) + + +def _reset_concurrency(): + instance.exec_in_container( + ["curl", "-s", f"http://localhost:{MOCK_PORT}/reset-concurrency"] + ) + + +def _observe_concurrency(call, concurrency, rows, expected_requests, extra_settings=None): + """Run `call` over `rows` rows at the given `ai_function_max_concurrent_requests_per_thread` and return + the high-water mark of requests the mock saw in flight at the same time.""" + _reset_concurrency() + qid = unique_query_id(f"ai_concurrency_{concurrency}") + instance.query( + f"SELECT {call} FROM numbers({rows}) FORMAT Null", + settings={ + "ai_function_max_concurrent_requests_per_thread": concurrency, + **(extra_settings or {}), + }, + query_id=qid, + ) + stats = _concurrency_stats() + assert stats["requests"] == expected_requests, ( + f"expected {expected_requests} requests, got {stats['requests']} at " + f"ai_function_max_concurrent_requests_per_thread={concurrency}" + ) + assert int(get_profile_events(qid)["api_calls"]) == expected_requests + return stats["max_concurrency"] + + +def test_max_concurrent_requests_controls_requests_in_flight(started_cluster): + """`ai_function_max_concurrent_requests_per_thread` decides how many provider requests a text AI function + has in flight, and nothing else does: the same query shape and the same data give one request + at a time at 1, and several at 8.""" + one = _observe_concurrency(SLOW_CHAT_CALL, concurrency=1, rows=16, expected_requests=16) + many = _observe_concurrency(SLOW_CHAT_CALL, concurrency=8, rows=16, expected_requests=16) + + assert one == 1, ( + f"ai_function_max_concurrent_requests_per_thread=1 must issue requests one at a time, saw {one} in flight" + ) + assert 2 <= many <= 8, ( + f"ai_function_max_concurrent_requests_per_thread=8 must overlap requests without exceeding the " + f"limit, saw {many} in flight" + ) + + +def test_max_concurrent_requests_applies_to_embeddings(started_cluster): + """The same setting governs the embedding path, where the unit in flight is a batch of texts + rather than a row: 16 rows at a batch size of 2 are 8 requests.""" + batching = {"ai_function_embedding_max_batch_size": 2} + one = _observe_concurrency( + SLOW_EMBED_CALL, concurrency=1, rows=16, expected_requests=8, extra_settings=batching + ) + many = _observe_concurrency( + SLOW_EMBED_CALL, concurrency=4, rows=16, expected_requests=8, extra_settings=batching + ) + + assert one == 1, ( + f"ai_function_max_concurrent_requests_per_thread=1 must issue batches one at a time, saw {one} in flight" + ) + assert 2 <= many <= 4, ( + f"ai_function_max_concurrent_requests_per_thread=4 must overlap batches without exceeding the " + f"limit, saw {many} in flight" + ) + + +def test_max_concurrent_requests_keeps_api_call_quota_exact(started_cluster): + """Concurrency must not let the API-call quota drift: a slot is reserved before each request is + dispatched, so the cap stays exact no matter how many requests are in flight.""" + _reset_concurrency() + qid = unique_query_id("ai_concurrency_quota") + instance.query( + f"SELECT {SLOW_CHAT_CALL} FROM numbers(64) FORMAT Null", + settings={ + "ai_function_max_concurrent_requests_per_thread": 8, + "ai_function_max_api_calls_per_query": 5, + "ai_function_throw_on_quota_exceeded": 0, + }, + query_id=qid, + ) + stats = _concurrency_stats() + assert stats["requests"] == 5, ( + f"the provider saw {stats['requests']} requests, but ai_function_max_api_calls_per_query = 5 " + "is an exact cap even with concurrent requests" + ) + assert int(get_profile_events(qid)["api_calls"]) == 5 + + +def test_concurrent_responses_map_to_their_rows(started_cluster): + """With several requests in flight and responses arriving in random order, every row still gets its + own response. The mock echoes the prompt after a random delay, so each output must equal its input. + Four blocks of 25 rows over two threads also run several function calls at once.""" + result = instance.query( + "SELECT countIf(answer = prompt), count() FROM " + "(SELECT concat('row-', toString(number)) AS prompt, " + "aiGenerate(prompt, map('credentials', 'ai_jitter')) AS answer FROM numbers_mt(100))", + settings={ + "ai_function_max_concurrent_requests_per_thread": 10, + "max_block_size": 25, + "max_threads": 2, + }, + ).strip() + assert result == "100\t100", f"expected every row to get its own response, got (matching, total) = {result}" + + +MULTI_CALL_QUERY = ( + "SELECT countIf(a = pa), countIf(b = pb), count(), sum(cityHash64(pa, e)) FROM " + "(SELECT concat('a-', toString(number)) AS pa, concat('b-', toString(number)) AS pb, " + "aiGenerate(pa, map('credentials', 'ai_jitter')) AS a, " + "aiGenerate(pb, map('credentials', 'ai_jitter')) AS b, " + "aiEmbed(pa, 'test-embed-model', map('credentials', 'ai_embed')) AS e " + "FROM numbers_mt(100))" +) +MULTI_CALL_SETTINGS = { + "max_block_size": 25, + "max_threads": 2, + "ai_function_embedding_max_batch_size": 1, +} + + +def test_multiple_ai_calls_in_one_query(started_cluster): + """Three AI function calls in one query share the request pool and the query's quota tracker. + Each call's outputs stay with its own rows, the embeddings match a run with one request at a time, + and `AIAPICalls` counts the requests of all three calls: 100 + 100 + 100 (one text per batch).""" + serial = instance.query( + MULTI_CALL_QUERY, settings={**MULTI_CALL_SETTINGS, "ai_function_max_concurrent_requests_per_thread": 1} + ).split("\t") + + qid = unique_query_id("ai_multi_call") + concurrent = instance.query( + MULTI_CALL_QUERY, + settings={**MULTI_CALL_SETTINGS, "ai_function_max_concurrent_requests_per_thread": 10}, + query_id=qid, + ).split("\t") + + assert concurrent[:3] == ["100", "100", "100"], ( + f"expected each aiGenerate call to answer its own rows, got (a matching, b matching, total) = {concurrent[:3]}" + ) + assert concurrent[3] == serial[3], "aiEmbed returned different row/embedding pairs with concurrent requests" + assert int(get_profile_events(qid)["api_calls"]) == 300 + + +def test_api_call_quota_is_shared_by_ai_calls_in_one_query(started_cluster): + """`ai_function_max_api_calls_per_query` is one allowance for all AI function calls of the query, + and stays exact with concurrent requests from three calls drawing on it at once.""" + qid = unique_query_id("ai_multi_call_quota") + instance.query( + MULTI_CALL_QUERY, + settings={ + **MULTI_CALL_SETTINGS, + "ai_function_max_concurrent_requests_per_thread": 10, + "ai_function_max_api_calls_per_query": 50, + "ai_function_throw_on_quota_exceeded": 0, + }, + query_id=qid, + ) + assert int(get_profile_events(qid)["api_calls"]) == 50 + + +def test_kill_query_stops_issuing_requests(started_cluster): + """`KILL QUERY` stops an AI function mid-block: no new request is sent after the kill. The 400 rows + form one block, which at 0.5s per request would otherwise take 200s and send all 400 requests.""" + _reset_concurrency() + qid = unique_query_id("ai_kill") + request = instance.get_query_request( + f"SELECT {SLOW_CHAT_CALL} FROM numbers(400) FORMAT Null", + settings={"ai_function_max_concurrent_requests_per_thread": 1}, + query_id=qid, + ) + + wait_condition( + _concurrency_stats, lambda stats: stats["requests"] >= 2, max_attempts=100, delay=0.1 + ) + instance.query(f"KILL QUERY WHERE query_id = '{qid}' SYNC") + + assert "Query was cancelled" in request.get_error() + requests = _concurrency_stats()["requests"] + assert requests < 40, f"the killed query kept sending requests: {requests} of 400 were sent" + + instance.query("SYSTEM FLUSH LOGS") + exception_code = instance.query( + f"SELECT exception_code FROM system.query_log " + f"WHERE query_id = '{qid}' AND type = 'ExceptionWhileProcessing'" + ).strip() + assert exception_code == "394" + assert int(get_profile_events(qid, "ExceptionWhileProcessing")["api_calls"]) == requests + + +def test_max_execution_time_stops_issuing_requests(started_cluster): + """`max_execution_time` stops an AI function mid-block like `KILL QUERY`: the 400 rows form one block + that would take 200s at 0.5s per request, and the query fails after 2s having sent a fraction of them.""" + _reset_concurrency() + error = instance.query_and_get_error( + f"SELECT countIf(r != '') FROM (SELECT {SLOW_CHAT_CALL} AS r FROM numbers(400))", + settings={"max_execution_time": 2, "ai_function_max_concurrent_requests_per_thread": 1}, + ) + assert "TIMEOUT_EXCEEDED" in error + requests = _concurrency_stats()["requests"] + assert requests < 40, f"the timed-out query kept sending requests: {requests} of 400 were sent" + + +def test_max_execution_time_break_stops_issuing_requests(started_cluster): + """With `timeout_overflow_mode = 'break'` the query returns without an error once the time limit is + reached, and no further requests are sent.""" + _reset_concurrency() + instance.query( + f"SELECT countIf(r != '') FROM (SELECT {SLOW_CHAT_CALL} AS r FROM numbers(400))", + settings={ + "max_execution_time": 2, + "timeout_overflow_mode": "break", + "ai_function_max_concurrent_requests_per_thread": 1, + }, + ) + requests = _concurrency_stats()["requests"] + assert requests < 40, f"the query kept sending requests past the time limit: {requests} of 400 were sent" + + +AI_THREAD_POOL_CONFIG = "/etc/clickhouse-server/config.d/ai_thread_pool.xml" + + +def _set_ai_thread_pool_size(size): + instance.replace_config( + AI_THREAD_POOL_CONFIG, + f"{size}", + ) + instance.query("SYSTEM RELOAD CONFIG") + + +def test_cancelled_query_does_not_wait_for_other_queries_requests(started_cluster): + """With a single pool thread, query A queues 64 requests. Query B's request queues behind them, and B + fails on `max_execution_time` before A's requests are served. Once A is killed, its queued requests + are not sent.""" + _set_ai_thread_pool_size(1) + qid_a = unique_query_id("ai_backlog") + try: + _reset_concurrency() + request_a = instance.get_query_request( + f"SELECT {SLOW_CHAT_CALL} FROM numbers(400) FORMAT Null", + settings={"ai_function_max_concurrent_requests_per_thread": 64}, + query_id=qid_a, + ) + wait_condition( + _concurrency_stats, lambda stats: stats["requests"] >= 1, max_attempts=100, delay=0.1 + ) + + error = instance.query_and_get_error( + f"SELECT {SLOW_CHAT_CALL} FROM numbers(1) FORMAT Null", + settings={"max_execution_time": 1}, + ) + assert "TIMEOUT_EXCEEDED" in error + served = _concurrency_stats()["requests"] + assert served < 64, f"the timed-out query waited for {served} requests of the other query" + + instance.query(f"KILL QUERY WHERE query_id = '{qid_a}' SYNC") + assert "Query was cancelled" in request_a.get_error() + sent = _concurrency_stats()["requests"] + wait_condition( + lambda: instance.query( + "SELECT value FROM system.metrics WHERE metric = 'AIRequestThreadsScheduled'" + ).strip(), + lambda value: value == "0", + max_attempts=100, + delay=0.1, + ) + # A request that passed its cancellation check before the kill may still arrive. + assert _concurrency_stats()["requests"] <= sent + 1 + finally: + # Fails with `NOTHING_TO_KILL` when the query was already killed above. + instance.query_and_get_answer_with_error(f"KILL QUERY WHERE query_id = '{qid_a}' SYNC") + _set_ai_thread_pool_size(1000) + + +def test_shutdown_after_cancelling_query_with_request_in_flight(started_cluster): + """A killed query does not wait for its request to `/v1/chat/hang`. The server shuts down right after, + while the request is still in flight, and must drain the AI request pool before destroying the context.""" + _reset_concurrency() + qid = unique_query_id("ai_hang") + request = instance.get_query_request( + "SELECT aiGenerate(toString(number), map('credentials', 'ai_hang')) FROM numbers(1) FORMAT Null", + settings={"ai_function_max_retries": 0}, + query_id=qid, + ) + wait_condition( + _concurrency_stats, lambda stats: stats["requests"] >= 1, max_attempts=100, delay=0.1 + ) + instance.query(f"KILL QUERY WHERE query_id = '{qid}' SYNC") + assert "Query was cancelled" in request.get_error() + + instance.restart_clickhouse() + assert instance.query("SELECT aiGenerate('x', map('credentials', 'ai_slow'))").strip() != "" diff --git a/tests/integration/test_allowed_url_from_config/configs/config_with_hosts.xml b/tests/integration/test_allowed_url_from_config/configs/config_with_hosts.xml index d4ac4986c22dc..ad199feefc1ed 100644 --- a/tests/integration/test_allowed_url_from_config/configs/config_with_hosts.xml +++ b/tests/integration/test_allowed_url_from_config/configs/config_with_hosts.xml @@ -2,6 +2,7 @@ host:80 ^[a-z]*\.ru$ + .*\.allowed\.example diff --git a/tests/integration/test_allowed_url_from_config/test.py b/tests/integration/test_allowed_url_from_config/test.py index 23c28067f9de4..b1786b7aa8011 100644 --- a/tests/integration/test_allowed_url_from_config/test.py +++ b/tests/integration/test_allowed_url_from_config/test.py @@ -81,6 +81,43 @@ def test_azure_host_filter_before_client_creation(start_cluster, query): assert node5.query(container_requests) == before +def test_object_storage_queue_move_destination_host_filter(start_cluster): + # The source is on an allowed host (`host:80`), so only the move destination can be rejected. `node1` has no + # Keeper, so a rejection with "not allowed" must come from the settings validation, which runs before any + # Keeper access. + s3_queue = ( + "CREATE TABLE s3queue_move_host (x UInt8) " + "ENGINE = S3Queue('http://host:80/bucket/data/', 'key', 'secret', 'CSV') " + "SETTINGS mode = 'unordered', after_processing = 'move', " + "after_processing_move_uri = '{}', " + "after_processing_move_access_key_id = 'key', after_processing_move_secret_access_key = 'secret'" + ) + azure_queue = ( + "CREATE TABLE azurequeue_move_host (x UInt8) " + "ENGINE = AzureQueue('http://host:80', 'container', '*', 'account', 'YQ==', 'CSV') " + "SETTINGS mode = 'unordered', after_processing = 'move', " + "after_processing_move_connection_string = " + "'DefaultEndpointsProtocol=http;AccountName=account;AccountKey=YQ==;BlobEndpoint=http://host:123/account;', " + "after_processing_move_container = 'dst'" + ) + + try: + error = node1.query_and_get_error(s3_queue.format("http://host:123/bucket/")) + assert "not allowed" in error, error + + error = node1.query_and_get_error( + azure_queue, settings={"azure_sdk_max_retries": 0} + ) + assert "not allowed" in error, error + + # Control: an allowed destination passes the host check and only fails later, on the missing Keeper. + error = node1.query_and_get_error(s3_queue.format("http://host:80/bucket/")) + assert "not allowed" not in error, error + finally: + node1.query("DROP TABLE IF EXISTS s3queue_move_host SYNC") + node1.query("DROP TABLE IF EXISTS azurequeue_move_host SYNC") + + def test_config_with_only_primary_hosts(start_cluster): assert ( node2.query( @@ -226,6 +263,34 @@ def test_mysql_dictionary_host_filter(start_cluster): node5.query("DROP DICTIONARY IF EXISTS default.test_mysql_host_filter") +def test_mysql_database_host_filter(start_cluster): + # node5 denies all hosts; the check runs before any connection attempt. + assert "not allowed" in node5.query_and_get_error( + "CREATE DATABASE test_mysql_db_filter ENGINE = MySQL('10.0.0.1:3306', 'db', 'u', 'p')" + ) + node5.query( + "CREATE NAMED COLLECTION IF NOT EXISTS test_mysql_db_filter_nc AS " + "host = '10.0.0.1', port = 3306, database = 'db', user = 'u', password = 'p'" + ) + assert "not allowed" in node5.query_and_get_error( + "CREATE DATABASE test_mysql_db_filter ENGINE = MySQL(test_mysql_db_filter_nc)" + ) + node5.query("DROP NAMED COLLECTION test_mysql_db_filter_nc") + + # node1's suffix rule would match the whole string, but the connector stops at the NUL; reject it. + assert "not allowed" in node1.query_and_get_error( + "CREATE DATABASE test_mysql_db_filter ENGINE = MySQL('127.0.0.1\\0.allowed.example:1', 'db', 'u', 'p')" + ) + node1.query( + "CREATE NAMED COLLECTION IF NOT EXISTS test_mysql_db_filter_nul_nc AS " + "host = '127.0.0.1\\0.allowed.example', port = 1, database = 'db', user = 'u', password = 'p'" + ) + assert "not allowed" in node1.query_and_get_error( + "CREATE DATABASE test_mysql_db_filter ENGINE = MySQL(test_mysql_db_filter_nul_nc)" + ) + node1.query("DROP NAMED COLLECTION test_mysql_db_filter_nul_nc") + + def test_mysql_dictionary_replica_host_filter(start_cluster): """MySQL dictionary source with REPLICA(...) syntax should respect RemoteHostFilter.""" # Covers the replica branch: if (config.has(settings_config_prefix + ".replica")) @@ -412,3 +477,54 @@ def test_schema_inference(start_cluster): error = node7.query_and_get_error("desc url('http://test.com', 'TSVRaw')") assert "not allowed" in error assert error.find("ReadWriteBufferFromHTTPBase") == -1 + + +def test_hive_catalog_host_filter(start_cluster): + query = ( + "CREATE DATABASE test_hive_host_filter ENGINE = DataLakeCatalog('thrift://localhost:1') " + "SETTINGS catalog_type = 'hive', warehouse = 'test_warehouse'" + ) + settings = {"allow_experimental_database_hms_catalog": 1} + try: + # Control first: `node4` declares no `remote_url_allow_hosts` at all, so its filter stays + # uninitialized and allows everything - the same statement gets past the check and fails at + # the network instead. Without this arm, a filter that rejected everything would also pass. + error = node4.query_and_get_error(query, settings=settings) + if "USE_HIVE" in error: + pytest.skip("ClickHouse was built without Hive support") + assert "UNACCEPTABLE_URL" not in error, error + + # `node5` declares an empty `remote_url_allow_hosts`, which allows nothing. + error = node5.query_and_get_error(query, settings=settings) + assert "UNACCEPTABLE_URL" in error, error + # The check must run before the connect, so the refusal carries no connection error. + assert "Connection refused" not in error, error + finally: + node4.query("DROP DATABASE IF EXISTS test_hive_host_filter") + node5.query("DROP DATABASE IF EXISTS test_hive_host_filter") + + +def test_hive_catalog_host_filter_rejects_nul(start_cluster): + def create(url): + return node1.query_and_get_error( + f"CREATE DATABASE test_hive_host_filter_nul ENGINE = DataLakeCatalog('{url}') " + "SETTINGS catalog_type = 'hive', warehouse = 'test_warehouse'", + settings={"allow_experimental_database_hms_catalog": 1}, + ) + + try: + # Control: the suffix rule in `config_with_hosts.xml` admits this host, so it gets past the + # filter and fails at name resolution instead. + error = create("thrift://hms.allowed.example:1") + if "USE_HIVE" in error: + pytest.skip("ClickHouse was built without Hive support") + assert "UNACCEPTABLE_URL" not in error, error + + # The same rule matches the whole string including the NUL, but `getaddrinfo` would stop at the + # NUL and connect to 127.0.0.1. The host must be refused before any connection attempt. + error = create("thrift://127.0.0.1\\0.allowed.example:1") + assert "BAD_ARGUMENTS" in error, error + assert "Unexpected character" in error, error + assert "Connection refused" not in error, error + finally: + node1.query("DROP DATABASE IF EXISTS test_hive_host_filter_nul") diff --git a/tests/integration/test_arrowflight_interface/test.py b/tests/integration/test_arrowflight_interface/test.py index 0f89533db0452..03a2dd8558baa 100644 --- a/tests/integration/test_arrowflight_interface/test.py +++ b/tests/integration/test_arrowflight_interface/test.py @@ -519,6 +519,43 @@ def test_doget_multiple_chunks(): assert actual[1].column("name").equals(pa.chunked_array([pa.array(["cde"], type=pa.string())])) +# A result is split across several Arrow arrays only when it exceeds the 32-bit offsets of the +# String/Binary/List buffers, so a small one must arrive as a single chunk per column for every type. +def test_doget_single_chunk_per_column(): + node.query( + """ + CREATE TABLE mytable ( + id Int64, + s String, + ns Nullable(String), + lc LowCardinality(String), + arr Array(String), + m Map(String, String), + t Tuple(String, String), + fs FixedString(3) + ) ORDER BY id + """ + ) + node.query( + """ + INSERT INTO mytable SELECT + number, toString(number), if(number % 2, NULL, toString(number)), toString(number % 2), + [toString(number)], map('k', toString(number)), (toString(number), 'x'), toFixedString('abc', 3) + FROM numbers(4) + """ + ) + + client, options = get_client() + + descriptor = flight.FlightDescriptor.for_command("SELECT * FROM mytable ORDER BY id") + flight_info = client.get_flight_info(descriptor, options) + table = client.do_get(flight_info.endpoints[0].ticket, options).read_all() + + assert table.num_rows == 4 + for name, column in zip(table.schema.names, table.columns): + assert column.num_chunks == 1, f"column {name} was split into {column.num_chunks} chunks" + + # We don't support any formats different from "Arrow". def test_doget_cmd_select_invalid_format(): node.query("CREATE TABLE mytable (id Int64, name String) ORDER BY id") diff --git a/tests/integration/test_backup_restore_azure_blob_storage/test.py b/tests/integration/test_backup_restore_azure_blob_storage/test.py index 36e4afda81fbe..2778e4dbb6b85 100644 --- a/tests/integration/test_backup_restore_azure_blob_storage/test.py +++ b/tests/integration/test_backup_restore_azure_blob_storage/test.py @@ -1,11 +1,17 @@ #!/usr/bin/env python3 +import datetime import io import os import time import pytest -from azure.storage.blob import BlobServiceClient +from azure.storage.blob import ( + AccountSasPermissions, + BlobServiceClient, + ResourceTypes, + generate_account_sas, +) from helpers.cluster import ClickHouseCluster @@ -936,6 +942,65 @@ def test_backup_legacy_disk_endpoint_settings_loaded(cluster): azure_query(node, "DROP TABLE test_legacy_native_copy_restored") +def test_connection_string_secrets_not_logged(cluster): + # The disk is configured by an account key connection string, which is also its description. The + # backup uses a SAS connection string, which the SDK appends to the blob service url as its query. + node = cluster.instances["node_legacy_native_copy"] + sas = generate_account_sas( + "devstoreaccount1", + "Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==", + resource_types=ResourceTypes(service=True, container=True, object=True), + permission=AccountSasPermissions( + read=True, write=True, delete=True, list=True, add=True, create=True + ), + expiry=datetime.datetime.now(datetime.timezone.utc) + + datetime.timedelta(hours=1), + ) + # URL-encoded, so it has no grep metacharacters. + sig = sas.split("sig=")[1].split("&")[0] + connection_string = f"BlobEndpoint={cluster.env_variables['AZURITE_STORAGE_ACCOUNT_URL']};SharedAccessSignature={sas}" + + azure_query(node, "DROP TABLE IF EXISTS test_secrets_not_logged SYNC") + azure_query( + node, + "CREATE TABLE test_secrets_not_logged(key UInt64, data String) Engine = MergeTree() ORDER BY tuple() SETTINGS storage_policy='blob_storage_policy_legacy_native_copy'", + ) + azure_query(node, "INSERT INTO test_secrets_not_logged VALUES (1, 'a')") + + backup_destination = ( + f"AzureBlobStorage('{connection_string}', 'cont', '{new_backup_name()}')" + ) + azure_query(node, f"BACKUP TABLE test_secrets_not_logged TO {backup_destination}") + azure_query(node, "DROP TABLE IF EXISTS test_secrets_not_logged_restored") + azure_query( + node, + f"RESTORE TABLE test_secrets_not_logged AS test_secrets_not_logged_restored FROM {backup_destination}", + ) + assert ( + azure_query(node, "SELECT * from test_secrets_not_logged_restored") == "1\ta\n" + ) + + function = f"azureBlobStorage('{connection_string}', 'cont', 'test_secrets_not_logged.csv'" + azure_query( + node, + f"INSERT INTO TABLE FUNCTION {function}, 'CSV', 'auto', 'x UInt64') SELECT 42 SETTINGS azure_truncate_on_insert = 1", + ) + assert azure_query(node, f"SELECT * FROM {function})") == "42\n" + sources = azure_query( + node, + "SELECT source FROM system.schema_inference_cache WHERE source LIKE '%test_secrets_not_logged.csv'", + ) + assert sources != "" and sig not in sources + + assert node.contains_in_log("BackupWriterAzureBlobStorage: Source description") + assert node.contains_in_log("BackupReaderAzureBlobStorage: Source description") + assert not node.contains_in_log("Source description.*Eby8vdM02xNO") + assert not node.contains_in_log(f"Source description.*{sig}") + + azure_query(node, "DROP TABLE test_secrets_not_logged") + azure_query(node, "DROP TABLE test_secrets_not_logged_restored") + + def test_reload_config_keeps_azure_endpoint_settings(cluster): # use_native_copy=true must survive SYSTEM RELOAD CONFIG (settings reloaded, not dropped). node = cluster.instances["node_native_copy"] diff --git a/tests/integration/test_backup_restore_s3/configs/disk_s3.xml b/tests/integration/test_backup_restore_s3/configs/disk_s3.xml index 6e82a13609704..53e114245a19a 100644 --- a/tests/integration/test_backup_restore_s3/configs/disk_s3.xml +++ b/tests/integration/test_backup_restore_s3/configs/disk_s3.xml @@ -52,6 +52,14 @@ + + +
+ disk_s3_plain +
+
+ true +
diff --git a/tests/integration/test_backup_restore_s3/test.py b/tests/integration/test_backup_restore_s3/test.py index 0a06846903071..2ea4a9f956d60 100644 --- a/tests/integration/test_backup_restore_s3/test.py +++ b/tests/integration/test_backup_restore_s3/test.py @@ -1278,6 +1278,187 @@ def test_backup_restore_s3_plain(cluster): instance.query("DROP TABLE sample_restored SYNC") +def test_backup_restore_s3_plain_backup_metadata_only(cluster): + # A table on a storage policy marked `backup_metadata_only` keeps its data on the storage: `BACKUP` stores only its + # metadata, and `RESTORE` recreates it with its UUID from the backup, so it finds its data on the storage again. + # An `Alias` of such a table keeps no data, so it is backed up and restored like any other `Alias`. Materialized views + # and `TimeSeries` tables with inner tables on such a policy are not supported. + instance = cluster.instances["node"] + backup_name = new_backup_name() + backup_destination = f"S3('http://minio1:9001/root/data/backups/{backup_name}', 'minio', '{minio_secret_key}')" + policy = "policy_s3_plain_backup_metadata_only" + + def new_backup_destination(): + return f"S3('http://minio1:9001/root/data/backups/{new_backup_name()}', 'minio', '{minio_secret_key}')" + + instance.query( + """ + DROP TABLE IF EXISTS sample_alias SYNC; + DROP TABLE IF EXISTS sample_alias_copy SYNC; + DROP TABLE IF EXISTS sample_structure_only SYNC; + DROP TABLE IF EXISTS ts SYNC; + DROP TABLE IF EXISTS sample SYNC; + DROP TABLE IF EXISTS mv SYNC; + DROP TABLE IF EXISTS src SYNC; + CREATE TABLE sample (key Int, value String) ENGINE = MergeTree PARTITION BY key % 2 ORDER BY key + AS SELECT number, concat('name_', toString(number)) FROM numbers(100); + """ + ) + + def get_uuid(table): + return instance.query( + f"SELECT uuid FROM system.tables WHERE database = 'default' AND name = '{table}'" + ).strip() + + sample_uuid = get_uuid("sample") + + # Copy the parts to the read-only plain disk, where a table with the same UUID finds them. + minio = cluster.minio_client + table_data_path = ( + instance.query(f"SELECT data_paths[1] FROM system.tables WHERE uuid = '{sample_uuid}'") + .strip() + .replace("/var/lib/clickhouse/", "") + .strip("/") + ) + remote_blob_path = f"data/disks/disk_s3_plain/{table_data_path}" + remove_directory(minio, cluster.minio_bucket, remote_blob_path) + upload_directory( + minio, + cluster.minio_bucket, + os.path.join(instance.path, "database", table_data_path), + remote_blob_path, + use_relpath=True, + ) + + instance.query( + f""" + DROP TABLE sample SYNC; + ATTACH TABLE sample UUID '{sample_uuid}' (key Int, value String) ENGINE = MergeTree PARTITION BY key % 2 ORDER BY key + SETTINGS storage_policy = '{policy}'; + """ + ) + instance.query("CREATE TABLE sample_alias ENGINE = Alias(sample)") + assert instance.query("SELECT count() FROM sample") == "100\n" + assert instance.query("SELECT count() FROM sample_alias") == "100\n" + + instance.query(f"BACKUP TABLE sample, TABLE sample_alias TO {backup_destination}") + + # The backup has the definitions of the tables, but none of their data. + backup_manifest = ( + minio.get_object(cluster.minio_bucket, f"data/backups/{backup_name}/.backup").read().decode() + ) + assert "metadata/default/sample.sql" in backup_manifest, backup_manifest + assert "data/" not in backup_manifest, backup_manifest + + # Only the definition is backed up, so a backup of some partitions would restore as the whole table. + err = instance.query_and_get_error(f"BACKUP TABLE sample PARTITIONS 0 TO {new_backup_destination()}") + assert "CANNOT_BACKUP_TABLE" in err and "Cannot backup specific partitions" in err, err + + # The inner tables of a materialized view and a `TimeSeries` table on such a policy are not supported: the flag + # is ignored for them and they are backed up as usual. Master also warns, naming the inner table that carries + # the flag; that warning is left out on this branch, so only the backup itself is asserted here. + instance.query( + f""" + CREATE TABLE src (key Int) ENGINE = Null; + CREATE MATERIALIZED VIEW mv ENGINE = MergeTree ORDER BY key SETTINGS storage_policy = '{policy}' AS SELECT key FROM src; + """ + ) + instance.query(f"BACKUP TABLE mv TO {new_backup_destination()}") + + instance.query("DROP TABLE mv SYNC") + instance.query("DROP TABLE src SYNC") + + # The restored table keeps its UUID, so it can't be restored next to the source table. + err = instance.query_and_get_error( + f"RESTORE TABLE sample AS sample_copy FROM {backup_destination}" + ) + assert "CANNOT_RESTORE_TABLE" in err and "another table already has this UUID" in err, err + + # A restore that leaves out the data has nothing to find on the storage, so it keeps no UUID either: it is an + # ordinary empty table and fits next to the source table. + instance.query( + f"RESTORE TABLE sample AS sample_structure_only FROM {backup_destination} SETTINGS structure_only = 1" + ) + assert get_uuid("sample_structure_only") != sample_uuid + assert instance.query("SELECT count() FROM sample_structure_only") == "0\n" + instance.query("DROP TABLE sample_structure_only SYNC") + + # The other table can be in a database the restoring user may not see, so the error doesn't name it. + instance.query( + """ + CREATE DATABASE private_db; + RENAME TABLE sample TO private_db.secret_table; + CREATE DATABASE restore_db; + CREATE USER restore_user; + GRANT CREATE TABLE, INSERT ON restore_db.* TO restore_user; + GRANT READ ON S3 TO restore_user; + """ + ) + try: + err = instance.query_and_get_error( + f"RESTORE TABLE sample AS restore_db.sample FROM {backup_destination}", user="restore_user" + ) + assert "CANNOT_RESTORE_TABLE" in err and "another table already has this UUID" in err, err + assert "private_db" not in err and "secret_table" not in err, err + finally: + instance.query( + """ + RENAME TABLE private_db.secret_table TO sample; + DROP DATABASE private_db; + DROP DATABASE restore_db; + DROP USER restore_user; + """ + ) + + instance.query("DROP TABLE sample_alias SYNC") + instance.query("DROP TABLE sample SYNC") + + # The table shows all the data of its storage, so a partition filter can't be applied. + err = instance.query_and_get_error( + f"RESTORE TABLE sample PARTITIONS 0 FROM {backup_destination}" + ) + assert "CANNOT_RESTORE_TABLE" in err and "partition filter cannot be applied" in err, err + instance.query("DROP TABLE IF EXISTS sample SYNC") + + # A table in an `Ordinary` database has no UUID: its data would be under a path derived from its name. + instance.query( + "CREATE DATABASE ordinary_db ENGINE = Ordinary", + settings={"allow_deprecated_database_ordinary": 1}, + ) + err = instance.query_and_get_error( + f"RESTORE TABLE sample AS ordinary_db.sample FROM {backup_destination}" + ) + assert "CANNOT_RESTORE_TABLE" in err and "its database doesn't give tables UUIDs" in err, err + instance.query("DROP DATABASE ordinary_db SYNC") + + # An existing table with another UUID doesn't see the data of the table from the backup. + instance.query( + f""" + ATTACH TABLE sample UUID '{uuid.uuid4()}' (key Int, value String) ENGINE = MergeTree PARTITION BY key % 2 ORDER BY key + SETTINGS storage_policy = '{policy}' + """ + ) + err = instance.query_and_get_error( + f"RESTORE TABLE sample FROM {backup_destination} SETTINGS create_table = 'must-exist'" + ) + assert "CANNOT_RESTORE_TABLE" in err and f"the table with UUID {sample_uuid} in the backup has UUID" in err, err + instance.query("DROP TABLE sample SYNC") + + instance.query(f"RESTORE TABLE sample FROM {backup_destination}") + assert get_uuid("sample") == sample_uuid + assert instance.query("SELECT count(), sum(key) FROM sample") == "100\t4950\n" + + # The `Alias` gets a new UUID like any table without data on such a policy, also when it is restored under another name. + instance.query(f"RESTORE TABLE sample_alias FROM {backup_destination}") + instance.query(f"RESTORE TABLE sample_alias AS sample_alias_copy FROM {backup_destination}") + assert instance.query("SELECT count(), sum(key) FROM sample_alias") == "100\t4950\n" + assert instance.query("SELECT count(), sum(key) FROM sample_alias_copy") == "100\t4950\n" + + instance.query("DROP TABLE sample_alias SYNC") + instance.query("DROP TABLE sample_alias_copy SYNC") + instance.query("DROP TABLE sample SYNC") + + @pytest.mark.parametrize( "to_disk", [ diff --git a/tests/integration/test_backup_restore_s3/test_backup_metadata_only.py b/tests/integration/test_backup_restore_s3/test_backup_metadata_only.py new file mode 100644 index 0000000000000..6018762638d4b --- /dev/null +++ b/tests/integration/test_backup_restore_s3/test_backup_metadata_only.py @@ -0,0 +1,134 @@ +import uuid + +import pytest + +from helpers.cluster import ClickHouseCluster +from helpers.config_cluster import minio_secret_key + +cluster = ClickHouseCluster(__file__) +node1 = cluster.add_instance( + "node1", + main_configs=["configs/disk_s3.xml"], + macros={"shard": "s1", "replica": "r1"}, + with_minio=True, + with_zookeeper=True, +) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/disk_s3.xml"], + macros={"shard": "s1", "replica": "r2"}, + with_minio=True, + with_zookeeper=True, +) + + +@pytest.fixture(scope="module", autouse=True) +def start_cluster(): + try: + cluster.start() + yield + finally: + cluster.shutdown() + + +def test_partitions_of_table_not_created_on_replica(start_cluster): + # A replica of a `Replicated` database backs up a table from its definition in Keeper before the table is created on + # the replica. A backup of some partitions of a table on a storage policy marked `backup_metadata_only` is refused from + # the definition then, as it would be restored as the whole table. + policy = "policy_s3_plain_backup_metadata_only" + zookeeper_path = f"/clickhouse/databases/rdb_{uuid.uuid4().hex}" + for node, replica in [(node1, "r1"), (node2, "r2")]: + node.query(f"CREATE DATABASE rdb ENGINE = Replicated('{zookeeper_path}', 's1', '{replica}')") + + node2.query("SYSTEM ENABLE FAILPOINT database_replicated_stop_entry_execution") + try: + node1.query( + f""" + CREATE TABLE rdb.t (key Int, value String) ENGINE = MergeTree PARTITION BY key % 2 ORDER BY key + SETTINGS storage_policy = '{policy}' + """, + settings={"distributed_ddl_task_timeout": 0}, + ) + assert node2.query("SELECT count() FROM system.tables WHERE database = 'rdb' AND name = 't'") == "0\n" + + err = node2.query_and_get_error( + f"BACKUP TABLE rdb.t PARTITIONS 0 TO S3('http://minio1:9001/root/data/backups/{uuid.uuid4().hex}', 'minio', '{minio_secret_key}')" + ) + assert "CANNOT_BACKUP_TABLE" in err and "Cannot backup specific partitions" in err, err + + # The flag is ignored for a `ReplicatedMergeTree` table known only by its definition too, which is backed up as usual. + node1.query( + f"CREATE TABLE rdb.t_replicated (key Int) ENGINE = ReplicatedMergeTree ORDER BY key SETTINGS storage_policy = '{policy}'", + settings={"distributed_ddl_task_timeout": 0}, + ) + assert node2.query("SELECT count() FROM system.tables WHERE database = 'rdb' AND name = 't_replicated'") == "0\n" + backup_destination = f"S3('http://minio1:9001/root/data/backups/{uuid.uuid4().hex}', 'minio', '{minio_secret_key}')" + node2.query(f"BACKUP TABLE rdb.t_replicated TO {backup_destination}") + finally: + node2.query("SYSTEM DISABLE FAILPOINT database_replicated_stop_entry_execution") + for node in [node1, node2]: + node.query("DROP DATABASE IF EXISTS rdb SYNC") + + +def test_disk_made_writable_by_config_reload(start_cluster): + # The policy is checked when it is created, and the reload keeps it when its new version fails the check, but the + # disk can still be made writable by the reload. Then the data written to it would be left out of the backup. + policy = "policy_s3_plain_backup_metadata_only" + config = "/etc/clickhouse-server/config.d/disk_s3.xml" + node1.query( + f"CREATE TABLE t_reload (key Int) ENGINE = MergeTree ORDER BY key SETTINGS storage_policy = '{policy}'" + ) + node1.replace_in_config(config, "true", "false") + try: + node1.query("SYSTEM RELOAD CONFIG") + err = node1.query_and_get_error( + f"BACKUP TABLE t_reload TO S3('http://minio1:9001/root/data/backups/{uuid.uuid4().hex}', 'minio', '{minio_secret_key}')" + ) + assert "INVALID_CONFIG_PARAMETER" in err and "is not read-only" in err, err + finally: + node1.replace_in_config(config, "false", "true") + node1.query("SYSTEM RELOAD CONFIG") + node1.query("DROP TABLE IF EXISTS t_reload SYNC") + + +def test_replicated_merge_tree_ignores_the_flag(start_cluster): + # `RESTORE` creates a `ReplicatedMergeTree` table in a mode that refuses the parts it finds on the storage, so such a + # table on a storage policy marked `backup_metadata_only` couldn't be restored from a backup of its definition: the flag + # is ignored for it, and it is backed up and restored as usual, with a new UUID. Master also warns, naming the engine; + # that warning is left out on this branch, so only the behavior is asserted. + policy = "policy_s3_plain_backup_metadata_only" + node1.query( + f""" + CREATE TABLE t_replicated (key Int) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{uuid.uuid4().hex}', 'r1') + ORDER BY key SETTINGS storage_policy = '{policy}' + """ + ) + try: + backup_destination = f"S3('http://minio1:9001/root/data/backups/{uuid.uuid4().hex}', 'minio', '{minio_secret_key}')" + node1.query(f"BACKUP TABLE t_replicated TO {backup_destination}") + uuid_before = node1.query("SELECT uuid FROM system.tables WHERE name = 't_replicated'").strip() + node1.query("DROP TABLE t_replicated SYNC") + node1.query(f"RESTORE TABLE t_replicated FROM {backup_destination} SETTINGS structure_only = 1") + assert node1.query("SELECT uuid FROM system.tables WHERE name = 't_replicated'").strip() != uuid_before + finally: + node1.query("DROP TABLE IF EXISTS t_replicated SYNC") + + +def test_log_family_keeps_the_normal_uuid_path(start_cluster): + # `storage_policy` is a builtin setting of the `Log` family too, but those engines keep no data on a storage policy + # and never take the metadata-only data-skip path. They keep their ordinary backup and restore behavior, which + # includes giving a restored copy a new UUID: preserving the source's would collide with the table it was taken from. + policy = "policy_s3_plain_backup_metadata_only" + node1.query(f"CREATE TABLE t_log (key Int) ENGINE = Log SETTINGS storage_policy = '{policy}'") + try: + backup_destination = f"S3('http://minio1:9001/root/data/backups/{uuid.uuid4().hex}', 'minio', '{minio_secret_key}')" + node1.query(f"BACKUP TABLE t_log TO {backup_destination}") + # The source is left in place on purpose: a preserved UUID is refused as already taken. + node1.query(f"RESTORE TABLE t_log AS t_log_copy FROM {backup_destination} SETTINGS structure_only = 1") + distinct_uuids = node1.query( + "SELECT countDistinct(uuid) FROM system.tables WHERE database = 'default' AND name IN ('t_log', 't_log_copy')" + ).strip() + assert distinct_uuids == "2", distinct_uuids + finally: + node1.query("DROP TABLE IF EXISTS t_log SYNC") + node1.query("DROP TABLE IF EXISTS t_log_copy SYNC") diff --git a/tests/integration/test_checking_s3_blobs_paranoid/test.py b/tests/integration/test_checking_s3_blobs_paranoid/test.py index 805e2b8e827e4..4e302288cf945 100644 --- a/tests/integration/test_checking_s3_blobs_paranoid/test.py +++ b/tests/integration/test_checking_s3_blobs_paranoid/test.py @@ -308,6 +308,7 @@ def test_when_error_is_retried(cluster, broken_s3, action_and_message): SETTINGS s3_max_single_part_upload_size=100, s3_min_upload_part_size=100, + s3_max_inflight_parts_for_one_file=1, s3_check_objects_after_upload=0 """, query_id=insert_query_id, @@ -316,6 +317,10 @@ def test_when_error_is_retried(cluster, broken_s3, action_and_message): assert "Code: 499" in error, error assert message in error, error + _, _, s3_errors = get_multipart_counters(node, insert_query_id) + # s3_retry_attempts + 1 attempts of the failing part, plus the failed abort of the fake upload + assert s3_errors == 5 + 1 + 1, s3_errors + def test_when_s3_broken_pipe_at_upload_is_retried(cluster, broken_s3): node = cluster.instances["node"] @@ -389,6 +394,49 @@ def test_when_s3_broken_pipe_at_upload_is_retried(cluster, broken_s3): ), error +def test_early_error_response_at_upload_is_not_retried(cluster, broken_s3): + node = cluster.instances["node"] + + broken_s3.setup_fake_multpartuploads() + broken_s3.setup_at_part_upload( + count=1000, + after=0, + action="no_such_upload", + action_args=["1"], + ) + + insert_query_id = randomize_query_id("TEST_EARLY_ERROR_RESPONSE_AT_UPLOAD") + error = node.query_and_get_error( + f""" + INSERT INTO + TABLE FUNCTION s3( + 'http://resolver:8083/root/data/test_early_error_response_at_upload_is_not_retried', + 'minio', '{minio_secret_key}', + 'CSV', auto, 'none' + ) + SELECT number, randomString(1000) FROM numbers(40000) + SETTINGS + s3_max_single_part_upload_size=100, + s3_min_upload_part_size=33554432, + s3_max_inflight_parts_for_one_file=1, + max_remote_write_network_bandwidth=1000000, + s3_check_objects_after_upload=0 + """, + query_id=insert_query_id, + ) + + assert "Code: 499" in error, error + assert "The specified upload does not exist" in error, error + + create_multipart, upload_parts, s3_errors = get_multipart_counters( + node, insert_query_id + ) + assert create_multipart == 1 + assert upload_parts == 1 + # the early 404 and the failed abort of the fake upload + assert s3_errors == 2, s3_errors + + @pytest.mark.parametrize("send_something", [True, False]) def test_when_s3_connection_reset_by_peer_at_upload_is_retried( cluster, broken_s3, send_something diff --git a/tests/integration/test_concurrent_threads_soft_limit/test.py b/tests/integration/test_concurrent_threads_soft_limit/test.py index b06f90664a4cd..3d39a8c2e9d7a 100644 --- a/tests/integration/test_concurrent_threads_soft_limit/test.py +++ b/tests/integration/test_concurrent_threads_soft_limit/test.py @@ -333,3 +333,72 @@ def limit_reached(): assert count < 6 node4.query("KILL QUERY WHERE query_id = 'background_query' SYNC") background_thread.join() + + +# A query that starts while the limit is reached gets only its non-competing slot. When the other +# query finishes, the released slots are granted to it and it must start using them, even if its +# pipeline never pushes new tasks to the executor's global queue (each stream is an independent +# chain whose steps are slow, as in object storage scans). +def test_concurrent_threads_soft_limit_upscale_after_release(started_cluster): + background_id = "background_query_upscale" + foreground_id = "test_concurrent_threads_soft_limit_5" + errors = {} + + def run(name, query, query_id): + try: + node4.query(query, query_id=query_id) + except QueryRuntimeException as e: + errors[name] = e + + def wait_for(condition, what, timeout=60): + deadline = time.monotonic() + timeout + while not condition(): + assert time.monotonic() < deadline, f"Timed out waiting for {what}, errors: {errors}" + time.sleep(0.1) + + def foreground_event(profile_event): + value = node4.query( + f"SELECT ProfileEvents['{profile_event}'] FROM system.processes WHERE query_id = '{foreground_id}'" + ).strip() + return int(value) if value else None + + background = threading.Thread( + target=run, + args=("background", "SELECT count(*) FROM numbers_mt(1e11) SETTINGS max_threads=100", background_id), + ) + # 8 streams; every step of a stream sleeps 1 s, so a thread that runs only local tasks pushes to the + # global queue only after more than 20 s. The query runs until it is killed. + foreground = threading.Thread( + target=run, + args=( + "foreground", + "SELECT count() FROM numbers_mt(1e9) WHERE NOT ignore(sleepEachRow(0.02)) " + "SETTINGS max_threads=8, max_block_size=50", + foreground_id, + ), + ) + try: + background.start() + wait_for( + lambda: int(node4.query("SELECT sum(length(thread_ids)) FROM system.processes").strip() or 0) >= 11, + "the background query to take all slots", + ) + + foreground.start() + wait_for(lambda: foreground_event("ConcurrencyControlSlotsDelayed"), "the foreground query to wait for slots") + + # Release all competing slots. The foreground query has to start threads for them at the next + # step (~1 s), long before its threads push to the global queue on their own. + node4.query(f"KILL QUERY WHERE query_id = '{background_id}' SYNC") + wait_for( + lambda: (foreground_event("ConcurrencyControlSlotsAcquired") or 0) >= 4, + "the foreground query to use the released slots", + timeout=8, + ) + finally: + try: + node4.query(f"KILL QUERY WHERE query_id IN ('{background_id}', '{foreground_id}') SYNC") + finally: + for thread in (background, foreground): + if thread.ident is not None: + thread.join() diff --git a/tests/integration/test_crash_log/test.py b/tests/integration/test_crash_log/test.py index 8d88e7afb79fe..fbe94ccc39932 100644 --- a/tests/integration/test_crash_log/test.py +++ b/tests/integration/test_crash_log/test.py @@ -69,9 +69,14 @@ def test_crash_log_synchronous(started_node): wait_for_clickhouse_stop(started_node) started_node.restart_clickhouse() crashes_count += 1 + # Report the rows that are there. Without them a lost row leaves only `'1' == '2'` behind, + # which says nothing about which of the two crashes failed to record itself. assert ( started_node.query("SELECT COUNT(*) FROM system.crash_log") == f"{crashes_count}\n" + ), started_node.query( + "SELECT event_time_microseconds, signal, signal_description, query " + "FROM system.crash_log ORDER BY event_time_microseconds FORMAT Vertical" ) diff --git a/tests/integration/test_create_union_system_log_tables/configs/prepare_on_startup.xml b/tests/integration/test_create_union_system_log_tables/configs/prepare_on_startup.xml new file mode 100644 index 0000000000000..af9d6b3ee639e --- /dev/null +++ b/tests/integration/test_create_union_system_log_tables/configs/prepare_on_startup.xml @@ -0,0 +1,4 @@ + + + 1 + diff --git a/tests/integration/test_create_union_system_log_tables/test.py b/tests/integration/test_create_union_system_log_tables/test.py index c170b9b90a0fc..87628d14645fe 100644 --- a/tests/integration/test_create_union_system_log_tables/test.py +++ b/tests/integration/test_create_union_system_log_tables/test.py @@ -34,6 +34,17 @@ stay_alive=True, ) +# Starts with the first release that has union tables and is upgraded. The log tables are prepared +# during startup, and drops are asynchronous (the default). +node4 = cluster.add_instance( + "node4", + main_configs=["configs/union_merge.xml", "configs/prepare_on_startup.xml"], + image="clickhouse/clickhouse-server", + tag="26.8", + with_installed_binary=True, + stay_alive=True, +) + @pytest.fixture(scope="module", autouse=True) def start_cluster(): @@ -152,6 +163,19 @@ def test_stable_across_restarts(start_cluster): assert uuid_before == uuid_after +def test_replaced_during_upgrade(start_cluster): + node4.query("SYSTEM FLUSH LOGS trace_log") + old_create_query = node4.query("SHOW CREATE TABLE system.all_trace_log FORMAT TSVRaw") + + # `all_trace_log` contains the build id, so the first start of another build replaces it, and it does so + # before loading the tables left in `metadata_dropped` by previous runs. The replaced table is dropped + # by this run and must not be loaded as one of them. + node4.restart_with_latest_version() + + assert node4.query("SHOW CREATE TABLE system.all_trace_log FORMAT TSVRaw") != old_create_query + assert not node4.contains_in_log("UUID collision", from_host=True) + + def test_cluster(start_cluster): node1.query("SELECT 'test_cluster_marker_node1'") node1.query("SYSTEM FLUSH LOGS query_log") diff --git a/tests/integration/test_dotnet_client/test.py b/tests/integration/test_dotnet_client/test.py index 4d23f3296f047..149e57eefa5c3 100644 --- a/tests/integration/test_dotnet_client/test.py +++ b/tests/integration/test_dotnet_client/test.py @@ -49,11 +49,14 @@ def test_dotnet_client(started_cluster, dotnet_container): reference = fp.read() code, (stdout, stderr) = dotnet_container.exec_run( - "dotnet run --host {host} --port {port} --user default --password 123 --database default".format( + "dotnet /client/out/clickhouse.test.dll --host {host} --port {port} --user default --password 123 --database default".format( host=started_cluster.get_instance_ip("node"), port=8123 ), demux=True, ) - assert code == 0 - assert stdout == reference + # These two streams exist nowhere else: the client runs in its own container and nothing in the + # job collects its output. Reporting them here is what makes a client that died on a signal + # diagnosable at all, instead of leaving a bare exit code behind. + assert code == 0, f"stdout:\n{stdout!r}\nstderr:\n{stderr!r}" + assert stdout == reference, f"stderr:\n{stderr!r}" diff --git a/tests/integration/test_git_import/test.py b/tests/integration/test_git_import/test.py index 51558ea4ee377..dc7add2acfc8c 100644 --- a/tests/integration/test_git_import/test.py +++ b/tests/integration/test_git_import/test.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import logging +import re import time import pytest @@ -40,8 +41,24 @@ def clone_git_repository(repo, dir, commit=None): time.sleep(1) +# `ClickHouseCluster` gives the whole container the running server's profraw merge pool +# (`LLVM_PROFILE_FILE=/debug/it-%c%4m.profraw`), which every `docker exec`-ed process inherits. +# A second writer cannot merge into a continuous-mode profile - it records its own writer's counter +# bias - so in a coverage build the profiling runtime writes +# `LLVM Profile Warning: Unable to merge profile data: source profile file is not compatible.` and +# two `File exists` errors to stderr. Give this one-shot process a file of its own: its coverage is +# still collected and it no longer touches the server's pool. +GIT_IMPORT_PROFILE_FILE = "/debug/git-import-%p.profraw" + +# One progress line per processed commit, which `git-import` writes to stderr. +COMMIT_PROGRESS_RE = re.compile(r"^\d+% \d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2} [0-9a-f]{40} ") + + def run_git_import(dir): - command = f"cd {dir} && /usr/bin/clickhouse git-import 2>&1" + command = ( + f"cd {dir} && LLVM_PROFILE_FILE={GIT_IMPORT_PROFILE_FILE} " + "/usr/bin/clickhouse git-import 2>&1" + ) return node.exec_in_container(["bash", "-c", command]) @@ -176,7 +193,15 @@ def test_git_import(): create_tables() insert_into_tables(dir) - assert output.count("\n") == 26 + # `git-import` prints the `git log` command it runs and the commit count to stdout, and one + # progress line per commit to stderr, which the `2>&1` above merges into the same stream. Match + # those shapes instead of counting every newline, so that a line written by anything other than + # `git-import` - a profiling or sanitizer runtime, a loader warning - fails the thing it + # actually broke rather than this count. + lines = output.splitlines() + assert "git log --reverse --no-merges --pretty=%H" in lines, output + assert "Total 24 commits to process." in lines, output + assert sum(1 for line in lines if COMMIT_PROGRESS_RE.match(line)) == 24, output assert node.query("SELECT count() FROM commits") == "24\n" assert node.query("SELECT count() FROM file_changes") == "35\n" assert node.query("SELECT count(), round(avg(indent), 1) FROM line_changes") == TSV( diff --git a/tests/integration/test_jemalloc_percpu_arena/test.py b/tests/integration/test_jemalloc_percpu_arena/test.py index 761d5357349cc..7829fddc9f8c6 100755 --- a/tests/integration/test_jemalloc_percpu_arena/test.py +++ b/tests/integration/test_jemalloc_percpu_arena/test.py @@ -52,6 +52,19 @@ def run_with_cpu_limit(cmd, *args): return run_command_in_container(cmd, *args) +def run_with_cpu_possible_limit(cmd, possible, *args): + with NamedTemporaryFile() as possible_cpu: + possible_cpu.write(f"{possible}".encode()) + possible_cpu.flush() + + args += ( + "--volume", + f"{possible_cpu.name}:/sys/devices/system/cpu/possible:ro", + ) + + return run_command_in_container(cmd, *args) + + def skip_if_jemalloc_disabled(): output = run_command_in_container( """clickhouse local -q " @@ -81,28 +94,33 @@ def test_jemalloc_percpu_arena(): all_cpus = int(run_with_cpu_limit("getconf _NPROCESSORS_CONF")) assert all_cpus == multiprocessing.cpu_count(), all_cpus - # implicitly disable percpu arena + # smoke under the online override result = run_with_cpu_limit( 'clickhouse local -q "select 1"', - # NOTE: explicitly disable, since it is enabled by default in debug build - # (and even though debug builds are not in CI let's state this). "--env", "MALLOC_CONF=abort_conf:false", ) assert int(result) == int(1), result - # jemalloc compares sysconf(_SC_NPROCESSORS_ONLN) and sysconf(_SC_NPROCESSORS_CONF) to determine the CPU topology info - # Apparently sometimes _SC_NPROCESSORS_ONLN SOMETIMES ignores the override of /sys/devices/system/cpu/online and it should be equal to - # _SC_NPROCESSORS_CONF - # Latter it also checks sched_getaffinity() + CPU_COUNT(&set), so, we rely also on --cpuset-cpus (taskset) in order to trigger the - # second condition - with pytest.raises(subprocess.CalledProcessError): - # should fail because of abort_conf:true but: - run_with_cpu_limit( - 'clickhouse local -q "select 1"', - "--cpuset-cpus", f"{CPU_ID}", - "--env", "MALLOC_CONF=abort_conf:true", - ) + # Restricted affinity (cpuset) on top of the online override: percpu arenas + # are sized by the possible CPU count (nr_cpu_ids), so this must not abort + # even with abort_conf:true. + # NOTE: abort:false to make it compatible with debug build + result = run_with_cpu_limit( + 'clickhouse local -q "select 1"', + "--cpuset-cpus", f"{CPU_ID}", + "--env", "MALLOC_CONF=abort_conf:true,abort:false", + ) + assert int(result) == int(1), result + + # ... and percpu arena must stay enabled (the old behavior was to disable + # it with "Number of CPUs detected is not deterministic"). + stats = run_with_cpu_limit( + '''clickhouse local -q "select 1" 2>&1 | grep -F "opt.percpu_arena"''', + "--cpuset-cpus", f"{CPU_ID}", + "--env", "MALLOC_CONF=abort_conf:true,abort:false,stats_print:true", + ) + assert b'"percpu"' in stats, stats # should not fail even with abort_conf:true, due to explicit narenas # NOTE: abort:false to make it compatible with debug build @@ -112,6 +130,25 @@ def test_jemalloc_percpu_arena(): ) +# If /sys/devices/system/cpu/possible is masked and does not cover the CPUs in +# the affinity mask, it is lying and percpu arena cannot be sized safely: +# jemalloc must fail close (disable percpu arena, abort with abort_conf:true) +# instead of indexing arenas out of range. +def test_jemalloc_percpu_arena_inconsistent_possible(): + skip_if_jemalloc_disabled() + + assert multiprocessing.cpu_count() > CPU_ID + + with pytest.raises(subprocess.CalledProcessError): + run_with_cpu_possible_limit( + 'clickhouse local -q "select 1"', + "0", + "--cpuset-cpus", f"{CPU_ID}", + "--env", "MALLOC_CONF=abort_conf:true", + ) + + # For manual run. if __name__ == "__main__": test_jemalloc_percpu_arena() + test_jemalloc_percpu_arena_inconsistent_possible() diff --git a/tests/integration/test_keeper_persistent_watches/test.py b/tests/integration/test_keeper_persistent_watches/test.py index 733e77063d9c5..c32d713242677 100644 --- a/tests/integration/test_keeper_persistent_watches/test.py +++ b/tests/integration/test_keeper_persistent_watches/test.py @@ -4,6 +4,7 @@ import os import threading import re +import struct import helpers.keeper_utils as keeper_utils from helpers.cluster import ClickHouseCluster @@ -341,3 +342,57 @@ def cb(_): destroy_zk_client(client) data = keeper_utils.send_4lw_cmd(cluster, node1, cmd="mntr") assert "zk_watch_count\t0" in data + +CHECK_WATCH, REMOVE_WATCH, ADD_WATCH = 17, 18, 106 +ZOK, ZBADARGUMENTS, ZNOWATCHER = 0, -8, -121 +# (opnum, type or mode, expected error): types are 1..5 and modes 0..1, so the last request of each op is valid. +UNKNOWN_TYPE_REQUESTS = [ + (CHECK_WATCH, 0, ZBADARGUMENTS), + (CHECK_WATCH, 6, ZBADARGUMENTS), + (CHECK_WATCH, 1, ZNOWATCHER), + (REMOVE_WATCH, 0, ZBADARGUMENTS), + (REMOVE_WATCH, 6, ZBADARGUMENTS), + (REMOVE_WATCH, 1, ZNOWATCHER), + (ADD_WATCH, -1, ZBADARGUMENTS), + (ADD_WATCH, 2, ZBADARGUMENTS), + (ADD_WATCH, 0, ZOK), +] + +def recv_exactly(sock, size): + data = b"" + while len(data) < size: + chunk = sock.recv(size - len(data)) + if not chunk: + raise EOFError("Keeper closed the connection") + data += chunk + return data + +def send_unknown_type_requests(): + with get_keeper_socket(node1.name) as sock: + handshake = struct.pack(">iqiqi16s", 0, 0, 30000, 0, 16, b"\0" * 16) + sock.sendall(struct.pack(">i", len(handshake)) + handshake) + reply = recv_exactly(sock, struct.unpack(">i", recv_exactly(sock, 4))[0]) + assert struct.unpack(">q", reply[8:16])[0] != 0 + path = b"/testUnknownWatchType" + for xid, (opnum, value, expected) in enumerate(UNKNOWN_TYPE_REQUESTS, start=1): + body = struct.pack(">iii", xid, opnum, len(path)) + path + struct.pack(">i", value) + sock.sendall(struct.pack(">i", len(body)) + body) + reply = recv_exactly(sock, struct.unpack(">i", recv_exactly(sock, 4))[0]) + reply_xid, _, error = struct.unpack(">iqi", reply[:16]) + assert (reply_xid, error) == (xid, expected), (opnum, value) + +def test_watch_requests_with_unknown_type(started_cluster): + # Through Raft (`quorum_reads`), then a restart that replays the logged requests, then through the local read path. + config = "/etc/clickhouse-server/config.d/keeper_config1.xml" + try: + node1.replace_in_config(config, "", "1") + node1.restart_clickhouse() + keeper_utils.wait_until_connected(cluster, node1) + assert "quorum_reads=true" in keeper_utils.send_4lw_cmd(cluster, node1, cmd="conf") + send_unknown_type_requests() + finally: + node1.replace_in_config(config, "1", "") + node1.restart_clickhouse(kill=True) + keeper_utils.wait_until_connected(cluster, node1) + assert "quorum_reads=false" in keeper_utils.send_4lw_cmd(cluster, node1, cmd="conf") + send_unknown_type_requests() diff --git a/tests/integration/test_mask_sensitive_info/test.py b/tests/integration/test_mask_sensitive_info/test.py index 24b95f0764938..031e83f12ea19 100644 --- a/tests/integration/test_mask_sensitive_info/test.py +++ b/tests/integration/test_mask_sensitive_info/test.py @@ -725,13 +725,13 @@ def test_table_functions_object_storage_cluster(): f"icebergS3Cluster('{ch_cluster}', '{s3_url}', '{s3_access_key_id}', '{s3_secret_access_key}')", f"icebergS3Cluster('{ch_cluster}', {named_collection}, url = '{s3_url}', access_key_id = '{s3_access_key_id}', secret_access_key = '{s3_secret_access_key}')", f"azureBlobStorageCluster('{ch_cluster}', '{azure_storage_account_url}', 'test', 'test', '{azure_account_name}', '{azure_account_key}')", - f"azureBlobStorageCluster('{ch_cluster}', {named_collection}, storage_account_url = '{azure_storage_account_url}', container_name = 'test', blobpath = 'test', account_name = '{azure_account_name}', account_key = '{azure_account_key}')", + f"azureBlobStorageCluster('{ch_cluster}', {named_collection}, storage_account_url = '{azure_storage_account_url}', container = 'test', blob_path = 'test', account_name = '{azure_account_name}', account_key = '{azure_account_key}')", f"azureBlobStorageCluster('{ch_cluster}', '{azure_connection_string}', 'test', 'test')", - f"azureBlobStorageCluster('{ch_cluster}', {named_collection}, connection_string = '{azure_connection_string}', container_name = 'test', blobpath = 'test')", + f"azureBlobStorageCluster('{ch_cluster}', {named_collection}, connection_string = '{azure_connection_string}', container = 'test', blob_path = 'test')", f"icebergAzureCluster('{ch_cluster}', '{azure_storage_account_url}', 'test', 'test', '{azure_account_name}', '{azure_account_key}')", - f"icebergAzureCluster('{ch_cluster}', {named_collection}, storage_account_url = '{azure_storage_account_url}', container_name = 'test', blobpath = 'test', account_name = '{azure_account_name}', account_key = '{azure_account_key}')", + f"icebergAzureCluster('{ch_cluster}', {named_collection}, storage_account_url = '{azure_storage_account_url}', container = 'test', blob_path = 'test', account_name = '{azure_account_name}', account_key = '{azure_account_key}')", f"icebergAzureCluster('{ch_cluster}', '{azure_connection_string}', 'test', 'test')", - f"icebergAzureCluster('{ch_cluster}', {named_collection}, connection_string = '{azure_connection_string}', container_name = 'test', blobpath = 'test')", + f"icebergAzureCluster('{ch_cluster}', {named_collection}, connection_string = '{azure_connection_string}', container = 'test', blob_path = 'test')", ] # DeltaLake table functions are not available under Memory Sanitizer (Rust code is not instrumented) @@ -742,9 +742,9 @@ def test_table_functions_object_storage_cluster(): f"deltaLakeS3Cluster('{ch_cluster}', '{s3_url}', '{s3_access_key_id}', '{s3_secret_access_key}')", f"deltaLakeS3Cluster('{ch_cluster}', {named_collection}, url = '{s3_url}', access_key_id = '{s3_access_key_id}', secret_access_key = '{s3_secret_access_key}')", f"deltaLakeAzureCluster('{ch_cluster}', '{azure_storage_account_url}', 'test', 'test', '{azure_account_name}', '{azure_account_key}')", - f"deltaLakeAzureCluster('{ch_cluster}', {named_collection}, storage_account_url = '{azure_storage_account_url}', container_name = 'test', blobpath = 'test', account_name = '{azure_account_name}', account_key = '{azure_account_key}')", + f"deltaLakeAzureCluster('{ch_cluster}', {named_collection}, storage_account_url = '{azure_storage_account_url}', container = 'test', blob_path = 'test', account_name = '{azure_account_name}', account_key = '{azure_account_key}')", f"deltaLakeAzureCluster('{ch_cluster}', '{azure_connection_string}', 'test', 'test')", - f"deltaLakeAzureCluster('{ch_cluster}', {named_collection}, connection_string = '{azure_connection_string}', container_name = 'test', blobpath = 'test')", + f"deltaLakeAzureCluster('{ch_cluster}', {named_collection}, connection_string = '{azure_connection_string}', container = 'test', blob_path = 'test')", ] if has_paimon: @@ -764,13 +764,13 @@ def test_table_functions_object_storage_cluster(): f"icebergS3Cluster('{ch_cluster}', '{s3_url}', '{s3_access_key_id}', '[HIDDEN]')", f"icebergS3Cluster('{ch_cluster}', {named_collection}, url = '{s3_url}', access_key_id = '{s3_access_key_id}', secret_access_key = '[HIDDEN]')", f"azureBlobStorageCluster('{ch_cluster}', '{azure_storage_account_url}', 'test', 'test', '{azure_account_name}', '[HIDDEN]')", - f"azureBlobStorageCluster('{ch_cluster}', {named_collection}, storage_account_url = '{azure_storage_account_url}', container_name = 'test', blobpath = 'test', account_name = '{azure_account_name}', account_key = '[HIDDEN]')", + f"azureBlobStorageCluster('{ch_cluster}', {named_collection}, storage_account_url = '{azure_storage_account_url}', container = 'test', blob_path = 'test', account_name = '{azure_account_name}', account_key = '[HIDDEN]')", f"azureBlobStorageCluster('{ch_cluster}', '{masked_azure_connection_string}', 'test', 'test')", - f"azureBlobStorageCluster('{ch_cluster}', {named_collection}, connection_string = '{masked_azure_connection_string}', container_name = 'test', blobpath = 'test')", + f"azureBlobStorageCluster('{ch_cluster}', {named_collection}, connection_string = '{masked_azure_connection_string}', container = 'test', blob_path = 'test')", f"icebergAzureCluster('{ch_cluster}', '{azure_storage_account_url}', 'test', 'test', '{azure_account_name}', '[HIDDEN]')", - f"icebergAzureCluster('{ch_cluster}', {named_collection}, storage_account_url = '{azure_storage_account_url}', container_name = 'test', blobpath = 'test', account_name = '{azure_account_name}', account_key = '[HIDDEN]')", + f"icebergAzureCluster('{ch_cluster}', {named_collection}, storage_account_url = '{azure_storage_account_url}', container = 'test', blob_path = 'test', account_name = '{azure_account_name}', account_key = '[HIDDEN]')", f"icebergAzureCluster('{ch_cluster}', '{masked_azure_connection_string}', 'test', 'test')", - f"icebergAzureCluster('{ch_cluster}', {named_collection}, connection_string = '{masked_azure_connection_string}', container_name = 'test', blobpath = 'test')", + f"icebergAzureCluster('{ch_cluster}', {named_collection}, connection_string = '{masked_azure_connection_string}', container = 'test', blob_path = 'test')", ] if has_delta_lake: @@ -780,9 +780,9 @@ def test_table_functions_object_storage_cluster(): f"deltaLakeS3Cluster('{ch_cluster}', '{s3_url}', '{s3_access_key_id}', '[HIDDEN]')", f"deltaLakeS3Cluster('{ch_cluster}', {named_collection}, url = '{s3_url}', access_key_id = '{s3_access_key_id}', secret_access_key = '[HIDDEN]')", f"deltaLakeAzureCluster('{ch_cluster}', '{azure_storage_account_url}', 'test', 'test', '{azure_account_name}', '[HIDDEN]')", - f"deltaLakeAzureCluster('{ch_cluster}', {named_collection}, storage_account_url = '{azure_storage_account_url}', container_name = 'test', blobpath = 'test', account_name = '{azure_account_name}', account_key = '[HIDDEN]')", + f"deltaLakeAzureCluster('{ch_cluster}', {named_collection}, storage_account_url = '{azure_storage_account_url}', container = 'test', blob_path = 'test', account_name = '{azure_account_name}', account_key = '[HIDDEN]')", f"deltaLakeAzureCluster('{ch_cluster}', '{masked_azure_connection_string}', 'test', 'test')", - f"deltaLakeAzureCluster('{ch_cluster}', {named_collection}, connection_string = '{masked_azure_connection_string}', container_name = 'test', blobpath = 'test')", + f"deltaLakeAzureCluster('{ch_cluster}', {named_collection}, connection_string = '{masked_azure_connection_string}', container = 'test', blob_path = 'test')", ] if has_paimon: diff --git a/tests/integration/test_parallel_replicas_insert_select/configs/remote_servers.xml b/tests/integration/test_parallel_replicas_insert_select/configs/remote_servers.xml index 89438a1ee76c9..35f7e776d71fc 100644 --- a/tests/integration/test_parallel_replicas_insert_select/configs/remote_servers.xml +++ b/tests/integration/test_parallel_replicas_insert_select/configs/remote_servers.xml @@ -34,6 +34,19 @@ + + + true + + node1 + 9000 + + + node2 + 1234 + + + diff --git a/tests/integration/test_parallel_replicas_insert_select/test.py b/tests/integration/test_parallel_replicas_insert_select/test.py index d92065f46bd02..c56ce0759dcf2 100644 --- a/tests/integration/test_parallel_replicas_insert_select/test.py +++ b/tests/integration/test_parallel_replicas_insert_select/test.py @@ -328,3 +328,123 @@ def test_insert_select_where(start_cluster, max_parallel_replicas, parallel_repl settings={"skip_unavailable_shards": 1}, ) assert (int(number_of_queries) > 1) + + +def create_replicated_database(db): + for i, node in enumerate([node1, node2, node3], start=1): + node.query( + f"CREATE DATABASE {db} ENGINE = Replicated('/clickhouse/databases/{db}', 'shard1', 'node{i}')" + ) + + +def sync_database(db): + for node in [node1, node2, node3]: + node.query(f"SYSTEM SYNC DATABASE REPLICA {db}") + + +def drop_database(db): + for node in [node1, node2, node3]: + node.query(f"DROP DATABASE IF EXISTS {db} SYNC") + + +def create_replicated_source(db): + node1.query(f"CREATE TABLE {db}.src (id UInt64) ENGINE = ReplicatedMergeTree ORDER BY id") + sync_database(db) + node1.query(f"INSERT INTO {db}.src SELECT number FROM numbers(1000)") + for node in [node2, node3]: + node.query(f"SYSTEM SYNC REPLICA {db}.src") + + +def populate_settings(cluster_name, max_parallel_replicas, parallel_distributed_insert_select=2): + return { + "enable_parallel_replicas": 1, + "max_parallel_replicas": max_parallel_replicas, + "parallel_distributed_insert_select": parallel_distributed_insert_select, + "parallel_replicas_local_plan": 0, + "cluster_for_parallel_replicas": cluster_name, + } + + +def query_per_node(query): + return [node.query(query) for node in [node1, node2, node3]] + + +# Every replica of a Replicated database populates its own view, not the view of another replica. +def test_replicated_database_populate(start_cluster): + db = f"db_{uuid.uuid4().hex}" + try: + create_replicated_database(db) + create_replicated_source(db) + node1.query( + f"CREATE MATERIALIZED VIEW {db}.mv ENGINE = MergeTree ORDER BY id POPULATE AS SELECT id FROM {db}.src", + settings=populate_settings("test_1_shard_2_replicas_1_unavailable", 2), + ) + sync_database(db) + assert query_per_node(f"SELECT count(), uniqExact(id), sum(id) FROM {db}.mv") == ["1000\t1000\t499500\n"] * 3 + finally: + drop_database(db) + + +# The initiator populates its view while the other replicas have not created theirs yet. +def test_replicated_database_populate_before_other_replicas(start_cluster): + db = f"db_{uuid.uuid4().hex}" + try: + create_replicated_database(db) + create_replicated_source(db) + for node in [node2, node3]: + node.query("SYSTEM ENABLE FAILPOINT database_replicated_stop_entry_execution") + node1.query( + f"CREATE MATERIALIZED VIEW {db}.mv ENGINE = MergeTree ORDER BY id POPULATE AS SELECT id FROM {db}.src", + settings={**populate_settings("test_1_shard_3_replicas", 3), "distributed_ddl_task_timeout": 0}, + ) + for node in [node2, node3]: + node.query("SYSTEM DISABLE FAILPOINT database_replicated_stop_entry_execution") + sync_database(db) + assert query_per_node(f"SELECT count(), uniqExact(id), sum(id) FROM {db}.mv") == ["1000\t1000\t499500\n"] * 3 + finally: + for node in [node2, node3]: + node.query("SYSTEM DISABLE FAILPOINT database_replicated_stop_entry_execution") + drop_database(db) + + +# `CREATE TABLE ... AS SELECT` fills the table of every replica the same way. +def test_replicated_database_create_as_select(start_cluster): + db = f"db_{uuid.uuid4().hex}" + try: + create_replicated_database(db) + create_replicated_source(db) + node1.query( + f"CREATE TABLE {db}.t ENGINE = Memory AS SELECT id FROM {db}.src", + settings=populate_settings("test_1_shard_2_replicas_1_unavailable", 2), + ) + sync_database(db) + assert query_per_node(f"SELECT count(), uniqExact(id), sum(id) FROM {db}.t") == ["1000\t1000\t499500\n"] * 3 + finally: + drop_database(db) + + +# The population reads the source of the replica executing it, also when the view's own `SELECT` enables parallel replicas. +def test_replicated_database_populate_reads_local_source(start_cluster): + db = f"db_{uuid.uuid4().hex}" + try: + create_replicated_database(db) + node1.query(f"CREATE TABLE {db}.src_local (id UInt64) ENGINE = MergeTree ORDER BY id") + sync_database(db) + for i, node in enumerate([node1, node2, node3]): + node.query(f"INSERT INTO {db}.src_local SELECT {i * 1000} + number FROM numbers(1000)") + node1.query( + f"CREATE MATERIALIZED VIEW {db}.mv ENGINE = MergeTree ORDER BY id POPULATE AS SELECT id FROM {db}.src_local " + "SETTINGS allow_experimental_parallel_reading_from_replicas = 1", + settings={ + **populate_settings("test_1_shard_2_replicas_1_unavailable", 2, parallel_distributed_insert_select=0), + "parallel_replicas_for_non_replicated_merge_tree": 1, + }, + ) + sync_database(db) + assert query_per_node(f"SELECT count(), min(id), max(id) FROM {db}.mv") == [ + "1000\t0\t999\n", + "1000\t1000\t1999\n", + "1000\t2000\t2999\n", + ] + finally: + drop_database(db) diff --git a/tests/integration/test_refreshable_mv_definer_interserver_secret/__init__.py b/tests/integration/test_refreshable_mv_definer_interserver_secret/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_refreshable_mv_definer_interserver_secret/configs/remote_servers.xml b/tests/integration/test_refreshable_mv_definer_interserver_secret/configs/remote_servers.xml new file mode 100644 index 0000000000000..362c39c90ed8c --- /dev/null +++ b/tests/integration/test_refreshable_mv_definer_interserver_secret/configs/remote_servers.xml @@ -0,0 +1,11 @@ + + + + foo + + node2 + 9000 + + + + diff --git a/tests/integration/test_refreshable_mv_definer_interserver_secret/test.py b/tests/integration/test_refreshable_mv_definer_interserver_secret/test.py new file mode 100644 index 0000000000000..f94ef84ab15b3 --- /dev/null +++ b/tests/integration/test_refreshable_mv_definer_interserver_secret/test.py @@ -0,0 +1,99 @@ +import pytest + +from helpers.cluster import ClickHouseCluster, QueryRuntimeException + +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance( + "node1", + main_configs=["configs/remote_servers.xml"], + with_zookeeper=True, +) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/remote_servers.xml"], + with_zookeeper=True, +) + +TENANT = {"user": "tenant", "password": "tpass"} + + +@pytest.fixture(scope="module") +def started_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def test_scheduled_refresh_of_definer_view_carries_definer_to_remote_shard( + started_cluster, +): + # Protected data lives only on node2. The tenant exists on both nodes but has no grant on it. + node2.query( + """ + CREATE DATABASE lab; + CREATE USER tenant IDENTIFIED WITH plaintext_password BY 'tpass'; + CREATE TABLE lab.secrets (id UInt64, payload String) ENGINE = MergeTree ORDER BY id; + INSERT INTO lab.secrets VALUES (1, 'a'), (2, 'b'), (3, 'c'); + """ + ) + node1.query( + """ + CREATE DATABASE tenant_db; + CREATE USER tenant IDENTIFIED WITH plaintext_password BY 'tpass'; + GRANT CREATE TABLE, SELECT, INSERT ON tenant_db.* TO tenant; + GRANT TABLE ENGINE ON Distributed TO tenant; + """ + ) + + node1.query( + """ + CREATE TABLE tenant_db.dist (id UInt64, payload String) + ENGINE = Distributed('secure', 'lab', 'secrets', rand()); + CREATE TABLE tenant_db.target + (id UInt64, payload String, current_user String, authenticated_user String) + ENGINE = MergeTree ORDER BY id; + """, + **TENANT, + ) + + # A direct read through the Distributed table is denied on the remote shard. + assert "ACCESS_DENIED" in node1.query_and_get_error( + "SELECT * FROM tenant_db.dist", **TENANT + ) + + # The shard also checks the allowed hosts of the definer against `initial_address`. A scheduled + # refresh has no client, so it must present itself as coming from the local host of the + # initiator rather than from the unspecified address, which only `HOST ANY` users accept. + node2.query("ALTER USER tenant HOST LOCAL") + + # The refresh runs under the definer's identity and must be denied on the remote shard too. + # Before the fix, the refresh query was sent with an empty `initial_user`, which the remote + # shard treated as interserver mode and executed without any access checks. + node1.query( + """ + CREATE MATERIALIZED VIEW tenant_db.rmv REFRESH EVERY 1 YEAR APPEND TO tenant_db.target + DEFINER = CURRENT_USER SQL SECURITY DEFINER + AS SELECT id, payload, currentUser() AS current_user, authenticatedUser() AS authenticated_user + FROM tenant_db.dist; + """, + **TENANT, + ) + + with pytest.raises(QueryRuntimeException, match="ACCESS_DENIED"): + node1.query("SYSTEM WAIT VIEW tenant_db.rmv") + assert node1.query("SELECT count() FROM tenant_db.target") == "0\n" + assert "ACCESS_DENIED" in node1.query( + "SELECT exception FROM system.view_refreshes WHERE database = 'tenant_db' AND view = 'rmv'" + ) + + # Once the definer is granted access on the remote shard, the refresh succeeds and the remote + # shard sees the definer as the current and authenticated user. + node2.query("GRANT SELECT ON lab.secrets TO tenant") + node1.query("SYSTEM REFRESH VIEW tenant_db.rmv") + node1.query("SYSTEM WAIT VIEW tenant_db.rmv") + assert node1.query("SELECT * FROM tenant_db.target ORDER BY id", **TENANT) == ( + "1\ta\ttenant\ttenant\n2\tb\ttenant\ttenant\n3\tc\ttenant\ttenant\n" + ) diff --git a/tests/integration/test_restore_replica_metadata_version/test.py b/tests/integration/test_restore_replica_metadata_version/test.py index b35af4075c11c..4fc23ce116ae7 100644 --- a/tests/integration/test_restore_replica_metadata_version/test.py +++ b/tests/integration/test_restore_replica_metadata_version/test.py @@ -36,6 +36,79 @@ def start_cluster(): cluster.shutdown() +def test_alter_after_concurrent_restore(start_cluster): + zk = cluster.get_kazoo_client("zoo1") + table = "test_concurrent_restore" + zk_path = f"/clickhouse/tables/{table}" + failpoint = "rmt_restore_replica_pause_before_metadata_version_raise" + + for node in [node_1, node_2]: + node.query(f"DROP TABLE IF EXISTS {table} SYNC") + for node in [node_1, node_2]: + node.query( + f"CREATE TABLE {table} (n UInt32) " + f"ENGINE = ReplicatedMergeTree('{zk_path}/', '{node.name}') ORDER BY n" + ) + + # Two ALTERs bring the metadata version of the table, and of the part inserted after them, to 2 + for column in ["a", "b"]: + node_1.query( + f"ALTER TABLE {table} ADD COLUMN {column} UInt32", + settings={"alter_sync": 2}, + ) + node_1.query(f"INSERT INTO {table} VALUES (1, 1, 1)") + node_2.query(f"SYSTEM SYNC REPLICA {table}") + for node in [node_1, node_2]: + assert ( + node.query( + f"SELECT metadata_version FROM system.tables WHERE name = '{table}'" + ) + == "2\n" + ) + + zk_rmr_with_retries(zk, zk_path) + for node in [node_1, node_2]: + node.query(f"SYSTEM RESTART REPLICA {table}") + + # Restore both replicas at the same time, as ON CLUSTER does: node_1 stops after it has + # restored the table in Keeper, before it raises the metadata version to the one of its part, + # and node_2 restores in the meantime by copying node_1 + node_1.query(f"SYSTEM ENABLE FAILPOINT {failpoint}") + restore = node_1.get_query_request(f"SYSTEM RESTORE REPLICA {table}") + node_1.query(f"SYSTEM WAIT FAILPOINT {failpoint} PAUSE") + node_2.query(f"SYSTEM RESTORE REPLICA {table}") + assert ( + node_2.query(f"SELECT is_readonly FROM system.replicas WHERE table = '{table}'") + == "0\n" + ) + node_1.query(f"SYSTEM DISABLE FAILPOINT {failpoint}") + restore.get_answer() + + for node in [node_1, node_2]: + node.query(f"SYSTEM SYNC REPLICA {table}") + + _, stat = zk.get(f"{zk_path}/metadata") + assert stat.version == 2 + for node in [node_1, node_2]: + version, _ = zk.get(f"{zk_path}/replicas/{node.name}/metadata_version") + assert version == b"2" + assert ( + node.query( + f"SELECT metadata_version FROM system.tables WHERE name = '{table}'" + ) + == "2\n" + ) + + # A replica left below the metadata version of the table rejects every ALTER with CANNOT_ASSIGN_ALTER + node_2.query(f"ALTER TABLE {table} ADD COLUMN c UInt32", settings={"alter_sync": 2}) + node_1.query(f"ALTER TABLE {table} ADD COLUMN d UInt32", settings={"alter_sync": 2}) + for node in [node_1, node_2]: + assert node.query(f"SELECT n, a, b, c, d FROM {table}") == "1\t1\t1\t0\t0\n" + + for node in [node_1, node_2]: + node.query(f"DROP TABLE IF EXISTS {table} SYNC") + + def test_fix_metadata_version_on_attach_part_after_restore(start_cluster): zk = cluster.get_kazoo_client("zoo1") diff --git a/tests/integration/test_scheduler_cpu_preemptive/test.py b/tests/integration/test_scheduler_cpu_preemptive/test.py index 6f647f8c9b24e..ec986552e3a18 100644 --- a/tests/integration/test_scheduler_cpu_preemptive/test.py +++ b/tests/integration/test_scheduler_cpu_preemptive/test.py @@ -41,6 +41,8 @@ def clear_workloads_and_resources(): drop workload if exists production; drop workload if exists development; drop workload if exists admin; + drop workload if exists victim; + drop workload if exists competitor; drop workload if exists all; drop resource if exists cpu; """ @@ -628,3 +630,78 @@ def test_create_workload_under_load(): development2.stop() assert development2.get_errors() == 0, "Errors occurred in development2 workload" assert production2.get_errors() == 0, "Errors occurred in production2 workload" + + +# A worker whose preempted CPU lease is not granted back in time is downscaled. When the competing query +# finishes and the capacity is back, the same query starts a thread for it again, without waiting for its +# threads to push to the executor's global queue (each step here takes ~1 s of CPU, and a thread that runs +# only local tasks pushes after 128 of them). +@pytest.mark.parametrize( + "with_custom_config", + [{"node": {"cpu_slot_preemption_timeout_ms": 100}}], + indirect=True, +) +def test_worker_replaced_after_downscale(with_custom_config): + if node.is_built_with_address_sanitizer() or node.is_built_with_thread_sanitizer() or node.is_built_with_llvm_coverage(): + pytest.skip("doesn't fit in timeouts due to heavy workload") + + node.query( + """ + create resource cpu (master thread, worker thread); + create workload all settings max_concurrent_threads=4; + create workload victim in all; + create workload competitor in all; + """ + ) + + victim_id = "test_worker_replaced_after_downscale_victim" + competitor_id = "test_worker_replaced_after_downscale_competitor" + # CPU-bound steps of ~1 s each with little memory; the query runs until it is killed. + hashes = "number" + for _ in range(150): + hashes = f"cityHash64({hashes})" + heavy = f"select count() from numbers_mt(1000000000000) where not ignore({hashes}) settings max_block_size=5000000, max_threads=4" + errors = {} + + def run(name, query_id, workload): + try: + node.query(f"{heavy}, workload='{workload}'", query_id=query_id) + except QueryRuntimeException as e: + errors[name] = e + + def wait_for(condition, what, timeout=60): + deadline = time.monotonic() + timeout + while not condition(): + assert time.monotonic() < deadline, f"Timed out waiting for {what}, errors: {errors}" + time.sleep(0.1) + + def victim_event(profile_event): + value = node.query( + f"select ProfileEvents['{profile_event}'] from system.processes where query_id = '{victim_id}'" + ).strip() + return int(value) if value else 0 + + victim = threading.Thread(target=run, args=("victim", victim_id, "victim")) + competitor = threading.Thread(target=run, args=("competitor", competitor_id, "competitor")) + try: + victim.start() + wait_for(lambda: victim_event("ConcurrencyControlSlotsAcquired") >= 4, "the victim to take all slots") + + # The competitor takes its share, so the victim's workers are preempted and downscaled after the timeout. + competitor.start() + wait_for(lambda: victim_event("ConcurrencyControlDownscales") >= 1, "the victim to be downscaled") + + upscales = victim_event("ConcurrencyControlUpscales") + node.query(f"kill query where query_id = '{competitor_id}' sync") + wait_for( + lambda: victim_event("ConcurrencyControlUpscales") > upscales, + "the victim to start a thread for the released capacity", + timeout=15, + ) + finally: + try: + node.query(f"kill query where query_id in ('{victim_id}', '{competitor_id}') sync") + finally: + for thread in (victim, competitor): + if thread.ident is not None: + thread.join() diff --git a/tests/integration/test_storage_iceberg_no_spark/test_manifest_length_cache_weight.py b/tests/integration/test_storage_iceberg_no_spark/test_manifest_length_cache_weight.py new file mode 100644 index 0000000000000..d90720f797bd4 --- /dev/null +++ b/tests/integration/test_storage_iceberg_no_spark/test_manifest_length_cache_weight.py @@ -0,0 +1,99 @@ +""" +An Iceberg manifest-list entry may declare any `manifest_length`. Reading such a table must not +depend on that number: it used to become the metadata cache cell weight and terminate the server. +""" +import os +import shutil +import tempfile + +from helpers.iceberg_utils import get_uuid_str +from test_storage_iceberg_no_spark.test_local_path_traversal import ( + _download_table_from_container, + _get_manifest_paths_from_chain, + _modify_avro_file, +) + +NOCACHE = { + "use_iceberg_metadata_files_cache": False, + "use_parquet_metadata_cache": False, +} +CACHE = {"use_iceberg_metadata_files_cache": True} +HUGE_MANIFEST_LENGTH = 2**62 + + +def _make_table_with_manifest_length(instance, table_name, table_path, new_length): + """Build a well-formed IcebergLocal table with the cache off, then rewrite every + manifest-list entry's `manifest_length`. new_length=None rewrites the file without + changing the value. Returns the manifest files' real total size on disk.""" + instance.query( + f"CREATE TABLE {table_name} (c0 Int) ENGINE = IcebergLocal('{table_path}/', 'Parquet');", + settings={"allow_insert_into_iceberg": 1, **NOCACHE}, + ) + instance.query( + f"INSERT INTO {table_name} VALUES (1), (2), (3)", + settings={"allow_insert_into_iceberg": 1, **NOCACHE}, + ) + assert instance.query(f"SELECT count() FROM {table_name}", settings=NOCACHE).strip() == "3" + instance.query(f"DROP TABLE {table_name}") + + temp_dir = tempfile.mkdtemp() + try: + host_path = os.path.join(temp_dir, table_name) + os.makedirs(host_path, exist_ok=True) + _download_table_from_container(instance, table_path, host_path) + manifest_list_rel, manifest_rels = _get_manifest_paths_from_chain(host_path, table_path) + local_manifest_list = os.path.join(host_path, manifest_list_rel) + _modify_avro_file( + local_manifest_list, + ["manifest_length"], + (lambda v: v) if new_length is None else (lambda _: new_length), + ) + instance.copy_file_to_container(local_manifest_list, f"{table_path}/{manifest_list_rel}") + return sum(os.path.getsize(os.path.join(host_path, rel)) for rel in manifest_rels) + finally: + shutil.rmtree(temp_dir, ignore_errors=True) + + +def _read_back(instance, table_path): + """Read with the metadata cache ON, from a cold cache: the cache-miss path is the only + one that builds the cell whose weight the declared length used to set. Returns the + query results plus the cache weight those reads produced.""" + instance.query("SYSTEM DROP ICEBERG METADATA CACHE") + count = instance.query(f"SELECT count() FROM icebergLocal('{table_path}/')", settings=CACHE).strip() + total = instance.query(f"SELECT sum(c0) FROM icebergLocal('{table_path}/')", settings=CACHE).strip() + weight = instance.query( + "SELECT value FROM system.metrics WHERE metric = 'IcebergMetadataFilesCacheBytes'" + ).strip() + return count, total, int(weight) + + +def _assert_weighed_by_real_size(weight, manifest_bytes): + """A manifest is charged three times its size on storage, so the total must clear that + bound. Reading the size from the file rather than hardcoding it keeps this stable while + still failing if the weight ever comes from something other than the real size.""" + assert weight >= 3 * manifest_bytes, f"weight {weight} < 3 * {manifest_bytes}" + + +def test_iceberg_rewritten_manifest_list_is_readable(started_cluster_iceberg_no_spark): + """Control for the test below: the same Avro round-trip with the declared length + untouched. It runs first so that a regression of the fix, which aborts the server and + makes every later query in the package fail to connect, cannot mask this result.""" + instance = started_cluster_iceberg_no_spark.instances["node1"] + table_name = "test_iceberg_manifest_list_roundtrip" + get_uuid_str() + table_path = f"/var/lib/clickhouse/user_files/iceberg_data/default/{table_name}" + manifest_bytes = _make_table_with_manifest_length(instance, table_name, table_path, None) + count, total, weight = _read_back(instance, table_path) + assert (count, total) == ("3", "6") + _assert_weighed_by_real_size(weight, manifest_bytes) + + +def test_iceberg_huge_manifest_length_is_readable(started_cluster_iceberg_no_spark): + instance = started_cluster_iceberg_no_spark.instances["node1"] + table_name = "test_iceberg_huge_manifest_length" + get_uuid_str() + table_path = f"/var/lib/clickhouse/user_files/iceberg_data/default/{table_name}" + manifest_bytes = _make_table_with_manifest_length( + instance, table_name, table_path, HUGE_MANIFEST_LENGTH + ) + count, total, weight = _read_back(instance, table_path) + assert (count, total) == ("3", "6") + _assert_weighed_by_real_size(weight, manifest_bytes) diff --git a/tests/integration/test_storage_iceberg_with_spark/test_async_metadata_refresh.py b/tests/integration/test_storage_iceberg_with_spark/test_async_metadata_refresh.py index 1108f44ca08a1..d0129ce43dc94 100644 --- a/tests/integration/test_storage_iceberg_with_spark/test_async_metadata_refresh.py +++ b/tests/integration/test_storage_iceberg_with_spark/test_async_metadata_refresh.py @@ -7,6 +7,7 @@ write_iceberg_from_df, default_upload_directory, ) +from helpers.test_tools import assert_eq_with_retry _ASYNC_CACHE_REFRESH_CONFIG_PATH = "/etc/clickhouse-server/config.d/iceberg_async_cache_refresh.xml" @@ -260,30 +261,39 @@ def test_async_metadata_refresh(started_cluster_iceberg_with_spark, storage_type "SELECT value FROM system.events WHERE name = 'S3ReadRequestsCount' SETTINGS system_events_show_zero_values=1" )) - write_iceberg_from_df( - spark, - generate_data(spark, 100, 200), - TABLE_NAME, - mode="append", - ) - default_upload_directory( - started_cluster_iceberg_with_spark, - storage_type, - f"/iceberg_data/default/{TABLE_NAME}/", - "", + # Hold the refresher before it reads the latest metadata, so the next SELECT must see stale data + instance.query("SYSTEM ENABLE FAILPOINT iceberg_metadata_prefetch_pause") + try: + instance.query("SYSTEM WAIT FAILPOINT iceberg_metadata_prefetch_pause PAUSE") + write_iceberg_from_df( + spark, + generate_data(spark, 100, 200), + TABLE_NAME, + mode="append", + ) + default_upload_directory( + started_cluster_iceberg_with_spark, + storage_type, + f"/iceberg_data/default/{TABLE_NAME}/", + "", + ) + # the fresh metadata won't get pulled at SELECT, so we see stale data + assert int(instance.query(f"SELECT count() FROM {TABLE_NAME} SETTINGS iceberg_metadata_staleness_ms=600000")) == 100 + finally: + instance.query("SYSTEM DISABLE FAILPOINT iceberg_metadata_prefetch_pause") + + # we don't pull fresh metadata at SELECT, but the data gets up to date because of the async refresh + assert_eq_with_retry( + instance, + f"SELECT count() FROM {TABLE_NAME} SETTINGS iceberg_metadata_staleness_ms=600000", + "200", + retry_count=60, ) - # the fresh metadata won't get pulled at SELECT, so we see stale data - assert int(instance.query(f"SELECT count() FROM {TABLE_NAME} SETTINGS iceberg_metadata_staleness_ms=600000")) == 100 - # Wait for the background async refresher to pick up the new metadata (2 periods of ASYNC_METADATA_REFRESH_PERIOD_MS) - time.sleep(ASYNC_METADATA_REFRESH_PERIOD_MS/1000 * 2) - # we expect some background activity against S3 s3_reads_after = int(instance.query( "SELECT value FROM system.events WHERE name = 'S3ReadRequestsCount' SETTINGS system_events_show_zero_values=1" )) assert s3_reads_after > s3_reads_before - # we don't pull fresh metadata at SELECT, but the data is up to date because of the async refresh - assert int(instance.query(f"SELECT count() FROM {TABLE_NAME} SETTINGS iceberg_metadata_staleness_ms=600000")) == 200 @pytest.mark.parametrize("storage_type", ["s3"]) diff --git a/tests/integration/test_storage_kafka/configs/small_message_broker_pool.xml b/tests/integration/test_storage_kafka/configs/small_message_broker_pool.xml new file mode 100644 index 0000000000000..f94998de21f2a --- /dev/null +++ b/tests/integration/test_storage_kafka/configs/small_message_broker_pool.xml @@ -0,0 +1,3 @@ + + 2 + diff --git a/tests/integration/test_storage_kafka/test_batch_fast.py b/tests/integration/test_storage_kafka/test_batch_fast.py index f55249b5a7d63..f4d206a9099f5 100644 --- a/tests/integration/test_storage_kafka/test_batch_fast.py +++ b/tests/integration/test_storage_kafka/test_batch_fast.py @@ -1259,6 +1259,8 @@ def test_kafka_many_materialized_views(kafka_cluster, create_query_generator): consumer_group=f"{topic_name}-group", ) + # A streaming loop keeps the materialized views it started with, so one that began before the + # second view existed commits without it; detaching and re-attaching joins it before producing. instance.query(f""" DROP TABLE IF EXISTS test.{kafka_table}_view1; DROP TABLE IF EXISTS test.{kafka_table}_view2; @@ -1275,11 +1277,10 @@ def test_kafka_many_materialized_views(kafka_cluster, create_query_generator): SELECT * FROM test.{kafka_table}; CREATE MATERIALIZED VIEW test.{kafka_table}_consumer2 TO test.{kafka_table}_view2 AS SELECT * FROM test.{kafka_table}; - """) - # we have to wait > kafka_poll_timeout_ms before producing data, - # otherwise it is expected that data might go via the first MV only - time.sleep(3) + DETACH TABLE test.{kafka_table} SYNC; + ATTACH TABLE test.{kafka_table}; + """) messages = [] for i in range(50): @@ -1310,8 +1311,8 @@ def test_kafka_many_materialized_views(kafka_cluster, create_query_generator): DROP TABLE test.{kafka_table}_view2; """) - k.kafka_check_result(result1, True) - k.kafka_check_result(result2, True) + assert k.kafka_check_result(result1), f"view1 got: {result1!r}" + assert k.kafka_check_result(result2), f"view2 got: {result2!r}" @pytest.mark.parametrize( "create_query_generator", diff --git a/tests/integration/test_storage_kafka/test_message_broker_pool_resize.py b/tests/integration/test_storage_kafka/test_message_broker_pool_resize.py new file mode 100644 index 0000000000000..c62b7e42cc906 --- /dev/null +++ b/tests/integration/test_storage_kafka/test_message_broker_pool_resize.py @@ -0,0 +1,117 @@ +import time + +import pytest + +from helpers.cluster import ClickHouseCluster +import helpers.kafka.common as k + +cluster = ClickHouseCluster(__file__) +instance = cluster.add_instance( + "instance", + main_configs=["configs/small_message_broker_pool.xml"], + user_configs=["configs/users.xml"], + with_kafka=True, + with_zookeeper=True, # For Kafka2 + macros={ + "kafka_broker": "kafka1", + "kafka_format_json_each_row": "JSONEachRow", + }, + stay_alive=True, +) + +STARTUP_POOL_SIZE = 2 +RELOADED_POOL_SIZE = 8 +NUM_TABLES = 4 +OVERRIDE_PATH = "/etc/clickhouse-server/config.d/zz_message_broker_pool_size.xml" + + +@pytest.fixture(scope="module") +def kafka_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +@pytest.fixture(autouse=True) +def kafka_setup_teardown(): + k.clean_test_database_and_topics(instance, cluster) + yield + # The pool size can only grow at runtime, so a restart is needed to get back to the startup size. + instance.query("DROP DATABASE IF EXISTS test SYNC") + instance.exec_in_container(["bash", "-c", f"rm -f {OVERRIDE_PATH}"]) + instance.restart_clickhouse() + + +def metric(name): + return int( + instance.query( + f"SELECT value FROM system.metrics WHERE metric = '{name}'" + ).strip() + ) + + +def max_executing_tasks(duration_sec, stop_at=None): + # Only the Kafka tables of this test run tasks in the message broker pool. + deadline = time.monotonic() + duration_sec + result = 0 + while time.monotonic() < deadline: + result = max(result, metric("BackgroundMessageBrokerSchedulePoolTask")) + if stop_at is not None and result >= stop_at: + break + time.sleep(0.1) + return result + + +def pool_size_metric(): + return metric("BackgroundMessageBrokerSchedulePoolSize") + + +@pytest.mark.parametrize( + "create_query_generator", + [k.generate_old_create_table_query, k.generate_new_create_table_query], +) +def test_kafka_tasks_use_increased_message_broker_pool( + kafka_cluster, create_query_generator +): + admin_client = k.get_admin_client(kafka_cluster) + postfix = k.get_topic_postfix(create_query_generator) + topics = [f"pool_resize_{postfix}_{i}" for i in range(NUM_TABLES)] + for topic in topics: + k.kafka_create_topic(admin_client, topic) + + # The topics stay empty, so every streaming task spends most of its time + # blocked in poll, holding a pool thread. + for i, topic in enumerate(topics): + create_query = create_query_generator( + f"kafka_{i}", + "key UInt64, value UInt64", + topic_list=topic, + consumer_group=topic, + settings={ + "kafka_thread_per_consumer": k.must_use_thread_per_consumer( + create_query_generator + ), + "kafka_consumer_reschedule_ms": 100, + }, + ) + instance.query(f""" + {create_query}; + CREATE TABLE test.view_{i} (key UInt64, value UInt64) ENGINE = MergeTree ORDER BY key; + CREATE MATERIALIZED VIEW test.consumer_{i} TO test.view_{i} AS SELECT * FROM test.kafka_{i}; + """) + + assert pool_size_metric() == STARTUP_POOL_SIZE + assert max_executing_tasks(30, stop_at=STARTUP_POOL_SIZE) == STARTUP_POOL_SIZE + assert max_executing_tasks(5) == STARTUP_POOL_SIZE + + instance.replace_config( + OVERRIDE_PATH, + f"{RELOADED_POOL_SIZE}", + ) + instance.query("SYSTEM RELOAD CONFIG") + assert pool_size_metric() == RELOADED_POOL_SIZE + + # Before the fix the tasks of one type stayed capped at the startup pool size. + assert max_executing_tasks(60, stop_at=NUM_TABLES) == NUM_TABLES diff --git a/tests/integration/test_storage_nats/test_nats_core.py b/tests/integration/test_storage_nats/test_nats_core.py index 5edf9dd292854..905c12cf4103a 100644 --- a/tests/integration/test_storage_nats/test_nats_core.py +++ b/tests/integration/test_storage_nats/test_nats_core.py @@ -1030,10 +1030,42 @@ def test_nats_restore_failed_connection_without_losses_on_write(nats_cluster): time.sleep(4) nats_helpers.revive_nats(nats_cluster) - result = instance.query_with_retry("SELECT count(DISTINCT key) FROM test.view", check_callback = lambda num_rows: int(num_rows) == messages_num) + # Core NATS is at-most-once: the broker keeps no message it has not already handed to a + # subscriber, so whatever was still in flight when it was stopped is gone for good and the + # count of the first batch cannot be asserted. The no-loss guarantee belongs to JetStream and + # is asserted by the test of the same name in `test_nats_jet_stream.py`. What this test is + # named after is the connection, so what is asserted here is that publishing works again. + # + # A core subscription comes back only when the client reconnects and re-sends `SUB` - + # `NATSCoreConsumer` does not override `needsResubscribe`, so nothing on the ClickHouse side + # resubscribes it and nothing is logged when it happens - and a message published before that + # lands on a subject with no subscriber and is dropped. Waiting longer after publishing cannot + # recover it, so the same batch is published until it arrives: its keys are fixed, so + # `DISTINCT` collapses the duplicates. + probe_num = 1000 + probe_values = ",".join( + "({i}, {i})".format(i=i) for i in range(messages_num, messages_num + probe_num) + ) + probe_count = "SELECT count(DISTINCT key) FROM test.view WHERE key >= {}".format(messages_num) + + received = 0 + deadline = time.monotonic() + 120 + while received != probe_num and time.monotonic() < deadline: + instance.query_with_retry( + "INSERT INTO test.producer_reconnect VALUES {}".format(probe_values) + ) + received = int( + instance.query_with_retry( + probe_count, + retry_count=10, + sleep_time=0.5, + check_callback=lambda num_rows: int(num_rows) == probe_num, + ) + ) - assert int(result) == messages_num, "ClickHouse lost some messages: {}".format( - result + assert received == probe_num, ( + "ClickHouse did not restore the connection: {} of the {} messages republished after the " + "broker came back arrived within 120s".format(received, probe_num) ) diff --git a/tests/integration/test_storage_s3/test.py b/tests/integration/test_storage_s3/test.py index 96e94b2a89e89..04a553a14f299 100644 --- a/tests/integration/test_storage_s3/test.py +++ b/tests/integration/test_storage_s3/test.py @@ -2938,6 +2938,41 @@ def test_page_cache(started_cluster): assert 0 < int(gets) +def test_cache_key_includes_storage(started_cluster): + # The same key with the same content, so the same ETag, in another bucket or behind another endpoint: + # the filesystem cache and the page cache must not serve it the entry filled by the first object. + instance = started_cluster.instances["dummy"] + table_name = f"test_cache_key_includes_storage-{uuid.uuid4()}" + host = f"{started_cluster.minio_host}:{started_cluster.minio_port}" + ip = f"{started_cluster.minio_ip}:{started_cluster.minio_port}" + + for bucket in [started_cluster.minio_bucket, started_cluster.minio_bucket_2]: + instance.query( + f"insert into function s3('http://{host}/{bucket}/{table_name}.tsv', auto, 'x UInt64') select number from numbers(100) SETTINGS s3_truncate_on_insert=1" + ) + + urls = [ + f"http://{host}/{started_cluster.minio_bucket}/{table_name}.tsv", + f"http://{ip}/{started_cluster.minio_bucket}/{table_name}.tsv", + f"http://{host}/{started_cluster.minio_bucket_2}/{table_name}.tsv", + ] + for settings in ["filesystem_cache_name = 'cache1', enable_filesystem_cache=1", "use_page_cache_for_object_storage=1"]: + for url in urls: + query_id = f"{table_name}-{uuid.uuid4()}" + assert "100\n" == instance.query( + f"select sum(ignore(*) + 1) from s3('{url}', 'TSV', 'x UInt64') SETTINGS {settings}", + query_id=query_id, + ) + + instance.query("SYSTEM FLUSH LOGS") + + assert 0 < int( + instance.query( + f"SELECT ProfileEvents['S3GetObject'] FROM system.query_log WHERE query_id = '{query_id}' AND type = 'QueryFinish'" + ) + ) + + def test_archive(started_cluster): uuid.uuid4() diff --git a/tests/integration/test_storage_url/test.py b/tests/integration/test_storage_url/test.py index cd72d82b2fc10..f0bbc2824b716 100644 --- a/tests/integration/test_storage_url/test.py +++ b/tests/integration/test_storage_url/test.py @@ -807,7 +807,8 @@ def test_table_function_url_access_rights(): "DESCRIBE TABLE url('http://nginx:80/test_1', 'TSV')", user="u1" ) - node1.query("GRANT READ ON URL TO u1") + # Like `SELECT`, `DESCRIBE` of a table function also needs `CREATE TEMPORARY TABLE`. + node1.query("GRANT READ ON URL, CREATE TEMPORARY TABLE ON *.* TO u1") assert node1.query( "DESCRIBE TABLE url('http://nginx:80/test_1', 'TSV')", user="u1", diff --git a/tests/integration/test_streaming_engines_definer_interserver_secret/__init__.py b/tests/integration/test_streaming_engines_definer_interserver_secret/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_streaming_engines_definer_interserver_secret/configs/rabbitmq.xml b/tests/integration/test_streaming_engines_definer_interserver_secret/configs/rabbitmq.xml new file mode 100644 index 0000000000000..0b5411455b46b --- /dev/null +++ b/tests/integration/test_streaming_engines_definer_interserver_secret/configs/rabbitmq.xml @@ -0,0 +1,6 @@ + + + root + clickhouse + + diff --git a/tests/integration/test_streaming_engines_definer_interserver_secret/configs/remote_servers.xml b/tests/integration/test_streaming_engines_definer_interserver_secret/configs/remote_servers.xml new file mode 100644 index 0000000000000..af0b7ad21cab3 --- /dev/null +++ b/tests/integration/test_streaming_engines_definer_interserver_secret/configs/remote_servers.xml @@ -0,0 +1,14 @@ + + + + + foo + + node2 + 9000 + + + + diff --git a/tests/integration/test_streaming_engines_definer_interserver_secret/configs/users.xml b/tests/integration/test_streaming_engines_definer_interserver_secret/configs/users.xml new file mode 100644 index 0000000000000..ae00b3b543f9f --- /dev/null +++ b/tests/integration/test_streaming_engines_definer_interserver_secret/configs/users.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + ::/0 + + default + default + 1 + + + + + + + + diff --git a/tests/integration/test_streaming_engines_definer_interserver_secret/test.py b/tests/integration/test_streaming_engines_definer_interserver_secret/test.py new file mode 100644 index 0000000000000..6a0ed02eadf3c --- /dev/null +++ b/tests/integration/test_streaming_engines_definer_interserver_secret/test.py @@ -0,0 +1,296 @@ +# pylint: disable=unused-argument +# pylint: disable=redefined-outer-name +# pylint: disable=line-too-long + +""" +Pushes from streaming engines into a `Distributed` table must reach the shard under the definer. + +`Kafka`, `RabbitMQ` and the other streaming engines never write anywhere themselves: a consumed +batch is pushed only into the dependent materialized views, from a copy of the global context that +has no user. A view with `SQL SECURITY DEFINER` switches that push to the definer. When the view's +target is a `Distributed` table over a cluster with an interserver ``, the definer's name +must travel with the query: before the fix `Context::setUser` left the `ClientInfo` user names +empty, the shard received an empty `initial_user`, treated the query as interserver mode and +executed it with full access. A tenant that could create tables only in its own database could +chain `Kafka -> MATERIALIZED VIEW -> Distributed('secure', default, secret_data)` and write into +a table on the shard it had no `INSERT` grant on. + +The `secure` cluster consists of `node2` only, so a table on `node1` can reach `default.secret_data` +solely over the interserver connection. Every pipeline is built twice with the same statements: by +`tenant`, who has no grants on `node2` and whose rows must be rejected there, and by `writer`, who +has `INSERT` on the target and whose rows must arrive. In both cases the shard must see the +definer's name in `system.query_log`. + +The push is also checked with `distributed_foreground_insert = 0`, where the rows go through the +background directory queue of the `Distributed` table: the persisted insert must keep the definer's +name, although the streaming context has `query_kind = NO_QUERY`. +""" + +import json +import time + +import pika +import pytest + +import helpers.kafka.common as k +from helpers.cluster import ClickHouseCluster + +cluster = ClickHouseCluster(__file__) + +node1 = cluster.add_instance( + "node1", + main_configs=["configs/remote_servers.xml", "configs/rabbitmq.xml"], + user_configs=["configs/users.xml"], + with_kafka=True, + with_rabbitmq=True, +) +node2 = cluster.add_instance( + "node2", + main_configs=["configs/remote_servers.xml"], + user_configs=["configs/users.xml"], +) + +WRITER = "writer" +TENANT = "tenant" +CREDENTIALS = { + WRITER: {"user": WRITER, "password": "wpass"}, + TENANT: {"user": TENANT, "password": "tpass"}, +} +ROWS_PER_PIPELINE = 2 + +# How long a streaming engine gets to consume the messages and push them to the shard. Consumer +# group setup in Kafka alone can take several seconds. +DELIVERY_WAIT_SECONDS = 60 + +users = pytest.mark.parametrize("user", [WRITER, TENANT]) + + +def database(user): + return f"{user}_db" + + +def query_as(user, sql): + return node1.query(sql, **CREDENTIALS[user]) + + +def remote_rows(): + return int(node2.query("SELECT count() FROM default.secret_data")) + + +def shard_inserts(user, condition): + """Number of `INSERT` queries into the protected table that the shard executed under `user`.""" + node2.query("SYSTEM FLUSH LOGS") + return int( + node2.query( + f""" + SELECT count() FROM system.query_log + WHERE user = '{user}' AND query LIKE 'INSERT INTO default.secret_data%' AND {condition} + """ + ) + ) + + +def shard_finished_inserts(user): + return shard_inserts(user, "type = 'QueryFinish'") + + +def shard_denied_inserts(user): + # The access check happens before the pipeline starts, so the shard logs the denial as + # `ExceptionBeforeStart`. + return shard_inserts( + user, + "type IN ('ExceptionBeforeStart', 'ExceptionWhileProcessing') AND exception_code = 497", + ) + + +def wait_until(condition, timeout=DELIVERY_WAIT_SECONDS): + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + if condition(): + return True + time.sleep(1) + return False + + +@pytest.fixture(scope="module", autouse=True) +def started_cluster(): + try: + cluster.start() + + # The protected table lives on the shard only. Both users exist on both nodes, but on + # the shard only `writer` may insert into it. + node2.query( + """ + CREATE TABLE default.secret_data (key Int) ENGINE = MergeTree ORDER BY key; + CREATE USER writer IDENTIFIED WITH plaintext_password BY 'wpass'; + CREATE USER tenant IDENTIFIED WITH plaintext_password BY 'tpass'; + GRANT INSERT ON default.secret_data TO writer; + """ + ) + for user in (WRITER, TENANT): + db = database(user) + # Send to the shard synchronously by default, so that a push either delivers the rows + # or fails right away, without a background directory queue in between. + node1.query( + f""" + CREATE USER {user} IDENTIFIED WITH plaintext_password BY '{CREDENTIALS[user]["password"]}' + SETTINGS distributed_foreground_insert = 1; + CREATE DATABASE {db}; + GRANT ALL ON {db}.* TO {user}; + GRANT TABLE ENGINE ON Distributed, TABLE ENGINE ON Kafka, TABLE ENGINE ON RabbitMQ TO {user}; + """ + ) + query_as( + user, + f"CREATE TABLE {db}.dist (key Int) ENGINE = Distributed(secure, default, secret_data)", + ) + + yield cluster + finally: + cluster.shutdown() + + +@pytest.fixture(autouse=True) +def clean_remote_table(): + node2.query("TRUNCATE TABLE default.secret_data") + yield + + +def drop_pipeline(user, source): + """Stop the streaming table, so that a rejected push is not retried into the next test.""" + db = database(user) + query_as(user, f"DROP TABLE {db}.{source}_to_dist") + query_as(user, f"DROP TABLE {db}.{source} SYNC") + + +def create_definer_view(user, source, target="dist"): + """A view owned by `user` that pushes the source rows into the user's `Distributed` table.""" + db = database(user) + query_as( + user, + f""" + CREATE MATERIALIZED VIEW {db}.{source}_to_dist TO {db}.{target} + DEFINER = CURRENT_USER SQL SECURITY DEFINER + AS SELECT key FROM {db}.{source} + """, + ) + + +def check_delivery(user, finished_before, denied_before): + """The shard must have executed the push under the user's name, with the outcome the user's + grants dictate: the rows arrive for `writer` and are rejected for `tenant`.""" + if user == WRITER: + assert wait_until(lambda: remote_rows() == ROWS_PER_PIPELINE) + assert shard_finished_inserts(WRITER) > finished_before + else: + assert wait_until( + lambda: shard_denied_inserts(TENANT) > denied_before or remote_rows() > 0 + ) + assert remote_rows() == 0 + assert shard_denied_inserts(TENANT) > denied_before + + +def check_direct_insert(user): + """The control case: a direct `INSERT` carries the user's name to the shard, where it is checked. + + It passes without the fix as well, so it is not a separate test: bugfix validation requires every + new test to fail on the unfixed binary. It runs before a streaming pipeline instead.""" + db = database(user) + if user == WRITER: + query_as(WRITER, f"INSERT INTO {db}.dist VALUES (1)") + assert remote_rows() == 1 + else: + error = node1.query_and_get_error( + f"INSERT INTO {db}.dist VALUES (1)", **CREDENTIALS[TENANT] + ) + assert "ACCESS_DENIED" in error + assert remote_rows() == 0 + node2.query("TRUNCATE TABLE default.secret_data") + + +@users +@pytest.mark.parametrize("foreground", [1, 0]) +def test_kafka(user, foreground): + db = database(user) + topic = f"definer_interserver_{user}_{foreground}" + k.kafka_create_topic(k.get_admin_client(cluster), topic) + + query_as( + user, + f""" + CREATE TABLE {db}.kafka (key Int) ENGINE = Kafka + SETTINGS kafka_broker_list = 'kafka1:19092', + kafka_topic_list = '{topic}', + kafka_group_name = '{topic}', + kafka_format = 'JSONEachRow', + kafka_flush_interval_ms = 1000 + """, + ) + # A separate `Distributed` table per mode, so that a rejected file left in the background queue + # is dropped together with the table and is not retried into the next test. + target = f"dist_{foreground}" + query_as( + user, + f"CREATE TABLE {db}.{target} (key Int) ENGINE = Distributed(secure, default, secret_data)", + ) + # The definer's settings apply to the push. It is not set in the server's profile, because the + # settings of the context the push starts from would override the definer's ones. + node1.query(f"ALTER USER {user} SETTINGS distributed_foreground_insert = {foreground}") + create_definer_view(user, "kafka", target) + + try: + finished_before = shard_finished_inserts(user) + denied_before = shard_denied_inserts(user) + k.kafka_produce( + cluster, topic, [json.dumps({"key": i}) for i in range(ROWS_PER_PIPELINE)] + ) + check_delivery(user, finished_before, denied_before) + finally: + drop_pipeline(user, "kafka") + query_as(user, f"DROP TABLE {db}.{target} SYNC") + node1.query(f"ALTER USER {user} SETTINGS distributed_foreground_insert = 1") + + +@users +def test_rabbitmq(user): + check_direct_insert(user) + db = database(user) + exchange = f"definer_interserver_{user}" + + query_as( + user, + f""" + CREATE TABLE {db}.rabbitmq (key Int) ENGINE = RabbitMQ + SETTINGS rabbitmq_host_port = 'rabbitmq1:5672', + rabbitmq_exchange_name = '{exchange}', + rabbitmq_format = 'JSONEachRow', + rabbitmq_flush_interval_ms = 1000, + rabbitmq_max_block_size = 100 + """, + ) + create_definer_view(user, "rabbitmq") + + # The exchange and the queue are declared by the table in the background; messages + # published before that are dropped by RabbitMQ. + node1.wait_for_log_line(f"StorageRabbitMQ .{db}.rabbitmq.: RabbitMQ setup completed") + node1.wait_for_log_line(f"StorageRabbitMQ .{db}.rabbitmq.: Queue .* is declared") + + try: + finished_before = shard_finished_inserts(user) + denied_before = shard_denied_inserts(user) + + credentials = pika.PlainCredentials("root", "clickhouse") + parameters = pika.ConnectionParameters( + cluster.rabbitmq_ip, cluster.rabbitmq_port, "/", credentials + ) + connection = pika.BlockingConnection(parameters) + channel = connection.channel() + for i in range(ROWS_PER_PIPELINE): + channel.basic_publish( + exchange=exchange, routing_key="", body=json.dumps({"key": i}) + ) + connection.close() + + check_delivery(user, finished_before, denied_before) + finally: + drop_pipeline(user, "rabbitmq") diff --git a/tests/integration/test_text_log_level_after_reload/__init__.py b/tests/integration/test_text_log_level_after_reload/__init__.py new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/integration/test_text_log_level_after_reload/configs/logger.xml b/tests/integration/test_text_log_level_after_reload/configs/logger.xml new file mode 100644 index 0000000000000..811ec0c3320d3 --- /dev/null +++ b/tests/integration/test_text_log_level_after_reload/configs/logger.xml @@ -0,0 +1,9 @@ + + + information + /var/log/clickhouse-server/clickhouse-server.log + + + trace + + diff --git a/tests/integration/test_text_log_level_after_reload/test.py b/tests/integration/test_text_log_level_after_reload/test.py new file mode 100644 index 0000000000000..6bb8f1264f391 --- /dev/null +++ b/tests/integration/test_text_log_level_after_reload/test.py @@ -0,0 +1,65 @@ +import pytest + +from helpers.cluster import ClickHouseCluster + +# The file log is capped at `information`, text_log wants `trace`. +# The root logger level must stay at `trace` (the most verbose sink), both at +# startup and after the levels are re-applied on a configuration reload. +cluster = ClickHouseCluster(__file__) +node = cluster.add_instance("node", main_configs=["configs/logger.xml"]) + + +@pytest.fixture(scope="module") +def start_cluster(): + try: + cluster.start() + yield cluster + finally: + cluster.shutdown() + + +def trace_rows_since(node, since): + node.query("SYSTEM FLUSH LOGS") + return int( + node.query( + f"SELECT count() FROM system.text_log WHERE level = 'Trace' AND event_time_microseconds > '{since}'" + ) + ) + + +def now(node): + return node.query("SELECT now64(6)").strip() + + +def test_text_log_keeps_trace_after_reload(start_cluster): + # Startup: the root level must already include text_log.level when the loggers are + # built, before the first configuration reload, so early Trace messages are kept. + node.query("SYSTEM FLUSH LOGS") + assert ( + int( + node.query( + "SELECT count() FROM system.text_log WHERE level = 'Trace' AND logger_name = 'Application' AND message LIKE 'Initialized DateLUT%'" + ) + ) + > 0 + ) + + since = now(node) + for _ in range(5): + node.query("SELECT 1") + assert trace_rows_since(node, since) > 0 + + # Re-applies the logger levels from the (unchanged) configuration. + node.query("SYSTEM RELOAD CONFIG") + + since = now(node) + for _ in range(5): + node.query("SELECT 1") + assert trace_rows_since(node, since) > 0 + + # The file log must still be capped at `information`. + log = node.exec_in_container( + ["bash", "-c", "cat /var/log/clickhouse-server/clickhouse-server.log"] + ) + assert "" not in log + assert "" not in log diff --git a/tests/integration/test_ttl_move/test.py b/tests/integration/test_ttl_move/test.py index 926079afcaf09..8c003af149e26 100644 --- a/tests/integration/test_ttl_move/test.py +++ b/tests/integration/test_ttl_move/test.py @@ -1346,12 +1346,15 @@ def test_moves_after_merges(started_cluster, engine, request): """) # we want to control over when to MOVE to eliminate flakinness node1.query(f"SYSTEM STOP MOVES {table_name}") + # SYSTEM STOP MOVES does not stop a merge from applying the TTL move, so the TTL must outlive the OPTIMIZE below. + ttl_ts = int(time.time()) + 10 + f_ttl_datetime = lambda _: f'toDateTime({ttl_ts})' populate_with_records( node=node1, table_name=table_name, generators=[ - gen_pair_records(10, F_SMALL_STRING, F_SOON_TO_EXPIRE_DATETIME), - gen_pair_records(10, F_SMALL_STRING, F_SOON_TO_EXPIRE_DATETIME), + gen_pair_records(10, F_SMALL_STRING, f_ttl_datetime), + gen_pair_records(10, F_SMALL_STRING, f_ttl_datetime), ] ) @@ -1365,7 +1368,7 @@ def test_moves_after_merges(started_cluster, engine, request): # 2. Waiting for the data to expire and then expecting it at the target location node1.query(f"SYSTEM START MOVES {table_name}") - time.sleep(SOON_SECONDS) + time.sleep(max(0, ttl_ts - time.time())) wait_if_moving_then_assert_disks(node1, table_name, target_state={ 'external' : {'rows': 20} }) diff --git a/tests/performance/final_read_in_order_limit.xml b/tests/performance/final_read_in_order_limit.xml index e543444482307..7d96fb223413c 100644 --- a/tests/performance/final_read_in_order_limit.xml +++ b/tests/performance/final_read_in_order_limit.xml @@ -9,7 +9,13 @@ no filtering) skips the split. The others are controls that must keep the layer split: with a filter the query consumes limit / selectivity rows, which is unknown at planning time (for the rare-value filter it is the - whole table), and the last query consumes most of the ordered stream. + whole table), and the `LIMIT 4000000` query consumes most of the ordered + stream. + + With `read_in_order_use_virtual_row` each layer announces the key it starts + from, so the merge after the sort defers the layers the limit never reaches. + The reverse read with a mild filter is answered by the first layer alone; + the same query with the setting off is a control that must not move. --> CREATE TABLE final_inorder_limit (k UInt64, v UInt64, payload String) @@ -25,10 +31,12 @@ INSERT INTO final_inorder_limit SELECT number, cityHash64(number, 2) % 1000, toString(cityHash64(number, 102)) FROM numbers_mt(4000000) INSERT INTO final_inorder_limit SELECT number, cityHash64(number, 3) % 1000, toString(cityHash64(number, 103)) FROM numbers_mt(4000000) + 0 + SELECT k, payload FROM final_inorder_limit FINAL ORDER BY k LIMIT 100 FORMAT Null SELECT k, payload FROM final_inorder_limit FINAL WHERE v = 7 ORDER BY k LIMIT 100 FORMAT Null SELECT k, payload FROM final_inorder_limit FINAL WHERE v = 1000 ORDER BY k LIMIT 100 FORMAT Null SELECT k, payload FROM final_inorder_limit FINAL ORDER BY k LIMIT 4000000 FORMAT Null - + SELECT k, payload FROM final_inorder_limit FINAL WHERE v < 500 ORDER BY k DESC LIMIT 100 FORMAT Null DROP TABLE IF EXISTS final_inorder_limit diff --git a/tests/performance/group_by_two_level_few_keys.xml b/tests/performance/group_by_two_level_few_keys.xml new file mode 100644 index 0000000000000..e2684ff54d78b --- /dev/null +++ b/tests/performance/group_by_two_level_few_keys.xml @@ -0,0 +1,42 @@ + + + + 8 + 1 + + + + CREATE TABLE group_by_two_level_few_keys + ( + lc LowCardinality(Nullable(String)), + s Nullable(String), + a UInt64, + b UInt64, + c UInt64 + ) + ENGINE = MergeTree ORDER BY tuple() + + + INSERT INTO group_by_two_level_few_keys + SELECT v, v, intHash32(intDiv(number, 16)) % 13, 7, 11 + FROM + ( + SELECT + number, + multiIf( + intHash32(intDiv(number, 16)) % 100 < 40, 'category_value_01_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + intHash32(intDiv(number, 16)) % 100 < 70, 'category_value_02_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + intHash32(intDiv(number, 16)) % 100 < 95, 'category_value_03_cccccccccccccccccccccccccccccccccccccccc', + intHash32(intDiv(number, 16)) % 100 < 97, NULL, + concat('category_value_', toString((intHash32(number) % 9) + 4), '_dddddddddddddddddddddddddddddddddddddddd')) AS v + FROM numbers_mt(300000000) + ) + + OPTIMIZE TABLE group_by_two_level_few_keys FINAL + + SELECT lc FROM group_by_two_level_few_keys GROUP BY lc FORMAT Null + SELECT s FROM group_by_two_level_few_keys GROUP BY s FORMAT Null + SELECT a, b, c, count() FROM group_by_two_level_few_keys GROUP BY a, b, c FORMAT Null + + DROP TABLE IF EXISTS group_by_two_level_few_keys + diff --git a/tests/performance/json_typed_path_subcolumn_lookup.xml b/tests/performance/json_typed_path_subcolumn_lookup.xml new file mode 100644 index 0000000000000..2c12273924bb2 --- /dev/null +++ b/tests/performance/json_typed_path_subcolumn_lookup.xml @@ -0,0 +1,223 @@ + + + + 1 + + + + CREATE TABLE t_json_subcolumn_lookup_compact (id UInt64, json JSON(t0 String, t1 String, t2 String, t3 String, t4 String, t5 String, t6 String, t7 String, t8 String, t9 String, t10 String, t11 String, t12 String, t13 String, t14 String, t15 String, t16 String, t17 String, t18 String, t19 String, t20 String, t21 String, t22 String, t23 String, t24 String, t25 String, t26 String, t27 String, t28 String, t29 String, t30 String, t31 String, t32 String, t33 String, t34 String, t35 String, t36 String, t37 String, t38 String, t39 String, t40 String, t41 String, t42 String, t43 String, t44 String, t45 String, t46 String, t47 String, t48 String, t49 String, t50 String, t51 String, t52 String, t53 String, t54 String, t55 String, t56 String, t57 String, t58 String, t59 String, t60 String, t61 String, t62 String, t63 String, t64 String, t65 String, t66 String, t67 String, t68 String, t69 String, t70 String, t71 String, t72 String, t73 String, t74 String, t75 String, t76 String, t77 String, t78 String, t79 String, t80 String, t81 String, t82 String, t83 String, t84 String, t85 String, t86 String, t87 String, t88 String, t89 String, t90 String, t91 String, t92 String, t93 String, t94 String, t95 String, t96 String, t97 String, t98 String, t99 String, t100 String, t101 String, t102 String, t103 String, t104 String, t105 String, t106 String, t107 String, t108 String, t109 String, t110 String, t111 String, t112 String, t113 String, t114 String, t115 String, t116 String, t117 String, t118 String, t119 String, t120 String, t121 String, t122 String, t123 String, t124 String, t125 String, t126 String, t127 String, t128 String, t129 String, t130 String, t131 String, t132 String, t133 String, t134 String, t135 String, t136 String, t137 String, t138 String, t139 String, t140 String, t141 String, t142 String, t143 String, t144 String, t145 String, t146 String, t147 String, t148 String, t149 String, t150 String, t151 String, t152 String, t153 String, t154 String, t155 String, t156 String, t157 String, t158 String, t159 String, t160 String, t161 String, t162 String, t163 String, t164 String, t165 String, t166 String, t167 String, t168 String, t169 String, t170 String, t171 String, t172 String, t173 String, t174 String, t175 String, t176 String, t177 String, t178 String, t179 String, t180 String, t181 String, t182 String, t183 String, t184 String, t185 String, t186 String, t187 String, t188 String, t189 String, t190 String, t191 String, t192 String, t193 String, t194 String, t195 String, t196 String, t197 String, t198 String, t199 String, t200 String, t201 String, t202 String, t203 String, t204 String, t205 String, t206 String, t207 String, t208 String, t209 String, t210 String, t211 String, t212 String, t213 String, t214 String, t215 String, t216 String, t217 String, t218 String, t219 String, t220 String, t221 String, t222 String, t223 String, t224 String, t225 String, t226 String, t227 String, t228 String, t229 String, t230 String, t231 String, t232 String, t233 String, t234 String, t235 String, t236 String, t237 String, t238 String, t239 String, t240 String, t241 String, t242 String, t243 String, t244 String, t245 String, t246 String, t247 String, t248 String, t249 String, t250 String, t251 String, t252 String, t253 String, t254 String, t255 String, t256 String, t257 String, t258 String, t259 String, t260 String, t261 String, t262 String, t263 String, t264 String, t265 String, t266 String, t267 String, t268 String, t269 String, t270 String, t271 String, t272 String, t273 String, t274 String, t275 String, t276 String, t277 String, t278 String, t279 String, t280 String, t281 String, t282 String, t283 String, t284 String, t285 String, t286 String, t287 String, t288 String, t289 String, t290 String, t291 String, t292 String, t293 String, t294 String, t295 String, t296 String, t297 String, t298 String, t299 String, t300 String, t301 String, t302 String, t303 String, t304 String, t305 String, t306 String, t307 String, t308 String, t309 String, t310 String, t311 String, t312 String, t313 String, t314 String, t315 String, t316 String, t317 String, t318 String, t319 String, t320 String, t321 String, t322 String, t323 String, t324 String, t325 String, t326 String, t327 String, t328 String, t329 String, t330 String, t331 String, t332 String, t333 String, t334 String, t335 String, t336 String, t337 String, t338 String, t339 String, t340 String, t341 String, t342 String, t343 String, t344 String, t345 String, t346 String, t347 String, t348 String, t349 String, t350 String, t351 String, t352 String, t353 String, t354 String, t355 String, t356 String, t357 String, t358 String, t359 String, t360 String, t361 String, t362 String, t363 String, t364 String, t365 String, t366 String, t367 String, t368 String, t369 String, t370 String, t371 String, t372 String, t373 String, t374 String, t375 String, t376 String, t377 String, t378 String, t379 String, t380 String, t381 String, t382 String, t383 String, t384 String, t385 String, t386 String, t387 String, t388 String, t389 String, t390 String, t391 String, t392 String, t393 String, t394 String, t395 String, t396 String, t397 String, t398 String, t399 String, t400 String, t401 String, t402 String, t403 String, t404 String, t405 String, t406 String, t407 String, t408 String, t409 String, t410 String, t411 String, t412 String, t413 String, t414 String, t415 String, t416 String, t417 String, t418 String, t419 String, t420 String, t421 String, t422 String, t423 String, t424 String, t425 String, t426 String, t427 String, t428 String, t429 String, t430 String, t431 String, t432 String, t433 String, t434 String, t435 String, t436 String, t437 String, t438 String, t439 String, t440 String, t441 String, t442 String, t443 String, t444 String, t445 String, t446 String, t447 String, t448 String, t449 String, t450 String, t451 String, t452 String, t453 String, t454 String, t455 String, t456 String, t457 String, t458 String, t459 String, t460 String, t461 String, t462 String, t463 String, t464 String, t465 String, t466 String, t467 String, t468 String, t469 String, t470 String, t471 String, t472 String, t473 String, t474 String, t475 String, t476 String, t477 String, t478 String, t479 String, t480 String, t481 String, t482 String, t483 String, t484 String, t485 String, t486 String, t487 String, t488 String, t489 String, t490 String, t491 String, t492 String, t493 String, t494 String, t495 String, t496 String, t497 String, t498 String, t499 String)) + ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = '100G', min_rows_for_wide_part = 1000000000 + + + CREATE TABLE t_json_subcolumn_lookup_wide (id UInt64, json JSON(t0 String, t1 String, t2 String, t3 String, t4 String, t5 String, t6 String, t7 String, t8 String, t9 String, t10 String, t11 String, t12 String, t13 String, t14 String, t15 String, t16 String, t17 String, t18 String, t19 String, t20 String, t21 String, t22 String, t23 String, t24 String, t25 String, t26 String, t27 String, t28 String, t29 String, t30 String, t31 String, t32 String, t33 String, t34 String, t35 String, t36 String, t37 String, t38 String, t39 String, t40 String, t41 String, t42 String, t43 String, t44 String, t45 String, t46 String, t47 String, t48 String, t49 String, t50 String, t51 String, t52 String, t53 String, t54 String, t55 String, t56 String, t57 String, t58 String, t59 String, t60 String, t61 String, t62 String, t63 String, t64 String, t65 String, t66 String, t67 String, t68 String, t69 String, t70 String, t71 String, t72 String, t73 String, t74 String, t75 String, t76 String, t77 String, t78 String, t79 String, t80 String, t81 String, t82 String, t83 String, t84 String, t85 String, t86 String, t87 String, t88 String, t89 String, t90 String, t91 String, t92 String, t93 String, t94 String, t95 String, t96 String, t97 String, t98 String, t99 String, t100 String, t101 String, t102 String, t103 String, t104 String, t105 String, t106 String, t107 String, t108 String, t109 String, t110 String, t111 String, t112 String, t113 String, t114 String, t115 String, t116 String, t117 String, t118 String, t119 String, t120 String, t121 String, t122 String, t123 String, t124 String, t125 String, t126 String, t127 String, t128 String, t129 String, t130 String, t131 String, t132 String, t133 String, t134 String, t135 String, t136 String, t137 String, t138 String, t139 String, t140 String, t141 String, t142 String, t143 String, t144 String, t145 String, t146 String, t147 String, t148 String, t149 String, t150 String, t151 String, t152 String, t153 String, t154 String, t155 String, t156 String, t157 String, t158 String, t159 String, t160 String, t161 String, t162 String, t163 String, t164 String, t165 String, t166 String, t167 String, t168 String, t169 String, t170 String, t171 String, t172 String, t173 String, t174 String, t175 String, t176 String, t177 String, t178 String, t179 String, t180 String, t181 String, t182 String, t183 String, t184 String, t185 String, t186 String, t187 String, t188 String, t189 String, t190 String, t191 String, t192 String, t193 String, t194 String, t195 String, t196 String, t197 String, t198 String, t199 String, t200 String, t201 String, t202 String, t203 String, t204 String, t205 String, t206 String, t207 String, t208 String, t209 String, t210 String, t211 String, t212 String, t213 String, t214 String, t215 String, t216 String, t217 String, t218 String, t219 String, t220 String, t221 String, t222 String, t223 String, t224 String, t225 String, t226 String, t227 String, t228 String, t229 String, t230 String, t231 String, t232 String, t233 String, t234 String, t235 String, t236 String, t237 String, t238 String, t239 String, t240 String, t241 String, t242 String, t243 String, t244 String, t245 String, t246 String, t247 String, t248 String, t249 String, t250 String, t251 String, t252 String, t253 String, t254 String, t255 String, t256 String, t257 String, t258 String, t259 String, t260 String, t261 String, t262 String, t263 String, t264 String, t265 String, t266 String, t267 String, t268 String, t269 String, t270 String, t271 String, t272 String, t273 String, t274 String, t275 String, t276 String, t277 String, t278 String, t279 String, t280 String, t281 String, t282 String, t283 String, t284 String, t285 String, t286 String, t287 String, t288 String, t289 String, t290 String, t291 String, t292 String, t293 String, t294 String, t295 String, t296 String, t297 String, t298 String, t299 String, t300 String, t301 String, t302 String, t303 String, t304 String, t305 String, t306 String, t307 String, t308 String, t309 String, t310 String, t311 String, t312 String, t313 String, t314 String, t315 String, t316 String, t317 String, t318 String, t319 String, t320 String, t321 String, t322 String, t323 String, t324 String, t325 String, t326 String, t327 String, t328 String, t329 String, t330 String, t331 String, t332 String, t333 String, t334 String, t335 String, t336 String, t337 String, t338 String, t339 String, t340 String, t341 String, t342 String, t343 String, t344 String, t345 String, t346 String, t347 String, t348 String, t349 String, t350 String, t351 String, t352 String, t353 String, t354 String, t355 String, t356 String, t357 String, t358 String, t359 String, t360 String, t361 String, t362 String, t363 String, t364 String, t365 String, t366 String, t367 String, t368 String, t369 String, t370 String, t371 String, t372 String, t373 String, t374 String, t375 String, t376 String, t377 String, t378 String, t379 String, t380 String, t381 String, t382 String, t383 String, t384 String, t385 String, t386 String, t387 String, t388 String, t389 String, t390 String, t391 String, t392 String, t393 String, t394 String, t395 String, t396 String, t397 String, t398 String, t399 String, t400 String, t401 String, t402 String, t403 String, t404 String, t405 String, t406 String, t407 String, t408 String, t409 String, t410 String, t411 String, t412 String, t413 String, t414 String, t415 String, t416 String, t417 String, t418 String, t419 String, t420 String, t421 String, t422 String, t423 String, t424 String, t425 String, t426 String, t427 String, t428 String, t429 String, t430 String, t431 String, t432 String, t433 String, t434 String, t435 String, t436 String, t437 String, t438 String, t439 String, t440 String, t441 String, t442 String, t443 String, t444 String, t445 String, t446 String, t447 String, t448 String, t449 String, t450 String, t451 String, t452 String, t453 String, t454 String, t455 String, t456 String, t457 String, t458 String, t459 String, t460 String, t461 String, t462 String, t463 String, t464 String, t465 String, t466 String, t467 String, t468 String, t469 String, t470 String, t471 String, t472 String, t473 String, t474 String, t475 String, t476 String, t477 String, t478 String, t479 String, t480 String, t481 String, t482 String, t483 String, t484 String, t485 String, t486 String, t487 String, t488 String, t489 String, t490 String, t491 String, t492 String, t493 String, t494 String, t495 String, t496 String, t497 String, t498 String, t499 String)) + ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0 + + + SYSTEM STOP MERGES t_json_subcolumn_lookup_compact + SYSTEM STOP MERGES t_json_subcolumn_lookup_wide + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 0, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 1000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 2000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 3000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 4000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 5000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 6000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 7000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 8000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 9000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 10000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 11000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 12000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 13000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 14000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 15000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 16000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 17000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 18000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 19000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 20000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 21000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 22000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 23000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 24000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 25000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 26000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 27000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 28000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 29000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 30000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 31000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 32000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 33000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 34000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 35000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 36000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 37000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 38000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 39000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 40000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 41000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 42000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 43000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 44000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 45000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 46000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 47000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 48000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 49000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 50000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 51000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 52000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 53000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 54000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 55000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 56000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 57000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 58000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 59000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 60000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 61000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 62000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 63000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 64000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 65000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 66000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 67000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 68000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 69000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 70000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 71000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 72000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 73000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 74000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 75000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 76000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 77000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 78000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 79000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 80000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 81000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 82000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 83000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 84000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 85000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 86000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 87000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 88000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 89000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 90000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 91000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 92000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 93000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 94000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_compact SELECT number + 95000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 0, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 1000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 2000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 3000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 4000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 5000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 6000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 7000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 8000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 9000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 10000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 11000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 12000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 13000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 14000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 15000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 16000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 17000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 18000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 19000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 20000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 21000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 22000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 23000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 24000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 25000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 26000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 27000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 28000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 29000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 30000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 31000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 32000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 33000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 34000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 35000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 36000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 37000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 38000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 39000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 40000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 41000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 42000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 43000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 44000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 45000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 46000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 47000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 48000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 49000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 50000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 51000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 52000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 53000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 54000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 55000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 56000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 57000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 58000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 59000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 60000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 61000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 62000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 63000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 64000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 65000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 66000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 67000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 68000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 69000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 70000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 71000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 72000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 73000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 74000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 75000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 76000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 77000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 78000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 79000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 80000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 81000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 82000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 83000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 84000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 85000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 86000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 87000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 88000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 89000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 90000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 91000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 92000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 93000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 94000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + INSERT INTO t_json_subcolumn_lookup_wide SELECT number + 95000, toJSONString(map('t' || toString(number % 500), toString(number), 'd' || toString(number % 50), toString(number))) FROM numbers(1000) + + SELECT json.t250 FROM t_json_subcolumn_lookup_compact FORMAT Null + SELECT json.d25 FROM t_json_subcolumn_lookup_compact FORMAT Null + SELECT json.t1, json.t50, json.t100, json.t150, json.t200, json.t250, json.t300, json.t350, json.t400, json.t450 FROM t_json_subcolumn_lookup_compact FORMAT Null + SELECT json.t250 FROM t_json_subcolumn_lookup_compact WHERE id = 50000 FORMAT Null + SELECT json.t250 FROM t_json_subcolumn_lookup_wide FORMAT Null + SELECT json.d25 FROM t_json_subcolumn_lookup_wide FORMAT Null + SELECT json.t1, json.t50, json.t100, json.t150, json.t200, json.t250, json.t300, json.t350, json.t400, json.t450 FROM t_json_subcolumn_lookup_wide FORMAT Null + SELECT json.t250 FROM t_json_subcolumn_lookup_wide WHERE id = 50000 FORMAT Null + + DROP TABLE IF EXISTS t_json_subcolumn_lookup_compact + DROP TABLE IF EXISTS t_json_subcolumn_lookup_wide + diff --git a/tests/performance/map_bucketed_serialization.xml b/tests/performance/map_bucketed_serialization.xml index d86c681a4f3b6..e5016ad60d469 100644 --- a/tests/performance/map_bucketed_serialization.xml +++ b/tests/performance/map_bucketed_serialization.xml @@ -1,6 +1,6 @@ @@ -107,7 +107,7 @@ CREATE TABLE map_subcol_pw_mixed (id UInt64, tags Map(String, String), value UInt32) @@ -126,7 +126,7 @@ OPTIMIZE TABLE map_subcol_pw_mixed FINAL SETTINGS mutations_sync=1 - diff --git a/tests/performance/prewhere_subcolumn_read_steps.xml b/tests/performance/prewhere_subcolumn_read_steps.xml new file mode 100644 index 0000000000000..04cb68d9dd077 --- /dev/null +++ b/tests/performance/prewhere_subcolumn_read_steps.xml @@ -0,0 +1,19 @@ + + + 0 + + + CREATE TABLE t_prewhere_json (id UInt64, j JSON(max_dynamic_paths = 0)) ENGINE = MergeTree ORDER BY id + CREATE TABLE t_prewhere_map (id UInt64, m Map(String, String)) ENGINE = MergeTree ORDER BY id SETTINGS map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets', map_buckets_strategy = 'constant', max_buckets_in_map = 16, map_buckets_min_avg_size = 0 + + INSERT INTO t_prewhere_json SELECT number, concat('{{"kind":"a","n":', toString(number), ',"payload":"', repeat('x', 2000), '"}}') FROM numbers_mt(3000000) + INSERT INTO t_prewhere_map SELECT number, mapFromArrays(arrayMap(i -> 'k' || toString(i), range(16)), arrayMap(i -> repeat('v', 60), range(16))) FROM numbers_mt(10000000) + + + SELECT count() FROM t_prewhere_json WHERE j.kind = 'b' AND length(toString(j)) > 10 + SELECT count() FROM t_prewhere_json WHERE j.kind = 'b' AND length(toString(j.payload)) > 10 + SELECT count() FROM t_prewhere_map WHERE m['k1'] = 'x' AND length(toString(m)) > 10 + + DROP TABLE IF EXISTS t_prewhere_json + DROP TABLE IF EXISTS t_prewhere_map + diff --git a/tests/performance/small_block_contention_workload.xml b/tests/performance/small_block_contention_workload.xml new file mode 100644 index 0000000000000..089e89add6c49 --- /dev/null +++ b/tests/performance/small_block_contention_workload.xml @@ -0,0 +1,13 @@ + + + CREATE RESOURCE IF NOT EXISTS perf_small_block_cpu (WORKER THREAD) + CREATE WORKLOAD IF NOT EXISTS perf_small_block_all + CREATE WORKLOAD IF NOT EXISTS perf_small_block IN perf_small_block_all SETTINGS max_concurrent_threads = 4 + + select sum(number) from numbers_mt(200000) settings max_threads=100, max_block_size = 1, workload = 'perf_small_block' format Null + + DROP WORKLOAD IF EXISTS perf_small_block + DROP WORKLOAD IF EXISTS perf_small_block_all + DROP RESOURCE IF EXISTS perf_small_block_cpu + diff --git a/tests/performance/subcolumn_deserialization_tuple_map.xml b/tests/performance/subcolumn_deserialization_tuple_map.xml index 743f640a916a0..ca5c564b23408 100644 --- a/tests/performance/subcolumn_deserialization_tuple_map.xml +++ b/tests/performance/subcolumn_deserialization_tuple_map.xml @@ -84,10 +84,10 @@ OPTIMIZE TABLE map_basic_compact FINAL - SELECT m['k0'], m['k1'], m['k2'], m['k3'], m['k4'] FROM map_basic_wide WHERE skip = 0 FORMAT Null SETTINGS max_threads = 1, preferred_block_size_bytes = 1000000000000 - SELECT m, m.keys, m.values, m['k0'], m['k1'] FROM map_basic_wide WHERE skip = 0 FORMAT Null SETTINGS max_threads = 1, preferred_block_size_bytes = 1000000000000 - SELECT m['k0'], m['k1'], m['k2'], m['k3'], m['k4'] FROM map_basic_compact FORMAT Null SETTINGS max_threads = 1, preferred_block_size_bytes = 1000000000000 - SELECT m, m.keys, m.values, m['k0'], m['k1'] FROM map_basic_compact FORMAT Null SETTINGS max_threads = 1, preferred_block_size_bytes = 1000000000000 + SELECT m.key_k0, m.key_k1, m.key_k2, m.key_k3, m.key_k4 FROM map_basic_wide WHERE skip = 0 FORMAT Null SETTINGS max_threads = 1, preferred_block_size_bytes = 1000000000000 + SELECT m, m.keys, m.values, m.key_k0, m.key_k1 FROM map_basic_wide WHERE skip = 0 FORMAT Null SETTINGS max_threads = 1, preferred_block_size_bytes = 1000000000000 + SELECT m.key_k0, m.key_k1, m.key_k2, m.key_k3, m.key_k4 FROM map_basic_compact FORMAT Null SETTINGS max_threads = 1, preferred_block_size_bytes = 1000000000000 + SELECT m, m.keys, m.values, m.key_k0, m.key_k1 FROM map_basic_compact FORMAT Null SETTINGS max_threads = 1, preferred_block_size_bytes = 1000000000000 diff --git a/tests/queries/0_stateless/00900_long_parquet_load_2.sh b/tests/queries/0_stateless/00900_long_parquet_load_2.sh index 459655fda1ddc..35b74b839dea2 100755 --- a/tests/queries/0_stateless/00900_long_parquet_load_2.sh +++ b/tests/queries/0_stateless/00900_long_parquet_load_2.sh @@ -65,6 +65,10 @@ EXCLUDE=( # malformed on purpose, so loading it here would print an exception. 05035_delta_byte_array_zero_values.parquet 05035_delta_byte_array_decimal.parquet + # `INT96` timestamps outside the `DateTime64(9)` range (throws with the default `date_time_overflow_behavior`), tested by 05443. + 05443_int96_out_of_range.parquet + # `INT96` timestamp on the lower boundary day of the `DateTime64(9)` range, tested by 05444. + 05444_int96_lower_boundary.parquet ) for NAME in $(find "$DATA_DIR" -type f \( -iname '*.parquet' -o -iname '*.parquet.gz' \) -print0 | xargs -0 -n 1 basename | LC_ALL=C sort | grep -vFf <(printf '%s\n' "${EXCLUDE[@]}")); do diff --git a/tests/queries/0_stateless/01035_avg.reference b/tests/queries/0_stateless/01035_avg.reference index 1f47b3a8b7ea3..192bceec0560d 100644 --- a/tests/queries/0_stateless/01035_avg.reference +++ b/tests/queries/0_stateless/01035_avg.reference @@ -1,6 +1,6 @@ nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan nan --0.5 -0.5 -0.5 -0.5 -0.5 -0.5 127.493856 32355.57552 499999.5 499999.5 499999.5 499999.5 499999.5 499999.5 -0.000500002 0.49999949943727 -0.000005 -0.000004999999999999992 --2767.546272 -999999 --0.50000449943727 +-0.5 -0.5 -0.5 -0.5 -0.5 -0.5 127.467744 31470.715104 124999.5 124999.5 124999.5 124999.5 124999.5 124999.5 -0.0005000056 0.12499949943764 -0.000005000000000000001 -0.0000050000000000000334 +-3182.944672 +249999 +-0.12500449943764 1 1 \N \N 1 1 diff --git a/tests/queries/0_stateless/01035_avg.sql b/tests/queries/0_stateless/01035_avg.sql index 0f7baddaec577..a8aae4480e66b 100644 --- a/tests/queries/0_stateless/01035_avg.sql +++ b/tests/queries/0_stateless/01035_avg.sql @@ -27,7 +27,7 @@ SELECT avg(i8), avg(i16), avg(i32), avg(i64), avg(i128), avg(i256), avg(f32), avg(f64), avg(d32), avg(d64), avg(d128), avg(d256) FROM test_01035_avg; -INSERT INTO test_01035_avg (u64) SELECT number FROM system.numbers LIMIT 1000000; +INSERT INTO test_01035_avg (u64) SELECT number FROM system.numbers LIMIT 250000; SELECT avg(i8), avg(i16), avg(i32), avg(i64), avg(i128), avg(i256), avg(u8), avg(u16), avg(u32), avg(u64), avg(u128), avg(u256), diff --git a/tests/queries/0_stateless/01271_show_privileges.reference b/tests/queries/0_stateless/01271_show_privileges.reference index b06d70208ba21..c5a77060b902a 100644 --- a/tests/queries/0_stateless/01271_show_privileges.reference +++ b/tests/queries/0_stateless/01271_show_privileges.reference @@ -244,5 +244,14 @@ ARROW FLIGHT [] GLOBAL ALL BIGQUERY [] GLOBAL ALL DISK [] GLOBAL ALL SOURCES [] GLOBAL ALL +aiClassify [] GLOBAL AI FUNCTIONS +aiEmbed [] GLOBAL AI FUNCTIONS +aiExtract [] GLOBAL AI FUNCTIONS +aiFilter [] GLOBAL AI FUNCTIONS +aiGenerate [] GLOBAL AI FUNCTIONS +aiRedact [] GLOBAL AI FUNCTIONS +aiSimilarity [] GLOBAL AI FUNCTIONS +aiTranslate [] GLOBAL AI FUNCTIONS +AI FUNCTIONS [] \N ALL ALL ['ALL PRIVILEGES'] \N \N NONE ['USAGE','NO PRIVILEGES'] \N \N diff --git a/tests/queries/0_stateless/01308_orc_output_format_arrays.reference b/tests/queries/0_stateless/01308_orc_output_format_arrays.reference index d771371b26442..0eb78aef36483 100644 --- a/tests/queries/0_stateless/01308_orc_output_format_arrays.reference +++ b/tests/queries/0_stateless/01308_orc_output_format_arrays.reference @@ -1 +1 @@ -0bedc9eaf0b07ce14f59ffd25e1a6df6 - +75b5b0cbd2734c37f4f02bd726c4ea60 - diff --git a/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh b/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh index c1bd1e0e1fa14..f35ad99c1a859 100755 --- a/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh +++ b/tests/queries/0_stateless/01502_jemalloc_percpu_arena.sh @@ -9,8 +9,10 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) ncpus="$(getconf _NPROCESSORS_ONLN)" -# Disable them explicitly, to avoid failing on "Number of CPUs is not deterministic". -export MALLOC_CONF=abort_conf:false,abort:false +# percpu arenas are sized by the possible CPU count (nr_cpu_ids), so a +# restricted affinity mask must not disable them or abort, even with abort_conf:true. +# NOTE: abort:false for compatibility with debug builds. +export MALLOC_CONF=abort_conf:true,abort:false # Regression for: # @@ -22,6 +24,4 @@ taskset --cpu-list $((ncpus-1)) ${CLICKHOUSE_LOCAL} -q 'select 1' 2>&1 # just in case something more complicated taskset --cpu-list $((ncpus-1)) ${CLICKHOUSE_LOCAL} -q 'select count() from numbers_mt(100000000) settings max_threads=100' 2>&1 -# this command should not fail because we specify narenas explicitly -# (even with abort_conf:true) -MALLOC_CONF=abort_conf:true,abort:false,narenas:$((ncpus)) taskset --cpu-list $((ncpus-1)) ${CLICKHOUSE_LOCAL} -q 'select 1' 2>&1 +MALLOC_CONF=$MALLOC_CONF,narenas:$((ncpus)) taskset --cpu-list $((ncpus-1)) ${CLICKHOUSE_LOCAL} -q 'select 1' 2>&1 diff --git a/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh b/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh index 7fa0bb6f47eb5..2a124fe4c3dab 100755 --- a/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh +++ b/tests/queries/0_stateless/01505_pipeline_executor_UAF.sh @@ -5,9 +5,6 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CURDIR"/../shell_config.sh -# Avoid failures due to "Number of CPUs is not deterministic" error from jemalloc. -export MALLOC_CONF=abort_conf:false,abort:false - # Regression for UAF in ThreadPool. # (Triggered under TSAN) for _ in {1..10}; do diff --git a/tests/queries/0_stateless/01514_distributed_cancel_query_on_error.sh b/tests/queries/0_stateless/01514_distributed_cancel_query_on_error.sh index 8e3659d2d07cd..960f2f03174fe 100755 --- a/tests/queries/0_stateless/01514_distributed_cancel_query_on_error.sh +++ b/tests/queries/0_stateless/01514_distributed_cancel_query_on_error.sh @@ -1,10 +1,10 @@ #!/usr/bin/env bash # Tags: distributed, no-llvm-coverage -# no-llvm-coverage: the test needs the shard allocating `groupArray(repeat('a', 100000))` -# per row to hit `max_memory_usage=1G` and emit `Query memory limit exceeded` before the -# other shard cancels. LLVM source-based coverage instrumentation perturbs this -# memory-tracking cadence, so the expected exception sometimes does not surface on `127.3` -# before cancellation, leaving an empty `grep` match and a flaky FAIL. +# no-llvm-coverage: the test needs the shard accumulating `repeat('a', 100000)` per row to hit +# `max_memory_usage=1G` and emit `Query memory limit exceeded` before the other shard cancels. +# LLVM source-based coverage instrumentation perturbs this memory-tracking cadence, so the +# expected exception sometimes does not surface on `127.3` before cancellation, leaving an empty +# `grep` match and a flaky FAIL. CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -14,7 +14,11 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # 1 on 127.2 # 2 on 127.3 # max_block_size to fail faster -# max_memory_usage/_shard_num/repeat() will allow failure on the first shard earlier. +# `groupArrayIf` on `_shard_num` makes `127.3` the only shard that can run out of memory: on +# `127.2` the condition is false for every row, so its aggregate states never grow and it just +# streams `system.numbers` until the initiator cancels it. Before this, `127.2` accumulated +# one-byte strings and reached the same `max_memory_usage` only about 20% later than `127.3`, so +# which shard failed first was a race. opts=( "--max_memory_usage=1G" "--max_block_size=50" @@ -22,9 +26,30 @@ opts=( "--max_distributed_connections=2" "--max_bytes_before_external_group_by=0" "--max_bytes_ratio_before_external_group_by=0" + # The query reads `system.numbers` until a shard runs out of memory, so `max_memory_usage` + # has to be the only limit that can stop it. The test profile of the CI configuration + # (tests/config/users.d/limits.yaml) sets `max_rows_to_read = 20000000`, which `127.2` and + # the initiator reach at about the same time as `127.3` reaches `max_memory_usage`; the query + # then fails with `TOO_MANY_ROWS` instead of `MEMORY_LIMIT_EXCEEDED` and the `grep` below + # finds nothing. + "--max_rows_to_read=0" + # `127.2` now never fails on its own, so if the cancellation this test is about ever breaks, + # the query would stream forever and the test would be killed by the runner's timeout with no + # output. Fail it with `TIMEOUT_EXCEEDED` instead, far above the ~1s the expected exception + # needs. + "--max_execution_time=100" ) -${CLICKHOUSE_CLIENT} "${opts[@]}" -q "SELECT groupArray(repeat('a', if(_shard_num == 2, 100000, 1))), number%100000 k from remote('127.{2,3}', system.numbers) GROUP BY k LIMIT 10e6" |& { - # the query should fail earlier on 127.3 and 127.2 should not even go to the memory limit exceeded error. - grep -F -q "DB::Exception: Received from 127.3:${CLICKHOUSE_PORT_TCP}. DB::Exception: Query memory limit exceeded:" - # while if this will not correctly then it will got the exception from the 127.2:9000 and fail -} +LOG="$CLICKHOUSE_TMP/err-$CLICKHOUSE_DATABASE" +trap 'rm -f "$LOG"' EXIT + +${CLICKHOUSE_CLIENT} "${opts[@]}" -q "SELECT groupArrayIf(repeat('a', 100000), _shard_num == 2), number%100000 k from remote('127.{2,3}', system.numbers) GROUP BY k LIMIT 10e6" > "$LOG" 2>&1 +CODE=$? + +# the query should fail earlier on 127.3 and 127.2 should not even go to the memory limit exceeded error. +# while if this will not correctly then it will got the exception from the 127.2:9000 and fail +if ! grep -F -q "DB::Exception: Received from 127.3:${CLICKHOUSE_PORT_TCP}. DB::Exception: Query memory limit exceeded:" "$LOG"; then + # stderr only on the failing path: a test that writes there and exits 0 is a FAIL for the runner. + echo "Fail, Code: $CODE, client output:" >&2 + cat "$LOG" >&2 + exit 1 +fi diff --git a/tests/queries/0_stateless/01721_join_implicit_cast_long.sql.j2 b/tests/queries/0_stateless/01721_join_implicit_cast_long.sql.j2 index 8d73d1ce9e651..ec0e7ced42581 100644 --- a/tests/queries/0_stateless/01721_join_implicit_cast_long.sql.j2 +++ b/tests/queries/0_stateless/01721_join_implicit_cast_long.sql.j2 @@ -1,4 +1,8 @@ --- Tags: long +-- Tags: long, no-msan +-- no-msan: 5 join algorithms x ~78 statements, several of them deliberately spilling, average +-- ~183s under MSan against ~4s in a release build; the p99 is 73% of the 600s per-test budget, so +-- a run that gets killed on the timeout is an ordinary tail event. Join implicit-cast semantics +-- get no MSan-specific coverage, and the test still runs under ASan, UBSan and TSan. SET enable_analyzer = 1; DROP TABLE IF EXISTS t1; diff --git a/tests/queries/0_stateless/02302_defaults_in_columnar_formats.reference b/tests/queries/0_stateless/02302_defaults_in_columnar_formats.reference index 9942759a9480b..2d3bd2d2ebfd9 100644 --- a/tests/queries/0_stateless/02302_defaults_in_columnar_formats.reference +++ b/tests/queries/0_stateless/02302_defaults_in_columnar_formats.reference @@ -1,3 +1,5 @@ 1 10 42 53 1 10 42 53 1 10 42 53 +1 +1 0 0 0 diff --git a/tests/queries/0_stateless/02302_defaults_in_columnar_formats.sql b/tests/queries/0_stateless/02302_defaults_in_columnar_formats.sql index c49bf392895bb..31bf6c5e6524a 100644 --- a/tests/queries/0_stateless/02302_defaults_in_columnar_formats.sql +++ b/tests/queries/0_stateless/02302_defaults_in_columnar_formats.sql @@ -6,3 +6,13 @@ insert into function file(currentDatabase() || '_data_02302.orc') select 1 as x, select * from file(currentDatabase() || '_data_02302.orc', auto, 'x UInt8, xx UInt8 default 10, y default 42, z default x + xx + y') settings input_format_orc_allow_missing_columns=1; insert into function file(currentDatabase() || '_data_02302.arrow') select 1 as x, null::Nullable(UInt8) as xx settings engine_file_truncate_on_insert=1; select * from file(currentDatabase() || '_data_02302.arrow', auto, 'x UInt8, xx UInt8 default 10, y default 42, z default x + xx + y') settings input_format_arrow_allow_missing_columns=1; + +-- The next two queries test the native Parquet reader, so they pin it (some release branches still randomize it off). +-- A count-only read reads just the single smallest declared column, so `d` must stay strictly +-- narrower than every other column for this to exercise a defaulted one. The count cache would +-- answer before a reader exists, and the two count-only optimizations are randomized off in some runs, so all three are pinned. +-- Missing columns are rejected so that a read which is not count-only fails here instead of +-- synthesizing `d` into a correctly sized mask and returning the same count with no guard in place. +select count() from file(currentDatabase() || '_data_02302.parquet', auto, 'x UInt64, d UInt8 default 10') settings use_cache_for_count_from_files = 0, optimize_count_from_files = 1, optimize_trivial_count_query = 1, input_format_parquet_allow_missing_columns = 0, input_format_parquet_use_native_reader_v3 = 1; +-- With the setting off no `default` expression is applied, so the columns keep their type defaults. +select * from file(currentDatabase() || '_data_02302.parquet', auto, 'x UInt8, xx UInt8 default 10, y default 42, z default x + xx + y') settings input_format_parquet_allow_missing_columns=1, input_format_defaults_for_omitted_fields=0, input_format_parquet_use_native_reader_v3=1; diff --git a/tests/queries/0_stateless/02346_text_index_add_share_nested_offsets_disabled.reference b/tests/queries/0_stateless/02346_text_index_add_share_nested_offsets_disabled.reference new file mode 100644 index 0000000000000..7dd2827664547 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_add_share_nested_offsets_disabled.reference @@ -0,0 +1,4 @@ +id +n +n.a +idx (n) diff --git a/tests/queries/0_stateless/02346_text_index_add_share_nested_offsets_disabled.sql b/tests/queries/0_stateless/02346_text_index_add_share_nested_offsets_disabled.sql new file mode 100644 index 0000000000000..4dff81df89552 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_add_share_nested_offsets_disabled.sql @@ -0,0 +1,16 @@ +-- The `ADD INDEX` permission preflight applies the commands as the table does, so with +-- `share_nested_offsets = 0` a column `n` is independent of `n.a` and is added before the index uses it. + +DROP TABLE IF EXISTS t; + +CREATE TABLE t (id UInt64, `n.a` Array(UInt32)) +ENGINE = MergeTree ORDER BY id SETTINGS share_nested_offsets = 0; + +ALTER TABLE t + ADD COLUMN IF NOT EXISTS n UInt32, + ADD INDEX idx(n) TYPE minmax GRANULARITY 1; + +SELECT name FROM system.columns WHERE database = currentDatabase() AND table = 't' ORDER BY name; +SELECT name, expr FROM system.data_skipping_indices WHERE database = currentDatabase() AND table = 't'; + +DROP TABLE t; diff --git a/tests/queries/0_stateless/02346_text_index_add_with_invalid_alter_command.reference b/tests/queries/0_stateless/02346_text_index_add_with_invalid_alter_command.reference new file mode 100644 index 0000000000000..0f9d2713f87d5 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_add_with_invalid_alter_command.reference @@ -0,0 +1 @@ +idx text diff --git a/tests/queries/0_stateless/02346_text_index_add_with_invalid_alter_command.sql b/tests/queries/0_stateless/02346_text_index_add_with_invalid_alter_command.sql new file mode 100644 index 0000000000000..a629b362ee8eb --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_add_with_invalid_alter_command.sql @@ -0,0 +1,24 @@ +-- An `ALTER` that adds an index is authorized against the schema the whole statement produces. +-- That must not skip the validation of its other commands and run into an exception. + +SET enable_full_text_index = 1; + +DROP TABLE IF EXISTS tab; +CREATE TABLE tab (id UInt64, val String) ENGINE = MergeTree ORDER BY id; + +ALTER TABLE tab + RENAME COLUMN missing TO renamed, + ADD INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha') GRANULARITY 1; -- { serverError NOT_FOUND_COLUMN_IN_BLOCK } + +ALTER TABLE tab + RENAME COLUMN val TO a, + RENAME COLUMN a TO b, + ADD INDEX idx(id) TYPE minmax GRANULARITY 1; -- { serverError NOT_IMPLEMENTED } + +ALTER TABLE tab + RENAME COLUMN IF EXISTS missing TO renamed, + ADD INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = lower(val)) GRANULARITY 1; + +SELECT name, type FROM system.data_skipping_indices WHERE database = currentDatabase() AND table = 'tab'; + +DROP TABLE tab; diff --git a/tests/queries/0_stateless/02346_text_index_transform_access.reference b/tests/queries/0_stateless/02346_text_index_transform_access.reference new file mode 100644 index 0000000000000..3da5718983273 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_transform_access.reference @@ -0,0 +1,24 @@ +1. Direct call denied. +2. Denied in a preprocessor. +3. Denied in a postprocessor. +4. Denied when nested under a String result. +5. Denied via ALTER ADD INDEX. +6. Unprivileged expression unaffected. +1 +7. Accepted with the privilege, so not a blocklist. +1 +8. Existing index stays readable without the privilege. +1 +9. Unrelated ALTER does not re-authorize. +1 +10. Neither does DETACH/ATTACH. +1 +11. An ALIAS column hiding a privileged function is denied. +12. So is an alias renamed by an earlier command of the same ALTER. +13. Redeclaring with a privileged function is denied. +14. The transform preflight does not reject what a plain ALTER accepts. +idx_pre +15. Replacing a privileged alias in the same statement is allowed. +idx_replaced +16. A statement without a transform is not analyzed at all. +idx_minmax diff --git a/tests/queries/0_stateless/02346_text_index_transform_access.sql b/tests/queries/0_stateless/02346_text_index_transform_access.sql new file mode 100644 index 0000000000000..fb873cb6ebebc --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_transform_access.sql @@ -0,0 +1,167 @@ +-- Text index `preprocessor`/`postprocessor` expressions must only use functions the user declaring +-- the index is permitted to run. +-- +-- The test needs a function the current user may not run. Introspection functions fit: they are +-- refused both without the grant and, as here, with `allow_introspection_functions = 0`. Using the +-- setting keeps the test in plain SQL, with no second user to create. + +DROP TABLE IF EXISTS tab; + +-- The test configuration sets this to 1. +SET allow_introspection_functions = 0; + +SELECT '1. Direct call denied.'; + +SELECT demangle('_ZNK2DB7Context9getAccessEv'); -- { serverError FUNCTION_NOT_ALLOWED } + +SELECT '2. Denied in a preprocessor.'; + +CREATE TABLE tab +( + id UInt64, + val String, + INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = demangle(val)) +) +ENGINE = MergeTree ORDER BY id; -- { serverError FUNCTION_NOT_ALLOWED } + +SELECT '3. Denied in a postprocessor.'; + +CREATE TABLE tab +( + id UInt64, + val String, + INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', postprocessor = demangle(val)) +) +ENGINE = MergeTree ORDER BY id; -- { serverError FUNCTION_NOT_ALLOWED } + +SELECT '4. Denied when nested under a String result.'; + +CREATE TABLE tab +( + id UInt64, + val String, + INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = concat(val, demangle(val))) +) +ENGINE = MergeTree ORDER BY id; -- { serverError FUNCTION_NOT_ALLOWED } + +SELECT '5. Denied via ALTER ADD INDEX.'; + +CREATE TABLE tab (id UInt64, val String) ENGINE = MergeTree ORDER BY id; + +ALTER TABLE tab ADD INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = demangle(val)); -- { serverError FUNCTION_NOT_ALLOWED } + +DROP TABLE tab; + +SELECT '6. Unprivileged expression unaffected.'; + +CREATE TABLE tab +( + id UInt64, + val String, + INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = lower(val)) +) +ENGINE = MergeTree ORDER BY id; + +INSERT INTO tab VALUES (1, 'Hello World'); +SELECT count() FROM tab WHERE hasAllTokens(val, ['hello']); + +DROP TABLE tab; + +SELECT '7. Accepted with the privilege, so not a blocklist.'; + +SET allow_introspection_functions = 1; + +-- `demangle` returns a non-mangled argument unchanged. +CREATE TABLE tab +( + id UInt64, + val String, + INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', postprocessor = demangle(val)) +) +ENGINE = MergeTree ORDER BY id; + +INSERT INTO tab VALUES (1, 'hello world'); +SELECT count() FROM tab WHERE hasAllTokens(val, ['hello']); + +SELECT '8. Existing index stays readable without the privilege.'; + +SET allow_introspection_functions = 0; + +SELECT count() FROM tab WHERE hasAllTokens(val, ['hello']); + +SELECT '9. Unrelated ALTER does not re-authorize.'; + +ALTER TABLE tab ADD COLUMN extra UInt8 DEFAULT 0; +ALTER TABLE tab RENAME COLUMN val TO val2; +SELECT count() FROM tab WHERE hasAllTokens(val2, ['hello']); + +SELECT '10. Neither does DETACH/ATTACH.'; + +DETACH TABLE tab; +ATTACH TABLE tab; +SELECT count() FROM tab WHERE hasAllTokens(val2, ['hello']); + +SELECT '11. An ALIAS column hiding a privileged function is denied.'; + +-- Its own table: a column may carry only one text index, and that is checked before permissions are. +SET allow_introspection_functions = 1; +CREATE TABLE tab_alias (id UInt64, val String, mangled String ALIAS demangle(val)) ENGINE = MergeTree ORDER BY id; +SET allow_introspection_functions = 0; + +ALTER TABLE tab_alias ADD INDEX idx_alias(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = mangled) GRANULARITY 1; -- { serverError FUNCTION_NOT_ALLOWED } + +SELECT '12. So is an alias renamed by an earlier command of the same ALTER.'; + +-- The rename is applied before the index is added, so `renamed` is what expands to `demangle(val)`. +ALTER TABLE tab_alias RENAME COLUMN mangled TO renamed, + ADD INDEX idx_renamed(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = renamed) GRANULARITY 1; -- { serverError FUNCTION_NOT_ALLOWED } + +DROP TABLE tab_alias; + +SELECT '13. Redeclaring with a privileged function is denied.'; + +ALTER TABLE tab DROP INDEX idx, + ADD INDEX idx(val2) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = demangle(val2)) GRANULARITY 1; -- { serverError FUNCTION_NOT_ALLOWED } + +DROP TABLE tab; + +SELECT '14. The transform preflight does not reject what a plain ALTER accepts.'; + +CREATE TABLE tab_preflight (id UInt64, val String, num UInt64) +ENGINE = MergeTree ORDER BY id SETTINGS auto_statistics_types = 'minmax'; + +-- The preflight must drop implicit statistics too, or `ADD STATISTICS` looks like a duplicate. +ALTER TABLE tab_preflight ADD INDEX idx_pre(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = lower(val)) GRANULARITY 1, + ADD STATISTICS num TYPE minmax; + +SELECT name FROM system.data_skipping_indices +WHERE database = currentDatabase() AND table = 'tab_preflight' ORDER BY name; + +DROP TABLE tab_preflight; + +SELECT '15. Replacing a privileged alias in the same statement is allowed.'; + +SET allow_introspection_functions = 1; +CREATE TABLE tab_replace (id UInt64, val String, transformed String ALIAS demangle(val)) +ENGINE = MergeTree ORDER BY id; +SET allow_introspection_functions = 0; + +-- The installed transform is `lower(val)`; the alias it replaces must not be demanded. +ALTER TABLE tab_replace MODIFY COLUMN transformed String ALIAS lower(val), + ADD INDEX idx_replaced(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = transformed) GRANULARITY 1; + +SELECT name FROM system.data_skipping_indices +WHERE database = currentDatabase() AND table = 'tab_replace' ORDER BY name; + +DROP TABLE tab_replace; + +SELECT '16. A statement without a transform is not analyzed at all.'; + +SET allow_statistics = 1; +CREATE TABLE tab_plain (x UInt64) ENGINE = MergeTree ORDER BY x SETTINGS auto_statistics_types = 'basic'; + +ALTER TABLE tab_plain ADD INDEX idx_minmax(x) TYPE minmax GRANULARITY 1, ADD STATISTICS x TYPE basic; + +SELECT name FROM system.data_skipping_indices WHERE database = currentDatabase() AND table = 'tab_plain'; + +DROP TABLE tab_plain; diff --git a/tests/queries/0_stateless/02346_text_index_transform_access_replicated_database.reference b/tests/queries/0_stateless/02346_text_index_transform_access_replicated_database.reference new file mode 100644 index 0000000000000..3a9622171b277 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_transform_access_replicated_database.reference @@ -0,0 +1,4 @@ +1. ALTER ADD INDEX is denied. +2. CREATE TABLE is denied. +3. An alias renamed by an earlier command of the same ALTER is denied. +4. Accepted with the privilege. diff --git a/tests/queries/0_stateless/02346_text_index_transform_access_replicated_database.sql b/tests/queries/0_stateless/02346_text_index_transform_access_replicated_database.sql new file mode 100644 index 0000000000000..492afc0a12edd --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_transform_access_replicated_database.sql @@ -0,0 +1,54 @@ +-- Tags: zookeeper, no-replicated-database +-- no-replicated-database: the test creates its own Replicated database + +-- A Replicated database hands the DDL to replicas that apply it without the submitting user, so the +-- permission check happens on the node it was submitted to. See 02346_text_index_transform_access +-- for the checks themselves, and 02346_text_index_transform_on_cluster for `ON CLUSTER`. + +SET allow_introspection_functions = 0; + +-- {CLICKHOUSE_DATABASE_2} is shared by every test in the client, hence the DROP. +DROP DATABASE IF EXISTS {CLICKHOUSE_DATABASE_2:Identifier}; +CREATE DATABASE {CLICKHOUSE_DATABASE_2:Identifier} +ENGINE = Replicated('/clickhouse/02346_text_index_transform_access/{database}', 'shard1', 'replica1') FORMAT Null; + +CREATE TABLE {CLICKHOUSE_DATABASE_2:Identifier}.tab (id UInt64, val String) +ENGINE = MergeTree ORDER BY id FORMAT Null; + +SELECT '1. ALTER ADD INDEX is denied.'; + +ALTER TABLE {CLICKHOUSE_DATABASE_2:Identifier}.tab + ADD INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = demangle(val)) GRANULARITY 1; -- { serverError FUNCTION_NOT_ALLOWED } + +SELECT '2. CREATE TABLE is denied.'; + +CREATE TABLE {CLICKHOUSE_DATABASE_2:Identifier}.tab_denied +( + id UInt64, + val String, + INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = demangle(val)) +) +ENGINE = MergeTree ORDER BY id; -- { serverError FUNCTION_NOT_ALLOWED } + +SELECT '3. An alias renamed by an earlier command of the same ALTER is denied.'; + +-- The initiator has to authorize against the schema the whole statement produces: the hosts run it +-- without the user, and only after the rename does `renamed` expand to `demangle(val)`. +SET allow_introspection_functions = 1; +CREATE TABLE {CLICKHOUSE_DATABASE_2:Identifier}.tab_alias (id UInt64, val String, mangled String ALIAS demangle(val)) +ENGINE = MergeTree ORDER BY id FORMAT Null; +CREATE TABLE tab_on_cluster_alias (id UInt64, val String, mangled String ALIAS demangle(val)) ENGINE = MergeTree ORDER BY id; +SET allow_introspection_functions = 0; + +ALTER TABLE {CLICKHOUSE_DATABASE_2:Identifier}.tab_alias + RENAME COLUMN mangled TO renamed, + ADD INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = renamed) GRANULARITY 1; -- { serverError FUNCTION_NOT_ALLOWED } + +SELECT '4. Accepted with the privilege.'; + +SET allow_introspection_functions = 1; + +ALTER TABLE {CLICKHOUSE_DATABASE_2:Identifier}.tab + ADD INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = demangle(val)) GRANULARITY 1 FORMAT Null; + +DROP DATABASE {CLICKHOUSE_DATABASE_2:Identifier} FORMAT Null; diff --git a/tests/queries/0_stateless/02346_text_index_transform_alter_segments.reference b/tests/queries/0_stateless/02346_text_index_transform_alter_segments.reference new file mode 100644 index 0000000000000..78c5393ccd90a --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_transform_alter_segments.reference @@ -0,0 +1,4 @@ +1. An ordinary index added after a mutation segment, with a rename in each metadata segment. +idx +2. The same with a text index transform, on the local host. +idx_text diff --git a/tests/queries/0_stateless/02346_text_index_transform_alter_segments.sh b/tests/queries/0_stateless/02346_text_index_transform_alter_segments.sh new file mode 100755 index 0000000000000..9039ed2f53914 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_transform_alter_segments.sh @@ -0,0 +1,37 @@ +#!/usr/bin/env bash +# Tags: no-replicated-database +# Tag no-replicated-database: a single `ALTER` of several types is prohibited there, which is exactly what this test needs. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The text index transform authorization of an `ALTER` must not reject a statement the executor accepts. + +echo '1. An ordinary index added after a mutation segment, with a rename in each metadata segment.' +$CLICKHOUSE_CLIENT -n -q " + CREATE TABLE seg (id UInt64, a String, m UInt64 MATERIALIZED 1) ENGINE = MergeTree ORDER BY id; + ALTER TABLE seg + RENAME COLUMN a TO b, + MATERIALIZE COLUMN m, + RENAME COLUMN b TO c, + ADD INDEX idx(c) TYPE minmax GRANULARITY 1; + SELECT name FROM system.data_skipping_indices WHERE database = currentDatabase() AND table = 'seg'; +" + +echo '2. The same with a text index transform, on the local host.' +$CLICKHOUSE_CLIENT -n -q " + SET enable_full_text_index = 1; + CREATE TABLE seg_text (id UInt64, a String, m UInt64 MATERIALIZED 1) ENGINE = MergeTree ORDER BY id; + ALTER TABLE seg_text + RENAME COLUMN a TO b, + MATERIALIZE COLUMN m, + RENAME COLUMN b TO c, + ADD INDEX idx_text(c) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = lower(c)); + SELECT name FROM system.data_skipping_indices WHERE database = currentDatabase() AND table = 'seg_text'; +" + +$CLICKHOUSE_CLIENT -n -q " + DROP TABLE seg; + DROP TABLE seg_text; +" diff --git a/tests/queries/0_stateless/02346_text_index_transform_data_access.reference b/tests/queries/0_stateless/02346_text_index_transform_data_access.reference new file mode 100644 index 0000000000000..3954d852fafaa --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_transform_data_access.reference @@ -0,0 +1,6 @@ +1. A set from a table. +2. A subquery. +3. A dictionary lookup. +4. The same via ALTER ADD INDEX. +5. A literal set is still accepted. +1 diff --git a/tests/queries/0_stateless/02346_text_index_transform_data_access.sql b/tests/queries/0_stateless/02346_text_index_transform_data_access.sql new file mode 100644 index 0000000000000..839daa8bca690 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_transform_data_access.sql @@ -0,0 +1,68 @@ +-- Text index transforms are analysed under the global context, so anything that reads a table or a +-- dictionary while being analysed would do it with full access, and constant folding could reveal +-- what it read. A transform must be a pure function of the indexed value, so these are rejected +-- before any analysis. See 02346_text_index_transform_access for the function permission checks. + +DROP TABLE IF EXISTS tab; +DROP TABLE IF EXISTS set_source; +DROP DICTIONARY IF EXISTS dict; +DROP TABLE IF EXISTS dict_source; + +CREATE TABLE set_source (x String) ENGINE = Set; +INSERT INTO set_source VALUES ('secret'); + +CREATE TABLE dict_source (id UInt64, value String) ENGINE = MergeTree ORDER BY id; +INSERT INTO dict_source VALUES (1, 'secret'); +CREATE DICTIONARY dict (id UInt64, value String) PRIMARY KEY id +SOURCE(CLICKHOUSE(TABLE 'dict_source')) LAYOUT(FLAT()) LIFETIME(0); + +SELECT '1. A set from a table.'; + +CREATE TABLE tab +( + id UInt64, + val String, + INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', postprocessor = if(val IN set_source, val, '')) +) +ENGINE = MergeTree ORDER BY id; -- { serverError INCORRECT_QUERY } + +SELECT '2. A subquery.'; + +CREATE TABLE tab +( + id UInt64, + val String, + INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = concat(val, (SELECT any(value) FROM dict_source))) +) +ENGINE = MergeTree ORDER BY id; -- { serverError INCORRECT_QUERY } + +SELECT '3. A dictionary lookup.'; + +CREATE TABLE tab +( + id UInt64, + val String, + INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', + preprocessor = concat(val, toString(toUInt64(dictGetString(currentDatabase() || '.dict', 'value', toUInt64(1)))))) +) +ENGINE = MergeTree ORDER BY id; -- { serverError INCORRECT_QUERY } + +SELECT '4. The same via ALTER ADD INDEX.'; + +CREATE TABLE tab (id UInt64, val String) ENGINE = MergeTree ORDER BY id; + +ALTER TABLE tab ADD INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', postprocessor = if(val IN set_source, val, '')); -- { serverError INCORRECT_QUERY } +ALTER TABLE tab ADD INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', postprocessor = if(val IN (SELECT x FROM set_source), val, '')); -- { serverError INCORRECT_QUERY } +ALTER TABLE tab ADD INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', + preprocessor = concat(val, toString(toUInt64(dictGetString(currentDatabase() || '.dict', 'value', toUInt64(1)))))); -- { serverError INCORRECT_QUERY } + +SELECT '5. A literal set is still accepted.'; + +ALTER TABLE tab ADD INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', postprocessor = if(val IN ('hello', 'world'), val, '')); +INSERT INTO tab VALUES (1, 'hello there'); +SELECT count() FROM tab WHERE hasAllTokens(val, ['hello']); + +DROP TABLE tab; +DROP DICTIONARY dict; +DROP TABLE dict_source; +DROP TABLE set_source; diff --git a/tests/queries/0_stateless/02346_text_index_transform_on_cluster.reference b/tests/queries/0_stateless/02346_text_index_transform_on_cluster.reference new file mode 100644 index 0000000000000..91fa6db492144 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_transform_on_cluster.reference @@ -0,0 +1,14 @@ +1. ALTER ADD INDEX with a transform is refused. +1 +2. CREATE with a transform is refused. +1 +3. Refused on the legacy entry-format path too, which normalizes on the host. +1 +4. Indices inherited by CREATE AS on the legacy path are refused. +1 +5. An index without a transform is unaffected. +idx_plain +6. CREATE AS with an explicit engine and no privilege on the source. +SHOW COLUMNS ON default.tab +7. The same for a source that does not exist. +SHOW COLUMNS ON default.missing diff --git a/tests/queries/0_stateless/02346_text_index_transform_on_cluster.sh b/tests/queries/0_stateless/02346_text_index_transform_on_cluster.sh new file mode 100755 index 0000000000000..15b49c8bf03b8 --- /dev/null +++ b/tests/queries/0_stateless/02346_text_index_transform_on_cluster.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Tags: zookeeper, no-replicated-database +# no-replicated-database: ON CLUSTER is not allowed for a Replicated database + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The hosts of an `ON CLUSTER` query resolve a text index transform against their own schema and +# without the submitting user, so it is refused unless they run as that user. + +USER="user_${CLICKHOUSE_DATABASE}" + +$CLICKHOUSE_CLIENT -n -q " + DROP USER IF EXISTS $USER; + CREATE TABLE tab (id UInt64, val String, lowered String ALIAS lower(val)) ENGINE = MergeTree ORDER BY id; +" + +echo '1. ALTER ADD INDEX with a transform is refused.' +$CLICKHOUSE_CLIENT --distributed_ddl_output_mode none -q " + ALTER TABLE tab ON CLUSTER test_shard_localhost + ADD INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = lowered) +" 2>&1 | grep -c -m1 "distributed_ddl_use_initial_user_and_roles" + +echo '2. CREATE with a transform is refused.' +$CLICKHOUSE_CLIENT --distributed_ddl_output_mode none -q " + CREATE TABLE ${CLICKHOUSE_DATABASE}.dst_transform ON CLUSTER test_shard_localhost + (id UInt64, val String, INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = lower(val))) + ENGINE = MergeTree ORDER BY id +" 2>&1 | grep -c -m1 "distributed_ddl_use_initial_user_and_roles" + +echo '3. Refused on the legacy entry-format path too, which normalizes on the host.' +$CLICKHOUSE_CLIENT --distributed_ddl_entry_format_version 2 --distributed_ddl_output_mode none -q " + CREATE TABLE ${CLICKHOUSE_DATABASE}.dst_legacy ON CLUSTER test_shard_localhost + (id UInt64, val String, INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = lower(val))) + ENGINE = MergeTree ORDER BY id +" 2>&1 | grep -c -m1 "distributed_ddl_use_initial_user_and_roles" + +echo '4. Indices inherited by CREATE AS on the legacy path are refused.' +$CLICKHOUSE_CLIENT --allow_introspection_functions 1 -q " + CREATE TABLE ${CLICKHOUSE_DATABASE}.src_transform (id UInt64, val String, + INDEX idx(val) TYPE text(tokenizer = 'splitByNonAlpha', preprocessor = demangle(val))) ENGINE = MergeTree ORDER BY id" +$CLICKHOUSE_CLIENT --allow_introspection_functions 1 --distributed_ddl_entry_format_version 2 --distributed_ddl_output_mode none -q " + CREATE TABLE ${CLICKHOUSE_DATABASE}.dst_inherited ON CLUSTER test_shard_localhost + AS ${CLICKHOUSE_DATABASE}.src_transform ENGINE = MergeTree ORDER BY id +" 2>&1 | grep -c -m1 "distributed_ddl_use_initial_user_and_roles" + +echo '5. An index without a transform is unaffected.' +$CLICKHOUSE_CLIENT --distributed_ddl_output_mode none -q " + ALTER TABLE tab ON CLUSTER test_shard_localhost + ADD INDEX idx_plain(val) TYPE text(tokenizer = 'splitByNonAlpha') +" +$CLICKHOUSE_CLIENT -q "SELECT name FROM system.data_skipping_indices WHERE database = currentDatabase() AND table = 'tab' ORDER BY name" + +# With an explicit engine `setEngine` does not look at `AS src`, and the inherited indices are inspected +# on the initiator, so describing the source has to be authorized first. +$CLICKHOUSE_CLIENT -n -q " + CREATE USER $USER; + GRANT CREATE TABLE ON ${CLICKHOUSE_DATABASE}.dst TO $USER; + GRANT TABLE ENGINE ON MergeTree TO $USER; +" + +echo '6. CREATE AS with an explicit engine and no privilege on the source.' +$CLICKHOUSE_CLIENT --user "$USER" --distributed_ddl_entry_format_version 2 --distributed_ddl_output_mode none -q " + CREATE TABLE ${CLICKHOUSE_DATABASE}.dst ON CLUSTER test_shard_localhost AS ${CLICKHOUSE_DATABASE}.tab ENGINE = MergeTree ORDER BY tuple() +" 2>&1 | grep -o -m1 "SHOW COLUMNS ON ${CLICKHOUSE_DATABASE}.tab" + +echo '7. The same for a source that does not exist.' +$CLICKHOUSE_CLIENT --user "$USER" --distributed_ddl_entry_format_version 2 --distributed_ddl_output_mode none -q " + CREATE TABLE ${CLICKHOUSE_DATABASE}.dst ON CLUSTER test_shard_localhost AS ${CLICKHOUSE_DATABASE}.missing ENGINE = MergeTree ORDER BY tuple() +" 2>&1 | grep -o -m1 "SHOW COLUMNS ON ${CLICKHOUSE_DATABASE}.missing" + +$CLICKHOUSE_CLIENT -n -q " + DROP USER $USER; + DROP TABLE tab; + DROP TABLE src_transform; +" diff --git a/tests/queries/0_stateless/02372_analyzer_join.sql.j2 b/tests/queries/0_stateless/02372_analyzer_join.sql.j2 index c9dc41d157090..201e69b64f00d 100644 --- a/tests/queries/0_stateless/02372_analyzer_join.sql.j2 +++ b/tests/queries/0_stateless/02372_analyzer_join.sql.j2 @@ -1,4 +1,4 @@ --- Tags: long +-- Tags: long, no-msan SET enable_analyzer = 1; SET single_join_prefer_left_table = 0; diff --git a/tests/queries/0_stateless/02426_orc_bug.reference b/tests/queries/0_stateless/02426_orc_bug.reference index 4f76072e70d1c..d5f93b43aa8dd 100644 --- a/tests/queries/0_stateless/02426_orc_bug.reference +++ b/tests/queries/0_stateless/02426_orc_bug.reference @@ -1 +1 @@ -64a50db9526e0e3aa8c969939922658d - +f8d66b83061a9b0ce960c0b9ce32a64c - diff --git a/tests/queries/0_stateless/02884_authentication_quota.reference b/tests/queries/0_stateless/02884_authentication_quota.reference index d8ed6d97b5051..c412693811857 100644 --- a/tests/queries/0_stateless/02884_authentication_quota.reference +++ b/tests/queries/0_stateless/02884_authentication_quota.reference @@ -6,7 +6,7 @@ > Login to the user account using the wrong password. password is incorrect > Quota is exceeded 1 >= 1. Login with correct password should fail. -QUOTA_EXCEEDED +password is incorrect > Check the failed_sequential_authentications, max_failed_sequential_authentications fields. 2 1 > Alter the quota with MAX FAILED SEQUENTIAL AUTHENTICATIONS = 4 @@ -18,9 +18,9 @@ password is incorrect password is incorrect password is incorrect password is incorrect -QUOTA_EXCEEDED +password is incorrect > Also try to login with correct password. Quota should stay exceeded. -QUOTA_EXCEEDED +password is incorrect > Check the failed_sequential_authentications, max_failed_sequential_authentications fields. 6 4 > Reset the quota by increasing MAX FAILED SEQUENTIAL AUTHENTICATIONS and successful login @@ -32,7 +32,7 @@ QUOTA_EXCEEDED > Login to the user account using the wrong password. password is incorrect > Quota is exceeded 1 >= 1. Login with correct password should fail. -QUOTA_EXCEEDED +password is incorrect > Check the failed_sequential_authentications, max_failed_sequential_authentications fields. 2 1 > Alter the quota with MAX FAILED SEQUENTIAL AUTHENTICATIONS = 4 @@ -44,9 +44,9 @@ password is incorrect password is incorrect password is incorrect password is incorrect -QUOTA_EXCEEDED +password is incorrect > Also try to login with correct password. Quota should stay exceeded. -QUOTA_EXCEEDED +password is incorrect > Check the failed_sequential_authentications, max_failed_sequential_authentications fields. 6 4 > Reset the quota by increasing MAX FAILED SEQUENTIAL AUTHENTICATIONS and successful login diff --git a/tests/queries/0_stateless/02884_authentication_quota.sh b/tests/queries/0_stateless/02884_authentication_quota.sh index bca24bf6cc275..b092e0b35359a 100755 --- a/tests/queries/0_stateless/02884_authentication_quota.sh +++ b/tests/queries/0_stateless/02884_authentication_quota.sh @@ -18,7 +18,7 @@ function login_test() ${CLICKHOUSE_CLIENT} --user ${USER} --password "wrong_pass" --query "select 1 format Null" 2>&1 | grep -m1 -o 'password is incorrect' echo "> Quota is exceeded 1 >= 1. Login with correct password should fail." - ${CLICKHOUSE_CLIENT} --user ${USER} --password "pass" --query "select 1 format Null" 2>&1 | grep -m1 -o 'QUOTA_EXCEEDED' + ${CLICKHOUSE_CLIENT} --user ${USER} --password "pass" --query "select 1 format Null" 2>&1 | grep -m1 -o 'password is incorrect' echo "> Check the failed_sequential_authentications, max_failed_sequential_authentications fields." ${CLICKHOUSE_CLIENT} -q "SELECT failed_sequential_authentications, max_failed_sequential_authentications FROM system.quotas_usage WHERE quota_name = '${QUOTA}'" @@ -37,10 +37,10 @@ function login_test() ${CLICKHOUSE_CLIENT} --user ${USER} --password "wrong_pass" --query "select 1 format Null" 2>&1 | grep -m1 -o 'password is incorrect' ${CLICKHOUSE_CLIENT} --user ${USER} --password "wrong_pass" --query "select 1 format Null" 2>&1 | grep -m1 -o 'password is incorrect' ${CLICKHOUSE_CLIENT} --user ${USER} --password "wrong_pass" --query "select 1 format Null" 2>&1 | grep -m1 -o 'password is incorrect' - ${CLICKHOUSE_CLIENT} --user ${USER} --password "wrong_pass" --query "select 1 format Null" 2>&1 | grep -m1 -o 'QUOTA_EXCEEDED' + ${CLICKHOUSE_CLIENT} --user ${USER} --password "wrong_pass" --query "select 1 format Null" 2>&1 | grep -m1 -o 'password is incorrect' echo "> Also try to login with correct password. Quota should stay exceeded." - ${CLICKHOUSE_CLIENT} --user ${USER} --password "pass" --query "select 1 format Null" 2>&1 | grep -m1 -o 'QUOTA_EXCEEDED' + ${CLICKHOUSE_CLIENT} --user ${USER} --password "pass" --query "select 1 format Null" 2>&1 | grep -m1 -o 'password is incorrect' echo "> Check the failed_sequential_authentications, max_failed_sequential_authentications fields." ${CLICKHOUSE_CLIENT} -q "SELECT failed_sequential_authentications, max_failed_sequential_authentications FROM system.quotas_usage WHERE quota_name = '${QUOTA}'" diff --git a/tests/queries/0_stateless/03100_lwu_58_pruning_empty_partition_zookeeper.reference b/tests/queries/0_stateless/03100_lwu_58_pruning_empty_partition_zookeeper.reference new file mode 100644 index 0000000000000..8c21354a38e1b --- /dev/null +++ b/tests/queries/0_stateless/03100_lwu_58_pruning_empty_partition_zookeeper.reference @@ -0,0 +1,6 @@ +1 +0 +0 +1 +0 +0 diff --git a/tests/queries/0_stateless/03100_lwu_58_pruning_empty_partition_zookeeper.sh b/tests/queries/0_stateless/03100_lwu_58_pruning_empty_partition_zookeeper.sh new file mode 100755 index 0000000000000..bc088b046271c --- /dev/null +++ b/tests/queries/0_stateless/03100_lwu_58_pruning_empty_partition_zookeeper.sh @@ -0,0 +1,140 @@ +#!/usr/bin/env bash +# Tags: zookeeper, no-fasttest, no-parallel, no-replicated-database, no-shared-merge-tree +# no-parallel: uses a global failpoint and two replicas with stopped replication queues. +# no-fasttest: a test that arms a global failpoint must run alone, and such tests are kept out of the fast test. +# no-replicated-database: the test controls replication queues of individual tables. +# no-shared-merge-tree: the failpoint is in the `ReplicatedMergeTree` block allocation path. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +set -e + +function cleanup_tables() +{ + $CLICKHOUSE_CLIENT --query "SYSTEM DISABLE FAILPOINT rmt_lightweight_update_sleep_after_block_allocation" 2>/dev/null || true + $CLICKHOUSE_CLIENT --query "SYSTEM START REPLICATION QUEUES $R1" 2>/dev/null || true + $CLICKHOUSE_CLIENT --query "SYSTEM START REPLICATION QUEUES $R2" 2>/dev/null || true + wait || true + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS $R2 SYNC" 2>/dev/null || true + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS $R1 SYNC" 2>/dev/null || true +} + +function cleanup() +{ + cleanup_tables +} +trap cleanup EXIT + +# Runs a lightweight update of partition 2 whose rows arrive from the other replica while the update +# waits for committing inserts, and checks that the update has patched all of them. +function run_case() +{ + local name=$1 + local assignment=$2 + R1=t_lwu_prune_empty_03100_58_${name}_r1 + R2=t_lwu_prune_empty_03100_58_${name}_r2 + + $CLICKHOUSE_CLIENT --query " + SET insert_keeper_fault_injection_probability = 0.0; + + DROP TABLE IF EXISTS $R2 SYNC; + DROP TABLE IF EXISTS $R1 SYNC; + + CREATE TABLE $R1 (p UInt8, x UInt64, v UInt64) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/t_lwu_prune_empty_03100_58_$name', 'r1') + PARTITION BY p ORDER BY x + SETTINGS + enable_block_number_column = 1, + enable_block_offset_column = 1, + remove_empty_parts = 0; + + CREATE TABLE $R2 (p UInt8, x UInt64, v UInt64) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/t_lwu_prune_empty_03100_58_$name', 'r2') + PARTITION BY p ORDER BY x + SETTINGS + enable_block_number_column = 1, + enable_block_offset_column = 1, + remove_empty_parts = 0; + + INSERT INTO $R1 SELECT 1, number, 1 FROM numbers(50); + INSERT INTO $R1 SELECT 2, number, 1 FROM numbers(50); + SYSTEM SYNC REPLICA $R2; + + ALTER TABLE $R1 DELETE WHERE p = 2 SETTINGS mutations_sync = 2; + + SYSTEM STOP REPLICATION QUEUES $R1; + + SELECT count() FROM system.parts + WHERE database = currentDatabase() AND table = '$R1' AND partition_id = '2' AND active AND rows = 0; + " + + $CLICKHOUSE_CLIENT --query "INSERT INTO $R2 SELECT 2, 1000000 + number, 1 FROM numbers(30)" + + block_numbers_path="/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/t_lwu_prune_empty_03100_58_$name/block_numbers/2" + block_count_before=$($CLICKHOUSE_CLIENT --query " + SYSTEM FLUSH LOGS zookeeper_log; + SELECT count() FROM system.zookeeper_log + WHERE type = 'Response' AND op_num = 'Create' AND path = '$block_numbers_path/block-' + AND startsWith(path_created, '$block_numbers_path/block-')") + + $CLICKHOUSE_CLIENT --query "SYSTEM ENABLE FAILPOINT rmt_lightweight_update_sleep_after_block_allocation" + + $CLICKHOUSE_CLIENT --query "UPDATE $R1 SET v = $assignment WHERE p = 2 SETTINGS enable_lightweight_update = 1, enable_shared_storage_snapshot_in_query = 1" & + + for _ in {0..300}; do + sleep 0.3 + block_count_after=$($CLICKHOUSE_CLIENT --query " + SYSTEM FLUSH LOGS zookeeper_log; + SELECT count() FROM system.zookeeper_log + WHERE type = 'Response' AND op_num = 'Create' AND path = '$block_numbers_path/block-' + AND startsWith(path_created, '$block_numbers_path/block-')") + if [[ "$block_count_after" -gt "$block_count_before" ]]; then + break + fi + done + + if [[ "$block_count_after" -le "$block_count_before" ]]; then + echo "Failed to observe a new block allocation for lightweight update at $block_numbers_path" >&2 + exit 2 + fi + + $CLICKHOUSE_CLIENT --query "SYSTEM START REPLICATION QUEUES $R1" + + wait + + + result=$($CLICKHOUSE_CLIENT --query " + SYSTEM SYNC REPLICA $R1; + SYSTEM SYNC REPLICA $R2; + + SELECT count() FROM $R1 WHERE p = 2 AND v = 1 SETTINGS apply_patch_parts = 1; + SELECT count() FROM $R2 WHERE p = 2 AND v = 1 SETTINGS apply_patch_parts = 1") + + if [[ "$result" != $'0\n0' ]]; then + echo "Unexpected unpatched-row counts in case $name:" >&2 + echo "$result" >&2 + $CLICKHOUSE_CLIENT --query " + SELECT 'patch parts', name, rows FROM system.parts + WHERE database = currentDatabase() AND table IN ('$R1', '$R2') AND startsWith(name, 'patch') + ORDER BY table, name; + SELECT 'partition 2 parts', table, name, rows, min_block_number, max_block_number FROM system.parts + WHERE database = currentDatabase() AND table IN ('$R1', '$R2') AND partition_id = '2' + ORDER BY table, name; + SELECT 'frozen mutation version', table, mutation_id, block_numbers.partition_id, block_numbers.number + FROM system.mutations + WHERE database = currentDatabase() AND table = '$R1' + ORDER BY create_time DESC;" >&2 + exit 1 + fi + + echo "$result" + + cleanup_tables +} + +run_case plain 999 +# The `IN` operand names the updated table, so validating it in `InterpreterUpdateQuery` takes a storage +# snapshot of the table before the update waits for committing inserts. +run_case in_assignment "if((p, x, v) IN t_lwu_prune_empty_03100_58_in_assignment_r1, 999, 998)" diff --git a/tests/queries/0_stateless/03300_ai_functions.reference b/tests/queries/0_stateless/03300_ai_functions.reference index 7fd7da0270370..146bff8b777d6 100644 --- a/tests/queries/0_stateless/03300_ai_functions.reference +++ b/tests/queries/0_stateless/03300_ai_functions.reference @@ -47,7 +47,7 @@ ai_function_embedding_max_batch_size 100 ai_function_max_api_calls_per_query 0 ai_function_max_input_tokens_per_query 0 ai_function_max_output_tokens_per_query 0 -ai_function_max_retries 1 +ai_function_max_retries 3 ai_function_request_timeout_sec 60 ai_function_retry_initial_delay_ms 1000 ai_function_text_default_credentials diff --git a/tests/queries/0_stateless/03300_ai_functions.sql b/tests/queries/0_stateless/03300_ai_functions.sql index 5ed5773b0f8e2..f787e53a51b32 100644 --- a/tests/queries/0_stateless/03300_ai_functions.sql +++ b/tests/queries/0_stateless/03300_ai_functions.sql @@ -15,6 +15,9 @@ -- `ai_function_text_default_credentials` / `ai_function_embedding_default_credentials`. -- ============================================================================= +-- The requests below go to unreachable endpoints. One retry keeps each failing row fast. +SET ai_function_max_retries = 1; + -- Helper table: a String column with zero rows, used to test function behavior -- without triggering actual HTTP calls. A non-constant column prevents the -- optimizer from constant-folding the AI function during analysis. diff --git a/tests/queries/0_stateless/03784_bad_base_backup.sh b/tests/queries/0_stateless/03784_bad_base_backup.sh index ef0cdd18a1102..6ac1401f2feae 100755 --- a/tests/queries/0_stateless/03784_bad_base_backup.sh +++ b/tests/queries/0_stateless/03784_bad_base_backup.sh @@ -48,3 +48,15 @@ thread & thread & wait + +# The databases are created outside of the test database, so they must be dropped here: +# leftover `Replicated` databases break tests that enumerate all `Replicated` databases of the server, +# such as `05061_system_drop_database_replica_access`. Wait for the `ASYNC` backups to finish first. +for _ in $(seq 1 600) +do + [[ $($CLICKHOUSE_CLIENT --query "SELECT count() FROM system.backups WHERE status = 'CREATING_BACKUP' AND name LIKE '%${CLICKHOUSE_DATABASE}%'") == 0 ]] && break + sleep 0.1 +done + +$CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS d1_$CLICKHOUSE_DATABASE SYNC" +$CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS d2_$CLICKHOUSE_DATABASE SYNC" diff --git a/tests/queries/0_stateless/03821_drop_workload_during_query_race.sh b/tests/queries/0_stateless/03821_drop_workload_during_query_race.sh index e94aca50620da..95280b1f9347e 100755 --- a/tests/queries/0_stateless/03821_drop_workload_during_query_race.sh +++ b/tests/queries/0_stateless/03821_drop_workload_during_query_race.sh @@ -32,13 +32,16 @@ CALL_TIMEOUT=30 function thread_query() { local TIMELIMIT=$((SECONDS+TIMEOUT)) + local out while [ $SECONDS -lt "$TIMELIMIT" ] do # Run a short query that uses CPU scheduling # Use smaller numbers so queries finish quickly - # Ignore expected errors when workload is dropped during query - timeout "$CALL_TIMEOUT" $CLICKHOUSE_CLIENT --format Null -q "SELECT sum(number) FROM numbers(100000) SETTINGS workload = '$WORKLOAD_NAME'" 2>&1 \ - | { grep -v -e "RESOURCE_ACCESS_DENIED" -e "INVALID_SCHEDULER_NODE" -e "There is no resource" -e "^$" || true; } + # A dropped workload fails the query with an expected error; its report spans several lines, so drop it whole + out=$(timeout "$CALL_TIMEOUT" $CLICKHOUSE_CLIENT --format Null -q "SELECT sum(number) FROM numbers(100000) SETTINGS workload = '$WORKLOAD_NAME'" 2>&1) + if [ -n "$out" ] && ! grep -q -e "RESOURCE_ACCESS_DENIED" -e "INVALID_SCHEDULER_NODE" -e "There is no resource" <<< "$out"; then + echo "$out" + fi done } diff --git a/tests/queries/0_stateless/03822_file_function_read_on_file_grant.sh b/tests/queries/0_stateless/03822_file_function_read_on_file_grant.sh index 44023bb901007..90b5207bf2d70 100755 --- a/tests/queries/0_stateless/03822_file_function_read_on_file_grant.sh +++ b/tests/queries/0_stateless/03822_file_function_read_on_file_grant.sh @@ -19,7 +19,7 @@ EOF (( $(${CLICKHOUSE_CLIENT} --user $user --query "SELECT file('$missing_txt')" 2>&1 | grep -c "ACCESS_DENIED") >= 1 )) && echo "ACCESS_DENIED" || echo "UNEXPECTED"; (( $(${CLICKHOUSE_CLIENT} --user $user --query "DESCRIBE TABLE file('$missing_csv', 'CSV')" 2>&1 | grep -c "ACCESS_DENIED") >= 1 )) && echo "ACCESS_DENIED" || echo "UNEXPECTED"; -${CLICKHOUSE_CLIENT} --query "GRANT READ ON FILE TO $user"; +${CLICKHOUSE_CLIENT} --query "GRANT READ ON FILE, CREATE TEMPORARY TABLE ON *.* TO $user"; (( $(${CLICKHOUSE_CLIENT} --user $user --query "SELECT file('$missing_txt')" 2>&1 | grep -c "FILE_DOESNT_EXIST") >= 1 )) && echo "FILE_DOESNT_EXIST" || echo "UNEXPECTED"; (( $(${CLICKHOUSE_CLIENT} --user $user --query "DESCRIBE TABLE file('$missing_csv', 'CSV')" 2>&1 | grep -c "CANNOT_STAT") >= 1 )) && echo "CANNOT_STAT" || echo "UNEXPECTED"; diff --git a/tests/queries/0_stateless/03923_kafka2_keeper_offsets.sh b/tests/queries/0_stateless/03923_kafka2_keeper_offsets.sh index 8621731b831b2..fb6179f956a80 100755 --- a/tests/queries/0_stateless/03923_kafka2_keeper_offsets.sh +++ b/tests/queries/0_stateless/03923_kafka2_keeper_offsets.sh @@ -116,7 +116,7 @@ $CLICKHOUSE_CLIENT --allow_experimental_kafka_offsets_storage_in_keeper 1 -q " kafka_format = 'JSONEachRow', kafka_max_block_size = 100, kafka_keeper_path = '$KEEPER_PATH', - kafka_replica_name = 'r1'; + kafka_replica_name = '{replica}'; " # Create destination table diff --git a/tests/queries/0_stateless/03928_deferred_filters_after_final_in_explain.reference b/tests/queries/0_stateless/03928_deferred_filters_after_final_in_explain.reference index 297db7472339e..f7a363d0eab41 100644 --- a/tests/queries/0_stateless/03928_deferred_filters_after_final_in_explain.reference +++ b/tests/queries/0_stateless/03928_deferred_filters_after_final_in_explain.reference @@ -40,6 +40,10 @@ Positions: 3 0 1 Deferred filters (applied after FINAL) Deferred row level filter column: notEquals(y, \'ccc\'_String) Deferred prewhere filter column: notEquals(y, \'ccc\'_String) + Virtual row conversions + Actions: INPUT : 0 -> x UInt32 : 0 + ALIAS x :: 0 -> __table1.x UInt32 : 1 + Positions: 1 = full plan: nothing deferred = Expression (Project names) Actions: INPUT : 0 -> __table1.x UInt32 : 0 @@ -79,6 +83,10 @@ Positions: 3 0 1 COLUMN Const(String) -> \'ccc\'_String String : 1 FUNCTION notEquals(y : 0, \'ccc\'_String :: 1) -> notEquals(y, \'ccc\'_String) UInt8 : 2 Positions: 2 0 + Virtual row conversions + Actions: INPUT : 0 -> x UInt32 : 0 + ALIAS x :: 0 -> __table1.x UInt32 : 1 + Positions: 1 = row policy deferred = Deferred filters (applied after FINAL) Deferred row level filter column: notEquals(y, \'ccc\'_String) diff --git a/tests/queries/0_stateless/03928_deferred_filters_after_final_in_explain.sql b/tests/queries/0_stateless/03928_deferred_filters_after_final_in_explain.sql index 8b4b08f97b66d..352ad8ef5f8b3 100644 --- a/tests/queries/0_stateless/03928_deferred_filters_after_final_in_explain.sql +++ b/tests/queries/0_stateless/03928_deferred_filters_after_final_in_explain.sql @@ -14,6 +14,7 @@ INSERT INTO tab SELECT 1, 'ccc', 2; CREATE ROW POLICY pol1 ON tab USING y != 'ccc' TO ALL; SET enable_analyzer = 1; +SET read_in_order_use_virtual_row = 1; SELECT '= full plan: both deferred ='; EXPLAIN actions=1 SELECT * FROM tab FINAL PREWHERE y != 'ccc' ORDER BY x diff --git a/tests/queries/0_stateless/03989_map_subcolumn_function_to_subcolumns.reference b/tests/queries/0_stateless/03989_map_subcolumn_function_to_subcolumns.reference index 491b7765b057a..7d4cda3c23953 100644 --- a/tests/queries/0_stateless/03989_map_subcolumn_function_to_subcolumns.reference +++ b/tests/queries/0_stateless/03989_map_subcolumn_function_to_subcolumns.reference @@ -16,7 +16,7 @@ 1 1 1 --- Optimization also works with basic (non-bucketed) serialization +-- No optimization with basic (non-bucketed) serialization 1 -- Correctness: results with optimization 7 7 70 diff --git a/tests/queries/0_stateless/03989_map_subcolumn_function_to_subcolumns.sql b/tests/queries/0_stateless/03989_map_subcolumn_function_to_subcolumns.sql index b161c8d4067d8..71b870061030e 100644 --- a/tests/queries/0_stateless/03989_map_subcolumn_function_to_subcolumns.sql +++ b/tests/queries/0_stateless/03989_map_subcolumn_function_to_subcolumns.sql @@ -151,20 +151,19 @@ SELECT count() = 0 FROM (EXPLAIN actions = 1 SELECT id FROM t_map_subcolumns WHE -- ========================================== -- Section 8: basic serialization (not with_buckets) --- The optimization still applies because DataTypeMap exposes dynamic subcolumns --- regardless of serialization version. With basic serialization, the whole map is --- read and the key is extracted; with_buckets reads only the relevant bucket. +-- The rewrite is not applied: with basic serialization the key subcolumn reads the whole map anyway. -- ========================================== DROP TABLE t_map_subcolumns; CREATE TABLE t_map_subcolumns (id UInt64, m Map(String, UInt64)) -ENGINE = MergeTree ORDER BY id; +ENGINE = MergeTree ORDER BY id +SETTINGS map_serialization_version = 'basic', map_serialization_version_for_zero_level_parts = 'basic'; INSERT INTO t_map_subcolumns SELECT number, map('key1', number) FROM numbers(10); -SELECT '-- Optimization also works with basic (non-bucketed) serialization'; -SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT id FROM t_map_subcolumns WHERE m['key1'] > 5) WHERE explain LIKE '%m.key_key1%'; +SELECT '-- No optimization with basic (non-bucketed) serialization'; +SELECT count() = 0 FROM (EXPLAIN actions = 1 SELECT id FROM t_map_subcolumns WHERE m['key1'] > 5) WHERE explain LIKE '%m.key_key1%'; -- ========================================== -- Section 9: Correctness - verify actual query results after optimization diff --git a/tests/queries/0_stateless/04000_map_subcolumns_prewhere.reference b/tests/queries/0_stateless/04000_map_subcolumns_prewhere.reference index 89024e17dd7f3..a108bf0eea76d 100644 --- a/tests/queries/0_stateless/04000_map_subcolumns_prewhere.reference +++ b/tests/queries/0_stateless/04000_map_subcolumns_prewhere.reference @@ -54,7 +54,7 @@ 98 98 2 99 99 1 -- Section 4: PREWHERE with basic serialization --- Optimization still rewrites to subcolumn +-- No rewrite to subcolumn 1 -- Correctness: PREWHERE with basic serialization 96 96 960 diff --git a/tests/queries/0_stateless/04000_map_subcolumns_prewhere.sql b/tests/queries/0_stateless/04000_map_subcolumns_prewhere.sql index 3c2e43838788d..704ed0d09ef1e 100644 --- a/tests/queries/0_stateless/04000_map_subcolumns_prewhere.sql +++ b/tests/queries/0_stateless/04000_map_subcolumns_prewhere.sql @@ -134,7 +134,7 @@ DROP TABLE t_map_prewhere; -- ========================================== -- Section 4: PREWHERE with basic serialization (non-bucketed) --- Optimization still applies, but reads the whole map internally. +-- No rewrite to the key subcolumn, it would read the whole map anyway. -- ========================================== DROP TABLE IF EXISTS t_map_prewhere; @@ -142,14 +142,15 @@ CREATE TABLE t_map_prewhere (id UInt64, m Map(String, UInt64)) ENGINE = MergeTree ORDER BY id SETTINGS map_serialization_version = 'basic', + map_serialization_version_for_zero_level_parts = 'basic', min_bytes_for_wide_part = 1, min_rows_for_wide_part = 1; INSERT INTO t_map_prewhere SELECT number, map('key1', number, 'key2', number * 10) FROM numbers(100); SELECT '-- Section 4: PREWHERE with basic serialization'; -SELECT '-- Optimization still rewrites to subcolumn'; -SELECT count() > 0 FROM ( +SELECT '-- No rewrite to subcolumn'; +SELECT count() = 0 FROM ( EXPLAIN actions = 1 SELECT id, m FROM t_map_prewhere PREWHERE m['key1'] > 95 ) WHERE explain LIKE '%m.key_key1%'; diff --git a/tests/queries/0_stateless/04039_text_index_map_values_subcolumn.sql b/tests/queries/0_stateless/04039_text_index_map_values_subcolumn.sql index e12e21f125aad..170b94727c0e1 100644 --- a/tests/queries/0_stateless/04039_text_index_map_values_subcolumn.sql +++ b/tests/queries/0_stateless/04039_text_index_map_values_subcolumn.sql @@ -1,7 +1,7 @@ -- Tags: no-random-settings -- Tests that text indexes built on mapValues(m) work correctly when the analyzer --- rewrites arrayElement(m, 'key') into the map.key_* subcolumn form. +-- rewrites arrayElement(m, 'key') into the map.key_* subcolumn form, which it does only for bucketed Maps. SET explain_query_plan_default = 'legacy'; SET enable_analyzer = 1; @@ -17,7 +17,8 @@ CREATE TABLE tab ) ENGINE = MergeTree ORDER BY id -SETTINGS index_granularity = 1, index_granularity_bytes = '10M', min_bytes_for_wide_part = 0; +SETTINGS index_granularity = 1, index_granularity_bytes = '10M', min_bytes_for_wide_part = 0, + map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO tab VALUES (0, {'service':'web-api'}), (1, {'service':'backend'}), (2, {'service':'frontend'}); diff --git a/tests/queries/0_stateless/04040_commit_order_projection_merge_levels.sql b/tests/queries/0_stateless/04040_commit_order_projection_merge_levels.sql index 594092c7f3e61..77021b6f561a9 100644 --- a/tests/queries/0_stateless/04040_commit_order_projection_merge_levels.sql +++ b/tests/queries/0_stateless/04040_commit_order_projection_merge_levels.sql @@ -2,6 +2,11 @@ set enable_analyzer = 1; +-- Bound the wait, so that a scheduled merge that never runs fails this test with +-- `TIMEOUT_EXCEEDED` instead of polling until the harness kills it: `SYSTEM SYNC MERGES` treats +-- `max_execution_time = 0`, the default of a stateless test, as "wait forever". +set max_execution_time = 120; + drop table if exists mt_merge_levels sync; CREATE TABLE mt_merge_levels( @@ -29,6 +34,14 @@ SYSTEM SYNC MERGES mt_merge_levels; SYSTEM SCHEDULE MERGE mt_merge_levels PARTS 'all_1_2_1', 'all_3_4_1'; SYSTEM SYNC MERGES mt_merge_levels; +-- `SYSTEM SYNC MERGES` returns as soon as the merged part is among the active parts, which +-- `MergePlainMergeTreeTask::finish` makes true in `transaction.commit()` - before it pushes the +-- `MergeParts` row of that merge with `write_part_log`. Flushing `part_log` right after +-- `SYSTEM SYNC MERGES` can therefore miss the row of the last merge. Dropping the table first +-- closes that window: the drop waits for the merge task to run to completion, and `part_log` +-- keeps the database and table names as strings, so the queries below still find the rows. +drop table mt_merge_levels sync; + -- Check part_log ProfileEvents to verify rebuild vs merge behavior system flush logs part_log; @@ -43,5 +56,3 @@ select part_name, ProfileEvents['RebuiltProjections'] as rebuilt, ProfileEvents[ from system.part_log where database = currentDatabase() and table = 'mt_merge_levels' and event_type = 'MergeParts' and part_name = 'all_1_4_2' order by part_name; - -drop table mt_merge_levels sync; diff --git a/tests/queries/0_stateless/04142_ai_functions_named_collection_access.sh b/tests/queries/0_stateless/04142_ai_functions_named_collection_access.sh index 017b8258c6d40..811dd75331a9f 100755 --- a/tests/queries/0_stateless/04142_ai_functions_named_collection_access.sh +++ b/tests/queries/0_stateless/04142_ai_functions_named_collection_access.sh @@ -21,6 +21,7 @@ CREATE NAMED COLLECTION $collection_name AS api_key = 'fake-key'; CREATE USER $user_name IDENTIFIED WITH plaintext_password BY 'password'; +GRANT AI FUNCTIONS ON *.* TO $user_name; " # Runs the AI functions as the test user in one client invocation and emits one line @@ -31,7 +32,7 @@ CREATE USER $user_name IDENTIFIED WITH plaintext_password BY 'password'; # when no row reaches the function — otherwise an empty input could be used to bypass the grant. function check_access_both() { - $CLICKHOUSE_CLIENT --user "$user_name" --password "password" --multiquery --ignore-error -q " + $CLICKHOUSE_CLIENT --user "$user_name" --password "password" --ai_function_max_retries 1 --multiquery --ignore-error -q " SELECT aiGenerate('hi', map('credentials', '$collection_name')) FORMAT Null; SELECT 'SEP'; SELECT aiEmbed('hi', 'test-model', map('credentials', '$collection_name')) FORMAT Null; @@ -56,7 +57,7 @@ function check_access_both() # for the embedding functions) that must enforce the same NAMED_COLLECTION grant. function check_access_both_default() { - $CLICKHOUSE_CLIENT --user "$user_name" --password "password" --multiquery --ignore-error -q " + $CLICKHOUSE_CLIENT --user "$user_name" --password "password" --ai_function_max_retries 1 --multiquery --ignore-error -q " SET ai_function_text_default_credentials = '$collection_name'; SET ai_function_embedding_default_credentials = '$collection_name'; SELECT aiGenerate('hi') FORMAT Null; diff --git a/tests/queries/0_stateless/04207_alias_map_arrayelement_subcolumn.sql b/tests/queries/0_stateless/04207_alias_map_arrayelement_subcolumn.sql index 142d8cdc7548b..bd324ef1c67f8 100644 --- a/tests/queries/0_stateless/04207_alias_map_arrayelement_subcolumn.sql +++ b/tests/queries/0_stateless/04207_alias_map_arrayelement_subcolumn.sql @@ -6,6 +6,7 @@ -- -- Without the look-through, m_alias['k'] reads the entire Map while m['k'] reads -- only the matching key subcolumn, so the alias path was orders of magnitude slower. +-- The key subcolumn rewrite applies only to bucketed Maps, so the tables with Maps use `with_buckets`. SET explain_query_plan_default = 'legacy'; SET enable_analyzer = 1; @@ -32,7 +33,8 @@ CREATE TABLE t_alias_map a_bad ALIAS mapApply((k, v) -> (k, v + 1), m), a_after_bad ALIAS a_bad ) -ENGINE = MergeTree ORDER BY id; +ENGINE = MergeTree ORDER BY id +SETTINGS map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_alias_map (id, m) SELECT number, map('key1', number, 'key2', number + 1) FROM numbers(10); @@ -153,7 +155,8 @@ CREATE TABLE t_demo some_alias ALIAS attributes, chained_alias ALIAS some_alias ) -ENGINE = MergeTree ORDER BY tuple(); +ENGINE = MergeTree ORDER BY tuple() +SETTINGS map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_demo (attributes) VALUES (map('key1810', 'value-a', 'other', 'value-b')), @@ -182,7 +185,8 @@ CREATE TABLE t_alias_map_pk m Map(String, UInt64), a ALIAS m ) -ENGINE = MergeTree ORDER BY (id, m); +ENGINE = MergeTree ORDER BY (id, m) +SETTINGS map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_alias_map_pk (id, m) SELECT number, map('key1', number) FROM numbers(10); @@ -282,7 +286,8 @@ CREATE TABLE t_alias_correctness a3 ALIAS a2, a_bad ALIAS mapApply((k, v) -> (k, v + 1), m) ) -ENGINE = MergeTree ORDER BY id; +ENGINE = MergeTree ORDER BY id +SETTINGS map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_alias_correctness (id, m) SELECT number, map('key1', number, 'key2', number * 10) FROM numbers(5); @@ -324,7 +329,8 @@ CREATE TABLE t_array_join id UInt64, arr Array(Map(String, UInt64)) ) -ENGINE = MergeTree ORDER BY id; +ENGINE = MergeTree ORDER BY id +SETTINGS map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_array_join VALUES (1, [map('key1', 10, 'key2', 20), map('key1', 11)]), @@ -345,8 +351,10 @@ DROP TABLE t_array_join; -- both sides. The walk must stop at the ListNode and not try to rewrite the join'd Map. DROP TABLE IF EXISTS t_using_left; DROP TABLE IF EXISTS t_using_right; -CREATE TABLE t_using_left (id UInt64, m Map(String, UInt64)) ENGINE = MergeTree ORDER BY id; -CREATE TABLE t_using_right (id UInt64, m Map(String, UInt64)) ENGINE = MergeTree ORDER BY id; +CREATE TABLE t_using_left (id UInt64, m Map(String, UInt64)) ENGINE = MergeTree ORDER BY id +SETTINGS map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; +CREATE TABLE t_using_right (id UInt64, m Map(String, UInt64)) ENGINE = MergeTree ORDER BY id +SETTINGS map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_using_left VALUES (1, map('key1', 10)), (2, map('key1', 20)); INSERT INTO t_using_right VALUES (1, map('key1', 10)), (2, map('key1', 20)); @@ -371,7 +379,8 @@ CREATE TABLE t_subquery_inner id UInt64, m Map(String, UInt64) ) -ENGINE = MergeTree ORDER BY id; +ENGINE = MergeTree ORDER BY id +SETTINGS map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_subquery_inner VALUES (1, map('key1', 100, 'key2', 200)), diff --git a/tests/queries/0_stateless/04337_orc_schema_inference_deep_recursion.sh b/tests/queries/0_stateless/04337_orc_schema_inference_deep_recursion.sh index 6dfb8038211b1..1639da5845ab2 100755 --- a/tests/queries/0_stateless/04337_orc_schema_inference_deep_recursion.sh +++ b/tests/queries/0_stateless/04337_orc_schema_inference_deep_recursion.sh @@ -7,10 +7,9 @@ # grind (and could overflow the native stack), uncancellable. It must now be rejected as # TOO_DEEP_RECURSION once the nesting exceeds max_parser_depth. # -# 2. The Apache ORC C++ library (convertType) builds its own orc::Type tree by recursing over the -# file footer before any ClickHouse code runs. A pathologically deep file used to overflow the -# native stack there (crashing the process / triggering the OOM killer). The library now bounds -# the nesting depth and rejects such a file instead of crashing. +# 2. The ORC reader builds its type tree by recursing over the file footer before the schema +# inference runs. A pathologically deep file must not overflow the native stack there: the +# reader bounds the nesting depth and rejects such a file. CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh @@ -55,18 +54,24 @@ orc.write_table(pa.table({'x': pa.array([], type=t)}), '$2') " } -# (1) A moderate file (200 levels) that the ORC library builds fine, with max_parser_depth lowered +# (1) A moderate file (200 levels) that the ORC reader builds fine, with max_parser_depth lowered # below the nesting so parseORCType rejects it deterministically and build-independently. gen_orc 200 "${TMP_DIR}/moderate.orc" $CLICKHOUSE_LOCAL --query "DESC TABLE file('${TMP_DIR}/moderate.orc', ORC) SETTINGS max_parser_depth = 100" 2>&1 \ | expect_contains parser_depth_limit TOO_DEEP_RECURSION -# (2) A file nested past the ORC library's own limit. Even with max_parser_depth raised so high that -# parseORCType would accept it, the library must reject it (without crashing). The library throws +# (2) A file nested past the ORC reader's own limit. Even with max_parser_depth raised so high that +# parseORCType would accept it, the reader must reject it (without crashing). The reader throws # before recursing past its limit, so this is safe to run. gen_orc 3000 "${TMP_DIR}/deep.orc" -$CLICKHOUSE_LOCAL --query "DESC TABLE file('${TMP_DIR}/deep.orc', ORC) SETTINGS max_parser_depth = 100000" 2>&1 \ - | expect_contains orc_library_depth_limit "nesting depth exceeds the limit" +# Under TSan the stack size check allows only 5% of the stack, so it can reject the file before the depth limit. +out=$($CLICKHOUSE_LOCAL --query "DESC TABLE file('${TMP_DIR}/deep.orc', ORC) SETTINGS max_parser_depth = 100000" 2>&1) +if printf '%s\n' "$out" | grep -qF -e "nesting depth exceeds the limit" -e "TOO_DEEP_RECURSION"; then + echo "orc_library_depth_limit: nesting depth exceeds the limit" +else + echo "orc_library_depth_limit: expected 'nesting depth exceeds the limit', got:" + printf '%s\n' "$out" | head -3 +fi # (3) max_parser_depth=0 means unlimited (matching the SQL parser), so a moderately nested file must # still be inferred normally rather than rejected by the explicit limit. diff --git a/tests/queries/0_stateless/04489_virtual_row_disabled_with_final.reference b/tests/queries/0_stateless/04489_virtual_row_disabled_with_final.reference deleted file mode 100644 index 3aa5aa91e7cca..0000000000000 --- a/tests/queries/0_stateless/04489_virtual_row_disabled_with_final.reference +++ /dev/null @@ -1,7 +0,0 @@ -8192 16384 -0 2 -1 2 -2 2 -16384 -1 -1 diff --git a/tests/queries/0_stateless/04489_virtual_row_disabled_with_final.sql b/tests/queries/0_stateless/04489_virtual_row_disabled_with_final.sql deleted file mode 100644 index 31d2673b9f179..0000000000000 --- a/tests/queries/0_stateless/04489_virtual_row_disabled_with_final.sql +++ /dev/null @@ -1,46 +0,0 @@ --- Tags: no-random-merge-tree-settings - --- SELECT ... FINAL must not lose rows when the read-in-order virtual row is enabled. --- ORDER BY with a LIMIT is required: without it the read-in-order plan is not built at all. - -DROP TABLE IF EXISTS t_vrow_select_final; - -CREATE TABLE t_vrow_select_final (x UInt64, y UInt64) -ENGINE = ReplacingMergeTree(y) -ORDER BY x; - -SYSTEM STOP MERGES t_vrow_select_final; - -INSERT INTO t_vrow_select_final SELECT number, 1 FROM numbers(8192); -INSERT INTO t_vrow_select_final SELECT number, 2 FROM numbers(8192); - --- Every SETTINGS clause below sits on the outermost statement: one attached to an inner subquery --- loses to the same setting arriving as a client option. -SELECT count(), sum(y) -FROM (SELECT x, y FROM t_vrow_select_final FINAL ORDER BY x LIMIT 20000) -SETTINGS optimize_read_in_order = 1, read_in_order_use_virtual_row = 1, enable_parallel_replicas = 0; - -SELECT x, y FROM t_vrow_select_final FINAL ORDER BY x LIMIT 3 -SETTINGS optimize_read_in_order = 1, read_in_order_use_virtual_row = 1, enable_parallel_replicas = 0; - -SELECT count() -FROM (SELECT x FROM t_vrow_select_final ORDER BY x LIMIT 20000) -SETTINGS optimize_read_in_order = 1, read_in_order_use_virtual_row = 1, enable_parallel_replicas = 0; - --- The row counts above stay correct whenever the virtual row is never attempted, so assert as a --- pair that the in-order FINAL read carries no virtual row conversion while the same fixture --- without FINAL does build one. `ReadType:` is matched whitespace-insensitively: the label prints --- as `Read type: ` or `ReadType: ` depending on `pretty`. -SELECT countIf(replaceAll(explain, ' ', '') ILIKE '%ReadType:InOrder%') > 0 - AND countIf(explain ILIKE '%Virtual row conversions%') = 0 -FROM (EXPLAIN actions = 1 - SELECT count(), sum(y) FROM (SELECT x, y FROM t_vrow_select_final FINAL ORDER BY x LIMIT 20000) - SETTINGS optimize_read_in_order = 1, read_in_order_use_virtual_row = 1, enable_parallel_replicas = 0); - -SELECT countIf(replaceAll(explain, ' ', '') ILIKE '%ReadType:InOrder%') > 0 - AND countIf(explain ILIKE '%Virtual row conversions%') > 0 -FROM (EXPLAIN actions = 1 - SELECT count() FROM (SELECT x FROM t_vrow_select_final ORDER BY x LIMIT 20000) - SETTINGS optimize_read_in_order = 1, read_in_order_use_virtual_row = 1, enable_parallel_replicas = 0); - -DROP TABLE t_vrow_select_final; diff --git a/tests/queries/0_stateless/04500_group_by_limit_pushdown_no_order_by.sql b/tests/queries/0_stateless/04500_group_by_limit_pushdown_no_order_by.sql index f390f1617c80f..3e2569996bcbb 100644 --- a/tests/queries/0_stateless/04500_group_by_limit_pushdown_no_order_by.sql +++ b/tests/queries/0_stateless/04500_group_by_limit_pushdown_no_order_by.sql @@ -305,9 +305,9 @@ WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND log_comm -- LowCardinality keys cannot prune the hash table, but the heap still runs in -- skip-only mode and the synthesized sort discards evicted stragglers; the -- query must stay within the memory limit and external aggregation must not --- be force-disabled (~660MB working set without spilling). -SET max_memory_usage = 300000000; -SET max_bytes_before_external_group_by = 50000000; +-- be force-disabled (~660MB working set without spilling). The limits are +-- per-query, not session `SET`s: the verification query below reads +-- `system.query_log`, whose cost tracks the whole suite's log volume. SELECT 'LowCardinality key under a memory limit (skip-only heap + spill)'; SELECT count() FROM @@ -316,7 +316,7 @@ SELECT count() FROM FROM (SELECT toLowCardinality(toString(number % 3000000)) AS k FROM numbers(9000000)) GROUP BY k LIMIT 5 -) SETTINGS log_comment = '04500_lowcard_spill'; +) SETTINGS log_comment = '04500_lowcard_spill', max_memory_usage = 300000000, max_bytes_before_external_group_by = 50000000; SYSTEM FLUSH LOGS query_log; diff --git a/tests/queries/0_stateless/04501_group_by_limit_pushdown_heap_stress.sql b/tests/queries/0_stateless/04501_group_by_limit_pushdown_heap_stress.sql index afa4594c2ef67..5533ed40308f8 100644 --- a/tests/queries/0_stateless/04501_group_by_limit_pushdown_heap_stress.sql +++ b/tests/queries/0_stateless/04501_group_by_limit_pushdown_heap_stress.sql @@ -347,9 +347,9 @@ WHERE current_database = currentDatabase() -- new key and evicts an older one, so without reuse the arena grows by one -- state per distinct key seen (20M here) even though the hash table stays -- bounded, defeating the optimization's memory contract for non-`count` --- aggregates and failing the memory limit below. - -SET max_memory_usage = 100000000; +-- aggregates and failing the memory limit below. The limit is per-query, not a +-- session `SET`: the verification query below reads `system.query_log`, whose +-- cost tracks the whole suite's log volume rather than anything under test. SELECT 'arena slot reuse under eviction'; SELECT k, sum(v) FROM @@ -359,7 +359,7 @@ SELECT k, sum(v) FROM GROUP BY k ORDER BY k ASC LIMIT 10 -SETTINGS log_comment = '04501_state_arena_reuse'; +SETTINGS log_comment = '04501_state_arena_reuse', max_memory_usage = 100000000; SYSTEM FLUSH LOGS query_log; diff --git a/tests/queries/0_stateless/04513_prewhere_map_subcolumn_read_cost.sql b/tests/queries/0_stateless/04513_prewhere_map_subcolumn_read_cost.sql index 58a42ead810e4..8d0f1e6387a0a 100644 --- a/tests/queries/0_stateless/04513_prewhere_map_subcolumn_read_cost.sql +++ b/tests/queries/0_stateless/04513_prewhere_map_subcolumn_read_cost.sql @@ -1,7 +1,8 @@ -- Regression test: the PREWHERE optimizer must account for Map-key subcolumn read cost. -- When `optimize_functions_to_subcolumns` rewrites `h['k']` to the `h.key_k` subcolumn, --- the optimizer must see its actual on-disk size (the whole Map) via getSubcolumnSize --- and place the cheap `modality` filter before the expensive Map-key predicate. +-- the optimizer must see its actual on-disk size via getSubcolumnSize and place the cheap +-- `modality` filter before the expensive Map-key predicate. The rewrite applies only to bucketed +-- Maps, and with two keys per row the single bucket holds the whole Map. -- -- We disable statistics so that only columns_size drives the ordering, -- and use equality conditions on both sides so that both are "good" conditions @@ -17,7 +18,9 @@ SET explain_query_plan_default = 'legacy'; DROP TABLE IF EXISTS t_prewhere_map_cost; CREATE TABLE t_prewhere_map_cost (id UInt64, modality LowCardinality(String), h Map(String, String)) -ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +ENGINE = MergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 0, + map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_prewhere_map_cost SELECT number, if(number < 1000, 'active', ''), map('k', repeat('v', 300), 'k2', repeat('w', 300)) diff --git a/tests/queries/0_stateless/04546_map_element_duplicate_keys_first_match.sql b/tests/queries/0_stateless/04546_map_element_duplicate_keys_first_match.sql index c0ccfda377af6..e05d09fbc8aa9 100644 --- a/tests/queries/0_stateless/04546_map_element_duplicate_keys_first_match.sql +++ b/tests/queries/0_stateless/04546_map_element_duplicate_keys_first_match.sql @@ -2,6 +2,7 @@ -- map[key] must return the value of the FIRST occurrence of the key in the row, and the -- result must not depend on optimize_functions_to_subcolumns, on preceding rows in the -- block, or on whether a filter isolates the row (issue #111203). +-- The Maps are bucketed: only then is `m[key]` rewritten to the key subcolumn with optimize_functions_to_subcolumns. DROP TABLE IF EXISTS t_map_dup; @@ -9,7 +10,7 @@ DROP TABLE IF EXISTS t_map_dup; -- its match for '' sits at relative position 1. Row 2 has '' at positions 0 and 1, so a -- prediction of position 1 would wrongly return the second occurrence. CREATE TABLE t_map_dup (id UInt64, m Map(String, String)) -ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0, map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_map_dup VALUES (1, map('x', 'k', '', 'prime')), (2, map('', 'FIRST', '', 'SECOND')); SELECT 'wide, filter id=2, subcolumns=0', m[''] FROM t_map_dup WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 0; @@ -23,7 +24,7 @@ SELECT 'wide, count first value, subcolumns=1', count() FROM t_map_dup WHERE id DROP TABLE t_map_dup; -- Compact part (default min_bytes_for_wide_part) exercises the other reader path. -CREATE TABLE t_map_dup (id UInt64, m Map(String, String)) ENGINE = MergeTree ORDER BY id; +CREATE TABLE t_map_dup (id UInt64, m Map(String, String)) ENGINE = MergeTree ORDER BY id SETTINGS map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_map_dup VALUES (1, map('x', 'k', '', 'prime')), (2, map('', 'FIRST', '', 'SECOND')); SELECT 'compact, filter id=2, subcolumns=0', m[''] FROM t_map_dup WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 0; @@ -39,7 +40,7 @@ DROP TABLE IF EXISTS t_map_types; -- Nullable(String) value CREATE TABLE t_map_types (id UInt64, m Map(String, Nullable(String))) -ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0, map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_map_types VALUES (1, map('x', 'k', 'd', 'prime')), (2, map('d', 'FIRST', 'd', 'SECOND')); SELECT 'Nullable(String) subcolumns=0', m['d'] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 0; SELECT 'Nullable(String) subcolumns=1', m['d'] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 1; @@ -47,7 +48,7 @@ DROP TABLE t_map_types; -- LowCardinality(String) value CREATE TABLE t_map_types (id UInt64, m Map(String, LowCardinality(String))) -ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0, map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_map_types VALUES (1, map('x', 'k', 'd', 'prime')), (2, map('d', 'FIRST', 'd', 'SECOND')); SELECT 'LowCardinality(String) subcolumns=0', m['d'] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 0; SELECT 'LowCardinality(String) subcolumns=1', m['d'] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 1; @@ -55,7 +56,7 @@ DROP TABLE t_map_types; -- LowCardinality(Nullable(String)) value CREATE TABLE t_map_types (id UInt64, m Map(String, LowCardinality(Nullable(String)))) -ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0, map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_map_types VALUES (1, map('x', 'k', 'd', 'prime')), (2, map('d', 'FIRST', 'd', 'SECOND')); SELECT 'LowCardinality(Nullable(String)) subcolumns=0', m['d'] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 0; SELECT 'LowCardinality(Nullable(String)) subcolumns=1', m['d'] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 1; @@ -63,7 +64,7 @@ DROP TABLE t_map_types; -- UInt64 value CREATE TABLE t_map_types (id UInt64, m Map(String, UInt64)) -ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0, map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_map_types VALUES (1, map('x', 10, 'd', 20)), (2, map('d', 111, 'd', 222)); SELECT 'UInt64 subcolumns=0', m['d'] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 0; SELECT 'UInt64 subcolumns=1', m['d'] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 1; @@ -71,7 +72,7 @@ DROP TABLE t_map_types; -- Array(String) value CREATE TABLE t_map_types (id UInt64, m Map(String, Array(String))) -ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0, map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_map_types VALUES (1, map('x', ['k'], 'd', ['prime'])), (2, map('d', ['FIRST'], 'd', ['SECOND'])); SELECT 'Array(String) subcolumns=0', m['d'] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 0; SELECT 'Array(String) subcolumns=1', m['d'] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 1; @@ -81,7 +82,7 @@ DROP TABLE t_map_types; -- LowCardinality(String) key CREATE TABLE t_map_types (id UInt64, m Map(LowCardinality(String), String)) -ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0, map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_map_types VALUES (1, map('x', 'k', 'd', 'prime')), (2, map('d', 'FIRST', 'd', 'SECOND')); SELECT 'LowCardinality(String) key subcolumns=0', m['d'] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 0; SELECT 'LowCardinality(String) key subcolumns=1', m['d'] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 1; @@ -89,7 +90,7 @@ DROP TABLE t_map_types; -- UInt64 key CREATE TABLE t_map_types (id UInt64, m Map(UInt64, String)) -ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0; +ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0, map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO t_map_types VALUES (1, map(9, 'k', 7, 'prime')), (2, map(7, 'FIRST', 7, 'SECOND')); SELECT 'UInt64 key subcolumns=0', m[7] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 0; SELECT 'UInt64 key subcolumns=1', m[7] FROM t_map_types WHERE id = 2 SETTINGS optimize_functions_to_subcolumns = 1; diff --git a/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.reference b/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.reference index 078656c7dc9d3..62d1ad01856ce 100644 --- a/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.reference +++ b/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.reference @@ -1,5 +1,5 @@ -- Current defaults -false 0 1 +false 0 3 8 -- compatibility = 26.6 restores the legacy defaults -true 0 0 +true 0 0 1 0 diff --git a/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.sql b/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.sql index 1fe196405fc4c..cc2a7875a35e9 100644 --- a/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.sql +++ b/tests/queries/0_stateless/04628_ai_functions_compatibility_defaults.sql @@ -3,10 +3,11 @@ -- no-replicated-database: named collections are server-global, not database-scoped -- ============================================================================= --- Two AI function defaults were flipped in 26.8: `ai_function_allow_insecure_endpoint` from 1 --- to 0 and `ai_function_max_retries` from 0 to 1. `ai_function_max_api_calls_per_query` stays 0 --- (unlimited). `compatibility = 26.6` predates the flips and restores them, which pins the --- previous_value/new_value pairs in `SettingsChangesHistory`. +-- Three AI function settings changed their defaults in 26.8: `ai_function_allow_insecure_endpoint` +-- from 1 to 0, `ai_function_max_retries` from 0 to 3, and the new +-- `ai_function_max_concurrent_requests_per_thread` is 8 where requests used to go out one at a time (1). +-- `ai_function_max_api_calls_per_query` stays 0 (unlimited). `compatibility = 26.6` predates the +-- changes and restores them, which pins the previous_value/new_value pairs in `SettingsChangesHistory`. -- -- The endpoint check runs in `resolveAIParams`, before the zero-row early return -- in `executeImpl`, so an empty source table exercises it without any real HTTP @@ -21,12 +22,12 @@ CREATE NAMED COLLECTION ai_compat_remote_http AS provider = 'openai', endpoint = 'http://ai.example.com/v1/chat/completions', model = 'chat-model', api_key = 'fake-key'; SELECT '-- Current defaults'; -SELECT getSetting('ai_function_allow_insecure_endpoint'), getSetting('ai_function_max_api_calls_per_query'), getSetting('ai_function_max_retries'); +SELECT getSetting('ai_function_allow_insecure_endpoint'), getSetting('ai_function_max_api_calls_per_query'), getSetting('ai_function_max_retries'), getSetting('ai_function_max_concurrent_requests_per_thread'); SELECT aiGenerate(x, map('credentials', 'ai_compat_remote_http')) FROM tab; -- { serverError BAD_ARGUMENTS } SELECT '-- compatibility = 26.6 restores the legacy defaults'; SET compatibility = '26.6'; -SELECT getSetting('ai_function_allow_insecure_endpoint'), getSetting('ai_function_max_api_calls_per_query'), getSetting('ai_function_max_retries'); +SELECT getSetting('ai_function_allow_insecure_endpoint'), getSetting('ai_function_max_api_calls_per_query'), getSetting('ai_function_max_retries'), getSetting('ai_function_max_concurrent_requests_per_thread'); SELECT count() FROM (SELECT aiGenerate(x, map('credentials', 'ai_compat_remote_http')) AS r FROM tab); -- ============================================================================= diff --git a/tests/queries/0_stateless/04648_kafka2_race_creation_removal.sh b/tests/queries/0_stateless/04648_kafka2_race_creation_removal.sh index e56b48c954b4a..bc55b6ce957da 100755 --- a/tests/queries/0_stateless/04648_kafka2_race_creation_removal.sh +++ b/tests/queries/0_stateless/04648_kafka2_race_creation_removal.sh @@ -55,7 +55,7 @@ function create_table() kafka_group_name = '${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}', kafka_format = 'RawBLOB', kafka_keeper_path = '${ZK_PATH}', - kafka_replica_name = 'r1' + kafka_replica_name = '{replica}' " } diff --git a/tests/queries/0_stateless/04707_tuple_subcolumn_pushdown_orc.sh b/tests/queries/0_stateless/04707_tuple_subcolumn_pushdown_orc.sh index d9f7e0d839f97..e96ecaf629530 100755 --- a/tests/queries/0_stateless/04707_tuple_subcolumn_pushdown_orc.sh +++ b/tests/queries/0_stateless/04707_tuple_subcolumn_pushdown_orc.sh @@ -22,7 +22,7 @@ ${CLICKHOUSE_CLIENT} --query " FROM numbers(20000) SETTINGS engine_file_truncate_on_insert = 1, output_format_orc_row_index_stride = 2000" -# rows_read is the pruning oracle: liborc skips row index strides whose statistics exclude the +# rows_read is the pruning oracle: the ORC reader skips row index strides whose statistics exclude the # predicate, so a pruned read reports fewer rows than the file holds. run_and_report() { local label="$1" diff --git a/tests/queries/0_stateless/04757_refreshable_mv_randomize_for_overflow.reference b/tests/queries/0_stateless/04757_refreshable_mv_randomize_for_overflow.reference new file mode 100644 index 0000000000000..d00491fd7e5bb --- /dev/null +++ b/tests/queries/0_stateless/04757_refreshable_mv_randomize_for_overflow.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/04757_refreshable_mv_randomize_for_overflow.sh b/tests/queries/0_stateless/04757_refreshable_mv_randomize_for_overflow.sh new file mode 100755 index 0000000000000..b47c10f274496 --- /dev/null +++ b/tests/queries/0_stateless/04757_refreshable_mv_randomize_for_overflow.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# Tags: memory-engine, atomic-database + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# An absurd RANDOMIZE FOR caps the offset at 3e9 seconds. The first timeslot is 2170-01-01, so both +# signs of the offset stay in the future and the view is scheduled without refreshing. +$CLICKHOUSE_CLIENT -q " + create materialized view rmv refresh every 200 year randomize for 10000000000000000000 second + (x Int64) engine Memory as select 1 as x;" + +# next_refresh_time is a UInt32 DateTime, so the expected instants are compared modulo 2^32. +query=" + with toInt64(toDateTime64('2170-01-01 00:00:00', 0, 'UTC')) as slot + select status = 'Scheduled' and last_refresh_time is null + and toUInt32(next_refresh_time) in ((slot - 3000000000) % 4294967296, (slot + 3000000000) % 4294967296) + from system.view_refreshes where database = currentDatabase() and view = 'rmv'" + +for _ in {1..100}; do + [ "$($CLICKHOUSE_CLIENT -q "$query")" = 1 ] && break + sleep 0.1 +done +$CLICKHOUSE_CLIENT -q "$query" diff --git a/tests/queries/0_stateless/04856_orc_corrupt_stripe_footer.reference b/tests/queries/0_stateless/04856_orc_corrupt_stripe_footer.reference index 0072ff8938135..7bed391e68f39 100644 --- a/tests/queries/0_stateless/04856_orc_corrupt_stripe_footer.reference +++ b/tests/queries/0_stateless/04856_orc_corrupt_stripe_footer.reference @@ -1,5 +1,4 @@ INCORRECT_DATA -Read past EOF in DecompressionStream::readBuffer 0 0 1 1 2 2 diff --git a/tests/queries/0_stateless/04856_orc_corrupt_stripe_footer.sh b/tests/queries/0_stateless/04856_orc_corrupt_stripe_footer.sh index b9b83b3b2e369..57a8727696be6 100755 --- a/tests/queries/0_stateless/04856_orc_corrupt_stripe_footer.sh +++ b/tests/queries/0_stateless/04856_orc_corrupt_stripe_footer.sh @@ -5,11 +5,8 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh -# The stripe footer of this blob is truncated, so ORC's decompression stream reports the -# failure by throwing from inside the protobuf parser. The parse must surface as a regular -# INCORRECT_DATA exception; a protobuf has-bits assertion instead means the message was left -# inconsistent while the exception unwound. Only debug and sanitizer builds check that -# assertion, so the discriminating arm of this test is those builds. +# The stripe footer of this blob is truncated, so reading it must fail with a regular +# INCORRECT_DATA exception. BLOB='4f52431100000a061204080150005500000a280a06000000000000121e08014a1818b8fef997d7d63820b8fef997d7d63828c1fc1530c1fc1550000f0000780039604d293e0b00006e0007dda36300f00a0608061000180b0a0608061001182d0a0608011001180a0a060805100118081204080010001204080210001a03474d545100000a260a04080150000a1e08014a1818b8fef997d7d63820b8fef997d7d63828c1fc1530c1fc155000b500000803107e1a0a08031038181220342801220f080c1201011a0263302000280030002208080920002800300030013a04080150003a1e08014a1818b8fef997d7d63820b8fef997d7d63828c1fc1530c1fc15500040904e48016200085d1005188080102202000c282b300682f403034f524317' # The fixture must stay pure even-length lowercase hex: unhex's result for a non-hex character is @@ -21,8 +18,7 @@ if [[ ! $BLOB =~ ^[0-9a-f]+$ ]] || (( ${#BLOB} % 2 )); then fi $CLICKHOUSE_LOCAL -q "SELECT c0 FROM format(ORC, 'c0 DateTime64(3)', unhex('$BLOB'));" 2>&1 \ - | grep -oE "Code: [0-9]+.*Read past EOF in DecompressionStream::readBuffer|INCORRECT_DATA|Has bits mismatch|Check failure stack trace" \ - | sed -E 's/^Code: [0-9]+.*Read past EOF in DecompressionStream::readBuffer$/Read past EOF in DecompressionStream::readBuffer/' \ + | grep -oE "INCORRECT_DATA" \ | sort -u # Well-formed ORC must still read back unchanged. diff --git a/tests/queries/0_stateless/05036_orc_unimplemented_codec_no_abort.reference b/tests/queries/0_stateless/05036_orc_unimplemented_codec_no_abort.reference new file mode 100644 index 0000000000000..1f714f4b2082b --- /dev/null +++ b/tests/queries/0_stateless/05036_orc_unimplemented_codec_no_abort.reference @@ -0,0 +1,14 @@ +BROTLI compression in ORC files is not supported +BROTLI compression in ORC files is not supported +BROTLI compression in ORC files is not supported +BROTLI compression in ORC files is not supported +BROTLI compression in ORC files is not supported +BROTLI compression in ORC files is not supported +CANNOT_EXTRACT_TABLE_STRUCTURE +INCORRECT_DATA +0 +CANNOT_EXTRACT_TABLE_STRUCTURE +INCORRECT_DATA +0 +1 +2 diff --git a/tests/queries/0_stateless/05036_orc_unimplemented_codec_no_abort.sh b/tests/queries/0_stateless/05036_orc_unimplemented_codec_no_abort.sh new file mode 100755 index 0000000000000..dd54288e64f9c --- /dev/null +++ b/tests/queries/0_stateless/05036_orc_unimplemented_codec_no_abort.sh @@ -0,0 +1,59 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Truncating this fixture to 289 bytes leaves byte 6 in the postscript's compression field. Codec 6 +# is BROTLI, which the ORC format declares but this reader does not implement. The reader rejects it +# with a NOT_IMPLEMENTED exception; it must never be a logical error, which aborts in debug and +# sanitizer builds. +DATA_FILE="$CLICKHOUSE_TMP/05036_orc_codec6_${CLICKHOUSE_DATABASE}.orc" +head -c 289 "$CUR_DIR"/data_orc/orc_nested_union_type.orc > "$DATA_FILE" + +# Both the schema-inference path and the explicit-structure path reach the reader, so both are +# asserted here. Match on the error text, never on the exit status: ClickHouse exits with the error +# code truncated to a byte, so 636 and 124 are the same outcome. +for query in \ + "SELECT * FROM file('$DATA_FILE', ORC) FORMAT Null" \ + "SELECT * FROM file('$DATA_FILE', ORC, 'x Int32') FORMAT Null" \ + "SELECT count() FROM file('$DATA_FILE', ORC, 'x Int32')" \ + "DESCRIBE file('$DATA_FILE', ORC)" \ + "SELECT * FROM format(ORC, 'x Int32', (SELECT * FROM file('$DATA_FILE', RawBLOB))) FORMAT Null" +do + # An abort prints nothing at all, so classify explicitly rather than letting the arm go silent: + # a missing line is much harder to read in a diff than a named verdict. + OUT=$($CLICKHOUSE_LOCAL --query "$query" 2>&1) + if [ -z "$OUT" ]; then + echo "no output at all" + else + echo "$OUT" | grep -o -m1 'BROTLI compression in ORC files is not supported\|Logical error' || echo "unexpected: $OUT" + fi +done + +# The message must name the codec, and must not call the file corrupt: a well-formed +# Brotli-compressed ORC file takes this same path. +$CLICKHOUSE_LOCAL --query "SELECT * FROM file('$DATA_FILE', ORC, 'x Int32') FORMAT Null" 2>&1 \ + | grep -o -m1 'BROTLI compression in ORC files is not supported\|CORRUPTED\|INCORRECT_DATA' + +# Neighbouring truncations are genuinely corrupt and must keep reporting corruption rather than the +# codec gap. Schema inference wraps every non-retryable cause in one outer code, so that arm alone +# cannot tell the two apart; the explicit-structure arm bypasses the wrapper and surfaces the inner +# classification, and counting the codec error there pins them apart. +for length in 200 400 +do + NEIGHBOUR="$CLICKHOUSE_TMP/05036_orc_${length}_${CLICKHOUSE_DATABASE}.orc" + head -c $length "$CUR_DIR"/data_orc/orc_nested_union_type.orc > "$NEIGHBOUR" + $CLICKHOUSE_LOCAL --query "SELECT * FROM file('$NEIGHBOUR', ORC) FORMAT Null" 2>&1 \ + | grep -o -m1 'CANNOT_EXTRACT_TABLE_STRUCTURE\|Logical error' + EXPLICIT=$($CLICKHOUSE_LOCAL --query "SELECT * FROM file('$NEIGHBOUR', ORC, 'x Int32') FORMAT Null" 2>&1) + echo "$EXPLICIT" | grep -o -m1 'INCORRECT_DATA\|Logical error' || echo "unexpected: $EXPLICIT" + echo "$EXPLICIT" | grep -c 'BROTLI compression' + rm "$NEIGHBOUR" +done + +# Well-formed ORC must still read back unchanged. +$CLICKHOUSE_LOCAL --query "SELECT i FROM file('$CUR_DIR/data_orc/orc_union_type.orc', ORC) ORDER BY i" + +rm "$DATA_FILE" diff --git a/tests/queries/0_stateless/05045_format_schema_cache_leftover_temp_file.reference b/tests/queries/0_stateless/05045_format_schema_cache_leftover_temp_file.reference new file mode 100644 index 0000000000000..f619cfc86a8a5 --- /dev/null +++ b/tests/queries/0_stateless/05045_format_schema_cache_leftover_temp_file.reference @@ -0,0 +1,2 @@ +data Array(Array(Int32)) +leftover file kept diff --git a/tests/queries/0_stateless/05045_format_schema_cache_leftover_temp_file.sh b/tests/queries/0_stateless/05045_format_schema_cache_leftover_temp_file.sh new file mode 100755 index 0000000000000..24f329eb412f0 --- /dev/null +++ b/tests/queries/0_stateless/05045_format_schema_cache_leftover_temp_file.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A schema given as a string is published into the schema cache through a temporary file. A +# temporary file already sitting at that path belongs to another query storing the same schema, so +# storing must neither fail because of it nor delete it. + +WORKDIR="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}" +rm -rf "$WORKDIR" +mkdir -p "$WORKDIR" + +SCHEMA='@0x844f048b15c12dab;\nstruct M { data @0 :List(List(Int32)); }' +QUERY="DESC format(CapnProto, '') +SETTINGS + format_schema_source = 'string', + format_schema = '${SCHEMA}', + format_schema_message_name = 'M'" + +# The cache directory of clickhouse-local is relative to the working directory. +(cd "$WORKDIR" && ${CLICKHOUSE_LOCAL} --logger.console=0 --query "$QUERY" > /dev/null) + +CACHED=$(basename "$(ls "$WORKDIR"/__cache__/*.capnp)") + +if [ -z "$CACHED" ]; then + echo "FAIL: the first query stored no schema file to learn the name from" +fi + +# Drop the published schema and leave a temporary file of it behind, so the next query has to store +# the schema again while that temporary path is taken. +rm -f "$WORKDIR"/__cache__/*.capnp +echo 'leftover' > "$WORKDIR/__cache__/$CACHED.tmp" + +(cd "$WORKDIR" && ${CLICKHOUSE_LOCAL} --logger.console=0 --query "$QUERY") + +if [ -f "$WORKDIR/__cache__/$CACHED.tmp" ]; then + echo 'leftover file kept' +else + echo 'leftover file removed' +fi + +rm -rf "$WORKDIR" diff --git a/tests/queries/0_stateless/05053_replicated_max_logs_to_keep_zero.reference b/tests/queries/0_stateless/05053_replicated_max_logs_to_keep_zero.reference new file mode 100644 index 0000000000000..98fb6a6865639 --- /dev/null +++ b/tests/queries/0_stateless/05053_replicated_max_logs_to_keep_zero.reference @@ -0,0 +1,4 @@ +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/05053_replicated_max_logs_to_keep_zero.sh b/tests/queries/0_stateless/05053_replicated_max_logs_to_keep_zero.sh new file mode 100755 index 0000000000000..0e451abc81517 --- /dev/null +++ b/tests/queries/0_stateless/05053_replicated_max_logs_to_keep_zero.sh @@ -0,0 +1,42 @@ +#!/usr/bin/env bash +# Tags: zookeeper, replica, no-shared-merge-tree, no-replicated-database +# no-shared-merge-tree: the setting bounds the ReplicatedMergeTree log, which SharedMergeTree does not have +# no-replicated-database: the ZooKeeper path and the replica name are literal, so they collide between the hosts of the database + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# The oldest kept log record names the threshold that log pointers of inactive replicas are compared +# with, so a window that keeps no records at all has no meaning. +ERR="A setting's value has to be greater than 0" + +$CLICKHOUSE_CLIENT --query " + CREATE TABLE max_logs_zero (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r', '1') ORDER BY x + SETTINGS max_replicated_logs_to_keep = 0; +" 2>&1 | grep -F -q "$ERR" && echo 1 || echo 0 + +# An ATTACH that carries a full definition is user input too, and reaches the same check. +uuid=$($CLICKHOUSE_CLIENT --query "SELECT generateUUIDv4()") +$CLICKHOUSE_CLIENT --query " + ATTACH TABLE max_logs_zero UUID '$uuid' (x UInt64) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r', '1') ORDER BY x + SETTINGS max_replicated_logs_to_keep = 0; +" 2>&1 | grep -F -q "$ERR" && echo 1 || echo 0 + +$CLICKHOUSE_CLIENT --query " + CREATE TABLE max_logs (x UInt64) ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/r2', '1') ORDER BY x; +" +$CLICKHOUSE_CLIENT --query " + ALTER TABLE max_logs MODIFY SETTING max_replicated_logs_to_keep = 0; +" 2>&1 | grep -F -q "$ERR" && echo 1 || echo 0 + +# A positive value is still accepted and stored. +$CLICKHOUSE_CLIENT --query "ALTER TABLE max_logs MODIFY SETTING max_replicated_logs_to_keep = 1;" +$CLICKHOUSE_CLIENT --query " + SELECT count() FROM system.tables + WHERE database = currentDatabase() AND name = 'max_logs' + AND engine_full LIKE '%max_replicated_logs_to_keep = 1%'; +" + +$CLICKHOUSE_CLIENT --query "DROP TABLE max_logs SYNC" diff --git a/tests/queries/0_stateless/05054_postgresql_protocol_message_size.reference b/tests/queries/0_stateless/05054_postgresql_protocol_message_size.reference index ec35dc29b3b47..6be1fa0c53ad0 100644 --- a/tests/queries/0_stateless/05054_postgresql_protocol_message_size.reference +++ b/tests/queries/0_stateless/05054_postgresql_protocol_message_size.reference @@ -1,5 +1,9 @@ SASL mechanism longer than the message: error response password without a terminator: error response +password message at the Int32 maximum: error response +password message above the limit: error response +SASL initial response at the Int32 maximum: error response +SASL final response at the Int32 maximum: error response well-formed exchange: query result received query with a smuggled tail: error response query shorter than declared: error response diff --git a/tests/queries/0_stateless/05054_postgresql_protocol_message_size.sh b/tests/queries/0_stateless/05054_postgresql_protocol_message_size.sh index e5fd34c2db3ff..b5d3e9b7acef8 100755 --- a/tests/queries/0_stateless/05054_postgresql_protocol_message_size.sh +++ b/tests/queries/0_stateless/05054_postgresql_protocol_message_size.sh @@ -32,6 +32,32 @@ def connect(): return sock +def connect_short(): + """A connection whose client timeout is well below the server's `handshake_timeout_milliseconds` (30 + seconds by default), so a message the server waits for shows up as a client timeout. With a timeout + at or above the handshake timeout the server would answer the same error response on its own, and a + regression that only makes the server wait would not be noticed.""" + sock = socket.create_connection(("127.0.0.1", port), timeout=5) + sock.settimeout(5) + return sock + + +def read_message(sock): + """Reads one backend message: a type byte, a length that counts itself, and the body.""" + def recv_exactly(size): + data = b"" + while len(data) < size: + chunk = sock.recv(size - len(data)) + if not chunk: + raise AssertionError("the connection closed before a complete message") + data += chunk + return data + + message_type = recv_exactly(1) + (length,) = struct.unpack(">i", recv_exactly(4)) + return message_type, recv_exactly(length - 4) + + def startup(sock, user): payload = ("user\x00" + user + "\x00\x00").encode() sock.sendall(struct.pack(">ii", 8 + len(payload), 196608) + payload) @@ -93,6 +119,46 @@ except (BrokenPipeError, ConnectionResetError, socket.timeout): print("password without a terminator:", outcome(sock)) sock.close() +# The declared length of a message that is parsed before authentication is bounded by the server, because +# the client that declares it is not trusted yet. Otherwise the client declares the Int32 maximum and the +# server buffers up to 2 GiB of payload that never completes. +sock = connect_short() +startup(sock, user_plain) +read_message(sock) +sock.sendall(b"p" + struct.pack(">i", 0x7FFFFFFF)) +print("password message at the Int32 maximum:", outcome(sock)) +sock.close() + +# The same for a length just above the limit, which pins the boundary itself. +sock = connect_short() +startup(sock, user_plain) +read_message(sock) +sock.sendall(b"p" + struct.pack(">i", 10001)) +print("password message above the limit:", outcome(sock)) +sock.close() + +# The same for a `SASLInitialResponse`. +sock = connect_short() +startup(sock, user_scram) +read_message(sock) +sock.sendall(b"p" + struct.pack(">i", 0x7FFFFFFF)) +print("SASL initial response at the Int32 maximum:", outcome(sock)) +sock.close() + +# The same for a `SASLResponse`, the second SCRAM message. A well-formed client-first message is needed +# to get there, and its own declared lengths stay small. +sock = connect_short() +startup(sock, user_scram) +read_message(sock) +client_first = b"n,,n=,r=abcdefghijklmnopqrstuvwx" +body = b"SCRAM-SHA-256\x00" + struct.pack(">i", len(client_first)) + client_first +sock.sendall(b"p" + struct.pack(">i", 4 + len(body)) + body) +message_type, body = read_message(sock) +assert message_type == b"R" and struct.unpack(">i", body[:4])[0] == 11, body +sock.sendall(b"p" + struct.pack(">i", 0x7FFFFFFF)) +print("SASL final response at the Int32 maximum:", outcome(sock)) +sock.close() + # A well-formed exchange still works: authenticate and run a query. def read_until_ready(sock): data = b"" diff --git a/tests/queries/0_stateless/05055_arrow_split_large_record_batch.reference b/tests/queries/0_stateless/05055_arrow_split_large_record_batch.reference new file mode 100644 index 0000000000000..af13b7a9ddc76 --- /dev/null +++ b/tests/queries/0_stateless/05055_arrow_split_large_record_batch.reference @@ -0,0 +1,10 @@ +String: batches>1=1 64 2016 1 +Nullable(String): batches>1=1 64 2016 1 +LowCardinality: batches>1=1 64 2016 4 +Array(String): batches>1=1 64 2016 1 +Map: batches>1=1 64 2016 1 +Tuple: batches>1=1 64 2016 1 +Variant: batches>1=1 64 2016 33 +FixedString: batches>1=1 64 2016 1 +FixedString fixed-width: batches>1=0 64 2016 1 +UInt64: batches>1=0 64 2016 64 diff --git a/tests/queries/0_stateless/05055_arrow_split_large_record_batch.sh b/tests/queries/0_stateless/05055_arrow_split_large_record_batch.sh new file mode 100755 index 0000000000000..9e4dba6b14cdf --- /dev/null +++ b/tests/queries/0_stateless/05055_arrow_split_large_record_batch.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Arrow offsets its String/Binary/List buffers with 32-bit integers, so a block holding more than that has +# to be written as several record batches. `output_format_arrow_max_record_batch_size_bytes` lowers the +# bound, which exercises the same splitting with kilobytes instead of the 2 GiB a real overflow needs. +# +# Compression is off because older pyarrow cannot read the "uncompressed" sentinel. + +FILE="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_split.arrows" +trap 'rm -f "$FILE"' EXIT + +batches() { + python3 - "$FILE" <<'PY' +import sys +import pyarrow as pa +import pyarrow.ipc as ipc + +with pa.OSFile(sys.argv[1], "rb") as src: + print(sum(1 for _ in ipc.open_stream(src))) +PY +} + +run() { + local label="$1" expr="$2" limit="$3" extra="${4:-}" + $CLICKHOUSE_LOCAL --allow_suspicious_fixed_string_types 1 --query " + SELECT number AS n, ${expr} AS v + FROM numbers(64) + SETTINGS max_block_size = 64, + output_format_arrow_compression_method = 'none', + output_format_arrow_max_record_batch_size_bytes = ${limit}${extra} + FORMAT ArrowStream + " > "$FILE" + echo -n "${label}: batches>1=$([ "$(batches)" -gt 1 ] && echo 1 || echo 0) " + $CLICKHOUSE_LOCAL --query "SELECT count(), sum(n), uniqExact(v) FROM file('$FILE', ArrowStream)" +} + +# One block of 64 rows against a small bound, so each shape must split. +run "String" "repeat('x', 64)" 512 +run "Nullable(String)" "if(number % 3 = 0, NULL, repeat('y', 64))::Nullable(String)" 512 +run "LowCardinality" "toLowCardinality(repeat(toString(number % 4), 64))" 512 +run "Array(String)" "[repeat('z', 64)]" 512 +run "Map" "map('k', repeat('m', 64))" 512 +run "Tuple" "(repeat('p', 64), repeat('q', 64))" 512 +run "Variant" "if(number % 2, number, repeat('v', 64))::Variant(UInt64, String)" 512 +# Only the offsets mode gives `FixedString` a buffer the bound applies to; the default maps it to a +# fixed-width Arrow type instead. +run "FixedString" "toFixedString(repeat('f', 64), 64)" 512 ", output_format_arrow_fixed_string_as_fixed_byte_array = 0" + +# A fixed-width column has no offsets buffer to overflow, so the bound leaves it in one batch. +run "FixedString fixed-width" "toFixedString(repeat('f', 64), 64)" 512 +run "UInt64" "number" 64 diff --git a/tests/queries/0_stateless/05055_format_schema_source_query_background_access.sh b/tests/queries/0_stateless/05055_format_schema_source_query_background_access.sh index ff237fd5db5b5..20fc7c819b314 100755 --- a/tests/queries/0_stateless/05055_format_schema_source_query_background_access.sh +++ b/tests/queries/0_stateless/05055_format_schema_source_query_background_access.sh @@ -14,7 +14,7 @@ CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) BG_MESSAGE="M_bg_${CLICKHOUSE_DATABASE}" FG_MESSAGE="M_fg_${CLICKHOUSE_DATABASE}" -START=$(${CLICKHOUSE_CLIENT} --query "SELECT toString(now64(6))") +START=$(${CLICKHOUSE_CLIENT} --query "SELECT toUnixTimestamp64Micro(now64(6))") ${CLICKHOUSE_CLIENT} --query " CREATE TABLE dest_bg (s String) ENGINE = File(ProtobufSingle) @@ -33,10 +33,11 @@ do ${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS text_log" # The logger is checked too because this query is itself written to the text log, so otherwise # an iteration would match the previous one instead of the background exception. + # `event_date` is in the server time zone, and the session one is randomized. refused=$(${CLICKHOUSE_CLIENT} --query " SELECT count() > 0 FROM system.text_log - WHERE event_date >= toDate('${START}') - AND event_time_microseconds >= toDateTime64('${START}', 6) + WHERE event_date >= yesterday() + AND event_time_microseconds >= fromUnixTimestamp64Micro(${START}) AND logger_name LIKE '%StorageBuffer%' AND message LIKE '%can only be executed on behalf of a user%'") [ "$refused" = "1" ] && break diff --git a/tests/queries/0_stateless/05056_arrow_record_batch_split_boundaries.reference b/tests/queries/0_stateless/05056_arrow_record_batch_split_boundaries.reference new file mode 100644 index 0000000000000..fe1489a7d47eb --- /dev/null +++ b/tests/queries/0_stateless/05056_arrow_record_batch_split_boundaries.reference @@ -0,0 +1,4 @@ +low_cardinality_as_dictionary=0 record batches for 3 blocks: 3 +6 15 +low_cardinality_as_dictionary=1 record batches for 3 blocks: 3 +6 15 diff --git a/tests/queries/0_stateless/05056_arrow_record_batch_split_boundaries.sh b/tests/queries/0_stateless/05056_arrow_record_batch_split_boundaries.sh new file mode 100755 index 0000000000000..ed8ae006336a8 --- /dev/null +++ b/tests/queries/0_stateless/05056_arrow_record_batch_split_boundaries.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The writer splits a chunk across record batches only when it exceeds the 32-bit offsets of the Arrow +# buffers, so a small result must give exactly one batch per block for every type shape. A wrong row +# count from the split logic shows up only here: the data reads back correctly either way. +# +# `FORMAT ArrowStream` into a file rather than `INSERT INTO FUNCTION file`, whose path squashes the +# blocks into one. Compression is off because older pyarrow cannot read the "uncompressed" sentinel. + +FILE="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_batches.arrows" +trap 'rm -f "$FILE"' EXIT + +GEN="SELECT + toString(number) AS s, + if(number % 3 = 0, NULL, toString(number))::Nullable(String) AS ns, + toLowCardinality(toString(number % 2)) AS lc, + arrayMap(x -> toString(x), range(number % 3)) AS arr, + arrayMap(y -> arrayMap(x -> toString(x), range(y)), range(number % 3)) AS arr2, + map(toString(number), toString(number % 2)) AS m, + (toString(number), toString(number % 2)) AS t, + toFixedString(toString(number), 3) AS fs, + number::Variant(UInt64, String) AS v, + initializeAggregation('quantileState(0.5)', number) AS agg, + toBFloat16(number) AS bf, + number AS u +FROM numbers(6)" + +for lc_as_dict in 0 1; do + $CLICKHOUSE_LOCAL --query "${GEN} SETTINGS + max_block_size = 2, + output_format_arrow_compression_method = 'none', + output_format_arrow_fixed_string_as_fixed_byte_array = 0, + output_format_arrow_low_cardinality_as_dictionary = ${lc_as_dict} + FORMAT ArrowStream" > "$FILE" + + echo -n "low_cardinality_as_dictionary=${lc_as_dict} record batches for 3 blocks: " + python3 - "$FILE" <<'PY' +import sys +import pyarrow as pa +import pyarrow.ipc as ipc + +with pa.OSFile(sys.argv[1], "rb") as src: + print(sum(1 for _ in ipc.open_stream(src))) +PY + + $CLICKHOUSE_LOCAL --query "SELECT count(), sum(u) FROM file('$FILE', ArrowStream)" +done diff --git a/tests/queries/0_stateless/05059_convert_column_to_type_bool_identity.reference b/tests/queries/0_stateless/05059_convert_column_to_type_bool_identity.reference new file mode 100644 index 0000000000000..6a0543ef42352 --- /dev/null +++ b/tests/queries/0_stateless/05059_convert_column_to_type_bool_identity.reference @@ -0,0 +1,16 @@ +1 +1 +[1] +(1) +{'a':1} +0 +1 +0 +1 +true 1 +true 1 +[true] 1 +(true) 1 +{'a':true} 1 +1 +1 diff --git a/tests/queries/0_stateless/05059_convert_column_to_type_bool_identity.sql b/tests/queries/0_stateless/05059_convert_column_to_type_bool_identity.sql new file mode 100644 index 0000000000000..782fc765296fe --- /dev/null +++ b/tests/queries/0_stateless/05059_convert_column_to_type_bool_identity.sql @@ -0,0 +1,26 @@ +-- `Bool` is a custom-named `UInt8`, so `IDataType::equals` treats `Bool` and `UInt8` (and wrapper pairs +-- such as `Nullable(Bool)` and `Nullable(UInt8)`) as the same type. The constant conversion must still +-- normalize a raw `Bool` byte (2 here, produced by `reinterpret`) to the semantic `true` rather than +-- alias it byte-for-byte. +SELECT * FROM values('x UInt8', reinterpret(toUInt8(2), 'Bool')); +SELECT * FROM values('x Nullable(UInt8)', toNullable(reinterpret(toUInt8(2), 'Bool'))); +SELECT * FROM values('x Array(UInt8)', [reinterpret(toUInt8(2), 'Bool')]); +SELECT * FROM values('x Tuple(UInt8)', tuple(reinterpret(toUInt8(2), 'Bool'))); +SELECT * FROM values('x Map(String, UInt8)', map('a', reinterpret(toUInt8(2), 'Bool'))); + +-- The `IN` set builder converts each literal strictly: a `Bool` literal is normalized to `true` before +-- it is compared with a `UInt8`, and 2 is not a valid `Bool`. +SELECT toUInt8(2) IN (reinterpret(toUInt8(2), 'Bool')); +SELECT toUInt8(1) IN (reinterpret(toUInt8(2), 'Bool')); +SELECT reinterpret(toUInt8(2), 'Bool') IN (toUInt8(2)); +SELECT reinterpret(toUInt8(2), 'Bool') IN (toUInt8(1)); + +-- Identical types containing `Bool` are not a byte-for-byte no-op either: the raw byte must still be +-- normalized, as `CAST` does for `Bool -> Bool`. +SELECT x, x = true FROM values('x Bool', reinterpret(toUInt8(2), 'Bool')); +SELECT x, x = true FROM values('x Nullable(Bool)', toNullable(reinterpret(toUInt8(2), 'Bool'))); +SELECT x, x = [true] FROM values('x Array(Bool)', [reinterpret(toUInt8(2), 'Bool')]); +SELECT x, x = tuple(true) FROM values('x Tuple(Bool)', tuple(reinterpret(toUInt8(2), 'Bool'))); +SELECT x, x = map('a', true) FROM values('x Map(String, Bool)', map('a', reinterpret(toUInt8(2), 'Bool'))); +SELECT true IN (reinterpret(toUInt8(2), 'Bool')); +SELECT reinterpret(toUInt8(3), 'Bool') IN (reinterpret(toUInt8(2), 'Bool')); diff --git a/tests/queries/0_stateless/05141_settings_constraints_nested_settings_effective.reference b/tests/queries/0_stateless/05141_settings_constraints_nested_settings_effective.reference new file mode 100644 index 0000000000000..a5714250422e8 --- /dev/null +++ b/tests/queries/0_stateless/05141_settings_constraints_nested_settings_effective.reference @@ -0,0 +1,35 @@ +-- use_query_cache = 0 CONST: a nested use_query_cache = 1 runs but caches nothing +3 +0 0 +-- readonly = 1: a nested use_query_cache = 1 runs but caches nothing +3 +0 0 +-- unconstrained: a nested use_query_cache = 1 caches the subquery +3 +1 0 +-- unconstrained: a nested use_query_cache = 1 equal to the outer value still caches the subquery, next to the outer query +3 +1 1 +-- unconstrained: a nested use_query_cache = 'true' given as a string is applied +3 +1 0 +-- a nested unknown setting still throws +1 +-- a nested value the setting cannot take still throws +1 +-- a nested clause forwarded to the shards carries the clamped value +1 +-- the same query for an unconstrained user reads both shards +6 +-- a view with an invalid nested setting in its stored body fails on a direct read +1 +-- and on a remote shard through Distributed +1 +-- also when the shard inlines the view +1 +-- a parameterized view with an invalid nested setting fails on a direct read +1 +-- and on a remote shard +1 +-- the session settings are untouched +false diff --git a/tests/queries/0_stateless/05141_settings_constraints_nested_settings_effective.sh b/tests/queries/0_stateless/05141_settings_constraints_nested_settings_effective.sh new file mode 100755 index 0000000000000..8206835476c28 --- /dev/null +++ b/tests/queries/0_stateless/05141_settings_constraints_nested_settings_effective.sh @@ -0,0 +1,146 @@ +#!/usr/bin/env bash +# Tags: no-old-analyzer +# (the clamp is analyzer-path behaviour; the legacy interpreter throws on such a nested clause) +# A nested `SETTINGS` clause is clamped to the session's settings constraints, and the clamped clause is +# what the query node keeps. So a nested `use_query_cache = 1` the constraints dropped must not enable +# the query result cache in the `Planner` (issue #117226), a shard receives the clamped clause, and an +# unknown name or an uncastable value in a nested clause throws, as it does in a top-level clause - also +# when the clause sits in a view's stored body and the view is read on a remote shard. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Users and settings profiles are server-global, so their names carry the test database to keep this +# test safe against a concurrent copy of itself - which is how the flaky check runs it. +SUFFIX="05141_${CLICKHOUSE_DATABASE}" +CONST_USER="user_const_${SUFFIX}" +CONST_PROFILE="profile_const_${SUFFIX}" +RO_USER="user_ro_${SUFFIX}" +RO_PROFILE="profile_ro_${SUFFIX}" +FREE_USER="user_free_${SUFFIX}" +DIST_USER="user_dist_${SUFFIX}" +DIST_PROFILE="profile_dist_${SUFFIX}" +TABLE="t_05141" +DIST_TABLE="dist_05141" +VIEW="v_05141" +DIST_VIEW="dist_v_05141" +PARAM_VIEW="pv_05141" +# `system.query_cache` lists the entries of every user and every test, and the cache is never dropped +# here, so each cached subquery carries a per-run marker in its text and is looked up by it. +MARKER="marker_${SUFFIX}" + +${CLICKHOUSE_CLIENT} --query " +DROP USER IF EXISTS ${CONST_USER}, ${RO_USER}, ${FREE_USER}, ${DIST_USER}; +DROP SETTINGS PROFILE IF EXISTS ${CONST_PROFILE}, ${RO_PROFILE}, ${DIST_PROFILE}; +DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.${DIST_TABLE}; +DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.${TABLE}; +DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.${DIST_VIEW}; +DROP VIEW IF EXISTS ${CLICKHOUSE_DATABASE}.${VIEW}; +DROP VIEW IF EXISTS ${CLICKHOUSE_DATABASE}.${PARAM_VIEW}; + +CREATE TABLE ${CLICKHOUSE_DATABASE}.${TABLE} (id UInt32, s String) ENGINE = MergeTree ORDER BY id; +INSERT INTO ${CLICKHOUSE_DATABASE}.${TABLE} VALUES (1, 'a'), (2, 'b'), (3, 'c'); +CREATE TABLE ${CLICKHOUSE_DATABASE}.${DIST_TABLE} AS ${CLICKHOUSE_DATABASE}.${TABLE} + ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), ${TABLE}); + +CREATE SETTINGS PROFILE ${CONST_PROFILE} SETTINGS use_query_cache = 0 CONST; +CREATE SETTINGS PROFILE ${RO_PROFILE} SETTINGS readonly = 1 CONST; +CREATE SETTINGS PROFILE ${DIST_PROFILE} SETTINGS max_rows_to_read MAX 2; +CREATE USER ${CONST_USER} IDENTIFIED WITH no_password SETTINGS PROFILE ${CONST_PROFILE}; +CREATE USER ${RO_USER} IDENTIFIED WITH no_password SETTINGS PROFILE ${RO_PROFILE}; +CREATE USER ${FREE_USER} IDENTIFIED WITH no_password; +CREATE USER ${DIST_USER} IDENTIFIED WITH no_password SETTINGS PROFILE ${DIST_PROFILE}; +GRANT SELECT ON ${CLICKHOUSE_DATABASE}.* TO ${CONST_USER}, ${RO_USER}, ${FREE_USER}, ${DIST_USER}; +" + +# `CREATE` does not analyze the body of a view with an explicit column list nor of a parameterized view, so +# such definitions can carry an invalid nested setting. +${CLICKHOUSE_CLIENT} --query " +CREATE VIEW ${CLICKHOUSE_DATABASE}.${VIEW} (x UInt8) AS SELECT * FROM (SELECT 1 AS x SETTINGS max_threds = 1); +CREATE TABLE ${CLICKHOUSE_DATABASE}.${DIST_VIEW} (x UInt8) ENGINE = Distributed(test_cluster_two_shards, currentDatabase(), ${VIEW}); +CREATE VIEW ${CLICKHOUSE_DATABASE}.${PARAM_VIEW} AS SELECT * FROM (SELECT 1 AS x SETTINGS max_threds = 1) WHERE x = {p:UInt8}; +" + +# The bare binary, not ${CLICKHOUSE_CLIENT}: the harness's randomized settings would be rejected for the readonly user. +RESTRICTED_CLIENT="${CLICKHOUSE_CLIENT_BINARY} --host=${CLICKHOUSE_HOST} --port=${CLICKHOUSE_PORT_TCP} --database=${CLICKHOUSE_DATABASE}" +CONST="${RESTRICTED_CLIENT} --user=${CONST_USER}" +RO="${RESTRICTED_CLIENT} --user=${RO_USER}" +FREE="${RESTRICTED_CLIENT} --user=${FREE_USER}" +DIST="${RESTRICTED_CLIENT} --user=${DIST_USER}" + +# Cache entries whose text carries the marker: subquery entries, then top-level entries. +function cache_entries() +{ + ${CLICKHOUSE_CLIENT} --query "SELECT countIf(is_subquery), countIf(NOT is_subquery) FROM system.query_cache WHERE query LIKE '%$1%'" +} + +# The marker is compared against a real column so it survives constant folding and stays in the +# subquery text the cache entry is keyed and listed by. +echo "-- use_query_cache = 0 CONST: a nested use_query_cache = 1 runs but caches nothing" +${CONST} --query "SELECT count() FROM (SELECT id FROM ${TABLE} WHERE s != '${MARKER}_const' SETTINGS use_query_cache = 1)" +cache_entries "${MARKER}_const" + +echo "-- readonly = 1: a nested use_query_cache = 1 runs but caches nothing" +${RO} --query "SELECT count() FROM (SELECT id FROM ${TABLE} WHERE s != '${MARKER}_ro' SETTINGS use_query_cache = 1)" +cache_entries "${MARKER}_ro" + +echo "-- unconstrained: a nested use_query_cache = 1 caches the subquery" +${FREE} --query "SELECT count() FROM (SELECT id FROM ${TABLE} WHERE s != '${MARKER}_free' SETTINGS use_query_cache = 1)" +cache_entries "${MARKER}_free" + +echo "-- unconstrained: a nested use_query_cache = 1 equal to the outer value still caches the subquery, next to the outer query" +${FREE} --query "SELECT count() FROM (SELECT id FROM ${TABLE} WHERE s != '${MARKER}_both' SETTINGS use_query_cache = 1) SETTINGS use_query_cache = 1" +cache_entries "${MARKER}_both" + +echo "-- unconstrained: a nested use_query_cache = 'true' given as a string is applied" +${FREE} --query "SELECT count() FROM (SELECT id FROM ${TABLE} WHERE s != '${MARKER}_string' SETTINGS use_query_cache = 'true')" +cache_entries "${MARKER}_string" + +echo "-- a nested unknown setting still throws" +${CONST} --query "SELECT count() FROM (SELECT id FROM ${TABLE} SETTINGS max_threds = 1)" 2>&1 | grep -c -F "UNKNOWN_SETTING" + +echo "-- a nested value the setting cannot take still throws" +${CONST} --query "SELECT count() FROM (SELECT id FROM ${TABLE} SETTINGS max_threads = 'abc')" 2>&1 | grep -c -F "CANNOT_PARSE_INPUT_ASSERTION_FAILED" + +# The IN subquery runs on the shards, whose user has no constraints. It must arrive with the value the +# initiator clamped (2, so reading 3 rows fails there), not with the value as written. +echo "-- a nested clause forwarded to the shards carries the clamped value" +${DIST} --query "SELECT count() FROM ${DIST_TABLE} WHERE id IN (SELECT id FROM ${TABLE} SETTINGS max_rows_to_read = 100)" 2>&1 | grep -c -F "TOO_MANY_ROWS" + +echo "-- the same query for an unconstrained user reads both shards" +${FREE} --query "SELECT count() FROM ${DIST_TABLE} WHERE id IN (SELECT id FROM ${TABLE} SETTINGS max_rows_to_read = 100)" + +# The bare binary again: `${CLICKHOUSE_CLIENT}` adds `--send_logs_level`, so the server's error log line would +# be printed too and `grep -c` would count two lines. +echo "-- a view with an invalid nested setting in its stored body fails on a direct read" +${RESTRICTED_CLIENT} --query "SELECT * FROM ${VIEW}" 2>&1 | grep -c -F "UNKNOWN_SETTING" + +# `prefer_localhost_replica = 0` is explicit: the local shard is analyzed under the initiator's query kind +# and already throws, so only the remote path exercises the fix, and the harness randomizes this setting. +echo "-- and on a remote shard through Distributed" +${RESTRICTED_CLIENT} --query "SELECT * FROM ${DIST_VIEW} SETTINGS prefer_localhost_replica = 0" 2>&1 | grep -c -F "UNKNOWN_SETTING" + +echo "-- also when the shard inlines the view" +${RESTRICTED_CLIENT} --query "SELECT * FROM ${DIST_VIEW} SETTINGS prefer_localhost_replica = 0, analyzer_inline_views = 1" 2>&1 | grep -c -F "UNKNOWN_SETTING" + +echo "-- a parameterized view with an invalid nested setting fails on a direct read" +${RESTRICTED_CLIENT} --query "SELECT * FROM ${PARAM_VIEW}(p = 1)" 2>&1 | grep -c -F "UNKNOWN_SETTING" + +# 127.0.0.2 is the non-local shard of `test_cluster_two_shards`; the port is explicit so the test runs on +# a non-default port; the database is qualified because the remote session does not inherit the test database. +echo "-- and on a remote shard" +${RESTRICTED_CLIENT} --query "SELECT * FROM remote('127.0.0.2:${CLICKHOUSE_PORT_TCP}', view(SELECT * FROM ${CLICKHOUSE_DATABASE}.${PARAM_VIEW}(p = 1)))" 2>&1 | grep -c -F "UNKNOWN_SETTING" + +echo "-- the session settings are untouched" +${CONST} --query "SELECT getSetting('use_query_cache')" + +${CLICKHOUSE_CLIENT} --query " +DROP TABLE ${CLICKHOUSE_DATABASE}.${DIST_TABLE}; +DROP TABLE ${CLICKHOUSE_DATABASE}.${TABLE}; +DROP TABLE ${CLICKHOUSE_DATABASE}.${DIST_VIEW}; +DROP VIEW ${CLICKHOUSE_DATABASE}.${VIEW}; +DROP VIEW ${CLICKHOUSE_DATABASE}.${PARAM_VIEW}; +DROP USER ${CONST_USER}, ${RO_USER}, ${FREE_USER}, ${DIST_USER}; +DROP SETTINGS PROFILE ${CONST_PROFILE}, ${RO_PROFILE}, ${DIST_PROFILE}; +" diff --git a/tests/queries/0_stateless/05175_nested_settings_equal_value_marks_changed.reference b/tests/queries/0_stateless/05175_nested_settings_equal_value_marks_changed.reference new file mode 100644 index 0000000000000..95ca9df0fcece --- /dev/null +++ b/tests/queries/0_stateless/05175_nested_settings_equal_value_marks_changed.reference @@ -0,0 +1,18 @@ +-- control: the setting is not changed in the session +0 1 +-- control: compatibility = '26.7' in a nested clause moves the setting to its old default +1 0 +-- an equal nested assignment is applied and marked changed in the subquery's context, as at the top level +1 1 +-- and stays on the query node +1 +-- same clause: an equal assignment before compatibility keeps the current value +1 1 +-- same clause: an equal assignment before a profile that carries compatibility keeps the current value +1 1 +-- child scope: an equal assignment in the outer subquery protects the setting from compatibility in the inner one +1 +-- child scope: the same with a profile that carries compatibility in the inner one +1 +-- top level over HTTP: an equal assignment protects the setting from compatibility in a subquery +1 diff --git a/tests/queries/0_stateless/05175_nested_settings_equal_value_marks_changed.sh b/tests/queries/0_stateless/05175_nested_settings_equal_value_marks_changed.sh new file mode 100755 index 0000000000000..4f9000f02f8c0 --- /dev/null +++ b/tests/queries/0_stateless/05175_nested_settings_equal_value_marks_changed.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash +# Tags: no-old-analyzer + +# A nested `SETTINGS` change equal to the current value is applied and marks the setting as changed +# in the subquery's context, as at the top level. That flag is what `compatibility` respects +# ("changed manually" settings are left alone), also when `compatibility` sits in a profile or in +# a nested query, so the flag must not be dropped as a "no-op". + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Settings profiles are server-global, so the name carries the test database. +PROFILE="p_compat_05175_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} --query "DROP SETTINGS PROFILE IF EXISTS ${PROFILE}" +${CLICKHOUSE_CLIENT} --query "CREATE SETTINGS PROFILE ${PROFILE} SETTINGS compatibility = '26.7'" + +Q="SELECT changed, value FROM system.settings WHERE name = 'enable_group_by_top_k_optimization'" + +echo "-- control: the setting is not changed in the session" +${CLICKHOUSE_CLIENT} --query "${Q}" + +echo "-- control: compatibility = '26.7' in a nested clause moves the setting to its old default" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM (${Q} SETTINGS compatibility = '26.7')" + +echo "-- an equal nested assignment is applied and marked changed in the subquery's context, as at the top level" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM (${Q} SETTINGS enable_group_by_top_k_optimization = 1)" + +echo "-- and stays on the query node" +${CLICKHOUSE_CLIENT} --query "EXPLAIN QUERY TREE SELECT * FROM (${Q} SETTINGS enable_group_by_top_k_optimization = 1)" | grep -c "SETTINGS enable_group_by_top_k_optimization=" + +echo "-- same clause: an equal assignment before compatibility keeps the current value" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM (${Q} SETTINGS enable_group_by_top_k_optimization = 1, compatibility = '26.7')" + +echo "-- same clause: an equal assignment before a profile that carries compatibility keeps the current value" +${CLICKHOUSE_CLIENT} --query "SELECT * FROM (${Q} SETTINGS enable_group_by_top_k_optimization = 1, profile = '${PROFILE}')" + +echo "-- child scope: an equal assignment in the outer subquery protects the setting from compatibility in the inner one" +${CLICKHOUSE_CLIENT} --query "SELECT value FROM (SELECT * FROM (${Q} SETTINGS compatibility = '26.7') SETTINGS enable_group_by_top_k_optimization = 1)" + +echo "-- child scope: the same with a profile that carries compatibility in the inner one" +${CLICKHOUSE_CLIENT} --query "SELECT value FROM (SELECT * FROM (${Q} SETTINGS profile = '${PROFILE}') SETTINGS enable_group_by_top_k_optimization = 1)" + +echo "-- top level over HTTP: an equal assignment protects the setting from compatibility in a subquery" +${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}" -d "SELECT value FROM (${Q} SETTINGS compatibility = '26.7') SETTINGS enable_group_by_top_k_optimization = 1" + +${CLICKHOUSE_CLIENT} --query "DROP SETTINGS PROFILE ${PROFILE}" diff --git a/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.sql b/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.sql index 9a5833c380f14..b162f0674020b 100644 --- a/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.sql +++ b/tests/queries/0_stateless/05217_text_index_map_lowcardinality_subcolumn_cast.sql @@ -6,6 +6,7 @@ -- and the text index has to look through the cast: for an index on `mapValues(m)`. -- The same applies to every conversion that does not change a non-NULL value: adding or dropping `LowCardinality` or -- `Nullable` (adding also as `toNullable`, which `join_use_nulls` emits in pushed-down filters), at any depth of `Array`. +-- The rewrite applies only to bucketed Maps, so the tables with Maps use `with_buckets`. SET explain_query_plan_default = 'legacy'; SET enable_analyzer = 1; @@ -23,7 +24,8 @@ CREATE TABLE tab_values ) ENGINE = MergeTree ORDER BY id -SETTINGS index_granularity = 2, min_bytes_for_wide_part = 0; +SETTINGS index_granularity = 2, min_bytes_for_wide_part = 0, + map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO tab_values VALUES (1, {'level':'error','msg':'disk is full'}), (2, {'level':'warn','msg':'disk is slow'}), (3, {'level':'error','msg':'network is down'}), (4, {}); @@ -57,7 +59,8 @@ CREATE TABLE tab_values_n ) ENGINE = MergeTree ORDER BY id -SETTINGS index_granularity = 2, min_bytes_for_wide_part = 0; +SETTINGS index_granularity = 2, min_bytes_for_wide_part = 0, + map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; INSERT INTO tab_values_n VALUES (1, {'level':'error'}), (2, {'level':'error'}), (3, {'level':'warn'}), (4, {'level':NULL}); diff --git a/tests/queries/0_stateless/05218_attach_table_from_path_read_on_file_grant.reference b/tests/queries/0_stateless/05218_attach_table_from_path_read_on_file_grant.reference new file mode 100644 index 0000000000000..06604339ef491 --- /dev/null +++ b/tests/queries/0_stateless/05218_attach_table_from_path_read_on_file_grant.reference @@ -0,0 +1,7 @@ +ACCESS_DENIED: necessary to have the grant READ, WRITE ON FILE +tables named stolen: 0 +staged directory is still in place +ACCESS_DENIED: Missing permissions: WRITE ON FILE +tables named stolen: 0 +staged directory is still in place +424242 diff --git a/tests/queries/0_stateless/05218_attach_table_from_path_read_on_file_grant.sh b/tests/queries/0_stateless/05218_attach_table_from_path_read_on_file_grant.sh new file mode 100755 index 0000000000000..1fa58284b9aa2 --- /dev/null +++ b/tests/queries/0_stateless/05218_attach_table_from_path_read_on_file_grant.sh @@ -0,0 +1,67 @@ +#!/usr/bin/env bash +# Tags: no-object-storage, no-replicated-database +# no-object-storage: the query attaches a directory on the local disk, and there the table data lives on +# object storage. +# no-replicated-database: a Replicated database enqueues the query before the path is resolved against +# `user_files`, and the DDL worker then rejects the path as outside of `user_files`. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `ATTACH TABLE ... FROM ''` reads a directory in `user_files` as the data of the new table and moves it to +# the data path of the table, so it needs `READ ON FILE` like the `file` function and `WRITE ON FILE` like +# renaming files after processing. Without that check `CREATE TABLE` on a table of their own is enough for a +# user to read and consume data that somebody else staged in `user_files`. + +user="user_${CLICKHOUSE_TEST_UNIQUE_NAME}" +staged_root="${CLICKHOUSE_TEST_UNIQUE_NAME}_staged" +staged_table_dir="${staged_root}/data/staging/protected" + +# Stage a MergeTree table directory in `user_files` with clickhouse-local, as an admin or a migration +# workflow would do. An Ordinary database keeps the table directory at a known place. +${CLICKHOUSE_LOCAL} --path "${USER_FILES_PATH}/${staged_root}" --allow_deprecated_database_ordinary=1 --query " + CREATE DATABASE staging ENGINE = Ordinary; + CREATE TABLE staging.protected (id UInt64, secret UInt64) ENGINE = MergeTree ORDER BY id; + INSERT INTO staging.protected VALUES (1, 424242); +" + +${CLICKHOUSE_CLIENT} <&1 | grep -qF "$1"; then + echo "ACCESS_DENIED: $1" + else + echo "UNEXPECTED: the attach was not denied with: $1" + fi + ${CLICKHOUSE_CLIENT} --query "SELECT 'tables named stolen:', count() FROM system.tables WHERE database = currentDatabase() AND name = 'stolen'" + if [ -d "${USER_FILES_PATH}/${staged_table_dir}" ]; then + echo "staged directory is still in place" + else + echo "UNEXPECTED: the staged directory is gone" + fi +} +check_denied_and_untouched "necessary to have the grant READ, WRITE ON FILE" +${CLICKHOUSE_CLIENT} --query "GRANT READ ON FILE TO $user" +check_denied_and_untouched "Missing permissions: WRITE ON FILE" + +# With both grants the same query attaches the data. +${CLICKHOUSE_CLIENT} --query "GRANT WRITE ON FILE TO $user" +${CLICKHOUSE_CLIENT} --user "$user" --query "$attach_query" +${CLICKHOUSE_CLIENT} --user "$user" --query "SELECT secret FROM stolen" + +${CLICKHOUSE_CLIENT} <&1 \ + | tr '\t' ' ' | grep -oE "QUOTA_REQUIRES_CLIENT_KEY|^[[:alnum:]_]+ [[:digit:]]+$" +} + +echo "1. warm two cached quota contexts of the same user, without and with a quota key" +usage --user "${user}" +usage --user "${user}" --quota_key kb + +echo "2. a tightening that keeps the key type reaches the warm context" +${CLICKHOUSE_CLIENT} -q "ALTER QUOTA ${quota} FOR INTERVAL 100 YEAR MAX queries = 900" +usage --user "${user}" + +echo "3. switching the key type to client_key must reach both warm contexts" +${CLICKHOUSE_CLIENT} -q "ALTER QUOTA ${quota} KEYED BY client_key FOR INTERVAL 100 YEAR MAX queries = 800" +usage --user "${user}" +usage --user "${user}" --quota_key kb + +echo "4. a user whose context was never cached is rejected the same way" +usage --user "${control_user}" + +echo "5. reverting the key type lets the never-detached context resume" +${CLICKHOUSE_CLIENT} -q "ALTER QUOTA ${quota} KEYED BY user_name FOR INTERVAL 100 YEAR MAX queries = 700" +usage --user "${user}" + +${CLICKHOUSE_CLIENT} -q "DROP QUOTA ${quota}" +${CLICKHOUSE_CLIENT} -q "DROP USER ${user}, ${control_user}" diff --git a/tests/queries/0_stateless/05233_assume_not_null_nullable_partition_key_pruning.reference b/tests/queries/0_stateless/05233_assume_not_null_nullable_partition_key_pruning.reference new file mode 100644 index 0000000000000..02703d86a7bd1 --- /dev/null +++ b/tests/queries/0_stateless/05233_assume_not_null_nullable_partition_key_pruning.reference @@ -0,0 +1,43 @@ +the reported oracle mismatch: count(p) + count(not p) + count(p is null) = count(*) +1 +predicate, row-level truth, answer over the partitioned table +assumeNotNull(c0) = 0 3 3 +assumeNotNull(c0) < 3 4 4 +NOT (assumeNotNull(c0) > 0) 3 3 +assumeNotNull(c0) >= 0 5 5 +assumeNotNull(c0) IN (0) 3 3 +isNull and isNotNull compared with a constant +(c0 IS NULL) >= true 2 2 +(c0 IS NOT NULL) < true 2 2 +not(1 < (c0 IS NULL)) 5 5 +the partition step is the sole pruner here, and it now keeps the NULL partition +Min-Max +Condition: true +Parts: 4/4 +Partition +Condition: (isNull(c0) in [1, +Inf)) +Parts: 1/4 +ifNull and coalesce reach the same chain once the rewrite that usually hides them is off +ifNull(c0, 0) = 0 3 3 +coalesce(c0, 0) = 0 3 3 +NOT (ifNull(c0, 0) >= 7) 5 5 +NOT (coalesce(nullIf(c0, 5), 0) = 1) 4 4 +Min-Max +Parts: 4/4 +Partition +Parts: 3/4 +reverse key 3 3 +LowCardinality(Nullable) key 3 3 +SummingMergeTree, the query the fuzzer reported +1 +control: with no NULL row the partition step is the only pruner and still prunes +isZeroOrNull(c0) >= true 1 +Min-Max +Parts: 3/3 +Partition +Parts: 1/3 +control: the primary-key path already rejected the chain and is unchanged +assumeNotNull(c0) = 0 3 3 +control: a NULL-preserving wrapper maps the NULL row to NULL and still answers correctly +c0 + 1 = 1 1 1 +round(c0) = 0 1 1 diff --git a/tests/queries/0_stateless/05233_assume_not_null_nullable_partition_key_pruning.sql b/tests/queries/0_stateless/05233_assume_not_null_nullable_partition_key_pruning.sql new file mode 100644 index 0000000000000..917f89a126cc6 --- /dev/null +++ b/tests/queries/0_stateless/05233_assume_not_null_nullable_partition_key_pruning.sql @@ -0,0 +1,115 @@ +-- Tags: no-replicated-database, no-parallel-replicas +-- no-replicated-database: EXPLAIN output differs for replicated database. +-- no-parallel-replicas: EXPLAIN output differs for parallel replicas. +-- Test for https://github.com/ClickHouse/ClickHouse/issues/121133, https://github.com/ClickHouse/ClickHouse/issues/126369 and https://github.com/ClickHouse/ClickHouse/issues/127755 + +-- Randomized to 0 or 1, and the value changes the plan the EXPLAIN blocks below print. +SET parallel_replicas_local_plan = 1; +-- Statistics are a part-pruning source of their own; pinning both off keeps the Min-Max and +-- Partition steps the only pruners the EXPLAIN blocks below can attribute a Parts: count to. +SET use_statistics = 0; +SET use_statistics_for_part_pruning = 0; +SET explain_query_plan_default = 'legacy'; + +DROP TABLE IF EXISTS t_part; +DROP TABLE IF EXISTS t_flat; + +-- `t_flat` holds the same rows behind an unprunable key, so its answer is the row-level truth +-- that every other table below must reproduce. +CREATE TABLE t_part (c0 Nullable(Int)) ENGINE = MergeTree PARTITION BY (c0) ORDER BY tuple() SETTINGS allow_nullable_key = 1; +CREATE TABLE t_flat (c0 Nullable(Int)) ENGINE = MergeTree PARTITION BY tuple() ORDER BY tuple() SETTINGS allow_nullable_key = 1; +INSERT INTO t_part VALUES (NULL), (0), (1), (5), (NULL); +INSERT INTO t_flat VALUES (NULL), (0), (1), (5), (NULL); + +SELECT 'the reported oracle mismatch: count(p) + count(not p) + count(p is null) = count(*)'; +SELECT (SELECT count() FROM t_part WHERE assumeNotNull(c0) = 0) + + (SELECT count() FROM t_part WHERE NOT (assumeNotNull(c0) = 0)) + + (SELECT count() FROM t_part WHERE (assumeNotNull(c0) = 0) IS NULL) + = (SELECT count() FROM t_part) AS tlp_identity_holds; + +SELECT 'predicate, row-level truth, answer over the partitioned table'; +SELECT 'assumeNotNull(c0) = 0', (SELECT count() FROM t_flat WHERE assumeNotNull(c0) = 0), (SELECT count() FROM t_part WHERE assumeNotNull(c0) = 0); +SELECT 'assumeNotNull(c0) < 3', (SELECT count() FROM t_flat WHERE assumeNotNull(c0) < 3), (SELECT count() FROM t_part WHERE assumeNotNull(c0) < 3); +SELECT 'NOT (assumeNotNull(c0) > 0)', (SELECT count() FROM t_flat WHERE NOT (assumeNotNull(c0) > 0)), (SELECT count() FROM t_part WHERE NOT (assumeNotNull(c0) > 0)); +SELECT 'assumeNotNull(c0) >= 0', (SELECT count() FROM t_flat WHERE assumeNotNull(c0) >= 0), (SELECT count() FROM t_part WHERE assumeNotNull(c0) >= 0); +-- A set atom carries its own copy of the chain and is applied by MergeTreeSetIndex, not by KeyCondition. +SELECT 'assumeNotNull(c0) IN (0)', (SELECT count() FROM t_flat WHERE assumeNotNull(c0) IN (0)), (SELECT count() FROM t_part WHERE assumeNotNull(c0) IN (0)); + +SELECT 'isNull and isNotNull compared with a constant'; +SELECT '(c0 IS NULL) >= true', (SELECT count() FROM t_flat WHERE (c0 IS NULL) >= true), (SELECT count() FROM t_part WHERE (c0 IS NULL) >= true); +SELECT '(c0 IS NOT NULL) < true', (SELECT count() FROM t_flat WHERE (c0 IS NOT NULL) < true), (SELECT count() FROM t_part WHERE (c0 IS NOT NULL) < true); +SELECT 'not(1 < (c0 IS NULL))', (SELECT count() FROM t_flat WHERE not(1 < (c0 IS NULL))), (SELECT count() FROM t_part WHERE not(1 < (c0 IS NULL))); + +-- `isNull` reports no monotonicity, so only the single-point Partition step can use it: it is the sole +-- pruner and its Parts: denominator is the table's part count. +SELECT 'the partition step is the sole pruner here, and it now keeps the NULL partition'; +SELECT trimLeft(explain) FROM ( + EXPLAIN indexes = 1 SELECT c0 FROM t_part WHERE (c0 IS NULL) >= true + SETTINGS optimize_use_implicit_projections = 0, optimize_use_projections = 0 +) WHERE explain LIKE '%Condition%' OR explain LIKE '%Parts%' OR explain LIKE '%Min-Max%' OR explain LIKE '%Partition%'; + +SELECT 'ifNull and coalesce reach the same chain once the rewrite that usually hides them is off'; +SELECT 'ifNull(c0, 0) = 0', (SELECT count() FROM t_flat WHERE ifNull(c0, 0) = 0 SETTINGS allow_key_condition_coalesce_rewrite = 0), (SELECT count() FROM t_part WHERE ifNull(c0, 0) = 0 SETTINGS allow_key_condition_coalesce_rewrite = 0); +SELECT 'coalesce(c0, 0) = 0', (SELECT count() FROM t_flat WHERE coalesce(c0, 0) = 0 SETTINGS allow_key_condition_coalesce_rewrite = 0), (SELECT count() FROM t_part WHERE coalesce(c0, 0) = 0 SETTINGS allow_key_condition_coalesce_rewrite = 0); +-- The rewrite applies only to a non-negated condition, so a negated one reaches the chain with the rewrite on. +SELECT 'NOT (ifNull(c0, 0) >= 7)', (SELECT count() FROM t_flat WHERE NOT (ifNull(c0, 0) >= 7) SETTINGS allow_key_condition_coalesce_rewrite = 1), (SELECT count() FROM t_part WHERE NOT (ifNull(c0, 0) >= 7) SETTINGS allow_key_condition_coalesce_rewrite = 1); + +-- An inner link can also produce a real NULL: `nullIf` maps the exact point 5 to one, and the chain +-- leaves it untransformed just like an infinity bound. Such a bound compares as both inside and +-- outside every range, so the negated atom is the form that can wrongly prune it; keep it negated. +SELECT 'NOT (coalesce(nullIf(c0, 5), 0) = 1)', (SELECT count() FROM t_flat WHERE NOT (coalesce(nullIf(c0, 5), 0) = 1) SETTINGS allow_key_condition_coalesce_rewrite = 0), (SELECT count() FROM t_part WHERE NOT (coalesce(nullIf(c0, 5), 0) = 1) SETTINGS allow_key_condition_coalesce_rewrite = 0); +-- Partition 1 is still pruned, so such a bound costs its own partition and no other. +SELECT trimLeft(explain) FROM ( + EXPLAIN indexes = 1 SELECT c0 FROM t_part WHERE NOT (coalesce(nullIf(c0, 5), 0) = 1) + SETTINGS optimize_use_implicit_projections = 0, optimize_use_projections = 0, use_skip_indexes = 0, + allow_key_condition_coalesce_rewrite = 0 +) WHERE explain LIKE '%Parts%' OR explain LIKE '%Min-Max%' OR explain LIKE '%Partition%'; + +DROP TABLE IF EXISTS t_rev; +CREATE TABLE t_rev (c0 Nullable(Int)) ENGINE = MergeTree PARTITION BY (c0) ORDER BY (c0 DESC) +SETTINGS allow_nullable_key = 1; +INSERT INTO t_rev VALUES (NULL), (0), (1), (5), (NULL); +SELECT 'reverse key', (SELECT count() FROM t_flat WHERE assumeNotNull(c0) = 0), (SELECT count() FROM t_rev WHERE assumeNotNull(c0) = 0); + +DROP TABLE IF EXISTS t_lc; +SET allow_suspicious_low_cardinality_types = 1; +CREATE TABLE t_lc (c0 LowCardinality(Nullable(Int))) ENGINE = MergeTree PARTITION BY (c0) ORDER BY tuple() SETTINGS allow_nullable_key = 1; +INSERT INTO t_lc VALUES (NULL), (0), (1), (5), (NULL); +SELECT 'LowCardinality(Nullable) key', (SELECT count() FROM t_flat WHERE assumeNotNull(c0) = 0), (SELECT count() FROM t_lc WHERE assumeNotNull(c0) = 0); + +DROP TABLE IF EXISTS t_smt; +CREATE TABLE t_smt (c0 Nullable(Int)) ENGINE = SummingMergeTree PARTITION BY (c0) ORDER BY tuple() SETTINGS allow_nullable_key = 1; +INSERT INTO t_smt VALUES (NULL), (0), (1), (NULL); +SELECT 'SummingMergeTree, the query the fuzzer reported'; +SELECT (SELECT count() FROM t_smt WHERE assumeNotNull(t_smt.c0) = false) + + (SELECT count() FROM t_smt WHERE NOT (assumeNotNull(t_smt.c0) = false)) + + (SELECT count() FROM t_smt WHERE (assumeNotNull(t_smt.c0) = false) IS NULL) + = (SELECT count() FROM t_smt) AS tlp_identity_holds; + +SELECT 'control: with no NULL row the partition step is the only pruner and still prunes'; +DROP TABLE IF EXISTS t_nonull; +CREATE TABLE t_nonull (c0 Nullable(Int)) ENGINE = MergeTree PARTITION BY (c0) ORDER BY tuple() SETTINGS allow_nullable_key = 1; +INSERT INTO t_nonull VALUES (0), (1), (5); +SELECT 'isZeroOrNull(c0) >= true', count() FROM t_nonull WHERE isZeroOrNull(c0) >= true; +SELECT trimLeft(explain) FROM ( + EXPLAIN indexes = 1 SELECT c0 FROM t_nonull WHERE isZeroOrNull(c0) >= true + SETTINGS optimize_use_implicit_projections = 0, optimize_use_projections = 0 +) WHERE explain LIKE '%Parts%' OR explain LIKE '%Min-Max%' OR explain LIKE '%Partition%'; + +SELECT 'control: the primary-key path already rejected the chain and is unchanged'; +DROP TABLE IF EXISTS t_pk; +CREATE TABLE t_pk (c0 Nullable(Int)) ENGINE = MergeTree ORDER BY c0 SETTINGS allow_nullable_key = 1, index_granularity = 1; +INSERT INTO t_pk VALUES (NULL), (0), (1), (5), (NULL); +SELECT 'assumeNotNull(c0) = 0', (SELECT count() FROM t_flat WHERE assumeNotNull(c0) = 0), (SELECT count() FROM t_pk WHERE assumeNotNull(c0) = 0); + +SELECT 'control: a NULL-preserving wrapper maps the NULL row to NULL and still answers correctly'; +SELECT 'c0 + 1 = 1', (SELECT count() FROM t_flat WHERE c0 + 1 = 1), (SELECT count() FROM t_part WHERE c0 + 1 = 1); +SELECT 'round(c0) = 0', (SELECT count() FROM t_flat WHERE round(c0) = 0), (SELECT count() FROM t_part WHERE round(c0) = 0); + +DROP TABLE t_part; +DROP TABLE t_flat; +DROP TABLE t_rev; +DROP TABLE t_lc; +DROP TABLE t_smt; +DROP TABLE t_nonull; +DROP TABLE t_pk; diff --git a/tests/queries/0_stateless/05239_workload_drop_memory_reservation_in_use.reference b/tests/queries/0_stateless/05239_workload_drop_memory_reservation_in_use.reference new file mode 100644 index 0000000000000..d00491fd7e5bb --- /dev/null +++ b/tests/queries/0_stateless/05239_workload_drop_memory_reservation_in_use.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/05239_workload_drop_memory_reservation_in_use.sh b/tests/queries/0_stateless/05239_workload_drop_memory_reservation_in_use.sh new file mode 100755 index 0000000000000..ce5710669e693 --- /dev/null +++ b/tests/queries/0_stateless/05239_workload_drop_memory_reservation_in_use.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Tags: no-parallel + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +workload=w_$CLICKHOUSE_TEST_UNIQUE_NAME +query_id=q_$CLICKHOUSE_TEST_UNIQUE_NAME + +function cleanup() +{ + $CLICKHOUSE_CLIENT -nm -q "DROP WORKLOAD IF EXISTS $workload" >& /dev/null || : + $CLICKHOUSE_CLIENT -q "DROP RESOURCE IF EXISTS memory" >& /dev/null || : +} +trap cleanup EXIT + +$CLICKHOUSE_CLIENT -nm -q " +CREATE OR REPLACE RESOURCE memory (MEMORY RESERVATION); +CREATE WORKLOAD $workload SETTINGS max_memory = '1Gi'; +" + +# The query keeps the dropped workload's queue alive. DROP WORKLOAD fails the query's memory reservation, +# so the query may end on its own before KILL. +$CLICKHOUSE_CLIENT --query_id "$query_id" --workload "$workload" --max_block_size 1 -q "SELECT sleepEachRow(1) FROM numbers(300) FORMAT Null" 2>/dev/null & + +while [[ $($CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes WHERE query_id = '$query_id'") != 1 ]]; do + : +done + +$CLICKHOUSE_CLIENT -q "DROP WORKLOAD $workload" +$CLICKHOUSE_CLIENT -q "KILL QUERY WHERE query_id = '$query_id' SYNC FORMAT Null" +wait + +$CLICKHOUSE_CLIENT -q "SELECT 1" diff --git a/tests/queries/0_stateless/05258_ssd_cache_dictionaries_same_path.reference b/tests/queries/0_stateless/05258_ssd_cache_dictionaries_same_path.reference new file mode 100644 index 0000000000000..59252d9172177 --- /dev/null +++ b/tests/queries/0_stateless/05258_ssd_cache_dictionaries_same_path.reference @@ -0,0 +1,7 @@ +0 +0 +0 +0 +42 +symlink kept +65536 diff --git a/tests/queries/0_stateless/05258_ssd_cache_dictionaries_same_path.sh b/tests/queries/0_stateless/05258_ssd_cache_dictionaries_same_path.sh new file mode 100755 index 0000000000000..78bdeb18385ca --- /dev/null +++ b/tests/queries/0_stateless/05258_ssd_cache_dictionaries_same_path.sh @@ -0,0 +1,64 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +# Two ssd_cache dictionaries (and two complex_key_ssd_cache ones) with the same PATH each return their own values. + +path="$CLICKHOUSE_USER_FILES/${CLICKHOUSE_DATABASE}_same_path" +complex_path="$CLICKHOUSE_USER_FILES/${CLICKHOUSE_DATABASE}_same_path_complex" + +$CLICKHOUSE_CLIENT --query=" + CREATE TABLE src_a (id UInt64, value UInt64) ENGINE = TinyLog; + CREATE TABLE src_b (id UInt64, value UInt64) ENGINE = TinyLog; + INSERT INTO src_a SELECT number, 1000 + number FROM numbers(500); + INSERT INTO src_b SELECT number, 9000000 + number FROM numbers(500); + + CREATE DICTIONARY dict_a (id UInt64, value UInt64 DEFAULT 0) PRIMARY KEY id + SOURCE(CLICKHOUSE(TABLE 'src_a')) LIFETIME(MIN 1000 MAX 2000) + LAYOUT(SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 65536 WRITE_BUFFER_SIZE 4096 PATH '$path')); + CREATE DICTIONARY dict_b (id UInt64, value UInt64 DEFAULT 0) PRIMARY KEY id + SOURCE(CLICKHOUSE(TABLE 'src_b')) LIFETIME(MIN 1000 MAX 2000) + LAYOUT(SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 65536 WRITE_BUFFER_SIZE 4096 PATH '$path')); + + CREATE TABLE src_c (key String, value UInt64) ENGINE = TinyLog; + CREATE TABLE src_d (key String, value UInt64) ENGINE = TinyLog; + INSERT INTO src_c SELECT toString(number), 1000 + number FROM numbers(500); + INSERT INTO src_d SELECT toString(number), 9000000 + number FROM numbers(500); + + CREATE DICTIONARY dict_c (key String, value UInt64 DEFAULT 0) PRIMARY KEY key + SOURCE(CLICKHOUSE(TABLE 'src_c')) LIFETIME(MIN 1000 MAX 2000) + LAYOUT(COMPLEX_KEY_SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 65536 WRITE_BUFFER_SIZE 4096 PATH '$complex_path')); + CREATE DICTIONARY dict_d (key String, value UInt64 DEFAULT 0) PRIMARY KEY key + SOURCE(CLICKHOUSE(TABLE 'src_d')) LIFETIME(MIN 1000 MAX 2000) + LAYOUT(COMPLEX_KEY_SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 65536 WRITE_BUFFER_SIZE 4096 PATH '$complex_path')); + + SELECT sum(dictGet('dict_a', 'value', number)) FROM numbers(500) FORMAT Null; + SELECT sum(dictGet('dict_b', 'value', number)) FROM numbers(500) FORMAT Null; + SELECT sum(dictGet('dict_c', 'value', tuple(toString(number)))) FROM numbers(500) FORMAT Null; + SELECT sum(dictGet('dict_d', 'value', tuple(toString(number)))) FROM numbers(500) FORMAT Null; + + -- Only the caches can answer from here on. + TRUNCATE TABLE src_a; TRUNCATE TABLE src_b; TRUNCATE TABLE src_c; TRUNCATE TABLE src_d; + + SELECT countIf(dictGet('dict_a', 'value', number) != 1000 + number) FROM numbers(500); + SELECT countIf(dictGet('dict_b', 'value', number) != 9000000 + number) FROM numbers(500); + SELECT countIf(dictGet('dict_c', 'value', tuple(toString(number))) != 1000 + number) FROM numbers(500); + SELECT countIf(dictGet('dict_d', 'value', tuple(toString(number))) != 9000000 + number) FROM numbers(500); +" + +# A symlink at a dictionary's cache file is kept, and the file it points to becomes the cache. +ln -s "${CLICKHOUSE_DATABASE}_link_target.bin" "$CLICKHOUSE_USER_FILES/${CLICKHOUSE_DATABASE}_link.bin" + +$CLICKHOUSE_CLIENT --query=" + CREATE TABLE src_e (id UInt64, value UInt64) ENGINE = TinyLog; + INSERT INTO src_e VALUES (1, 42); + CREATE DICTIONARY dict_e (id UInt64, value UInt64 DEFAULT 0) PRIMARY KEY id + SOURCE(CLICKHOUSE(TABLE 'src_e')) LIFETIME(MIN 1000 MAX 2000) + LAYOUT(SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 65536 WRITE_BUFFER_SIZE 4096 PATH '$CLICKHOUSE_USER_FILES/${CLICKHOUSE_DATABASE}_link')); + SELECT dictGet('dict_e', 'value', toUInt64(1)); +" + +[ -L "$CLICKHOUSE_USER_FILES/${CLICKHOUSE_DATABASE}_link.bin" ] && echo "symlink kept" +wc -c < "$CLICKHOUSE_USER_FILES/${CLICKHOUSE_DATABASE}_link_target.bin" diff --git a/tests/queries/0_stateless/05291_azure_blob_storage_partition_strategy_key_value.reference b/tests/queries/0_stateless/05291_azure_blob_storage_partition_strategy_key_value.reference new file mode 100644 index 0000000000000..cf17f1f3312f6 --- /dev/null +++ b/tests/queries/0_stateless/05291_azure_blob_storage_partition_strategy_key_value.reference @@ -0,0 +1,11 @@ +t_implicit 1 +t_implicit 1 +t_kv_hive id +t_kv_hive id +t_kv_expr id +1 +2 +t_kv_none 1 +t_kv_none 1 +id UInt64 +t_positional_equals id diff --git a/tests/queries/0_stateless/05291_azure_blob_storage_partition_strategy_key_value.sql b/tests/queries/0_stateless/05291_azure_blob_storage_partition_strategy_key_value.sql new file mode 100644 index 0000000000000..c2b93e5231bdb --- /dev/null +++ b/tests/queries/0_stateless/05291_azure_blob_storage_partition_strategy_key_value.sql @@ -0,0 +1,106 @@ +-- Tags: no-fasttest +-- Tag no-fasttest: needs the AzureBlobStorage engine and Azurite, which the fast test does not have + +-- AzureBlobStorage engine arguments may end with key-value `partition_strategy` and +-- `partition_columns_in_data_file` arguments. The server itself writes `partition_strategy = 'none'` +-- into the table definition when `PARTITION BY` resolves to no strategy, so such a table must attach again. +-- The write arms use Azurite; the other statements use a SAS token in the URL and stay off the network. + +DROP TABLE IF EXISTS t_implicit; +DROP TABLE IF EXISTS t_kv_hive; +DROP TABLE IF EXISTS t_kv_expr; +DROP TABLE IF EXISTS t_kv_hive_write; +DROP TABLE IF EXISTS t_kv_hive_write_pcdf; +DROP TABLE IF EXISTS t_kv_none; +DROP TABLE IF EXISTS t_positional_equals; + +SET file_like_engine_default_partition_strategy = 'wildcard'; + +CREATE TABLE t_implicit (id UInt64, v String) +ENGINE = AzureBlobStorage('http://localhost:11111/devstoreaccount1?sig=X', 'cont', 'data.csv', 'CSV') +PARTITION BY id; +SELECT name, empty(partition_key) FROM system.tables WHERE database = currentDatabase() AND name = 't_implicit'; +DETACH TABLE t_implicit; +ATTACH TABLE t_implicit; +SELECT name, empty(partition_key) FROM system.tables WHERE database = currentDatabase() AND name = 't_implicit'; + +CREATE TABLE t_kv_hive (id UInt64, v String) +ENGINE = AzureBlobStorage('http://localhost:11111/devstoreaccount1?sig=X', 'cont', 'data', 'CSV', partition_strategy = 'hive') +PARTITION BY id; +SELECT name, partition_key FROM system.tables WHERE database = currentDatabase() AND name = 't_kv_hive'; +DETACH TABLE t_kv_hive; +ATTACH TABLE t_kv_hive; +SELECT name, partition_key FROM system.tables WHERE database = currentDatabase() AND name = 't_kv_hive'; + +-- The stored definition keeps the value of a key-value argument, not its expression. +SET use_hive_partitioning = 1; +CREATE TABLE t_kv_expr (id UInt64, v String) +ENGINE = AzureBlobStorage('http://localhost:11111/devstoreaccount1?sig=X', 'cont', 'data', 'CSV', partition_strategy = if(getSetting('use_hive_partitioning'), 'hive', 'none')) +PARTITION BY id; +DETACH TABLE t_kv_expr; +SET use_hive_partitioning = 0; +ATTACH TABLE t_kv_expr; +SELECT name, partition_key FROM system.tables WHERE database = currentDatabase() AND name = 't_kv_expr'; + +-- `num_columns` of the written file: a key-value `hive` keeps the partition column out of it by default. +CREATE TABLE t_kv_hive_write (id UInt64, v String) +ENGINE = AzureBlobStorage('DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;', 'cont05291', concat(currentDatabase(), '_kv_hive'), 'Parquet', partition_strategy = 'hive') +PARTITION BY id; +INSERT INTO t_kv_hive_write VALUES (1, 'a'); +SELECT DISTINCT num_columns FROM azureBlobStorage('DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;', 'cont05291', concat(currentDatabase(), '_kv_hive/**.parquet'), 'ParquetMetadata') SETTINGS use_hive_partitioning = 0; + +CREATE TABLE t_kv_hive_write_pcdf (id UInt64, v String) +ENGINE = AzureBlobStorage('DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;', 'cont05291', concat(currentDatabase(), '_kv_hive_pcdf'), 'Parquet', partition_strategy = 'hive', partition_columns_in_data_file = 1) +PARTITION BY id; +INSERT INTO t_kv_hive_write_pcdf VALUES (1, 'a'); +SELECT DISTINCT num_columns FROM azureBlobStorage('DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;', 'cont05291', concat(currentDatabase(), '_kv_hive_pcdf/**.parquet'), 'ParquetMetadata') SETTINGS use_hive_partitioning = 0; + +SET file_like_engine_default_partition_strategy = 'hive'; + +CREATE TABLE t_kv_none (id UInt64, v String) +ENGINE = AzureBlobStorage('http://localhost:11111/devstoreaccount1?sig=X', 'cont', 'data.csv', 'CSV', partition_strategy = 'none') +PARTITION BY id; +SELECT name, empty(partition_key) FROM system.tables WHERE database = currentDatabase() AND name = 't_kv_none'; +DETACH TABLE t_kv_none; +ATTACH TABLE t_kv_none; +SELECT name, empty(partition_key) FROM system.tables WHERE database = currentDatabase() AND name = 't_kv_none'; + +DESCRIBE TABLE azureBlobStorage('http://localhost:11111/devstoreaccount1?sig=X', 'cont', 'data.csv', 'CSV', 'auto', 'id UInt64', partition_strategy = 'none'); + +-- Cluster queries accept `extra_credentials` together with the key-value arguments, in both argument orders and +-- when parallel replicas turn `azureBlobStorage` into a cluster query. Reading then fails: its token is not sent over plain http. +SET s3_allow_server_credentials_in_user_queries = 1; +SELECT * FROM azureBlobStorageCluster('test_shard_localhost', 'http://localhost:11111/devstoreaccount1?sig=X', 'cont', 'data.csv', 'CSV', 'auto', 'id UInt64', extra_credentials(client_id = 'x', tenant_id = 'y'), partition_strategy = 'none'); -- { serverError STD_EXCEPTION } +SELECT * FROM azureBlobStorageCluster('test_shard_localhost', 'http://localhost:11111/devstoreaccount1?sig=X', 'cont', 'data.csv', 'CSV', 'auto', 'id UInt64', partition_strategy = 'none', extra_credentials(client_id = 'x', tenant_id = 'y')); -- { serverError STD_EXCEPTION } +SELECT * FROM azureBlobStorage('http://localhost:11111/devstoreaccount1?sig=X', 'cont', 'data.csv', 'CSV', 'auto', 'id UInt64', extra_credentials(client_id = 'x', tenant_id = 'y'), partition_strategy = 'none') +SETTINGS enable_parallel_replicas = 1, automatic_parallel_replicas_mode = 0, max_parallel_replicas = 3, parallel_replicas_for_cluster_engines = 1, + cluster_for_parallel_replicas = 'test_cluster_one_shard_three_replicas_localhost'; -- { serverError STD_EXCEPTION } + +-- Invalid key-value arguments. +CREATE TABLE t_bad (id UInt64, v String) +ENGINE = AzureBlobStorage('http://localhost:11111/devstoreaccount1?sig=X', 'cont', 'data.csv', 'CSV', format = 'CSV'); -- { serverError BAD_ARGUMENTS } +CREATE TABLE t_bad (id UInt64, v String) +ENGINE = AzureBlobStorage('http://localhost:11111/devstoreaccount1?sig=X', 'cont', 'data.csv', 'CSV', partition_strategy = 'unknown') +PARTITION BY id; -- { serverError BAD_ARGUMENTS } +CREATE TABLE t_bad (id UInt64, v String) +ENGINE = AzureBlobStorage('http://localhost:11111/devstoreaccount1?sig=X', 'cont', 'data', 'CSV', 'auto', 'hive', partition_strategy = 'hive') +PARTITION BY id; -- { serverError BAD_ARGUMENTS } +CREATE TABLE t_bad (id UInt64, v String) +ENGINE = AzureBlobStorage('http://localhost:11111/devstoreaccount1?sig=X', 'cont', 'data.csv', 'CSV', partition_strategy = 'none', partition_columns_in_data_file = 0) +PARTITION BY id; -- { serverError BAD_ARGUMENTS } +CREATE TABLE t_bad (id UInt64, v String) +ENGINE = AzureBlobStorage('http://localhost:11111/devstoreaccount1/cont/data.csv?sig=X', 'X', partition_strategy = 'none'); -- { serverError BAD_ARGUMENTS } + +-- A constant `1 = 1` stays a positional argument (`partition_columns_in_data_file`). +CREATE TABLE t_positional_equals (id UInt64, v String) +ENGINE = AzureBlobStorage('http://localhost:11111/devstoreaccount1?sig=X', 'cont', 'data', 'CSV', 'auto', 'hive', 1 = 1) +PARTITION BY id; +SELECT name, partition_key FROM system.tables WHERE database = currentDatabase() AND name = 't_positional_equals'; + +DROP TABLE t_implicit; +DROP TABLE t_kv_hive; +DROP TABLE t_kv_expr; +DROP TABLE t_kv_hive_write; +DROP TABLE t_kv_hive_write_pcdf; +DROP TABLE t_kv_none; +DROP TABLE t_positional_equals; diff --git a/tests/queries/0_stateless/05291_system_mutations_hide_secrets.reference b/tests/queries/0_stateless/05291_system_mutations_hide_secrets.reference new file mode 100644 index 0000000000000..c9a233584ace8 --- /dev/null +++ b/tests/queries/0_stateless/05291_system_mutations_hide_secrets.reference @@ -0,0 +1,5 @@ +t_05291 (DELETE WHERE id IN (SELECT CAST(c1, \'UInt64\') FROM s3(\'http://127.0.0.1:1/nope.csv\', \'AKIAFAKEKEYID\', \'[HIDDEN]\', \'CSV\', \'c1 String\'))) +t_05291 (UPDATE v = 0 WHERE id IN (SELECT CAST(c1, \'UInt64\') FROM s3(\'http://127.0.0.1:1/nope.csv\', \'AKIAFAKEKEYID\', \'[HIDDEN]\', \'CSV\', \'c1 String\'))) +t_05291 (UPDATE _row_exists = 0 WHERE id IN (SELECT CAST(c1, \'UInt64\') FROM s3(\'http://127.0.0.1:1/nope.csv\', \'AKIAFAKEKEYID\', \'[HIDDEN]\', \'CSV\', \'c1 String\'))) +t_05291_replicated (DELETE WHERE id IN (SELECT CAST(c1, \'UInt64\') FROM s3(\'http://127.0.0.1:1/nope.csv\', \'AKIAFAKEKEYID\', \'[HIDDEN]\', \'CSV\', \'c1 String\'))) +0 diff --git a/tests/queries/0_stateless/05291_system_mutations_hide_secrets.sql b/tests/queries/0_stateless/05291_system_mutations_hide_secrets.sql new file mode 100644 index 0000000000000..58aa5021b160c --- /dev/null +++ b/tests/queries/0_stateless/05291_system_mutations_hide_secrets.sql @@ -0,0 +1,47 @@ +-- Tags: no-fasttest +-- no-fasttest: the `s3` table function is not in the fast test build. + +-- A mutation entry keeps the real text of its commands, because the mutation is executed from it (on every +-- replica, for replicated tables). `system.mutations.command` must still hide the credentials of the table +-- functions in the mutation subqueries, the way the query log does. + +SET mutations_sync = 0; +SET lightweight_deletes_sync = 0; +SET allow_nondeterministic_mutations = 1; + +DROP TABLE IF EXISTS t_05291; +DROP TABLE IF EXISTS t_05291_replicated; + +CREATE TABLE t_05291 (id UInt64, v UInt64) ENGINE = MergeTree ORDER BY id; +CREATE TABLE t_05291_replicated (id UInt64, v UInt64) +ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/t_05291_replicated', 'r1') ORDER BY id; + +INSERT INTO t_05291 SELECT number, number FROM numbers(10); +INSERT INTO t_05291_replicated SELECT number, number FROM numbers(10); + +-- Keep the mutations pending, so that they stay in `system.mutations`. +SYSTEM STOP MERGES t_05291; +SYSTEM STOP MERGES t_05291_replicated; + +ALTER TABLE t_05291 DELETE WHERE id IN + (SELECT c1::UInt64 FROM s3('http://127.0.0.1:1/nope.csv', 'AKIAFAKEKEYID', 'SECRET05291DELETE', 'CSV', 'c1 String')); +ALTER TABLE t_05291 UPDATE v = 0 WHERE id IN + (SELECT c1::UInt64 FROM s3('http://127.0.0.1:1/nope.csv', 'AKIAFAKEKEYID', 'SECRET05291UPDATE', 'CSV', 'c1 String')); +DELETE FROM t_05291 WHERE id IN + (SELECT c1::UInt64 FROM s3('http://127.0.0.1:1/nope.csv', 'AKIAFAKEKEYID', 'SECRET05291LIGHTWEIGHT', 'CSV', 'c1 String')); +ALTER TABLE t_05291_replicated DELETE WHERE id IN + (SELECT c1::UInt64 FROM s3('http://127.0.0.1:1/nope.csv', 'AKIAFAKEKEYID', 'SECRET05291REPLICATED', 'CSV', 'c1 String')); + +-- The replica loads the mutations from ZooKeeper asynchronously; `PULL` fetches them synchronously. +SYSTEM SYNC REPLICA t_05291_replicated PULL; + +SELECT table, command FROM system.mutations WHERE database = currentDatabase() ORDER BY table, mutation_id; + +-- A secret cannot be probed with a filter on the column either. +SELECT count() FROM system.mutations WHERE database = currentDatabase() AND position(command, 'SECRET05291') > 0; + +-- `KILL MUTATION` parses the command that it reads from `system.mutations` to check the access rights. +KILL MUTATION WHERE database = currentDatabase() AND table = 't_05291' FORMAT Null; + +DROP TABLE t_05291; +DROP TABLE t_05291_replicated; diff --git a/tests/queries/0_stateless/05316_ai_functions_last_in_prewhere.reference b/tests/queries/0_stateless/05316_ai_functions_last_in_prewhere.reference new file mode 100644 index 0000000000000..ef26043724e37 --- /dev/null +++ b/tests/queries/0_stateless/05316_ai_functions_last_in_prewhere.reference @@ -0,0 +1,15 @@ +aiGenerate: in PREWHERE, last +aiClassify: in PREWHERE, last +aiExtract: in PREWHERE, last +aiTranslate: in PREWHERE, last +aiFilter: in PREWHERE, last +aiRedact: in PREWHERE, last +aiEmbed: in PREWHERE, last +aiSimilarity: in PREWHERE, last +AI inside a lambda: in PREWHERE, last +AI and a cheap condition on the same column: in PREWHERE, last +AI inside OR: in PREWHERE, last +AI condition alone: in PREWHERE, last +AI behind a condition left in WHERE: not in PREWHERE +Reordering disabled, AI written first: in PREWHERE, not last +Reordering disabled, AI written last: in PREWHERE, last diff --git a/tests/queries/0_stateless/05316_ai_functions_last_in_prewhere.sh b/tests/queries/0_stateless/05316_ai_functions_last_in_prewhere.sh new file mode 100755 index 0000000000000..e9c830efb2d0d --- /dev/null +++ b/tests/queries/0_stateless/05316_ai_functions_last_in_prewhere.sh @@ -0,0 +1,92 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# An AI function issues one request per row. PREWHERE evaluates its conditions as read +# steps in order, each on the rows that passed the previous ones, so a condition with an +# AI function is moved to PREWHERE after all other conditions. +# +# The order is taken from the PREWHERE filter column of `EXPLAIN json = 1`, which lists the +# conditions as `and(...)` arguments in their final order. The AI condition is last if the +# AI function appears after every cheap condition (`flag = 1`, `text LIKE`). +# +# Only `EXPLAIN` is used, so no HTTP call is made. + +CHAT_CREDS="ai_creds_${CLICKHOUSE_DATABASE}" +VEC_CREDS="ai_vec_creds_${CLICKHOUSE_DATABASE}" + +$CLICKHOUSE_CLIENT -q " + CREATE TABLE tab (flag UInt8, text String, other String) ENGINE = MergeTree ORDER BY tuple(); + INSERT INTO tab SELECT number % 2, 'row ' || toString(number), 'x' FROM numbers(16); + CREATE NAMED COLLECTION ${CHAT_CREDS} AS provider = 'openai', endpoint = 'http://localhost:1/v1/chat/completions', model = 'test-model', api_key = 'test-key'; + CREATE NAMED COLLECTION ${VEC_CREDS} AS provider = 'openai', endpoint = 'http://localhost:1/v1/embeddings', api_key = 'test-key'; +" + +CHAT="map('credentials', '${CHAT_CREDS}')" +VEC="map('credentials', '${VEC_CREDS}')" + +# Offset of the last match of a pattern in a string, empty if there is none. +function last_offset() +{ + grep -boE "$2" <<< "$1" | tail -1 | cut -d: -f1 +} + +# Prints whether the AI condition is in PREWHERE and whether it comes after the cheap ones. +# The settings that decide whether and how conditions are moved are pinned rather than left +# to test randomization. +function ai_position() +{ + local label=$1 where=$2 allow_reorder=${3:-1} + local column + column=$($CLICKHOUSE_CLIENT -q " + EXPLAIN json = 1, actions = 1 SELECT other FROM tab WHERE ${where} + SETTINGS enable_analyzer = 1, optimize_move_to_prewhere = 1, query_plan_enable_optimizations = 1, query_plan_optimize_prewhere = 1, + move_all_conditions_to_prewhere = 1, allow_reorder_prewhere_conditions = ${allow_reorder}" \ + | grep -oE '"Prewhere filter column": "[^"]*"') + + local ai cheap + ai=$(grep -boE 'ai[A-Z]' <<< "$column" | head -1 | cut -d: -f1) + if [[ -z $ai ]]; then echo "${label}: not in PREWHERE"; return; fi + for pattern in 'equals\(__table1\.flag' 'like\(__table1\.text'; do + cheap=$(last_offset "$column" "$pattern") + if [[ -n $cheap ]] && (( cheap > ai )); then echo "${label}: in PREWHERE, not last"; return; fi + done + echo "${label}: in PREWHERE, last" +} + +ai_position 'aiGenerate' "aiGenerate(text, ${CHAT}) != '' AND flag = 1" +ai_position 'aiClassify' "aiClassify(text, ['a', 'b'], ${CHAT}) = 'a' AND flag = 1" +ai_position 'aiExtract' "aiExtract(text, 'the topic', ${CHAT}) != '' AND flag = 1" +ai_position 'aiTranslate' "aiTranslate(text, 'French', ${CHAT}) != '' AND flag = 1" +ai_position 'aiFilter' "aiFilter(text, 'matches', ${CHAT}) AND flag = 1" +ai_position 'aiRedact' "aiRedact(text, ['email'], ${CHAT}) != '' AND flag = 1" +ai_position 'aiEmbed' "length(aiEmbed(text, 'test-model', ${VEC})) != 0 AND flag = 1" +ai_position 'aiSimilarity' "aiSimilarity(text, 'reference', 'test-model', ${VEC}) != 0 AND flag = 1" + +# A lambda body is not among the condition's children, so the capture reports the trait. +ai_position 'AI inside a lambda' "arrayExists(x -> aiFilter(x, 'matches', ${CHAT}), [text]) AND flag = 1" + +# Conditions on the same columns are grouped and moved together. The AI condition is kept +# out of the group, otherwise it would move ahead of `flag = 1` with `text LIKE`. +ai_position 'AI and a cheap condition on the same column' "aiFilter(text, 'matches', ${CHAT}) AND text LIKE '%1%' AND flag = 1" + +# An `OR` is not split, so the whole disjunction is expensive and moved last. +ai_position 'AI inside OR' "(flag = 1 OR aiFilter(text, 'matches', ${CHAT})) AND text LIKE '%1%'" + +# With no other condition the AI condition is still moved. +ai_position 'AI condition alone' "aiFilter(text, 'matches', ${CHAT})" + +# A condition using every queried column is not moved, so the AI condition stays in WHERE behind it. +ai_position 'AI behind a condition left in WHERE' "length(text) = flag + length(other) AND aiFilter(text, 'matches', ${CHAT})" + +# Without reordering the written order is kept, the AI condition included. +ai_position 'Reordering disabled, AI written first' "aiFilter(text, 'matches', ${CHAT}) AND flag = 1" 0 +ai_position 'Reordering disabled, AI written last' "flag = 1 AND aiFilter(text, 'matches', ${CHAT})" 0 + +$CLICKHOUSE_CLIENT -q " + DROP NAMED COLLECTION ${CHAT_CREDS}; + DROP NAMED COLLECTION ${VEC_CREDS}; + DROP TABLE tab; +" diff --git a/tests/queries/0_stateless/05317_orc_corrupt_stripe_footer_stream_column.reference b/tests/queries/0_stateless/05317_orc_corrupt_stripe_footer_stream_column.reference new file mode 100644 index 0000000000000..88b9566a6b12d --- /dev/null +++ b/tests/queries/0_stateless/05317_orc_corrupt_stripe_footer_stream_column.reference @@ -0,0 +1,3 @@ +column 20 is out of range, the file has 5 columns +column 20 is out of range, the file has 5 columns +100 4950 diff --git a/tests/queries/0_stateless/05317_orc_corrupt_stripe_footer_stream_column.sh b/tests/queries/0_stateless/05317_orc_corrupt_stripe_footer_stream_column.sh new file mode 100755 index 0000000000000..27b41467be726 --- /dev/null +++ b/tests/queries/0_stateless/05317_orc_corrupt_stripe_footer_stream_column.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: Depends on S3 + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A 569-byte ORC file (id Int64, name String, v Float64, part Int32) with one row, whose stripe footer declares a +# row index stream for column 20, but the file has 5 columns. Two readers indexed the selected-column mask with +# that id and aborted the server; both must report a malformed file instead: +# - remote reads prebuffer stripes (`ORC::Reader::preBuffer`), so the file is read through s3; +# - predicate pushdown loads the row indexes (`ORC::RowReader::loadStripeIndex`), also for local files. The +# predicate must keep the stripe (`id >= 0`; the row has id 1), or the stripe statistics skip it first. +ORC_B64=$(tr -d '\n' <<'B64' +T1JDEQAACgYSBAgBUAArAAAKEwoDAAAAEgwIARIGCAIQAhgCUAAzAAAKFwoFAAAAAAASDggBIggKAWESAWEYAlAAUwAACicKAgAA +EiEIARobCQAAAAAAAPg/EQAAAAAAAPg/GQAAAAAAAPg/UAArAAAKEwoDAAAAEgwIARIGCAAQABgAUAAHAABCAIADAABhBwAAQACA +EQAAAAAAAAAA+D8HAABAAACuAAAotS/9IGR1AgCEAwoGCAYQABgLCgYIBhABGBgCGBwDGCwUARABGAYCGAQKBggCEAIDARAEGAYS +AggAEgIIAgASAggCCAB7/hAam+nkirydJ0AbQIZdZQm6AAAotS/9IF+lAgBEBApdCgQIAVAACg4IARIGCAIQAhgCUABYBgoQCAEi +CAoBYRIBYQoKIwgBGhsJAPg/EQD4PxkA+D9QAFgLABAAGABQAFgGBQBKyAIaAUJwd1jtfAJ4AQAotS/9IOydBQBCyiYskLc5//// +///w6acgxiIELbemIDUZs4IjVzcRUBwT/pE0KZF9o02pZZNSphSQJ3okw0BbBghYiEDLlRuOvFiuU8aKKePElHHiyPYGVW7/bcVn +e6TQcm1Ni83htzOUa/ujDT2c/5Sg/4wO/hPqp6rnKrCqiyUKQ5//PDC42JMILQmsez1JVU0+1/o9wp3ZDoEFo2mF6HyRLSE6DwgA +SsgCGgFCcHdas32g06DT47p5Bgi/ARAFGICAECICAAwoYDAJgvQDA09SQxg= +B64 +) +FILE="05317_${CLICKHOUSE_DATABASE}.orc" + +$CLICKHOUSE_CLIENT --s3_truncate_on_insert 1 \ + --query "INSERT INTO FUNCTION s3(s3_conn, filename='$FILE', format='RawBLOB') SELECT base64Decode('$ORC_B64')" + +# Stripes are prebuffered only with `remote_filesystem_read_prefetch`, which CI randomizes. +$CLICKHOUSE_CLIENT --remote_filesystem_read_prefetch 1 \ + --query "SELECT * FROM s3(s3_conn, filename='$FILE', format='ORC') FORMAT Null" 2>&1 \ + | grep -o -m1 'column 20 is out of range, the file has 5 columns' + +# Predicate pushdown (on by default) on a local file, which is never prebuffered. +LOCAL_FILE="05317_${CLICKHOUSE_DATABASE}.orc" +$CLICKHOUSE_CLIENT --engine_file_truncate_on_insert 1 \ + --query "INSERT INTO FUNCTION file('$LOCAL_FILE', 'RawBLOB') SELECT base64Decode('$ORC_B64')" +$CLICKHOUSE_CLIENT --input_format_orc_filter_push_down 1 \ + --query "SELECT * FROM file('$LOCAL_FILE', 'ORC') WHERE id >= 0 FORMAT Null" 2>&1 \ + | grep -o -m1 'column 20 is out of range, the file has 5 columns' +rm -f "${USER_FILES_PATH:?}/$LOCAL_FILE" + +# The server is still alive, and a valid ORC file is still read through the same path. +VALID="05317_valid_${CLICKHOUSE_DATABASE}.orc" +$CLICKHOUSE_CLIENT --s3_truncate_on_insert 1 \ + --query "INSERT INTO FUNCTION s3(s3_conn, filename='$VALID', format='ORC') SELECT number AS id, toString(number) AS s FROM numbers(100)" +$CLICKHOUSE_CLIENT --query "SELECT count(), sum(id) FROM s3(s3_conn, filename='$VALID', format='ORC')" diff --git a/tests/queries/0_stateless/05325_read_in_order_final_virtual_row.reference b/tests/queries/0_stateless/05325_read_in_order_final_virtual_row.reference new file mode 100644 index 0000000000000..61aa4150afa3d --- /dev/null +++ b/tests/queries/0_stateless/05325_read_in_order_final_virtual_row.reference @@ -0,0 +1,144 @@ +--- ReplacingMergeTree: plan +-- INNER JOIN +Prefix sort description: k ASC +Read type: InOrder +Virtual row conversions +Read type: Default +--- ReplacingMergeTree: results +0 p4 +3 p1 +6 p1 +9 p1 +12 p1 +15 p1 +18 p1 +45 p1 +48 p1 +51 p2 +54 p2 +57 p2 +60 p4 +63 p2 +48 p1 +49 p1 +50 p4 +--- CollapsingMergeTree +Prefix sort description: k ASC +Read type: InOrder +Virtual row conversions +Read type: Default +0 b +3 a +9 a +12 b +15 a +21 a +--- VersionedCollapsingMergeTree +Prefix sort description: k ASC +Read type: InOrder +Virtual row conversions +Read type: Default +0 b +3 a +9 a +12 b +15 a +21 a +--- SummingMergeTree +Prefix sort description: k ASC +Read type: InOrder +Virtual row conversions +Read type: Default +45 11 +48 11 +51 111 +54 111 +57 111 +60 111 +--- AggregatingMergeTree +Prefix sort description: k ASC +Read type: InOrder +Virtual row conversions +Read type: Default +45 10 +48 10 +51 10 +54 10 +57 10 +60 10 +--- CoalescingMergeTree +Prefix sort description: k ASC +Read type: InOrder +Virtual row conversions +Read type: Default +45 1 x +48 1 x +51 2 x +54 2 x +57 2 x +60 2 x +--- layers +Prefix sort description: k ASC +Read type: InOrder +Virtual row conversions +Read type: Default +-- several FINAL merges, each followed by a virtual row +1 1 1 +0 p4 +3 p1 +6 p1 +9 p1 +12 p1 +15 p1 +18 p1 +735 p2 +738 p2 +741 p2 +744 p2 +747 p2 +750 p4 +1248 p3 +1249 p3 +1250 p4 +1251 p3 +1252 p3 +1253 p3 +1254 p3 +1255 p3 +738 p2 +739 p2 +740 p4 +741 p2 +742 p2 +--- ORDER BY a prefix of the primary key, which is a prefix of the sorting key +-- ORDER BY a +Prefix sort description: a ASC +Read type: InOrder +Virtual row conversions +Read type: Default +0 15 2 3 -1 1 +3 15 2 0 -1 1 +-- ORDER BY a, b +Prefix sort description: a ASC, b ASC +Read type: InOrder +Virtual row conversions +Read type: Default +0 0 3 0 3 -1 1 +0 1 3 0 0 -1 1 +0 2 3 1 0 -1 1 +-- ORDER BY a, b, c: beyond the primary key, no virtual row +Read type: Default +Read type: Default +0 0 -1 p3 +0 0 0 p3 +0 0 1 p3 +0 1 -1 p1 +0 1 0 p1 +0 1 1 p1 +0 2 -1 p1 +0 2 0 p2 +0 2 1 p1 +0 3 -1 p1 +--- GraphiteMergeTree: no virtual row +Prefix sort description: Path ASC, Time ASC +Read type: InOrder diff --git a/tests/queries/0_stateless/05325_read_in_order_final_virtual_row.sql b/tests/queries/0_stateless/05325_read_in_order_final_virtual_row.sql new file mode 100644 index 0000000000000..41b6bc3e8f130 --- /dev/null +++ b/tests/queries/0_stateless/05325_read_in_order_final_virtual_row.sql @@ -0,0 +1,162 @@ +-- Tags: no-parallel-replicas + +-- Virtual rows with FINAL: each FINAL merge announces the key its parts start from, so the read in +-- order through an INNER JOIN, which requires a virtual row, applies to FINAL as well, and the merge +-- after the join defers the FINAL merges the limit does not reach. + +SET optimize_read_in_order = 1, read_in_order_use_virtual_row = 1, read_in_order_use_virtual_row_per_block = 0; +SET query_plan_read_in_order_through_join = 1, max_bytes_ratio_before_external_join = 0, max_bytes_before_external_join = 0; +SET query_plan_optimize_join_order_limit = 1, query_plan_optimize_join_order_randomize = 0, query_plan_join_swap_table = 0; +SET join_algorithm = 'hash', enable_join_runtime_filters = 0, enable_parallel_replicas = 0; +SET max_threads = 3, max_final_threads = 3, optimize_on_insert = 0; +SET do_not_merge_across_partitions_select_final = 0; + +DROP TABLE IF EXISTS r; +CREATE TABLE r (k UInt64) ENGINE = MergeTree ORDER BY k; +INSERT INTO r SELECT number * 3 FROM numbers(700); + +SELECT '--- ReplacingMergeTree: plan'; +DROP TABLE IF EXISTS t_rep; +CREATE TABLE t_rep (k UInt64, ver UInt8, v String) ENGINE = ReplacingMergeTree(ver) ORDER BY k; +INSERT INTO t_rep SELECT number, 1, 'p1' FROM numbers(100); +INSERT INTO t_rep SELECT 50 + number, 2, 'p2' FROM numbers(100); +INSERT INTO t_rep SELECT 100 + number, 3, 'p3' FROM numbers(100); +INSERT INTO t_rep SELECT number * 10, 4, 'p4' FROM numbers(20); + +SELECT '-- INNER JOIN'; +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 + SELECT l.k, l.v FROM t_rep AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k LIMIT 7) WHERE e != ''; + +SELECT '--- ReplacingMergeTree: results'; +SELECT l.k, l.v FROM t_rep AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k LIMIT 7; +SELECT l.k, l.v FROM t_rep AS l FINAL INNER JOIN r ON l.k = r.k WHERE l.k >= 45 ORDER BY l.k LIMIT 7; +SELECT k, v FROM t_rep FINAL ORDER BY k LIMIT 3 OFFSET 48; + +SELECT '--- CollapsingMergeTree'; +DROP TABLE IF EXISTS t_col; +CREATE TABLE t_col (k UInt64, sign Int8, v String) ENGINE = CollapsingMergeTree(sign) ORDER BY k; +INSERT INTO t_col SELECT number, 1, 'a' FROM numbers(100); +INSERT INTO t_col SELECT number * 2, -1, 'a' FROM numbers(50); +INSERT INTO t_col SELECT number * 4, 1, 'b' FROM numbers(25); +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 + SELECT l.k, l.v FROM t_col AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k LIMIT 6) WHERE e != ''; +SELECT l.k, l.v FROM t_col AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k LIMIT 6; + +SELECT '--- VersionedCollapsingMergeTree'; +DROP TABLE IF EXISTS t_vcol; +CREATE TABLE t_vcol (k UInt64, sign Int8, ver UInt8, v String) ENGINE = VersionedCollapsingMergeTree(sign, ver) ORDER BY k; +INSERT INTO t_vcol SELECT number, 1, 1, 'a' FROM numbers(100); +INSERT INTO t_vcol SELECT number * 2, -1, 1, 'a' FROM numbers(50); +INSERT INTO t_vcol SELECT number * 4, 1, 2, 'b' FROM numbers(25); +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 + SELECT l.k, l.v FROM t_vcol AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k LIMIT 6) WHERE e != ''; +SELECT l.k, l.v FROM t_vcol AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k LIMIT 6; + +SELECT '--- SummingMergeTree'; +DROP TABLE IF EXISTS t_sum; +CREATE TABLE t_sum (k UInt64, s UInt64) ENGINE = SummingMergeTree ORDER BY k; +INSERT INTO t_sum SELECT number, 1 FROM numbers(100); +INSERT INTO t_sum SELECT number, 10 FROM numbers(100); +INSERT INTO t_sum SELECT 50 + number, 100 FROM numbers(100); +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 + SELECT l.k, l.s FROM t_sum AS l FINAL INNER JOIN r ON l.k = r.k WHERE l.k >= 45 ORDER BY l.k LIMIT 6) WHERE e != ''; +SELECT l.k, l.s FROM t_sum AS l FINAL INNER JOIN r ON l.k = r.k WHERE l.k >= 45 ORDER BY l.k LIMIT 6; + +SELECT '--- AggregatingMergeTree'; +DROP TABLE IF EXISTS t_agg; +CREATE TABLE t_agg (k UInt64, m SimpleAggregateFunction(max, UInt64), c AggregateFunction(count)) ENGINE = AggregatingMergeTree ORDER BY k; +INSERT INTO t_agg SELECT number, 1, countState() FROM numbers(100) GROUP BY number; +INSERT INTO t_agg SELECT number, 10, countState() FROM numbers(100) GROUP BY number; +INSERT INTO t_agg SELECT 50 + number, 5, countState() FROM numbers(100) GROUP BY number; +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 + SELECT l.k, l.m FROM t_agg AS l FINAL INNER JOIN r ON l.k = r.k WHERE l.k >= 45 ORDER BY l.k LIMIT 6) WHERE e != ''; +SELECT l.k, l.m FROM t_agg AS l FINAL INNER JOIN r ON l.k = r.k WHERE l.k >= 45 ORDER BY l.k LIMIT 6; + +SELECT '--- CoalescingMergeTree'; +DROP TABLE IF EXISTS t_coal; +CREATE TABLE t_coal (k UInt64, a Nullable(UInt64), b Nullable(String)) ENGINE = CoalescingMergeTree ORDER BY k; +INSERT INTO t_coal SELECT number, 1, NULL FROM numbers(100); +INSERT INTO t_coal SELECT number, NULL, 'x' FROM numbers(100); +INSERT INTO t_coal SELECT 50 + number, 2, NULL FROM numbers(100); +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 + SELECT l.k, l.a, l.b FROM t_coal AS l FINAL INNER JOIN r ON l.k = r.k WHERE l.k >= 45 ORDER BY l.k LIMIT 6) WHERE e != ''; +SELECT l.k, l.a, l.b FROM t_coal AS l FINAL INNER JOIN r ON l.k = r.k WHERE l.k >= 45 ORDER BY l.k LIMIT 6; + +-- Parts of several granules are split into PK-range layers, each merged by its own FINAL merge +-- and trimmed to its range. Each layer announces the key its parts start from, so the merge after +-- the join defers the layers the limit does not reach. +SELECT '--- layers'; +DROP TABLE IF EXISTS t_layers; +CREATE TABLE t_layers (k UInt64, ver UInt8, v String) ENGINE = ReplacingMergeTree(ver) ORDER BY k SETTINGS index_granularity = 8; +INSERT INTO t_layers SELECT number, 1, 'p1' FROM numbers(1000); +INSERT INTO t_layers SELECT 500 + number, 2, 'p2' FROM numbers(1000); +INSERT INTO t_layers SELECT 1000 + number, 3, 'p3' FROM numbers(1000); +INSERT INTO t_layers SELECT number * 10, 4, 'p4' FROM numbers(200); +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 + SELECT l.k, l.v FROM t_layers AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k LIMIT 7) WHERE e != ''; +SELECT '-- several FINAL merges, each followed by a virtual row'; +SELECT countIf(explain LIKE '%filter values in%') > 1, countIf(explain LIKE '%ReplacingSorted%') > 0, countIf(explain LIKE '%VirtualRowTransform%') > 0 +FROM (EXPLAIN PIPELINE SELECT l.k, l.v FROM t_layers AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k LIMIT 7); +-- the limit is reached within the first layer: far fewer rows read than the table has +SELECT l.k, l.v FROM t_layers AS l FINAL INNER JOIN r ON l.k = r.k ORDER BY l.k LIMIT 7 SETTINGS max_rows_to_read = 3200; +-- limits crossing a layer border (740 and 1252 for these parts with max_final_threads = 3) +SELECT l.k, l.v FROM t_layers AS l FINAL INNER JOIN r ON l.k = r.k WHERE l.k >= 735 ORDER BY l.k LIMIT 6; +SELECT l.k, l.v FROM t_layers AS l FINAL LEFT JOIN r ON l.k = r.k WHERE l.k >= 1248 ORDER BY l.k LIMIT 8; +SELECT k, v FROM t_layers FINAL ORDER BY k LIMIT 5 OFFSET 738; + +-- The query orders by a prefix of the primary key, itself a prefix of the sorting key. The virtual +-- row covers the primary key columns the sort reads; the FINAL merge compares the whole sorting key +-- and a sort beyond the primary key gets no virtual row, so the INNER JOIN keeps no order for it. +SELECT '--- ORDER BY a prefix of the primary key, which is a prefix of the sorting key'; +DROP TABLE IF EXISTS t_prefix; +CREATE TABLE t_prefix (a UInt64, b UInt64, c Int32, ver UInt8, v String) + ENGINE = ReplacingMergeTree(ver) ORDER BY (a, b, c) PRIMARY KEY (a, b) SETTINGS index_granularity = 4; +INSERT INTO t_prefix SELECT number % 10, intDiv(number, 10) % 5, toInt32(intDiv(number, 50)) - 1, 1, 'p1' FROM numbers(150); +INSERT INTO t_prefix SELECT number % 10, intDiv(number, 10) % 5, toInt32(intDiv(number, 50)) - 1, 2, 'p2' FROM numbers(150) WHERE number % 7 = 0; +INSERT INTO t_prefix SELECT number % 10, intDiv(number, 10) % 5, toInt32(intDiv(number, 50)) - 1, 3, 'p3' FROM numbers(150) WHERE number % 50 = 0; +SELECT '-- ORDER BY a'; +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 + SELECT l.a FROM t_prefix AS l FINAL INNER JOIN r ON l.a = r.k ORDER BY l.a LIMIT 30) WHERE e != ''; +SELECT a, count(), countIf(v = 'p2'), countIf(v = 'p3'), min(c), max(c) +FROM (SELECT l.a, l.b, l.c, l.v FROM t_prefix AS l FINAL INNER JOIN r ON l.a = r.k ORDER BY l.a LIMIT 30) +GROUP BY a ORDER BY a; +SELECT '-- ORDER BY a, b'; +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 + SELECT l.a, l.b FROM t_prefix AS l FINAL INNER JOIN r ON l.a = r.k ORDER BY l.a, l.b LIMIT 9) WHERE e != ''; +SELECT a, b, count(), countIf(v = 'p2'), countIf(v = 'p3'), min(c), max(c) +FROM (SELECT l.a, l.b, l.c, l.v FROM t_prefix AS l FINAL INNER JOIN r ON l.a = r.k ORDER BY l.a, l.b LIMIT 9) +GROUP BY a, b ORDER BY a, b; +SELECT '-- ORDER BY a, b, c: beyond the primary key, no virtual row'; +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 + SELECT l.a, l.b, l.c FROM t_prefix AS l FINAL INNER JOIN r ON l.a = r.k ORDER BY l.a, l.b, l.c LIMIT 10) WHERE e != ''; +SELECT l.a, l.b, l.c, l.v FROM t_prefix AS l FINAL INNER JOIN r ON l.a = r.k ORDER BY l.a, l.b, l.c LIMIT 10; + +SELECT '--- GraphiteMergeTree: no virtual row'; +DROP TABLE IF EXISTS t_graphite; +CREATE TABLE t_graphite (Path String, Time DateTime, Value Float64, Version UInt32, k UInt64 ALIAS Version) + ENGINE = GraphiteMergeTree('graphite_rollup') ORDER BY (Path, Time); +INSERT INTO t_graphite SELECT 'one_min.x', toDateTime('2020-01-01 00:00:00') + number, number, 1 FROM numbers(10); +SELECT extract(explain, '(?:Prefix sort description|Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 + SELECT Path, Time FROM t_graphite FINAL ORDER BY Path, Time LIMIT 3) WHERE e != ''; + +DROP TABLE t_rep; +DROP TABLE t_col; +DROP TABLE t_vcol; +DROP TABLE t_sum; +DROP TABLE t_agg; +DROP TABLE t_coal; +DROP TABLE t_layers; +DROP TABLE t_prefix; +DROP TABLE t_graphite; +DROP TABLE r; diff --git a/tests/queries/0_stateless/05339_avro_schema_inference_deep_named_type_chain.reference b/tests/queries/0_stateless/05339_avro_schema_inference_deep_named_type_chain.reference new file mode 100644 index 0000000000000..3af6e7be9ccaa --- /dev/null +++ b/tests/queries/0_stateless/05339_avro_schema_inference_deep_named_type_chain.reference @@ -0,0 +1,2 @@ +42 +too deep diff --git a/tests/queries/0_stateless/05339_avro_schema_inference_deep_named_type_chain.sh b/tests/queries/0_stateless/05339_avro_schema_inference_deep_named_type_chain.sh new file mode 100755 index 0000000000000..22ac566fbaaaf --- /dev/null +++ b/tests/queries/0_stateless/05339_avro_schema_inference_deep_named_type_chain.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: the Avro format is not available in the fast test build. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +DIR="$CLICKHOUSE_TMP/${CLICKHOUSE_TEST_UNIQUE_NAME}" +rm -rf "$DIR" +mkdir -p "$DIR" + +# A chain of named records A_k { x: A_(k-1) }, each defined inside its own array field f_k, so the schema +# JSON nests only a few levels deep. Schema inference follows the references: f_k is 2k + 1 levels deep. +function gen() +{ + python3 -c " +import json, sys +n = int(sys.argv[2]) +def zz(v): + v = (v << 1) ^ (v >> 63) + v &= (1 << 64) - 1 + out = bytearray() + while True: + b = v & 0x7f + v >>= 7 + out.append(b | 0x80 if v else b) + if not v: + break + return bytes(out) +def ab(b): + return zz(len(b)) + b +fields = [{'name': 'id', 'type': 'long'}, + {'name': 'f1', 'type': {'type': 'array', 'items': {'type': 'record', 'name': 'A1', 'fields': [{'name': 'v', 'type': 'long'}]}}}] +for k in range(2, n + 1): + fields.append({'name': 'f%d' % k, 'type': {'type': 'array', 'items': {'type': 'record', 'name': 'A%d' % k, 'fields': [{'name': 'x', 'type': 'A%d' % (k - 1)}]}}}) +fields.append({'name': 'last', 'type': ['null', 'A%d' % n]}) +schema = json.dumps({'type': 'record', 'name': 'root', 'fields': fields}, separators=(',', ':')).encode() +meta = zz(2) + ab(b'avro.schema') + ab(schema) + ab(b'avro.codec') + ab(b'null') + zz(0) +sync = bytes(16) +payload = zz(0) + b'\x00' * n + zz(0) +open(sys.argv[1], 'wb').write(b'Obj\x01' + meta + sync + zz(1) + zz(len(payload)) + payload + sync) +" "$@" +} + +gen "$DIR/chain_40.avro" 40 +gen "$DIR/chain_10000.avro" 10000 + +$CLICKHOUSE_LOCAL -q "DESC file('$DIR/chain_40.avro', Avro)" | wc -l + +# Inference builds every field's type before the deepest one overflows, so its cost grows with the stack +# size: a 1 MiB stack keeps this case small on every build. +(ulimit -s 1024; $CLICKHOUSE_LOCAL -q "DESC file('$DIR/chain_10000.avro', Avro)" 2>&1) | grep -q -F 'TOO_DEEP_RECURSION' && echo 'too deep' + +rm -rf "$DIR" diff --git a/tests/queries/0_stateless/05339_backup_table_function_table_without_source.reference b/tests/queries/0_stateless/05339_backup_table_function_table_without_source.reference new file mode 100644 index 0000000000000..8dd7f35b70168 --- /dev/null +++ b/tests/queries/0_stateless/05339_backup_table_function_table_without_source.reference @@ -0,0 +1,3 @@ +BACKUP_CREATED +RESTORED +t diff --git a/tests/queries/0_stateless/05339_backup_table_function_table_without_source.sh b/tests/queries/0_stateless/05339_backup_table_function_table_without_source.sh new file mode 100755 index 0000000000000..95c5a5cebd527 --- /dev/null +++ b/tests/queries/0_stateless/05339_backup_table_function_table_without_source.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# A table defined from a table function keeps no data of its own, so `BACKUP` and `RESTORE` store and recreate only its +# definition, without running the table function: they work while its source is missing. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +backup="Disk('backups', '${CLICKHOUSE_TEST_UNIQUE_NAME}')" + +$CLICKHOUSE_CLIENT -q " +DROP TABLE IF EXISTS t; +CREATE TABLE t (x UInt64) AS file('${CLICKHOUSE_TEST_UNIQUE_NAME}_missing.csv', 'CSV'); +" +$CLICKHOUSE_CLIENT -q "BACKUP TABLE t TO $backup" | grep -o "BACKUP_CREATED" +$CLICKHOUSE_CLIENT -q "DROP TABLE t" +$CLICKHOUSE_CLIENT -q "RESTORE TABLE t FROM $backup" | grep -o "RESTORED" +$CLICKHOUSE_CLIENT -q "SELECT name FROM system.tables WHERE database = currentDatabase() AND name = 't'" +$CLICKHOUSE_CLIENT -q "DROP TABLE t" diff --git a/tests/queries/0_stateless/05339_prewhere_subcolumn_read_steps.reference b/tests/queries/0_stateless/05339_prewhere_subcolumn_read_steps.reference new file mode 100644 index 0000000000000..7a770acba3a2d --- /dev/null +++ b/tests/queries/0_stateless/05339_prewhere_subcolumn_read_steps.reference @@ -0,0 +1,7 @@ +json_whole 1 +json_whole_prewhere 1 +json_other_path 1 +tuple_whole 1 +map_size_and_key 1 +maps_other_map 1 +map_key_and_whole 1 diff --git a/tests/queries/0_stateless/05339_prewhere_subcolumn_read_steps.sql b/tests/queries/0_stateless/05339_prewhere_subcolumn_read_steps.sql new file mode 100644 index 0000000000000..a87623566a816 --- /dev/null +++ b/tests/queries/0_stateless/05339_prewhere_subcolumn_read_steps.sql @@ -0,0 +1,79 @@ +-- Tags: no-parallel-replicas +-- no-parallel-replicas: read_bytes in the initiator's query_log does not include the replicas' reads. + +-- A condition on a cheap subcolumn gets its own PREWHERE read step, so the column it is combined with is read +-- only for the rows it keeps. No row passes the first condition, so a case reads about the bytes of that one alone. + +SET enable_analyzer = 1; +SET optimize_functions_to_subcolumns = 1; +SET optimize_move_to_prewhere = 1; +SET query_plan_optimize_prewhere = 1; +SET move_all_conditions_to_prewhere = 1; +SET enable_multiple_prewhere_read_steps = 1; +SET use_query_condition_cache = 0; + +DROP TABLE IF EXISTS t_json; +DROP TABLE IF EXISTS t_tuple; +DROP TABLE IF EXISTS t_map; +DROP TABLE IF EXISTS t_maps; + +CREATE TABLE t_json (id UInt64, j JSON) ENGINE = MergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; +CREATE TABLE t_tuple (id UInt64, t Tuple(kind String, payload String)) ENGINE = MergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; +-- Bucketed, with as many buckets as keys, so that `m['k1']` reads one bucket and not the whole Map. +CREATE TABLE t_map (id UInt64, m Map(String, String)) ENGINE = MergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0, + map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets', + map_buckets_strategy = 'constant', max_buckets_in_map = 16, map_buckets_min_avg_size = 0; +CREATE TABLE t_maps (id UInt64, c Tuple(left Map(String, String), right Map(String, String))) ENGINE = MergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; + +-- Random values, so that the size of a column on disk, by which the conditions are ordered, reflects the cost of reading it. +INSERT INTO t_json SELECT number, concat('{"kind":"a","payload":"', hex(randomString(500)), '"}') FROM numbers(20000); +INSERT INTO t_tuple SELECT number, tuple('a', hex(randomString(500))) FROM numbers(20000); +INSERT INTO t_map SELECT number, mapFromArrays(arrayMap(i -> 'k' || toString(i), range(16)), arrayMap(i -> hex(randomString(30)), range(16))) FROM numbers(20000); +INSERT INTO t_maps SELECT number, tuple(map('k', 'a'), map('k', hex(randomString(500)))) FROM numbers(20000); + +SELECT count() FROM t_json WHERE j.kind = 'b' FORMAT Null SETTINGS log_comment = 'json_cheap'; +SELECT count() FROM t_tuple WHERE t.kind = 'b' FORMAT Null SETTINGS log_comment = 'tuple_cheap'; +SELECT count() FROM t_map WHERE length(m) = 3 FORMAT Null SETTINGS log_comment = 'map_cheap'; +SELECT count() FROM t_map WHERE m['k1'] = 'x' FORMAT Null SETTINGS log_comment = 'map_key_cheap'; +SELECT count() FROM t_maps WHERE c.left['k'] = 'b' FORMAT Null SETTINGS log_comment = 'maps_cheap'; + +SELECT count() FROM t_json WHERE j.kind = 'b' AND length(toString(j)) > 10 FORMAT Null SETTINGS log_comment = 'json_whole'; +SELECT count() FROM t_json PREWHERE j.kind = 'b' AND length(toString(j)) > 10 FORMAT Null SETTINGS log_comment = 'json_whole_prewhere'; +SELECT count() FROM t_json WHERE j.kind = 'b' AND length(toString(j.payload)) > 10 FORMAT Null SETTINGS log_comment = 'json_other_path'; +SELECT count() FROM t_tuple WHERE t.kind = 'b' AND length(toString(t)) > 10 FORMAT Null SETTINGS log_comment = 'tuple_whole'; +-- PREWHERE keeps the written order, which the WHERE optimizer may change based on the column sizes. +SELECT count() FROM t_map PREWHERE length(m) = 3 AND m['k1'] != '' FORMAT Null SETTINGS log_comment = 'map_size_and_key'; +SELECT count() FROM t_maps WHERE c.left['k'] = 'b' AND c.right['k'] != '' FORMAT Null SETTINGS log_comment = 'maps_other_map'; +SELECT count() FROM t_map WHERE m['k1'] = 'x' AND length(toString(m)) > 10 FORMAT Null SETTINGS log_comment = 'map_key_and_whole'; + +SYSTEM FLUSH LOGS query_log; + +SELECT c.case_name, q.bytes < 2 * base.bytes +FROM VALUES('idx UInt8, case_name String, base_name String', + (1, 'json_whole', 'json_cheap'), + (2, 'json_whole_prewhere', 'json_cheap'), + (3, 'json_other_path', 'json_cheap'), + (4, 'tuple_whole', 'tuple_cheap'), + (5, 'map_size_and_key', 'map_cheap'), + (6, 'maps_other_map', 'maps_cheap'), + (7, 'map_key_and_whole', 'map_key_cheap')) AS c +INNER JOIN +( + SELECT log_comment, max(read_bytes) AS bytes FROM system.query_log + WHERE current_database = currentDatabase() AND type = 'QueryFinish' GROUP BY log_comment +) AS q ON q.log_comment = c.case_name +INNER JOIN +( + SELECT log_comment, max(read_bytes) AS bytes FROM system.query_log + WHERE current_database = currentDatabase() AND type = 'QueryFinish' GROUP BY log_comment +) AS base ON base.log_comment = c.base_name +ORDER BY c.idx; + +DROP TABLE t_json; +DROP TABLE t_tuple; +DROP TABLE t_map; +DROP TABLE t_maps; diff --git a/tests/queries/0_stateless/05340_map_element_to_subcolumn_bucketed_only.reference b/tests/queries/0_stateless/05340_map_element_to_subcolumn_bucketed_only.reference new file mode 100644 index 0000000000000..fcc2c16d9ebd0 --- /dev/null +++ b/tests/queries/0_stateless/05340_map_element_to_subcolumn_bucketed_only.reference @@ -0,0 +1,30 @@ +bucketed MergeTree +1 +MergeTree with bucketed zero-level parts +1 +basic MergeTree +1 +Memory +1 +explicit key subcolumn of a basic Map +8 8 +9 9 +results +8 8 +9 9 +8 8 +9 9 +8 8 +9 9 +Merge over a bucketed table +1 +MaterializedView with a bucketed target +1 +Buffer with a bucketed destination +1 +Merge over a bucketed and a basic table +1 +8 8 +8 8 +9 9 +9 9 diff --git a/tests/queries/0_stateless/05340_map_element_to_subcolumn_bucketed_only.sql b/tests/queries/0_stateless/05340_map_element_to_subcolumn_bucketed_only.sql new file mode 100644 index 0000000000000..f1fdfafeb55a6 --- /dev/null +++ b/tests/queries/0_stateless/05340_map_element_to_subcolumn_bucketed_only.sql @@ -0,0 +1,76 @@ +-- `m['key']` is rewritten to the Map key subcolumn `m.key_` only when the table serializes Maps +-- with buckets, because only then can a single key be read without the whole Map. +-- Both serialization settings are named in every table, clickhouse-test randomizes them. + +SET enable_analyzer = 1; +SET optimize_functions_to_subcolumns = 1; +SET explain_query_plan_default = 'legacy'; + +DROP TABLE IF EXISTS t_map_buffer; +DROP TABLE IF EXISTS t_map_mv; +DROP TABLE IF EXISTS t_map_merge; +DROP TABLE IF EXISTS t_map_memory; +DROP TABLE IF EXISTS t_map_basic; +DROP TABLE IF EXISTS t_map_zero_level_bucketed; +DROP TABLE IF EXISTS t_map_bucketed; + +CREATE TABLE t_map_bucketed (id UInt64, m Map(String, UInt64)) ENGINE = MergeTree ORDER BY id +SETTINGS map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets'; +CREATE TABLE t_map_zero_level_bucketed (id UInt64, m Map(String, UInt64)) ENGINE = MergeTree ORDER BY id +SETTINGS map_serialization_version = 'basic', map_serialization_version_for_zero_level_parts = 'with_buckets'; +CREATE TABLE t_map_basic (id UInt64, m Map(String, UInt64)) ENGINE = MergeTree ORDER BY id +SETTINGS map_serialization_version = 'basic', map_serialization_version_for_zero_level_parts = 'basic'; +CREATE TABLE t_map_memory (id UInt64, m Map(String, UInt64)) ENGINE = Memory; + +INSERT INTO t_map_bucketed SELECT number, map('k', number) FROM numbers(10); +INSERT INTO t_map_zero_level_bucketed SELECT number, map('k', number) FROM numbers(10); +INSERT INTO t_map_basic SELECT number, map('k', number) FROM numbers(10); +INSERT INTO t_map_memory SELECT number, map('k', number) FROM numbers(10); + +SELECT 'bucketed MergeTree'; +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT id FROM t_map_bucketed WHERE m['k'] > 7) WHERE explain LIKE '%m.key_k%'; +SELECT 'MergeTree with bucketed zero-level parts'; +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT id FROM t_map_zero_level_bucketed WHERE m['k'] > 7) WHERE explain LIKE '%m.key_k%'; +SELECT 'basic MergeTree'; +SELECT count() = 0 FROM (EXPLAIN actions = 1 SELECT id FROM t_map_basic WHERE m['k'] > 7) WHERE explain LIKE '%m.key_k%'; +SELECT 'Memory'; +SELECT count() = 0 FROM (EXPLAIN actions = 1 SELECT id FROM t_map_memory WHERE m['k'] > 7) WHERE explain LIKE '%m.key_k%'; + +SELECT 'explicit key subcolumn of a basic Map'; +SELECT id, m.key_k FROM t_map_basic WHERE m.key_k > 7 ORDER BY id; + +SELECT 'results'; +SELECT id, m['k'] FROM t_map_bucketed WHERE m['k'] > 7 ORDER BY id; +SELECT id, m['k'] FROM t_map_basic WHERE m['k'] > 7 ORDER BY id; +SELECT id, m['k'] FROM t_map_memory WHERE m['k'] > 7 ORDER BY id; + +-- Engines that pass the analyzed query to other tables answer for those tables. +CREATE TABLE t_map_merge (id UInt64, m Map(String, UInt64)) ENGINE = Merge(currentDatabase(), '^t_map_bucketed$'); +CREATE MATERIALIZED VIEW t_map_mv (id UInt64, m Map(String, UInt64)) ENGINE = MergeTree ORDER BY id +SETTINGS map_serialization_version = 'with_buckets', map_serialization_version_for_zero_level_parts = 'with_buckets' +AS SELECT id, m FROM t_map_bucketed; +CREATE TABLE t_map_buffer (id UInt64, m Map(String, UInt64)) +ENGINE = Buffer(currentDatabase(), t_map_bucketed, 1, 10, 100, 10000, 1000000, 10000000, 100000000); + +SELECT 'Merge over a bucketed table'; +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT id FROM t_map_merge WHERE m['k'] > 7) WHERE explain LIKE '%m.key_k%'; +SELECT 'MaterializedView with a bucketed target'; +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT id FROM t_map_mv WHERE m['k'] > 7) WHERE explain LIKE '%m.key_k%'; +SELECT 'Buffer with a bucketed destination'; +SELECT count() > 0 FROM (EXPLAIN actions = 1 SELECT id FROM t_map_buffer WHERE m['k'] > 7) WHERE explain LIKE '%m.key_k%'; + +-- The query is sent to every table of a Merge, so all of them must be bucketed. +DROP TABLE t_map_merge; +CREATE TABLE t_map_merge (id UInt64, m Map(String, UInt64)) ENGINE = Merge(currentDatabase(), '^t_map_(bucketed|basic)$'); + +SELECT 'Merge over a bucketed and a basic table'; +SELECT count() = 0 FROM (EXPLAIN actions = 1 SELECT id FROM t_map_merge WHERE m['k'] > 7) WHERE explain LIKE '%m.key_k%'; +SELECT id, m['k'] FROM t_map_merge WHERE m['k'] > 7 ORDER BY id; + +DROP TABLE t_map_buffer; +DROP TABLE t_map_mv; +DROP TABLE t_map_merge; +DROP TABLE t_map_memory; +DROP TABLE t_map_basic; +DROP TABLE t_map_zero_level_bucketed; +DROP TABLE t_map_bucketed; diff --git a/tests/queries/0_stateless/05353_ai_functions_allow_sql_named_collections.reference b/tests/queries/0_stateless/05353_ai_functions_allow_sql_named_collections.reference new file mode 100644 index 0000000000000..af474ec4afd6e --- /dev/null +++ b/tests/queries/0_stateless/05353_ai_functions_allow_sql_named_collections.reference @@ -0,0 +1,34 @@ +== default +-- config collection, credentials parameter +0 +0 +0 +-- SQL collection, credentials parameter +0 +0 +0 +-- config collection, default-credentials settings +0 +0 +0 +-- SQL collection, default-credentials settings +0 +0 +0 +== ai_function_allow_sql_named_collections = 0 +-- config collection, credentials parameter +0 +0 +0 +-- SQL collection, credentials parameter +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +-- config collection, default-credentials settings +0 +0 +0 +-- SQL collection, default-credentials settings +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED diff --git a/tests/queries/0_stateless/05353_ai_functions_allow_sql_named_collections.sh b/tests/queries/0_stateless/05353_ai_functions_allow_sql_named_collections.sh new file mode 100755 index 0000000000000..e093d6f0d3918 --- /dev/null +++ b/tests/queries/0_stateless/05353_ai_functions_allow_sql_named_collections.sh @@ -0,0 +1,80 @@ +#!/usr/bin/env bash + +# The server setting `ai_function_allow_sql_named_collections = 0` restricts AI functions to named +# collections defined in the server configuration file. Collections created with `CREATE NAMED COLLECTION` +# are rejected, whether passed in the `credentials` parameter or via the default-credentials settings. +# The check runs in `resolveAIParams`, before the zero-row early return, so empty inputs exercise it +# without any HTTP call. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +function config() +{ + echo " + $1 + + + openai + https://ai.example.com/v1/chat/completions + chat-model + fake-key + + + openai + https://ai.example.com/v1/embeddings + fake-key + + + " +} + +SETUP=" +CREATE NAMED COLLECTION ai_sql_chat AS + provider = 'openai', endpoint = 'https://ai.example.com/v1/chat/completions', model = 'chat-model', api_key = 'fake-key'; +CREATE NAMED COLLECTION ai_sql_embed AS + provider = 'openai', endpoint = 'https://ai.example.com/v1/embeddings', api_key = 'fake-key'; +CREATE TABLE tab (x String) ENGINE = Memory; +" + +# Runs one query in its own process and prints its result, or the code name of its error. +# `--ignore-error` is not used: older `clickhouse-local` versions do not print errors with it. +function query() +{ + $CLICKHOUSE_LOCAL --config-file <(config "$1") -q "$SETUP $2" 2>&1 \ + | sed -E 's/.*Code: [0-9]+\. DB::Exception: .*\(([A-Z_]+)\)$/\1/' +} + +# $1 is the server config fragment, $2 and $3 are the chat and embedding collections. +function queries_with_credentials() +{ + query "$1" "SELECT count() FROM (SELECT aiGenerate(x, map('credentials', '$2')) AS r FROM tab)" + query "$1" "SELECT count() FROM (SELECT aiEmbed(x, 'embed-model', map('credentials', '$3')) AS r FROM tab)" + query "$1" "SELECT count() FROM (SELECT aiSimilarity(x, x, 'embed-model', map('credentials', '$3')) AS r FROM tab)" +} + +function queries_with_default_credentials() +{ + local set="SET ai_function_text_default_credentials = '$2', ai_function_embedding_default_credentials = '$3';" + query "$1" "$set SELECT count() FROM (SELECT aiGenerate(x) AS r FROM tab)" + query "$1" "$set SELECT count() FROM (SELECT aiEmbed(x, 'embed-model') AS r FROM tab)" + query "$1" "$set SELECT count() FROM (SELECT aiSimilarity(x, x, 'embed-model') AS r FROM tab)" +} + +function run() +{ + echo "-- config collection, credentials parameter" + queries_with_credentials "$1" ai_config_chat ai_config_embed + echo "-- SQL collection, credentials parameter" + queries_with_credentials "$1" ai_sql_chat ai_sql_embed + echo "-- config collection, default-credentials settings" + queries_with_default_credentials "$1" ai_config_chat ai_config_embed + echo "-- SQL collection, default-credentials settings" + queries_with_default_credentials "$1" ai_sql_chat ai_sql_embed +} + +echo "== default" +run "" +echo "== ai_function_allow_sql_named_collections = 0" +run "0" diff --git a/tests/queries/0_stateless/05385_ipv6_all_zero_literal_host_ip.reference b/tests/queries/0_stateless/05385_ipv6_all_zero_literal_host_ip.reference new file mode 100644 index 0000000000000..ffa35be221a6d --- /dev/null +++ b/tests/queries/0_stateless/05385_ipv6_all_zero_literal_host_ip.reference @@ -0,0 +1,3 @@ +CREATE USER u_05385_a IDENTIFIED WITH no_password HOST IP \'::\' +CREATE USER u_05385_b IDENTIFIED WITH no_password HOST IP \'::\' +CREATE USER u_05385_c IDENTIFIED WITH no_password HOST IP \'::\' diff --git a/tests/queries/0_stateless/05385_ipv6_all_zero_literal_host_ip.sql b/tests/queries/0_stateless/05385_ipv6_all_zero_literal_host_ip.sql new file mode 100644 index 0000000000000..48e9885892979 --- /dev/null +++ b/tests/queries/0_stateless/05385_ipv6_all_zero_literal_host_ip.sql @@ -0,0 +1,13 @@ +-- Every spelling of the IPv6 unspecified address is an address. The peer `::6` of a proxy is +-- serialized as `::`, and a secondary query is rejected if `::` is not parsed back. +DROP USER IF EXISTS u_05385_a, u_05385_b, u_05385_c; + +CREATE USER u_05385_a HOST IP '::'; +CREATE USER u_05385_b HOST IP '0:0:0:0:0:0:0:0'; +CREATE USER u_05385_c HOST IP '::0.0.0.0'; + +SHOW CREATE USER u_05385_a; +SHOW CREATE USER u_05385_b; +SHOW CREATE USER u_05385_c; + +DROP USER u_05385_a, u_05385_b, u_05385_c; diff --git a/tests/queries/0_stateless/05394_ai_functions_retry_setting_limits.reference b/tests/queries/0_stateless/05394_ai_functions_retry_setting_limits.reference new file mode 100644 index 0000000000000..139597f9cb07c --- /dev/null +++ b/tests/queries/0_stateless/05394_ai_functions_retry_setting_limits.reference @@ -0,0 +1,2 @@ + + diff --git a/tests/queries/0_stateless/05394_ai_functions_retry_setting_limits.sql b/tests/queries/0_stateless/05394_ai_functions_retry_setting_limits.sql new file mode 100644 index 0000000000000..363215546ca78 --- /dev/null +++ b/tests/queries/0_stateless/05394_ai_functions_retry_setting_limits.sql @@ -0,0 +1,31 @@ +-- Tags: no-parallel, no-replicated-database +-- no-parallel: creates and drops global named collections +-- no-replicated-database: named collections are server-global, not database-scoped + +-- `ai_function_max_retries` is at most 5 and `ai_function_retry_initial_delay_ms` at most 10000. The limits are +-- checked before any request is sent, so the unreachable endpoint below is never contacted for a rejected query. + +DROP NAMED COLLECTION IF EXISTS ai_retry_limits; +DROP NAMED COLLECTION IF EXISTS ai_retry_limits_embed; +CREATE NAMED COLLECTION ai_retry_limits AS + provider = 'openai', endpoint = 'http://localhost:1/v1/chat/completions', model = 'test-model', api_key = 'fake-key'; +CREATE NAMED COLLECTION ai_retry_limits_embed AS + provider = 'openai', endpoint = 'http://localhost:1/v1/embeddings', api_key = 'fake-key'; + +DROP TABLE IF EXISTS t_retry_limits; +CREATE TABLE t_retry_limits (x String) ENGINE = Memory; +INSERT INTO t_retry_limits VALUES ('a'); + +SELECT aiGenerate(x, map('credentials', 'ai_retry_limits')) FROM t_retry_limits SETTINGS ai_function_max_retries = 6; -- { serverError BAD_ARGUMENTS } +SELECT aiGenerate(x, map('credentials', 'ai_retry_limits')) FROM t_retry_limits SETTINGS ai_function_retry_initial_delay_ms = 10001; -- { serverError BAD_ARGUMENTS } +SELECT aiEmbed(x, 'test-model', map('credentials', 'ai_retry_limits_embed')) FROM t_retry_limits SETTINGS ai_function_max_retries = 6; -- { serverError BAD_ARGUMENTS } + +-- The maximums themselves are accepted: the request fails against the unreachable endpoint and the row gets an empty result. +SELECT aiGenerate(x, map('credentials', 'ai_retry_limits')) FROM t_retry_limits +SETTINGS ai_function_max_retries = 5, ai_function_retry_initial_delay_ms = 1, ai_function_throw_on_error = 0, ai_function_request_timeout_sec = 3; +SELECT aiGenerate(x, map('credentials', 'ai_retry_limits')) FROM t_retry_limits +SETTINGS ai_function_max_retries = 0, ai_function_retry_initial_delay_ms = 10000, ai_function_throw_on_error = 0, ai_function_request_timeout_sec = 3; + +DROP TABLE t_retry_limits; +DROP NAMED COLLECTION ai_retry_limits; +DROP NAMED COLLECTION ai_retry_limits_embed; diff --git a/tests/queries/0_stateless/05401_subcolumn_size_without_reading_prefixes.sql b/tests/queries/0_stateless/05401_subcolumn_size_without_reading_prefixes.sql index 573667b7263e1..81c53c24c3247 100644 --- a/tests/queries/0_stateless/05401_subcolumn_size_without_reading_prefixes.sql +++ b/tests/queries/0_stateless/05401_subcolumn_size_without_reading_prefixes.sql @@ -33,7 +33,7 @@ INSERT INTO t_subcolumn_size_lc_map SELECT toString(number % 7), map('k' || toSt SELECT count() FROM system.parts WHERE database = currentDatabase() AND table = 't_subcolumn_size_lc_map' AND active AND part_type = 'Wide'; SELECT countIf(explain ILIKE '%Prewhere filter%') > 0, countIf(explain ILIKE '%m.key_k1%') > 0 -FROM (EXPLAIN actions = 1 SELECT count() FROM t_subcolumn_size_lc_map WHERE m['k1'] = '1' AND s = '3') +FROM (EXPLAIN actions = 1 SELECT count() FROM t_subcolumn_size_lc_map WHERE m.key_k1 = '1' AND s = '3') SETTINGS log_comment = '05401_subcolumn_size_without_reading_prefixes_explain'; SYSTEM FLUSH LOGS query_log; diff --git a/tests/queries/0_stateless/05406_iceberg_write_many_partitions_thread_pool.reference b/tests/queries/0_stateless/05406_iceberg_write_many_partitions_thread_pool.reference new file mode 100644 index 0000000000000..dfd940ddad714 --- /dev/null +++ b/tests/queries/0_stateless/05406_iceberg_write_many_partitions_thread_pool.reference @@ -0,0 +1,3 @@ +3000 30 +3000 4501500 +1000 499500 diff --git a/tests/queries/0_stateless/05406_iceberg_write_many_partitions_thread_pool.sh b/tests/queries/0_stateless/05406_iceberg_write_many_partitions_thread_pool.sh new file mode 100755 index 0000000000000..e213e850986de --- /dev/null +++ b/tests/queries/0_stateless/05406_iceberg_write_many_partitions_thread_pool.sh @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# - no-fasttest: requires `IcebergLocal` (USE_AVRO build option) + +# Each finished Iceberg data file used to keep its Parquet encoder threads until the commit, so writing +# 30 partitions with `max_threads = 16` needed 480 threads of the 300-thread global pool. +# https://github.com/ClickHouse/ClickHouse/issues/126508 + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +TABLE_DIR="${CLICKHOUSE_USER_FILES}/${CLICKHOUSE_TEST_UNIQUE_NAME}" +rm -rf "${TABLE_DIR}" +mkdir -p "${TABLE_DIR}" +trap 'rm -rf "${TABLE_DIR}"' EXIT + +COLS=$(for i in $(seq 16); do printf ', c%d Int64' "$i"; done) +EXPRS=$(for i in $(seq 16); do printf ', number * %d' "$i"; done) +SETTINGS="SET allow_insert_into_iceberg = 1, max_insert_threads = 1, max_threads = 16, output_format_parquet_parallel_encoding = 1" + +# INSERT, then ALTER TABLE ... UPDATE, which writes one position delete file and one data file per partition. +${CLICKHOUSE_LOCAL} --query " + ${SETTINGS}; + CREATE TABLE t (p Int32 ${COLS}) ENGINE = IcebergLocal('${TABLE_DIR}/update/') PARTITION BY p; + INSERT INTO t SELECT number % 30 ${EXPRS} FROM numbers(3000); + SELECT count(), uniqExact(_path) FROM t; + ALTER TABLE t UPDATE c1 = c1 + 1 WHERE 1; + SELECT count(), sum(c1) FROM t; +" -- --max_thread_pool_size=300 --thread_pool_queue_size=300 + +# ALTER TABLE ... DELETE writes two-column position delete files, so it needs more partitions to fill the pool. +${CLICKHOUSE_LOCAL} --query " + ${SETTINGS}; + CREATE TABLE t (p Int32, x Int64) ENGINE = IcebergLocal('${TABLE_DIR}/delete/') PARTITION BY p; + INSERT INTO t SELECT number % 80, number FROM numbers(2000); + ALTER TABLE t DELETE WHERE x >= 1000; + SELECT count(), sum(x) FROM t; +" -- --max_thread_pool_size=300 --thread_pool_queue_size=300 diff --git a/tests/queries/0_stateless/05406_join_swap_array_join_estimate.reference b/tests/queries/0_stateless/05406_join_swap_array_join_estimate.reference new file mode 100644 index 0000000000000..c826867912a98 --- /dev/null +++ b/tests/queries/0_stateless/05406_join_swap_array_join_estimate.reference @@ -0,0 +1,13 @@ +arrayJoin, empty input 1 0 +lowered arrayJoin, empty input, LEFT JOIN 1 0 +ARRAY JOIN, empty input 1 0 +lowered arrayJoin, 5 rows 1 0 +arrayJoin, 100 rows 0 1 +ARRAY JOIN, 100 rows 0 1 +0 +10 +2000 +ARRAY JOIN, element filter 1 0 1 +ARRAY JOIN, filter step 1 0 1 +arrayJoin, filter 1 0 1 +ARRAY JOIN, element filter, empty input 1 0 diff --git a/tests/queries/0_stateless/05406_join_swap_array_join_estimate.sql b/tests/queries/0_stateless/05406_join_swap_array_join_estimate.sql new file mode 100644 index 0000000000000..152d40621d340 --- /dev/null +++ b/tests/queries/0_stateless/05406_join_swap_array_join_estimate.sql @@ -0,0 +1,84 @@ +-- Random settings limits: query_plan_optimize_join_order_randomize=(0, 0); query_plan_optimize_join_order_limit=(1, None) + +-- A hash join compares the sides by their row estimates, and an array join input is estimated +-- as 100 rows per row of its input. + +SET enable_parallel_replicas = 0, join_algorithm = 'hash', query_plan_join_swap_table = 'auto'; + +DROP TABLE IF EXISTS aj_l; +DROP TABLE IF EXISTS aj_s; +DROP TABLE IF EXISTS aj_r; + +CREATE TABLE aj_l (id UInt64, arr Array(UInt64)) ENGINE = MergeTree ORDER BY id SETTINGS auto_statistics_types = ''; +CREATE TABLE aj_s (id UInt64, arr Array(UInt64)) ENGINE = MergeTree ORDER BY id SETTINGS auto_statistics_types = ''; +CREATE TABLE aj_r (id UInt64) ENGINE = MergeTree ORDER BY id SETTINGS auto_statistics_types = ''; + +INSERT INTO aj_l SELECT number, range(number, number + 20) FROM numbers(100); +INSERT INTO aj_s SELECT number, [number, number + 1] FROM numbers(5); +INSERT INTO aj_r SELECT number FROM numbers(1000); + +-- Each line counts the plans whose join reads `aj_r` first (`s` is the build side) and those reading `s` first. + +SELECT 'arrayJoin, empty input', countIf(explain ILIKE '%Join: aj_r%'), countIf(explain ILIKE '%Join: s%') +FROM (EXPLAIN actions = 1, keep_logical_steps = 1 + SELECT count() FROM (SELECT arrayJoin(arr) AS k FROM aj_l WHERE id > 1000) AS s INNER JOIN aj_r ON aj_r.id = s.k) +WHERE explain ILIKE '%Join:%'; + +SELECT 'lowered arrayJoin, empty input, LEFT JOIN', countIf(explain ILIKE '%Join: aj_r%'), countIf(explain ILIKE '%Join: s%') +FROM (EXPLAIN actions = 1, keep_logical_steps = 1 + SELECT count() FROM (SELECT arrayJoin(arr) AS k FROM aj_l WHERE id > 1000) AS s LEFT JOIN aj_r ON aj_r.id = s.k) +WHERE explain ILIKE '%Join:%'; + +SELECT 'ARRAY JOIN, empty input', countIf(explain ILIKE '%Join: aj_r%'), countIf(explain ILIKE '%Join: s%') +FROM (EXPLAIN actions = 1, keep_logical_steps = 1 + SELECT count() FROM (SELECT k FROM aj_l ARRAY JOIN arr AS k WHERE id > 1000) AS s INNER JOIN aj_r ON aj_r.id = s.k) +WHERE explain ILIKE '%Join:%'; + +-- 5 input rows are estimated as 500 rows, fewer than the 1000 rows of `aj_r`. +SELECT 'lowered arrayJoin, 5 rows', countIf(explain ILIKE '%Join: aj_r%'), countIf(explain ILIKE '%Join: s%') +FROM (EXPLAIN actions = 1, keep_logical_steps = 1 + SELECT count() FROM (SELECT arrayJoin(arr) AS k FROM aj_s) AS s INNER JOIN aj_r ON aj_r.id = s.k) +WHERE explain ILIKE '%Join:%'; + +-- 100 input rows are estimated as 10000 rows (2000 in fact), more than the 1000 rows of `aj_r`. +SELECT 'arrayJoin, 100 rows', countIf(explain ILIKE '%Join: aj_r%'), countIf(explain ILIKE '%Join: s%') +FROM (EXPLAIN actions = 1, keep_logical_steps = 1 + SELECT count() FROM (SELECT arrayJoin(arr) AS k FROM aj_l) AS s INNER JOIN aj_r ON aj_r.id = s.k) +WHERE explain ILIKE '%Join:%'; + +SELECT 'ARRAY JOIN, 100 rows', countIf(explain ILIKE '%Join: aj_r%'), countIf(explain ILIKE '%Join: s%') +FROM (EXPLAIN actions = 1, keep_logical_steps = 1 + SELECT count() FROM (SELECT k FROM aj_l ARRAY JOIN arr AS k) AS s INNER JOIN aj_r ON aj_r.id = s.k) +WHERE explain ILIKE '%Join:%'; + +SELECT count() FROM (SELECT arrayJoin(arr) AS k FROM aj_l WHERE id > 1000) AS s LEFT JOIN aj_r ON aj_r.id = s.k; +SELECT count() FROM (SELECT arrayJoin(arr) AS k FROM aj_s) AS s INNER JOIN aj_r ON aj_r.id = s.k; +SELECT count() FROM (SELECT k FROM aj_l ARRAY JOIN arr AS k) AS s INNER JOIN aj_r ON aj_r.id = s.k; + +-- A filter on the array-joined elements leaves 6 rows in `s`; the estimate of its input must not make `aj_r` the build side. +-- The third count is the plans that show no row estimate for `s`. +SELECT 'ARRAY JOIN, element filter', countIf(explain ILIKE '%Join: aj_r%'), countIf(explain ILIKE '%Join: s%'), countIf(explain ILIKE '%s[no_stats~?]%') +FROM (EXPLAIN actions = 1, keep_logical_steps = 1 + SELECT count() FROM aj_r INNER JOIN (SELECT id FROM aj_l ARRAY JOIN arr AS k WHERE k = 5) AS s ON aj_r.id = s.id) +WHERE explain ILIKE '%Join:%'; + +SELECT 'ARRAY JOIN, filter step', countIf(explain ILIKE '%Join: aj_r%'), countIf(explain ILIKE '%Join: s%'), countIf(explain ILIKE '%s[no_stats~?]%') +FROM (EXPLAIN actions = 1, keep_logical_steps = 1 + SELECT count() FROM aj_r INNER JOIN (SELECT id FROM aj_l ARRAY JOIN arr AS k WHERE k = 5) AS s ON aj_r.id = s.id) +WHERE explain ILIKE '%Join:%'; + +SELECT 'arrayJoin, filter', countIf(explain ILIKE '%Join: aj_r%'), countIf(explain ILIKE '%Join: s%'), countIf(explain ILIKE '%s[no_stats~?]%') +FROM (EXPLAIN actions = 1, keep_logical_steps = 1 + SELECT count() FROM aj_r INNER JOIN (SELECT id FROM (SELECT id, arrayJoin(arr) AS k FROM aj_l) WHERE k = 5) AS s ON aj_r.id = s.id + SETTINGS query_plan_merge_expressions = 0) +WHERE explain ILIKE '%Join:%'; + +-- A filter on the elements of an empty input keeps its estimate of 0 rows. +SELECT 'ARRAY JOIN, element filter, empty input', countIf(explain ILIKE '%Join: aj_r%'), countIf(explain ILIKE '%Join: s%') +FROM (EXPLAIN actions = 1, keep_logical_steps = 1 + SELECT count() FROM (SELECT id FROM aj_l ARRAY JOIN arr AS k WHERE id > 1000 AND k = 5) AS s INNER JOIN aj_r ON aj_r.id = s.id) +WHERE explain ILIKE '%Join:%'; + +DROP TABLE aj_l; +DROP TABLE aj_s; +DROP TABLE aj_r; diff --git a/tests/queries/0_stateless/05407_bool_force_setting_shorthand.reference b/tests/queries/0_stateless/05407_bool_force_setting_shorthand.reference new file mode 100644 index 0000000000000..98fb6a6865639 --- /dev/null +++ b/tests/queries/0_stateless/05407_bool_force_setting_shorthand.reference @@ -0,0 +1,4 @@ +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/05407_bool_force_setting_shorthand.sh b/tests/queries/0_stateless/05407_bool_force_setting_shorthand.sh new file mode 100755 index 0000000000000..a7f3ea6a2171b --- /dev/null +++ b/tests/queries/0_stateless/05407_bool_force_setting_shorthand.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +set -e + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `SET` shorthand changes both `false` and `force` to `true`. +$CLICKHOUSE_CLIENT --multiquery --query " + SET query_plan_read_in_order_through_join = 0; + SET query_plan_read_in_order_through_join; + SELECT value FROM system.settings WHERE name = 'query_plan_read_in_order_through_join'; + SET query_plan_read_in_order_through_join = 'force'; + SET query_plan_read_in_order_through_join; + SELECT value FROM system.settings WHERE name = 'query_plan_read_in_order_through_join'; +" + +# Query-level `SETTINGS` shorthand overrides a disabled session setting. +$CLICKHOUSE_CLIENT --query_plan_read_in_order_through_join=0 --query " + SELECT value FROM system.settings WHERE name = 'query_plan_read_in_order_through_join' + SETTINGS query_plan_read_in_order_through_join +" + +# A client option without a value means `true`. +$CLICKHOUSE_CLIENT --query_plan_read_in_order_through_join --query " + SELECT value FROM system.settings WHERE name = 'query_plan_read_in_order_through_join' +" diff --git a/tests/queries/0_stateless/05407_dynamic_deep_value_column_operations.reference b/tests/queries/0_stateless/05407_dynamic_deep_value_column_operations.reference new file mode 100644 index 0000000000000..636bda45df5e6 --- /dev/null +++ b/tests/queries/0_stateless/05407_dynamic_deep_value_column_operations.reference @@ -0,0 +1,4 @@ +[[[[NULL]]],[[[NULL]]]] +[NULL,NULL] +4 2 +server is alive diff --git a/tests/queries/0_stateless/05407_dynamic_deep_value_column_operations.sh b/tests/queries/0_stateless/05407_dynamic_deep_value_column_operations.sh new file mode 100755 index 0000000000000..81d6382e19109 --- /dev/null +++ b/tests/queries/0_stateless/05407_dynamic_deep_value_column_operations.sh @@ -0,0 +1,50 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# The fast-test build is unoptimized and re-runs a new test several times under a 60s cap, while +# materializing a deeply nested Dynamic value whose operation does not trip the recursion guard takes +# superlinear time. The optimized stateless configurations run it within budget. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Each repetition of 1E2B0801 is a value of type Array(Dynamic(max_types=8)) with one element, so the +# nesting depth of the Dynamic value comes from the data, not from the declared type. The second row is NULL. +nested_value() { echo "format(RowBinary, 'd Dynamic', unhex(concat(repeat('1E2B0801', $1), '0000')))"; } + +$CLICKHOUSE_CLIENT -q "SELECT arraySort([d, d]) FROM $(nested_value 3)" + +# Depending on the build and the stack size, a deep value is either processed or rejected with +# TOO_DEEP_RECURSION, but it must never crash the server or use memory quadratic in the depth. +# Parsing a value takes time quadratic in its depth (every level has a longer type name), so the deepest value, +# which exhausts the stack in the release build, is only used for the operations that recurse into the nested +# columns: `index`, `compareAt`, and `filter`. +check_query() { + out=$($CLICKHOUSE_CLIENT -q "$1 FORMAT Null" 2>&1) + [[ -z "$out" || "$out" == *TOO_DEEP_RECURSION* ]] || echo "$1: $out" +} +table=$(nested_value 1000) +for query in "SELECT [d] FROM $table" "SELECT arraySort([d, d]) FROM $table" "SELECT d, arrayJoin([1, 2]) FROM $table" "SELECT d FROM $table WHERE d IS NOT NULL"; do + check_query "$query" +done +table=$(nested_value 9000) +for query in "SELECT [d] FROM $table" "SELECT arraySort([d, d]) FROM $table" "SELECT d FROM $table WHERE d IS NOT NULL"; do + check_query "$query" +done + +# Writing a part and merging parts go through the dynamic structure helpers of the column. +# Writing a part takes time quadratic in the nesting depth (every level adds substreams with longer names), +# so a depth that exhausts the stack takes tens of seconds; use a small depth that checks the normal path +# and stays far from the stack limit in sanitizer builds, where depth 300 already hits it. +# The substream names of a deep value exceed the file name length limit of the filesystem in a wide part, +# so they must be replaced with hashes regardless of the randomized MergeTree settings. +$CLICKHOUSE_CLIENT -q "CREATE TABLE deep_dynamic (d Dynamic) ENGINE = MergeTree ORDER BY tuple() SETTINGS replace_long_file_name_to_hash = 1, max_file_name_length = 127" +table=$(nested_value 64) +for query in "INSERT INTO deep_dynamic SELECT d FROM $table" "INSERT INTO deep_dynamic SELECT d FROM $table" "OPTIMIZE TABLE deep_dynamic FINAL"; do + out=$($CLICKHOUSE_CLIENT -q "$query" 2>&1) + [[ -z "$out" ]] || echo "$query: $out" +done +$CLICKHOUSE_CLIENT -q "SELECT count(), countIf(d IS NULL) FROM deep_dynamic" +$CLICKHOUSE_CLIENT -q "DROP TABLE deep_dynamic" + +$CLICKHOUSE_CLIENT -q "SELECT 'server is alive'" diff --git a/tests/queries/0_stateless/05407_orc_timezone_traversal_abort.reference b/tests/queries/0_stateless/05407_orc_timezone_traversal_abort.reference new file mode 100644 index 0000000000000..fcc653eab3e4a --- /dev/null +++ b/tests/queries/0_stateless/05407_orc_timezone_traversal_abort.reference @@ -0,0 +1,5 @@ +writer: time zone is rejected +reader: time zone is rejected +reader file: time zone is rejected +--- stateless server is still alive --- +1 diff --git a/tests/queries/0_stateless/05407_orc_timezone_traversal_abort.sh b/tests/queries/0_stateless/05407_orc_timezone_traversal_abort.sh new file mode 100755 index 0000000000000..f8e04add8d562 --- /dev/null +++ b/tests/queries/0_stateless/05407_orc_timezone_traversal_abort.sh @@ -0,0 +1,137 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh +set -e + +# A malformed ORC timezone name used to reach an empty TZif file outside TZDIR and abort +# inside libc++. The time zones are now resolved only from the embedded time zone database, +# so such a name must be rejected without touching the file system. Keep each attempt in a child clickhouse-local process and disable core dumps. +ulimit -c 0 + +WORK_DIR="${CLICKHOUSE_TMP}/orc_timezone_traversal_${CLICKHOUSE_TEST_UNIQUE_NAME}" +rm -rf "${WORK_DIR}" +mkdir -p "${WORK_DIR}" +trap 'rm -rf "${WORK_DIR}"' EXIT + +regression_status=0 +python3 - "${WORK_DIR}" <<'PY' || regression_status=$? +import os +import pathlib +import shlex +import shutil +import subprocess +import sys + +work_dir = pathlib.Path(sys.argv[1]) +tz_dir = work_dir / "zoneinfo" / "a" / "b" +tz_dir.mkdir(parents=True) + +# Use real, minimal zone files so the valid setup and the ORC fixture can be loaded while +# TZDIR points at this private directory. The traversal resolves to this test's empty file. +system_zoneinfo = pathlib.Path("/usr/share/zoneinfo") +for zone in ("UTC", "America/Vancouver"): + source = system_zoneinfo / zone + if not source.is_file(): + raise SystemExit(f"required time zone file not found: {source}") + target = tz_dir / zone + target.parent.mkdir(parents=True, exist_ok=True) + shutil.copyfile(source, target) + +(work_dir / "outside").write_bytes(b"") +if (tz_dir / "../../../outside").resolve() != (work_dir / "outside").resolve(): + raise SystemExit("private timezone traversal fixture is misconfigured") +(work_dir / "dev").mkdir() +(work_dir / "dev" / "null").write_bytes(b"") +if (tz_dir / "../../../dev/null").resolve() != (work_dir / "dev" / "null").resolve(): + raise SystemExit("private ORC metadata traversal fixture is misconfigured") + +env = os.environ.copy() +env["TZDIR"] = str(tz_dir) +env["TZ"] = "UTC" +local_command = shlex.split(env["CLICKHOUSE_LOCAL"]) + + +def run_local(path, *args, query=None, input_data=None): + path.mkdir(parents=True, exist_ok=True) + command = local_command + ["--path", str(path), *args] + if query is not None: + command += ["--query", query] + return subprocess.run( + command, + input=input_data, + env=env, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + check=False, + ) + + +safe_query = ( + "SELECT toDateTime(0, 'America/Vancouver') AS t FORMAT ORC " + "SETTINGS output_format_orc_writer_time_zone_name='America/Vancouver', " + "output_format_orc_compression_method='none'" +) +fixture = run_local(work_dir / "safe", query=safe_query) +if fixture.returncode != 0 or not fixture.stdout.startswith(b"ORC"): + diagnostic = (fixture.stderr + fixture.stdout).decode(errors="replace") + raise SystemExit(f"could not create valid ORC fixture: {diagnostic[:1000]}") + +# Preserve the ORC file's byte lengths while replacing its writer timezone metadata +# with the path from the customer reproducer. This exercises a file supplied by a +# reader, without requiring a malicious query setting. +safe_name = b"America/Vancouver" +unsafe_name = b"../../../dev/null" +if len(safe_name) != len(unsafe_name) or fixture.stdout.count(safe_name) != 1: + raise SystemExit("valid ORC fixture has unexpected timezone metadata") +malicious_file = fixture.stdout.replace(safe_name, unsafe_name, 1) + +malicious_writer_query = ( + "SELECT toDateTime(0, 'America/Vancouver') AS t FORMAT ORC " + "SETTINGS output_format_orc_writer_time_zone_name='../../../outside', " + "output_format_orc_compression_method='none'" +) +writer = run_local(work_dir / "writer", query=malicious_writer_query) +reader = run_local( + work_dir / "reader", + "--input-format=ORC", + "--input_format_orc_reader_time_zone_name=../../../outside", + query="SELECT * FROM table", + input_data=fixture.stdout, +) +reader_file = run_local( + work_dir / "reader_file", + "--input-format=ORC", + query="SELECT * FROM table", + input_data=malicious_file, +) + +failed = False +for label, result in (("writer", writer), ("reader", reader), ("reader file", reader_file)): + if result.returncode < 0: + print(f"{label}: terminated by signal {-result.returncode}") + failed = True + elif result.returncode == 0: + print(f"{label}: malformed timezone unexpectedly succeeded") + failed = True + elif any( + message in result.stdout + result.stderr + for message in (b"is not supported. Use a name from `system.time_zones`", b"Cannot load time zone ../../../") + ): + print(f"{label}: time zone is rejected") + else: + diagnostic = (result.stderr + result.stdout).decode(errors="replace") + first_line = next((line for line in diagnostic.splitlines() if line), "no diagnostic") + print(f"{label}: unexpected error: {first_line[:300]}") + failed = True + +if failed: + raise SystemExit(1) +PY + +echo '--- stateless server is still alive ---' +${CLICKHOUSE_CLIENT} --query "SELECT 1" +exit "${regression_status}" diff --git a/tests/queries/0_stateless/05410_parse_date_time_daylight_saving_time.reference b/tests/queries/0_stateless/05410_parse_date_time_daylight_saving_time.reference new file mode 100644 index 0000000000000..23ae65ac69b67 --- /dev/null +++ b/tests/queries/0_stateless/05410_parse_date_time_daylight_saving_time.reference @@ -0,0 +1,65 @@ +2026-10-25 01:00:00 +Europe/Berlin, clocks go back +2026-10-25 00:00:00 1792879200 1 +2026-10-25 00:30:00 1792881000 1 +2026-10-25 01:00:00 1792882800 1 +2026-10-25 01:30:00 1792884600 1 +2026-10-25 02:00:00 1792886400 1 +2026-10-25 02:30:00 1792888200 1 +2026-10-25 03:00:00 1792893600 1 +2026-10-25 03:30:00 1792895400 1 +2026-10-25 04:00:00 1792897200 1 +2026-10-25 04:30:00 1792899000 1 +Europe/Berlin, clocks go forward +2026-03-29 00:00:00 1774738800 1 +2026-03-29 00:30:00 1774740600 1 +2026-03-29 01:00:00 1774742400 1 +2026-03-29 01:30:00 1774744200 1 +2026-03-29 02:00:00 1774742400 1 +2026-03-29 02:30:00 1774744200 1 +2026-03-29 03:00:00 1774746000 1 +2026-03-29 03:30:00 1774747800 1 +2026-03-29 04:00:00 1774749600 1 +2026-03-29 04:30:00 1774751400 1 +America/New_York, clocks go back +2026-11-01 00:00:00 1793505600 1 +2026-11-01 00:30:00 1793507400 1 +2026-11-01 01:00:00 1793509200 1 +2026-11-01 01:30:00 1793511000 1 +2026-11-01 02:00:00 1793516400 1 +2026-11-01 02:30:00 1793518200 1 +2026-11-01 03:00:00 1793520000 1 +2026-11-01 03:30:00 1793521800 1 +2026-11-01 04:00:00 1793523600 1 +2026-11-01 04:30:00 1793525400 1 +other functions and formats +2026-10-25 01:00:00.500000 +2026-10-25 01:00:00 +2026-10-25 01:00:00 +2026-10-25 01:00:00 +2026-10-25 01:00:00 +2026-10-25 01:00:00 +2026-10-25 02:00:00 +America/New_York, clocks go forward, time zone name in the string +2026-03-08 00:00:00 1772946000 1 +2026-03-08 00:30:00 1772947800 1 +2026-03-08 01:00:00 1772949600 1 +2026-03-08 01:30:00 1772951400 1 +2026-03-08 02:00:00 1772949600 1 +2026-03-08 02:30:00 1772951400 1 +2026-03-08 03:00:00 1772953200 1 +2026-03-08 03:30:00 1772955000 1 +2026-03-08 04:00:00 1772956800 1 +2026-03-08 04:30:00 1772958600 1 +2026-03-08 06:30:00 +2026-10-25 00:30:00 +2026-10-25 00:30:00 +2026-10-25 01:30:00 +2026-10-25 01:30:00.5 +PM with a time zone name in the string +2026-10-25 12:30:00 +2026-10-25 11:30:00 +2026-10-24 22:30:00 +2026-10-25 12:30:00.5 +2026-03-10 13:30:00 +2026-10-25 12:30:00 diff --git a/tests/queries/0_stateless/05410_parse_date_time_daylight_saving_time.sql b/tests/queries/0_stateless/05410_parse_date_time_daylight_saving_time.sql new file mode 100644 index 0000000000000..465889c1a0cd2 --- /dev/null +++ b/tests/queries/0_stateless/05410_parse_date_time_daylight_saving_time.sql @@ -0,0 +1,51 @@ +-- `parseDateTime` converts a local time as `toDateTime` does, also next to a daylight saving time change. +-- In `Europe/Berlin` 02:00-02:59 happens twice on 2026-10-25 and does not exist on 2026-03-29. +-- https://github.com/ClickHouse/ClickHouse/issues/126667 + +SELECT parseDateTime('2026-10-25 01:00:00', '%Y-%m-%d %H:%i:%s', 'Europe/Berlin'); + +SELECT 'Europe/Berlin, clocks go back'; +SELECT s, toUnixTimestamp(parseDateTime(s, '%Y-%m-%d %H:%i:%s', 'Europe/Berlin')) AS t, t = toUnixTimestamp(toDateTime(s, 'Europe/Berlin')) +FROM (SELECT formatDateTime(toDateTime('2026-10-25 00:00:00', 'UTC') + number * 1800, '%Y-%m-%d %H:%i:%s', 'UTC') AS s FROM numbers(10)); + +SELECT 'Europe/Berlin, clocks go forward'; +SELECT s, toUnixTimestamp(parseDateTime(s, '%Y-%m-%d %H:%i:%s', 'Europe/Berlin')) AS t, t = toUnixTimestamp(toDateTime(s, 'Europe/Berlin')) +FROM (SELECT formatDateTime(toDateTime('2026-03-29 00:00:00', 'UTC') + number * 1800, '%Y-%m-%d %H:%i:%s', 'UTC') AS s FROM numbers(10)); + +-- West of UTC, the hours after the change were wrong. +SELECT 'America/New_York, clocks go back'; +SELECT s, toUnixTimestamp(parseDateTime(s, '%Y-%m-%d %H:%i:%s', 'America/New_York')) AS t, t = toUnixTimestamp(toDateTime(s, 'America/New_York')) +FROM (SELECT formatDateTime(toDateTime('2026-11-01 00:00:00', 'UTC') + number * 1800, '%Y-%m-%d %H:%i:%s', 'UTC') AS s FROM numbers(10)); + +SELECT 'other functions and formats'; +SELECT parseDateTime64('2026-10-25 01:00:00.5', '%Y-%m-%d %H:%i:%s.%f', 'Europe/Berlin'); +SELECT parseDateTimeInJodaSyntax('2026-10-25 01:00:00', 'yyyy-MM-dd HH:mm:ss', 'Europe/Berlin'); +SELECT parseDateTimeOrNull('2026-10-25 01:00:00', '%Y-%m-%d %H:%i:%s', 'Europe/Berlin'); +SELECT parseDateTime('2026-10-25 01:00:00 AM', '%Y-%m-%d %h:%i:%s %p', 'Europe/Berlin'); +SELECT parseDateTime('2026-298 01:00:00', '%Y-%j %H:%i:%s', 'Europe/Berlin'); +SELECT parseDateTimeInJodaSyntax('2026-43-7 01:00:00', 'xxxx-ww-e HH:mm:ss', 'Europe/Berlin'); + +-- An offset in the string is used as it is. +SELECT parseDateTime('2026-10-25 01:00:00 +0100', '%Y-%m-%d %H:%i:%s %z', 'Europe/Berlin'); + +-- A time zone name in the string gives the same results, also for a local time that does not exist. +SELECT 'America/New_York, clocks go forward, time zone name in the string'; +SELECT s, toUnixTimestamp(parseDateTimeInJodaSyntax(concat(s, ' America/New_York'), 'yyyy-MM-dd HH:mm:ss z', 'UTC')) AS t, + t = toUnixTimestamp(toDateTime(s, 'America/New_York')) +FROM (SELECT formatDateTime(toDateTime('2026-03-08 00:00:00', 'UTC') + number * 1800, '%Y-%m-%d %H:%i:%s', 'UTC') AS s FROM numbers(10)); +SELECT parseDateTime64InJodaSyntaxOrNull('2026-03-08 02:30:00 America/New_York', 'yyyy-MM-dd HH:mm:ss z', 'UTC'); +SELECT parseDateTimeInJodaSyntax('2026-10-25 02:30:00 Europe/Berlin', 'yyyy-MM-dd HH:mm:ss z', 'UTC'); + +-- An offset before the time zone name selects one of the two 02:30. +SELECT parseDateTimeInJodaSyntax('2026-10-25 02:30:00 +0200 Europe/Berlin', 'yyyy-MM-dd HH:mm:ss Z z', 'UTC'); +SELECT parseDateTimeInJodaSyntax('2026-10-25 02:30:00 +0100 Europe/Berlin', 'yyyy-MM-dd HH:mm:ss Z z', 'UTC'); +SELECT parseDateTime64InJodaSyntax('2026-10-25 02:30:00.5 +0100 Europe/Berlin', 'yyyy-MM-dd HH:mm:ss.S Z z', 'UTC'); + +-- A time zone name in the string builds the time once more, which added the 12 hours of PM twice. +SELECT 'PM with a time zone name in the string'; +SELECT parseDateTimeInJodaSyntax('2026-10-25 01:30:00 PM Europe/Berlin', 'yyyy-MM-dd hh:mm:ss a z', 'UTC'); +SELECT parseDateTimeInJodaSyntax('2026-10-25 12:30:00 PM Europe/Berlin', 'yyyy-MM-dd hh:mm:ss a z', 'UTC'); +SELECT parseDateTimeInJodaSyntax('2026-10-25 12:30:00 AM Europe/Berlin', 'yyyy-MM-dd hh:mm:ss a z', 'UTC'); +SELECT parseDateTime64InJodaSyntax('2026-10-25 01:30:00.5 PM Europe/Berlin', 'yyyy-MM-dd hh:mm:ss.S a z', 'UTC'); +SELECT parseDateTimeInJodaSyntax('2026-03-10 01:30:00 PM UTC', 'yyyy-MM-dd hh:mm:ss a z', 'UTC'); +SELECT parseDateTimeInJodaSyntax('2026-10-25 01:30:00 PM +0100 Europe/Berlin', 'yyyy-MM-dd hh:mm:ss a Z z', 'UTC'); diff --git a/tests/queries/0_stateless/05421_bloom_filter_text_index_total_size.reference b/tests/queries/0_stateless/05421_bloom_filter_text_index_total_size.reference new file mode 100644 index 0000000000000..b261da18d51a3 --- /dev/null +++ b/tests/queries/0_stateless/05421_bloom_filter_text_index_total_size.reference @@ -0,0 +1,2 @@ +1 +0 diff --git a/tests/queries/0_stateless/05421_bloom_filter_text_index_total_size.sql b/tests/queries/0_stateless/05421_bloom_filter_text_index_total_size.sql new file mode 100644 index 0000000000000..0a25cd09791d7 --- /dev/null +++ b/tests/queries/0_stateless/05421_bloom_filter_text_index_total_size.sql @@ -0,0 +1,32 @@ +-- A granule of a text bloom filter index allocates one filter per indexed column, +-- so the total size of filters (filter size multiplied by the number of columns) is limited by 1 GiB. + +DROP TABLE IF EXISTS t_bf_total_size; + +-- A single filter of the maximum size is still allowed. +CREATE TABLE t_bf_total_size (a String, INDEX idx a TYPE tokenbf_v1(1073741824, 3, 0) GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple(); +DROP TABLE t_bf_total_size; + +-- Two columns with filters of half of the maximum size are allowed. +CREATE TABLE t_bf_total_size (a String, b String, INDEX idx (a, b) TYPE tokenbf_v1(536870912, 3, 0) GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple(); +DROP TABLE t_bf_total_size; + +-- One byte more is too much. +CREATE TABLE t_bf_total_size (a String, b String, INDEX idx (a, b) TYPE tokenbf_v1(536870913, 3, 0) GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS } +CREATE TABLE t_bf_total_size (a String, b String, INDEX idx (a, b) TYPE ngrambf_v1(3, 536870913, 3, 0) GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS } +CREATE TABLE t_bf_total_size (a String, b String, c String, d String, e String, f String, g String, h String, + INDEX idx (a, b, c, d, e, f, g, h) TYPE tokenbf_v1(268435456, 3, 0) GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple(); -- { serverError BAD_ARGUMENTS } + +-- The same check is done for ALTER TABLE ... ADD INDEX. +CREATE TABLE t_bf_total_size (a String, b String) ENGINE = MergeTree ORDER BY tuple(); +ALTER TABLE t_bf_total_size ADD INDEX idx (a, b) TYPE tokenbf_v1(536870913, 3, 0) GRANULARITY 1; -- { serverError BAD_ARGUMENTS } +ALTER TABLE t_bf_total_size ADD INDEX idx (a, b) TYPE tokenbf_v1(1024, 3, 0) GRANULARITY 1; +INSERT INTO t_bf_total_size VALUES ('hello', 'world'); +SELECT count() FROM t_bf_total_size WHERE hasToken(a, 'hello'); +DROP TABLE t_bf_total_size; + +-- The memory of bloom filters is tracked, so a too large filter fails with an exception instead of exhausting the memory. +CREATE TABLE t_bf_total_size (a String, INDEX idx a TYPE tokenbf_v1(268435456, 3, 0) GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple(); +INSERT INTO t_bf_total_size SETTINGS max_memory_usage = '100Mi', async_insert = 0 VALUES ('hello world'); -- { serverError MEMORY_LIMIT_EXCEEDED } +SELECT count() FROM t_bf_total_size; +DROP TABLE t_bf_total_size; diff --git a/tests/queries/0_stateless/05422_bloom_filter_text_index_total_size_attach.reference b/tests/queries/0_stateless/05422_bloom_filter_text_index_total_size_attach.reference new file mode 100644 index 0000000000000..51aa977e86033 --- /dev/null +++ b/tests/queries/0_stateless/05422_bloom_filter_text_index_total_size_attach.reference @@ -0,0 +1,4 @@ +a +d +c +idx diff --git a/tests/queries/0_stateless/05422_bloom_filter_text_index_total_size_attach.sh b/tests/queries/0_stateless/05422_bloom_filter_text_index_total_size_attach.sh new file mode 100755 index 0000000000000..b1fd11bddf6d4 --- /dev/null +++ b/tests/queries/0_stateless/05422_bloom_filter_text_index_total_size_attach.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A text bloom filter index with the total size of filters over the limit cannot be created anymore, +# but a table created before the check still loads and can be altered, as long as its index does not grow. + +UUID=$($CLICKHOUSE_CLIENT -q "SELECT generateUUIDv4()") + +$CLICKHOUSE_CLIENT -m -q " +ATTACH TABLE t_bf_total_size_attach UUID '$UUID' (a String, b String, INDEX idx (a, b) TYPE tokenbf_v1(536870913, 3, 0) GRANULARITY 1) ENGINE = MergeTree ORDER BY tuple(); +ALTER TABLE t_bf_total_size_attach ADD COLUMN c String; +ALTER TABLE t_bf_total_size_attach RENAME COLUMN b TO d; +ALTER TABLE t_bf_total_size_attach ADD INDEX idx2 (a, c) TYPE tokenbf_v1(536870913, 3, 0) GRANULARITY 1; -- { serverError BAD_ARGUMENTS } +SELECT name FROM system.columns WHERE database = currentDatabase() AND table = 't_bf_total_size_attach' ORDER BY position; +SELECT name FROM system.data_skipping_indices WHERE database = currentDatabase() AND table = 't_bf_total_size_attach'; +DROP TABLE t_bf_total_size_attach; +" diff --git a/tests/queries/0_stateless/05427_azure_sas_url_parsing.reference b/tests/queries/0_stateless/05427_azure_sas_url_parsing.reference new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/tests/queries/0_stateless/05427_azure_sas_url_parsing.sql b/tests/queries/0_stateless/05427_azure_sas_url_parsing.sql new file mode 100644 index 0000000000000..329dfc62bcdae --- /dev/null +++ b/tests/queries/0_stateless/05427_azure_sas_url_parsing.sql @@ -0,0 +1,9 @@ +-- Tags: no-fasttest +-- no-fasttest: needs Azure Blob Storage support, which is not built in the fast test. + +-- The (url, sas_token) form of the Azure table functions parses a URL that ends right after the host like the same URL +-- with a trailing slash (both then fail at format detection), and rejects a malformed abfss URL with BAD_ARGUMENTS. +SELECT * FROM azureBlobStorage('https://acc.blob.core.windows.net', 'sv=2022'); -- { serverError CANNOT_DETECT_FORMAT } +SELECT * FROM azureBlobStorage('abfss://foo.net', 'sv=2022'); -- { serverError BAD_ARGUMENTS } +SELECT * FROM azureBlobStorage('abfss://foo.net/c/x.csv', 'sv=2022'); -- { serverError BAD_ARGUMENTS } +SELECT * FROM azureBlobStorage('abfss://c.x@acc.dfs.core.windows.net/p/x.csv', 'sv=2022'); -- { serverError BAD_ARGUMENTS } diff --git a/tests/queries/0_stateless/05427_direct_dictionary_read_truncate_source.reference b/tests/queries/0_stateless/05427_direct_dictionary_read_truncate_source.reference new file mode 100644 index 0000000000000..35ed03607e4a7 --- /dev/null +++ b/tests/queries/0_stateless/05427_direct_dictionary_read_truncate_source.reference @@ -0,0 +1,4 @@ +reader started: 1 +truncate: ok +reader alive after truncate: 1 +0 diff --git a/tests/queries/0_stateless/05427_direct_dictionary_read_truncate_source.sh b/tests/queries/0_stateless/05427_direct_dictionary_read_truncate_source.sh new file mode 100755 index 0000000000000..e5100ffe12526 --- /dev/null +++ b/tests/queries/0_stateless/05427_direct_dictionary_read_truncate_source.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, use-rocksdb + +# Reading a DIRECT dictionary as a table runs its source query inside the reading query. The source query +# must release its table locks only after its processors (and the RocksDB iterator) are destroyed, or a +# TRUNCATE of the source table closes the RocksDB database under the iterator. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -q " + CREATE TABLE rdb (k UInt64, v String) ENGINE = EmbeddedRocksDB PRIMARY KEY k; + INSERT INTO rdb SELECT number, toString(number) FROM numbers(300); + CREATE DICTIONARY dd (k UInt64, v String) PRIMARY KEY k + SOURCE(CLICKHOUSE(DB '${CLICKHOUSE_DATABASE}' TABLE 'rdb')) LAYOUT(DIRECT()); +" + +query_id="${CLICKHOUSE_DATABASE}_reader" + +# ORDER BY ... LIMIT consumes the whole dictionary before producing a row, so the source query has finished +# before the sleeps start; the reader then runs until it is killed below. +$CLICKHOUSE_CLIENT --query_id "${query_id}" -q " + SELECT sum(sleepEachRow(0.2)) FROM (SELECT k FROM dd ORDER BY k LIMIT 1000) + SETTINGS max_block_size = 1, max_threads = 1, function_sleep_max_microseconds_per_block = 100000000 +" > /dev/null 2>&1 & +reader_pid=$! + +started=0 +for _ in {1..200}; do + if [ "$($CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes WHERE query_id = '${query_id}' AND read_rows > 0")" = "1" ]; then + started=1 + break + fi + sleep 0.1 +done +echo "reader started: ${started}" + +$CLICKHOUSE_CLIENT -q "TRUNCATE TABLE rdb SETTINGS lock_acquire_timeout = 10" && echo "truncate: ok" +echo "reader alive after truncate: $($CLICKHOUSE_CLIENT -q "SELECT count() FROM system.processes WHERE query_id = '${query_id}'")" + +$CLICKHOUSE_CLIENT -q "KILL QUERY WHERE query_id = '${query_id}' SYNC" > /dev/null 2>&1 +wait "${reader_pid}" 2>/dev/null + +$CLICKHOUSE_CLIENT -q "SELECT count() FROM rdb" diff --git a/tests/queries/0_stateless/05428_mysqldump_create_without_columns.reference b/tests/queries/0_stateless/05428_mysqldump_create_without_columns.reference new file mode 100644 index 0000000000000..e838531bffe08 --- /dev/null +++ b/tests/queries/0_stateless/05428_mysqldump_create_without_columns.reference @@ -0,0 +1,51 @@ +LIKE `t0` +c1 Nullable(Int64) +c2 Nullable(String) +1 a +2 b +(KEY `k` (`x`)) ENGINE=InnoDB +c1 Nullable(Int64) +c2 Nullable(String) +1 a +2 b +(CONSTRAINT `c` CHECK (`x` > 0)) ENGINE=InnoDB +c1 Nullable(Int64) +c2 Nullable(String) +1 a +2 b +LIKE `t0` +1 a +2 b +1 a +2 b +3 c +1 a +2 b +3 c +(KEY `k` (`x`)) ENGINE=InnoDB +1 a +2 b +1 a +2 b +3 c +1 a +2 b +3 c +(CONSTRAINT `c` CHECK (`x` > 0)) ENGINE=InnoDB +1 a +2 b +1 a +2 b +3 c +1 a +2 b +3 c +(`x` int, `y` varchar(10)) ENGINE=InnoDB +1 a +2 b +1 a +2 b +3 c +1 a +2 b +3 c diff --git a/tests/queries/0_stateless/05428_mysqldump_create_without_columns.sh b/tests/queries/0_stateless/05428_mysqldump_create_without_columns.sh new file mode 100755 index 0000000000000..ab7e6079e8297 --- /dev/null +++ b/tests/queries/0_stateless/05428_mysqldump_create_without_columns.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A `CREATE TABLE` in a MySQL dump without column definitions (`CREATE TABLE ... LIKE`, or only indexes or constraints) +# gives no structure, so the structure is inferred from the data of the `INSERT`. + +function dump() +{ + printf 'CREATE TABLE `t1` %s;\n/*!40101 SET character_set_client = @saved_cs_client */;\nINSERT INTO `t1` VALUES (1,'"'"'a'"'"'),(2,'"'"'b'"'"');\n' "$1" +} + +for create in 'LIKE `t0`' '(KEY `k` (`x`)) ENGINE=InnoDB' '(CONSTRAINT `c` CHECK (`x` > 0)) ENGINE=InnoDB'; do + echo "$create" + dump "$create" | $CLICKHOUSE_LOCAL --input-format MySQLDump --query "DESCRIBE TABLE table" + dump "$create" | $CLICKHOUSE_LOCAL --input-format MySQLDump --query "SELECT * FROM table" +done + +# The `INSERT` immediately following the `CREATE TABLE` must not be skipped. + +function dump_single_insert() +{ + printf 'CREATE TABLE `t1` %s;\nINSERT INTO `t1` VALUES (1,'"'"'a'"'"'),(2,'"'"'b'"'"');\n' "$1" +} + +function dump_multiple_inserts() +{ + printf 'CREATE TABLE `t1` %s;\nINSERT INTO `t1` VALUES (1,'"'"'a'"'"');\nINSERT INTO `t1` VALUES (2,'"'"'b'"'"');\nINSERT INTO `t1` VALUES (3,'"'"'c'"'"');\n' "$1" +} + +for create in 'LIKE `t0`' '(KEY `k` (`x`)) ENGINE=InnoDB' '(CONSTRAINT `c` CHECK (`x` > 0)) ENGINE=InnoDB' '(`x` int, `y` varchar(10)) ENGINE=InnoDB'; do + echo "$create" + dump_single_insert "$create" | $CLICKHOUSE_LOCAL --input-format MySQLDump --query "SELECT * FROM table" + dump_multiple_inserts "$create" | $CLICKHOUSE_LOCAL --input-format MySQLDump --query "SELECT * FROM table" + dump_multiple_inserts "$create" | $CLICKHOUSE_LOCAL --input-format MySQLDump --structure 'x UInt32, y String' --query "SELECT * FROM table" +done diff --git a/tests/queries/0_stateless/05441_ai_functions_privileges.reference b/tests/queries/0_stateless/05441_ai_functions_privileges.reference new file mode 100644 index 0000000000000..5d861b92d7d0a --- /dev/null +++ b/tests/queries/0_stateless/05441_ai_functions_privileges.reference @@ -0,0 +1,4 @@ +--- no grant +--- GRANT aiGenerate +--- GRANT AI FUNCTIONS +--- REVOKE aiClassify diff --git a/tests/queries/0_stateless/05441_ai_functions_privileges.sql.j2 b/tests/queries/0_stateless/05441_ai_functions_privileges.sql.j2 new file mode 100644 index 0000000000000..133a07f3576b4 --- /dev/null +++ b/tests/queries/0_stateless/05441_ai_functions_privileges.sql.j2 @@ -0,0 +1,70 @@ +-- Tags: no-parallel +-- no-parallel: creates global users, a named collection and a function with fixed names + +-- All queries read zero rows, so no outbound request is made. + +{% set text_params = "map('credentials', 'nc_05441', 'model', 'm')" %} +{% set embed_params = "map('credentials', 'nc_05441')" %} +{% set fns = [ + ('aiClassify', "x, ['a'], " ~ text_params), + ('aiEmbed', "x, 'm', " ~ embed_params), + ('aiExtract', "x, 'i', " ~ text_params), + ('aiFilter', "x, 'c', " ~ text_params), + ('aiGenerate', "x, " ~ text_params), + ('aiRedact', "x, ['a'], " ~ text_params), + ('aiSimilarity', "x, x, 'm', " ~ embed_params), + ('aiTranslate', "x, 'fr', " ~ text_params), +] %} + +{% macro check(denied, invoker_denied) %} +{% for f, args in fns %} +EXECUTE AS u_05441 SELECT {{ f }}({{ args }}) FROM (SELECT '' AS x WHERE 0);{% if f in denied %} -- { serverError ACCESS_DENIED }{% endif %} + +{% endfor %} +EXECUTE AS u_05441 SELECT * FROM v_definer; +EXECUTE AS u_05441 SELECT * FROM v_invoker;{% if invoker_denied %} -- { serverError ACCESS_DENIED }{% endif %} + +EXECUTE AS u_05441 SELECT udf_05441(x) FROM (SELECT '' AS x WHERE 0);{% if invoker_denied %} -- { serverError ACCESS_DENIED }{% endif %} + +{% endmacro %} + +DROP USER IF EXISTS u_05441, definer_05441; +DROP NAMED COLLECTION IF EXISTS nc_05441; +DROP FUNCTION IF EXISTS udf_05441; + +CREATE NAMED COLLECTION nc_05441 AS + provider = 'openai', + endpoint = 'http://localhost:1/v1/chat/completions', + api_key = 'fake-key'; + +CREATE USER u_05441, definer_05441 IDENTIFIED WITH no_password; +GRANT NAMED COLLECTION ON nc_05441 TO u_05441, definer_05441; +GRANT SELECT ON * TO u_05441; +GRANT AI FUNCTIONS ON *.* TO definer_05441; + +CREATE VIEW v_definer DEFINER = definer_05441 SQL SECURITY DEFINER + AS SELECT aiGenerate(x, map('credentials', 'nc_05441', 'model', 'm')) FROM (SELECT '' AS x WHERE 0); +CREATE VIEW v_invoker SQL SECURITY INVOKER + AS SELECT aiGenerate(x, map('credentials', 'nc_05441', 'model', 'm')) FROM (SELECT '' AS x WHERE 0); +CREATE FUNCTION udf_05441 AS x -> aiGenerate(x, map('credentials', 'nc_05441', 'model', 'm')); + +SELECT '--- no grant'; +{{ check(fns | map(attribute=0) | list, true) }} + +SELECT '--- GRANT aiGenerate'; +GRANT aiGenerate ON *.* TO u_05441; +{{ check(fns | map(attribute=0) | reject('equalto', 'aiGenerate') | list, false) }} + +SELECT '--- GRANT AI FUNCTIONS'; +GRANT AI FUNCTIONS ON *.* TO u_05441; +{{ check([], false) }} + +SELECT '--- REVOKE aiClassify'; +REVOKE aiClassify ON *.* FROM u_05441; +{{ check(['aiClassify'], false) }} + +DROP VIEW v_definer; +DROP VIEW v_invoker; +DROP FUNCTION udf_05441; +DROP USER u_05441, definer_05441; +DROP NAMED COLLECTION nc_05441; diff --git a/tests/queries/0_stateless/05441_deep_refresh_cursor.reference b/tests/queries/0_stateless/05441_deep_refresh_cursor.reference new file mode 100644 index 0000000000000..2481fe970f6a6 --- /dev/null +++ b/tests/queries/0_stateless/05441_deep_refresh_cursor.reference @@ -0,0 +1 @@ +BAD_ARGUMENTS diff --git a/tests/queries/0_stateless/05441_deep_refresh_cursor.sh b/tests/queries/0_stateless/05441_deep_refresh_cursor.sh new file mode 100755 index 0000000000000..7d3019b268338 --- /dev/null +++ b/tests/queries/0_stateless/05441_deep_refresh_cursor.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# A cursor with an absurdly deep structure in `STREAM CURSOR` of a query must be rejected with an exception instead of exhausting the stack. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +DEEP_PATH="a$(printf '.a%.0s' {1..100000})" +echo "SELECT 1 FROM system.one STREAM CURSOR {'${DEEP_PATH}': 1}" | ${CLICKHOUSE_CURL} -sS "${CLICKHOUSE_URL}&max_query_size=10000000" --data-binary @- | grep -o -m1 'BAD_ARGUMENTS' diff --git a/tests/queries/0_stateless/05441_rawblob_aggregate_function.reference b/tests/queries/0_stateless/05441_rawblob_aggregate_function.reference new file mode 100644 index 0000000000000..98c80be8e58ba --- /dev/null +++ b/tests/queries/0_stateless/05441_rawblob_aggregate_function.reference @@ -0,0 +1,5 @@ +0300000000000000 +c9b15b7fd5143f1d8f677be3dc4daab9 - +c9b15b7fd5143f1d8f677be3dc4daab9 - +9038957ba26f89fe09ee1964aad98576 - +9038957ba26f89fe09ee1964aad98576 - diff --git a/tests/queries/0_stateless/05441_rawblob_aggregate_function.sh b/tests/queries/0_stateless/05441_rawblob_aggregate_function.sh new file mode 100755 index 0000000000000..d6493c3c6808f --- /dev/null +++ b/tests/queries/0_stateless/05441_rawblob_aggregate_function.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + + +# `RawBLOB` must write the serialized state of an aggregate function, the same as its conversion to `String`, +# not the pointer to the state in memory. + +$CLICKHOUSE_CLIENT -q "SELECT sumState(number) FROM numbers(3) FORMAT RawBLOB" | od -An -tx1 | tr -d ' \n' +echo + +$CLICKHOUSE_CLIENT -q "SELECT quantileState(number) FROM numbers(100000) FORMAT RawBLOB" | md5sum +$CLICKHOUSE_CLIENT -q "SELECT CAST(quantileState(number) AS String) FROM numbers(100000) FORMAT RawBLOB" | md5sum + +$CLICKHOUSE_CLIENT -q "SELECT uniqExactState(number) FROM numbers(10) GROUP BY number % 2 ORDER BY number % 2 FORMAT RawBLOB" | md5sum +$CLICKHOUSE_CLIENT -q "SELECT CAST(uniqExactState(number) AS String) FROM numbers(10) GROUP BY number % 2 ORDER BY number % 2 FORMAT RawBLOB" | md5sum diff --git a/tests/queries/0_stateless/05441_restore_into_existing_table_unknown_storage_policy.reference b/tests/queries/0_stateless/05441_restore_into_existing_table_unknown_storage_policy.reference new file mode 100644 index 0000000000000..632d3fe0d3f87 --- /dev/null +++ b/tests/queries/0_stateless/05441_restore_into_existing_table_unknown_storage_policy.reference @@ -0,0 +1,4 @@ +restore into an existing table whose policy this host does not configure +3 6 +creating the table still reports the unknown policy +1 diff --git a/tests/queries/0_stateless/05441_restore_into_existing_table_unknown_storage_policy.sh b/tests/queries/0_stateless/05441_restore_into_existing_table_unknown_storage_policy.sh new file mode 100755 index 0000000000000..615176a9b4774 --- /dev/null +++ b/tests/queries/0_stateless/05441_restore_into_existing_table_unknown_storage_policy.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# `RESTORE` into an existing table must not resolve the storage policy named in the backup: nothing is +# created for it. Two configurations, because only the source defines `extra_policy`. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +WORK_DIR="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}" +trap 'rm -rf "${WORK_DIR}"' EXIT +rm -rf "${WORK_DIR}" +mkdir -p "${WORK_DIR}/backups" + +cat > "${WORK_DIR}/source.xml" < + + + local${WORK_DIR}/extra/ + + +
extra
+
+
+ ${WORK_DIR}/backups + +XML + +cat > "${WORK_DIR}/target.xml" < + ${WORK_DIR}/backups + +XML + +${CLICKHOUSE_LOCAL} --config-file "${WORK_DIR}/source.xml" --path "${WORK_DIR}/source" -m -q " + CREATE DATABASE db ENGINE = Atomic; + CREATE TABLE db.t (x UInt64) ENGINE = MergeTree ORDER BY x SETTINGS storage_policy = 'extra_policy'; + INSERT INTO db.t VALUES (1), (2), (3); + BACKUP TABLE db.t TO File('${WORK_DIR}/backups/b') FORMAT Null; +" + +echo "restore into an existing table whose policy this host does not configure" +${CLICKHOUSE_LOCAL} --config-file "${WORK_DIR}/target.xml" --path "${WORK_DIR}/target" -m -q " + CREATE DATABASE db ENGINE = Atomic; + CREATE TABLE db.t (x UInt64) ENGINE = MergeTree ORDER BY x; + RESTORE TABLE db.t FROM File('${WORK_DIR}/backups/b') SETTINGS allow_different_table_def = 1 FORMAT Null; + SELECT count(), sum(x) FROM db.t; +" + +echo "creating the table still reports the unknown policy" +${CLICKHOUSE_LOCAL} --config-file "${WORK_DIR}/target.xml" --path "${WORK_DIR}/target_create" -m -q " + CREATE DATABASE db ENGINE = Atomic; + RESTORE TABLE db.t FROM File('${WORK_DIR}/backups/b') FORMAT Null; +" 2>&1 | grep -c -m1 "UNKNOWN_POLICY" diff --git a/tests/queries/0_stateless/05441_s3queue_move_destination_access.reference b/tests/queries/0_stateless/05441_s3queue_move_destination_access.reference new file mode 100644 index 0000000000000..e5046dab0a101 --- /dev/null +++ b/tests/queries/0_stateless/05441_s3queue_move_destination_access.reference @@ -0,0 +1,35 @@ +table created +-- ALTER with only ALTER SETTINGS +settings that are not a destination: allowed +-- ALTER with SELECT on the table, but no WRITE on S3 +-- ALTER ON CLUSTER is rejected by the initiator +-- ALTER ON CLUSTER of a table that does not exist on the initiator +-- ALTER with WRITE on S3 limited by a URL filter +-- ALTER with WRITE on S3 +destination with WRITE on S3: allowed +empty destination without WRITE on S3: allowed +-- ALTER with WRITE on S3, restricted by a row policy +empty destination with a row policy: allowed +-- ALTER through an Alias +settings that are not a destination through an alias: allowed +-- CREATE without WRITE on S3 or AZURE +table without a destination: created +-- CREATE AS with a destination, without WRITE on S3 +-- CREATE MATERIALIZED VIEW with a queue inner engine, without WRITE on S3 +-- CREATE AS ON CLUSTER with the legacy entry format, without WRITE on S3 +-- CREATE AS ON CLUSTER with the default entry format, without WRITE on S3 +-- CREATE ON CLUSTER with a queue engine and a destination, without WRITE on S3 +-- ATTACH with a full definition and a destination, without WRITE on S3 +-- RESTORE of a queue with a destination, without WRITE on S3 +0 +0 +0 +-- CREATE AS and CREATE MATERIALIZED VIEW with WRITE on S3 +CREATE AS with WRITE on S3: created +materialized view with WRITE on S3: created +RESTORE with WRITE on S3: restored +RESTORE ON CLUSTER with WRITE on S3: restored +CREATE ON CLUSTER with WRITE on S3: created +CREATE AS ON CLUSTER with WRITE on S3: created +ATTACH with WRITE on S3: attached +RESTORE TEMPORARY with WRITE on S3: restored 1 diff --git a/tests/queries/0_stateless/05441_s3queue_move_destination_access.sh b/tests/queries/0_stateless/05441_s3queue_move_destination_access.sh new file mode 100755 index 0000000000000..5474fde6a4fd2 --- /dev/null +++ b/tests/queries/0_stateless/05441_s3queue_move_destination_access.sh @@ -0,0 +1,374 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, zookeeper, no-replicated-database +# Tag no-fasttest: the `S3Queue` and `AzureQueue` engines are not built into the fast test build. +# Tag zookeeper: `S3Queue` keeps its state in Keeper, and `ON CLUSTER` queries use the distributed DDL queue. +# Tag no-replicated-database: with `access_control_improvements.enable_read_write_grants` disabled (the default), +# a replicated access storage round-trips the user through serialized SQL, which rewrites `WRITE ON S3('')` +# into the old-style whole-source `S3` grant without the filter. + +# The move destination of an `S3Queue` or `AzureQueue` table is the same data flow as +# `INSERT INTO FUNCTION s3('') SELECT * FROM
`: the table reads the processed files with its +# own credentials and writes them to the destination. So setting `after_processing_move_uri`, +# `after_processing_move_connection_string` or `after_processing_move_container` needs `WRITE` on the `S3` (`AZURE`) +# source without a URL filter, because the move writes keys built from the prefix and the source keys, not from the URI. +# `ALTER` needs `SELECT` on the table too. Settings that do not choose a destination need no extra grant. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +db="${CLICKHOUSE_DATABASE}" +user_alter="s3queue_move_alter_${db}" +user_create="s3queue_move_create_${db}" +keeper_path="/s3queue_move_destination_access_${db}" +src="http://localhost:11111/test/${db}_move_src/" +dst="http://localhost:11111/test/${db}_move_dst/" +backup="Disk('backups', '05441_${CLICKHOUSE_TEST_UNIQUE_NAME}')" +backup_tmp="Disk('backups', '05441_${CLICKHOUSE_TEST_UNIQUE_NAME}_tmp')" +keys="after_processing_move_access_key_id = 'clickhouse', after_processing_move_secret_access_key = 'clickhouse'" +attach_uuid=$($CLICKHOUSE_CLIENT -q "SELECT generateUUIDv4()") +azure_connection_string="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;" + +$CLICKHOUSE_CLIENT -q " + DROP TABLE IF EXISTS ${db}.t_alias SYNC; + DROP TABLE IF EXISTS ${db}.t SYNC; + DROP TABLE IF EXISTS ${db}.t_create SYNC; + DROP TABLE IF EXISTS ${db}.t_as SYNC; + DROP TABLE IF EXISTS ${db}.t_as_cluster SYNC; + DROP TABLE IF EXISTS ${db}.t_as_cluster_new SYNC; + DROP TABLE IF EXISTS ${db}.t_create_cluster SYNC; + DROP TABLE IF EXISTS ${db}.t_attach SYNC; + DROP TABLE IF EXISTS ${db}.t_backup SYNC; + DROP TABLE IF EXISTS ${db}.t_backup_cluster SYNC; + DROP TABLE IF EXISTS ${db}.mv SYNC; + DROP ROW POLICY IF EXISTS p_${db} ON ${db}.t; + DROP USER IF EXISTS ${user_alter}, ${user_create}; +" + +$CLICKHOUSE_CLIENT -q " + CREATE TABLE ${db}.t (x UInt8) ENGINE = S3Queue('${src}', 'clickhouse', 'clickhouse', 'CSV') + SETTINGS mode = 'unordered', keeper_path = '${keeper_path}', after_processing = 'keep' +" +echo "table created" + +$CLICKHOUSE_CLIENT -q " + CREATE USER ${user_alter}; + GRANT ALTER SETTINGS ON ${db}.t TO ${user_alter}; + CREATE USER ${user_create}; + GRANT CREATE TABLE ON ${db}.* TO ${user_create}; + GRANT TABLE ENGINE ON S3Queue TO ${user_create}; + GRANT TABLE ENGINE ON AzureQueue TO ${user_create}; +" + +echo "-- ALTER with only ALTER SETTINGS" + +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t MODIFY SETTING after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t MODIFY SETTING s3queue_after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" + +# Changing only the container redirects the move to another container of the same account. +# The check reads only the query, so it fails before the storage is involved, although `t` is an `S3Queue`. +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t MODIFY SETTING after_processing_move_container = 'dst' + -- { serverError ACCESS_DENIED } +" + +# Settings that do not choose a destination need only ALTER SETTINGS. +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t MODIFY SETTING polling_max_timeout_ms = 20000, after_processing_move_prefix = 'prefix_${db}' +" && echo "settings that are not a destination: allowed" + +echo "-- ALTER with SELECT on the table, but no WRITE on S3" + +$CLICKHOUSE_CLIENT -q "GRANT SELECT ON ${db}.t TO ${user_alter}" +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t MODIFY SETTING after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t MODIFY SETTING s3queue_after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" + +# Changing only the container is a destination change that needs `WRITE` on `AZURE`. +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t MODIFY SETTING after_processing_move_container = 'dst' + -- { serverError ACCESS_DENIED } +" + +echo "-- ALTER ON CLUSTER is rejected by the initiator" + +$CLICKHOUSE_CLIENT -q "GRANT CLUSTER ON *.* TO ${user_alter}" +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t ON CLUSTER test_shard_localhost MODIFY SETTING after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" + +echo "-- ALTER ON CLUSTER of a table that does not exist on the initiator" + +# The grants come from the settings in the query, not from the table, so the initiator checks them without the table. +# With the grant, the query reaches the host and fails there. +$CLICKHOUSE_CLIENT -q "GRANT ALTER SETTINGS, SELECT ON ${db}.t_missing TO ${user_alter}" +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t_missing ON CLUSTER test_shard_localhost MODIFY SETTING after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" +$CLICKHOUSE_CLIENT -q "GRANT WRITE ON S3 TO ${user_alter}" +$CLICKHOUSE_CLIENT --user "${user_alter}" --distributed_ddl_output_mode none -q " + ALTER TABLE ${db}.t_missing ON CLUSTER test_shard_localhost MODIFY SETTING after_processing_move_uri = '${dst}', ${keys} + -- { serverError UNKNOWN_TABLE } +" +$CLICKHOUSE_CLIENT -q "REVOKE WRITE ON S3 FROM ${user_alter}" + +echo "-- ALTER with WRITE on S3 limited by a URL filter" + +# The move writes keys built from the prefix and the source keys, so a URL filter cannot authorize them. +$CLICKHOUSE_CLIENT -q "GRANT WRITE ON S3('${dst}.*') TO ${user_alter}" +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t MODIFY SETTING after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" +$CLICKHOUSE_CLIENT -q "REVOKE WRITE ON S3('${dst}.*') FROM ${user_alter}" + +echo "-- ALTER with WRITE on S3" + +$CLICKHOUSE_CLIENT -q "GRANT WRITE ON S3 TO ${user_alter}" +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t MODIFY SETTING after_processing_move_uri = '${dst}', ${keys} +" && echo "destination with WRITE on S3: allowed" + +# Resetting the destination disables the move, so it needs no grant for a destination. +$CLICKHOUSE_CLIENT -q "REVOKE WRITE ON S3 FROM ${user_alter}" +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t MODIFY SETTING after_processing_move_uri = '' +" && echo "empty destination without WRITE on S3: allowed" + +echo "-- ALTER with WRITE on S3, restricted by a row policy" + +# The move copies whole files, so a user whose row policy hides rows of the table cannot set a destination. +$CLICKHOUSE_CLIENT -q " + GRANT WRITE ON S3 TO ${user_alter}; + CREATE ROW POLICY p_${db} ON ${db}.t USING x = 1 TO ${user_alter}; +" +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t MODIFY SETTING after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t ON CLUSTER test_shard_localhost MODIFY SETTING after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" +# Turning the move off needs no check. +$CLICKHOUSE_CLIENT --user "${user_alter}" -q " + ALTER TABLE ${db}.t MODIFY SETTING after_processing_move_uri = '' +" && echo "empty destination with a row policy: allowed" +$CLICKHOUSE_CLIENT -q " + DROP ROW POLICY p_${db} ON ${db}.t; + REVOKE WRITE ON S3 FROM ${user_alter}; +" + +echo "-- ALTER through an Alias" + +# The export checks see only the name in the query, which is the alias, so the change is refused for every user. +$CLICKHOUSE_CLIENT --allow_experimental_alias_table_engine 1 -q "CREATE TABLE ${db}.t_alias ENGINE = Alias('${db}', 't')" +$CLICKHOUSE_CLIENT -q " + ALTER TABLE ${db}.t_alias MODIFY SETTING after_processing_move_uri = '${dst}', ${keys} + -- { serverError NOT_IMPLEMENTED } +" +$CLICKHOUSE_CLIENT -q " + ALTER TABLE ${db}.t_alias MODIFY SETTING polling_max_timeout_ms = 20000 +" && echo "settings that are not a destination through an alias: allowed" +$CLICKHOUSE_CLIENT -q "DROP TABLE ${db}.t_alias SYNC" + +echo "-- CREATE without WRITE on S3 or AZURE" + +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + CREATE TABLE ${db}.t_create (x UInt8) ENGINE = S3Queue('${src}', 'clickhouse', 'clickhouse', 'CSV') + SETTINGS mode = 'unordered', keeper_path = '${keeper_path}_create', after_processing = 'move', + after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + CREATE TABLE ${db}.t_create (x UInt8) ENGINE = S3Queue('${src}', 'clickhouse', 'clickhouse', 'CSV') + SETTINGS mode = 'unordered', keeper_path = '${keeper_path}_create', after_processing = 'move', + s3queue_after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + CREATE TABLE ${db}.t_create (x UInt8) ENGINE = AzureQueue('https://account.blob.core.windows.net', 'container', '*', 'CSV') + SETTINGS mode = 'unordered', keeper_path = '${keeper_path}_create', after_processing = 'move', + after_processing_move_connection_string = '${azure_connection_string}', after_processing_move_container = 'dst' + -- { serverError ACCESS_DENIED } +" + +# The same CREATE without a destination, or with an empty one, needs no grant for it. +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + CREATE TABLE ${db}.t_create (x UInt8) ENGINE = S3Queue('${src}', 'clickhouse', 'clickhouse', 'CSV') + SETTINGS mode = 'unordered', keeper_path = '${keeper_path}_create', after_processing = 'keep', + after_processing_move_uri = '' +" && echo "table without a destination: created" + +echo "-- CREATE AS with a destination, without WRITE on S3" + +# The engine and the settings of `AS` are inherited after the query is parsed, so the check has to see the final definition. +# `SELECT` is required because the definition of the source has secret parts. +$CLICKHOUSE_CLIENT -q "GRANT SHOW COLUMNS, SELECT ON ${db}.t TO ${user_create}" +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + CREATE TABLE ${db}.t_as AS ${db}.t + SETTINGS keeper_path = '${keeper_path}_as', after_processing = 'move', after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" + +echo "-- CREATE MATERIALIZED VIEW with a queue inner engine, without WRITE on S3" + +# The engine is an inner engine of the view, and the inner table is created as an internal query, which checks nothing. +$CLICKHOUSE_CLIENT -q "GRANT CREATE VIEW ON ${db}.* TO ${user_create}" +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + CREATE MATERIALIZED VIEW ${db}.mv ENGINE = S3Queue('${src}', 'clickhouse', 'clickhouse', 'CSV') + SETTINGS mode = 'unordered', keeper_path = '${keeper_path}_mv', after_processing = 'move', + after_processing_move_uri = '${dst}', ${keys} + AS SELECT 1::UInt8 AS x + -- { serverError ACCESS_DENIED } +" + +echo "-- CREATE AS ON CLUSTER with the legacy entry format, without WRITE on S3" + +# This format ships the query as written and the workers resolve `AS` with no user. +$CLICKHOUSE_CLIENT -q "GRANT CLUSTER ON *.* TO ${user_create}" +$CLICKHOUSE_CLIENT --user "${user_create}" --distributed_ddl_entry_format_version 2 -q " + CREATE TABLE ${db}.t_as_cluster ON CLUSTER test_shard_localhost AS ${db}.t + SETTINGS keeper_path = '${keeper_path}_as_cluster', after_processing = 'move', after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" + +echo "-- CREATE AS ON CLUSTER with the default entry format, without WRITE on S3" + +# This format builds the definition on the initiator. +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + CREATE TABLE ${db}.t_as_cluster_new ON CLUSTER test_shard_localhost AS ${db}.t + SETTINGS keeper_path = '${keeper_path}_as_cluster_new', after_processing = 'move', after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" + +echo "-- CREATE ON CLUSTER with a queue engine and a destination, without WRITE on S3" + +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + CREATE TABLE ${db}.t_create_cluster ON CLUSTER test_shard_localhost (x UInt8) ENGINE = S3Queue('${src}', 'clickhouse', 'clickhouse', 'CSV') + SETTINGS mode = 'unordered', keeper_path = '${keeper_path}_create_cluster', after_processing = 'move', + after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" + +echo "-- ATTACH with a full definition and a destination, without WRITE on S3" + +# A full definition in `ATTACH` is a new definition, like `CREATE`. An `Atomic` database needs a `UUID` for it. +# The server warns that a full definition in `ATTACH` is not recommended, so only errors are sent to the client. +$CLICKHOUSE_CLIENT --user "${user_create}" --send_logs_level=error -q " + ATTACH TABLE ${db}.t_attach UUID '${attach_uuid}' (x UInt8) ENGINE = S3Queue('${src}', 'clickhouse', 'clickhouse', 'CSV') + SETTINGS mode = 'unordered', keeper_path = '${keeper_path}_attach', after_processing = 'move', + after_processing_move_uri = '${dst}', ${keys} + -- { serverError ACCESS_DENIED } +" + +echo "-- RESTORE of a queue with a destination, without WRITE on S3" + +# The restored definition comes from the backup, and the table is created by an internal query, +# so the restore has to authorize the destination before it creates anything. +$CLICKHOUSE_CLIENT -q " + CREATE TABLE ${db}.t_backup (x UInt8) ENGINE = S3Queue('${src}', 'clickhouse', 'clickhouse', 'CSV') + SETTINGS mode = 'unordered', keeper_path = '${keeper_path}_backup', after_processing = 'move', + after_processing_move_uri = '${dst}', ${keys} +" +$CLICKHOUSE_CLIENT -q "BACKUP TABLE ${db}.t_backup TO ${backup} FORMAT Null" +$CLICKHOUSE_CLIENT -q "DROP TABLE ${db}.t_backup SYNC" +$CLICKHOUSE_CLIENT -q "GRANT READ ON DISK TO ${user_create}" +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + RESTORE TABLE ${db}.t_backup FROM ${backup} SETTINGS structure_only = 1 FORMAT Null + -- { serverError ACCESS_DENIED } +" +$CLICKHOUSE_CLIENT -q "SELECT count() FROM system.tables WHERE database = '${db}' AND name = 't_backup'" + +# The hosts create the tables with no user, so the initiator must authorize the destination. +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + RESTORE TABLE ${db}.t_backup AS ${db}.t_backup_cluster ON CLUSTER test_shard_localhost FROM ${backup} + SETTINGS structure_only = 1 FORMAT Null + -- { serverError ACCESS_DENIED } +" +$CLICKHOUSE_CLIENT -q "SELECT count() FROM system.tables WHERE database = '${db}' AND name = 't_backup_cluster'" + +# A temporary table keeps its engine, even with `restore_replace_external_engines_to_null`, so the same check applies. +# Temporary tables live in a session, so the table is created and backed up in one client. +$CLICKHOUSE_CLIENT -m -q " + CREATE TEMPORARY TABLE t_tmp_backup (x UInt8) ENGINE = S3Queue('${src}', 'clickhouse', 'clickhouse', 'CSV') + SETTINGS mode = 'unordered', keeper_path = '${keeper_path}_tmp_backup', after_processing = 'move', + after_processing_move_uri = '${dst}', ${keys}; + BACKUP TEMPORARY TABLE t_tmp_backup TO ${backup_tmp} FORMAT Null; +" +$CLICKHOUSE_CLIENT -q "GRANT CREATE TEMPORARY TABLE ON *.* TO ${user_create}" +$CLICKHOUSE_CLIENT --user "${user_create}" -m -q " + RESTORE TEMPORARY TABLE t_tmp_backup FROM ${backup_tmp} SETTINGS structure_only = 1 FORMAT Null; -- { serverError ACCESS_DENIED } + RESTORE TEMPORARY TABLE t_tmp_backup FROM ${backup_tmp} + SETTINGS structure_only = 1, restore_replace_external_engines_to_null = 1 FORMAT Null; -- { serverError ACCESS_DENIED } + SELECT count() FROM system.tables WHERE is_temporary AND name = 't_tmp_backup'; +" + +echo "-- CREATE AS and CREATE MATERIALIZED VIEW with WRITE on S3" + +# The same queries succeed with the grant, so `WRITE` on `S3` was the only missing one. +$CLICKHOUSE_CLIENT -q "GRANT WRITE ON S3 TO ${user_create}" +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + CREATE TABLE ${db}.t_as AS ${db}.t + SETTINGS keeper_path = '${keeper_path}_as', after_processing = 'move', after_processing_move_uri = '${dst}', ${keys} +" && echo "CREATE AS with WRITE on S3: created" +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + CREATE MATERIALIZED VIEW ${db}.mv ENGINE = S3Queue('${src}', 'clickhouse', 'clickhouse', 'CSV') + SETTINGS mode = 'unordered', keeper_path = '${keeper_path}_mv', after_processing = 'move', + after_processing_move_uri = '${dst}', ${keys} + AS SELECT 1::UInt8 AS x +" && echo "materialized view with WRITE on S3: created" +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + RESTORE TABLE ${db}.t_backup FROM ${backup} SETTINGS structure_only = 1 FORMAT Null +" && echo "RESTORE with WRITE on S3: restored" +$CLICKHOUSE_CLIENT --user "${user_create}" -q " + RESTORE TABLE ${db}.t_backup AS ${db}.t_backup_cluster ON CLUSTER test_shard_localhost FROM ${backup} + SETTINGS structure_only = 1 FORMAT Null +" && echo "RESTORE ON CLUSTER with WRITE on S3: restored" +$CLICKHOUSE_CLIENT --user "${user_create}" --distributed_ddl_output_mode none -q " + CREATE TABLE ${db}.t_create_cluster ON CLUSTER test_shard_localhost (x UInt8) ENGINE = S3Queue('${src}', 'clickhouse', 'clickhouse', 'CSV') + SETTINGS mode = 'unordered', keeper_path = '${keeper_path}_create_cluster', after_processing = 'move', + after_processing_move_uri = '${dst}', ${keys} +" && echo "CREATE ON CLUSTER with WRITE on S3: created" +$CLICKHOUSE_CLIENT --user "${user_create}" --distributed_ddl_output_mode none -q " + CREATE TABLE ${db}.t_as_cluster_new ON CLUSTER test_shard_localhost AS ${db}.t + SETTINGS keeper_path = '${keeper_path}_as_cluster_new', after_processing = 'move', after_processing_move_uri = '${dst}', ${keys} +" && echo "CREATE AS ON CLUSTER with WRITE on S3: created" +$CLICKHOUSE_CLIENT --user "${user_create}" --send_logs_level=error -q " + ATTACH TABLE ${db}.t_attach UUID '${attach_uuid}' (x UInt8) ENGINE = S3Queue('${src}', 'clickhouse', 'clickhouse', 'CSV') + SETTINGS mode = 'unordered', keeper_path = '${keeper_path}_attach', after_processing = 'move', + after_processing_move_uri = '${dst}', ${keys} +" && echo "ATTACH with WRITE on S3: attached" +$CLICKHOUSE_CLIENT --user "${user_create}" -m -q " + RESTORE TEMPORARY TABLE t_tmp_backup FROM ${backup_tmp} SETTINGS structure_only = 1 FORMAT Null; + SELECT 'RESTORE TEMPORARY with WRITE on S3: restored', count() FROM system.tables WHERE is_temporary AND name = 't_tmp_backup'; +" + +$CLICKHOUSE_CLIENT -q " + DROP TABLE IF EXISTS ${db}.t_alias SYNC; + DROP TABLE IF EXISTS ${db}.t SYNC; + DROP TABLE IF EXISTS ${db}.t_create SYNC; + DROP TABLE IF EXISTS ${db}.t_as SYNC; + DROP TABLE IF EXISTS ${db}.t_as_cluster SYNC; + DROP TABLE IF EXISTS ${db}.t_as_cluster_new SYNC; + DROP TABLE IF EXISTS ${db}.t_create_cluster SYNC; + DROP TABLE IF EXISTS ${db}.t_attach SYNC; + DROP TABLE IF EXISTS ${db}.t_backup SYNC; + DROP TABLE IF EXISTS ${db}.t_backup_cluster SYNC; + DROP TABLE IF EXISTS ${db}.mv SYNC; + DROP ROW POLICY IF EXISTS p_${db} ON ${db}.t; + DROP USER IF EXISTS ${user_alter}, ${user_create}; +" diff --git a/tests/queries/0_stateless/05442_ai_functions_privileges_stored_expressions.reference b/tests/queries/0_stateless/05442_ai_functions_privileges_stored_expressions.reference new file mode 100644 index 0000000000000..573541ac9702d --- /dev/null +++ b/tests/queries/0_stateless/05442_ai_functions_privileges_stored_expressions.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/05442_ai_functions_privileges_stored_expressions.sql b/tests/queries/0_stateless/05442_ai_functions_privileges_stored_expressions.sql new file mode 100644 index 0000000000000..bbebd33778dc9 --- /dev/null +++ b/tests/queries/0_stateless/05442_ai_functions_privileges_stored_expressions.sql @@ -0,0 +1,56 @@ +-- Tags: no-parallel, no-replicated-database +-- no-parallel: creates a global user with a fixed name +-- no-replicated-database: DDL in a `Replicated` database is executed with a context without the user, so it is not checked + +-- Expressions stored in a table definition are checked against the user who defines the table. +-- AI functions are executed only in queries run by a user, so they are rejected there even with the privilege. + +DROP USER IF EXISTS u_05442; +CREATE USER u_05442 IDENTIFIED WITH no_password; +GRANT CREATE TABLE, INSERT, SELECT, ALTER ON * TO u_05442; +GRANT TABLE ENGINE ON MergeTree TO u_05442; + +EXECUTE AS u_05442 CREATE TABLE t1 (x String, d DateTime) ENGINE = MergeTree ORDER BY x TTL d + INTERVAL 1 DAY DELETE WHERE aiFilter(x, 'c'); -- { serverError ACCESS_DENIED } +EXECUTE AS u_05442 CREATE TABLE t2 (x String) ENGINE = MergeTree ORDER BY length(aiEmbed(x, 'm')); -- { serverError ACCESS_DENIED } +EXECUTE AS u_05442 CREATE TABLE t3 (x String) ENGINE = MergeTree PARTITION BY length(aiEmbed(x, 'm')) ORDER BY x; -- { serverError ACCESS_DENIED } +EXECUTE AS u_05442 CREATE TABLE t4 (x String, INDEX i length(aiEmbed(x, 'm')) TYPE minmax) ENGINE = MergeTree ORDER BY x; -- { serverError ACCESS_DENIED } +EXECUTE AS u_05442 CREATE TABLE t5 (x String, PROJECTION p (SELECT aiGenerate(x) ORDER BY x)) ENGINE = MergeTree ORDER BY x; -- { serverError ACCESS_DENIED } +EXECUTE AS u_05442 CREATE TABLE t8 (k String, d DateTime, x String) ENGINE = MergeTree ORDER BY (k, d) TTL d + INTERVAL 1 DAY GROUP BY k SET x = any(aiGenerate(x)); -- { serverError ACCESS_DENIED } + +EXECUTE AS u_05442 CREATE TABLE t7 (x String, d DateTime) ENGINE = MergeTree ORDER BY x; +EXECUTE AS u_05442 ALTER TABLE t7 MODIFY TTL d + INTERVAL 1 DAY DELETE WHERE aiFilter(x, 'c'); -- { serverError ACCESS_DENIED } +EXECUTE AS u_05442 ALTER TABLE t7 ADD INDEX i length(aiEmbed(x, 'm')) TYPE minmax; -- { serverError ACCESS_DENIED } +EXECUTE AS u_05442 ALTER TABLE t7 ADD PROJECTION p (SELECT aiGenerate(x) ORDER BY x); -- { serverError ACCESS_DENIED } +EXECUTE AS u_05442 ALTER TABLE t7 ADD COLUMN y String, MODIFY ORDER BY (x, length(aiEmbed(y, 'm'))); -- { serverError ACCESS_DENIED } + +-- Applies to all function privileges, e.g. `demangle`. +SET allow_introspection_functions = 1; +EXECUTE AS u_05442 CREATE TABLE t10 (x String) ENGINE = MergeTree ORDER BY demangle(x); -- { serverError ACCESS_DENIED } +GRANT demangle ON *.* TO u_05442; +EXECUTE AS u_05442 CREATE TABLE t10 (x String) ENGINE = MergeTree ORDER BY demangle(x); +DROP TABLE t10; + +GRANT AI FUNCTIONS ON *.* TO u_05442; +EXECUTE AS u_05442 CREATE TABLE t1 (x String, d DateTime) ENGINE = MergeTree ORDER BY x TTL d + INTERVAL 1 DAY DELETE WHERE aiFilter(x, 'c'); -- { serverError NOT_IMPLEMENTED } +EXECUTE AS u_05442 CREATE TABLE t2 (x String) ENGINE = MergeTree ORDER BY length(aiEmbed(x, 'm')); -- { serverError NOT_IMPLEMENTED } +EXECUTE AS u_05442 CREATE TABLE t8 (k String, d DateTime, x String) ENGINE = MergeTree ORDER BY (k, d) TTL d + INTERVAL 1 DAY GROUP BY k SET x = any(aiGenerate(x)); -- { serverError NOT_IMPLEMENTED } +EXECUTE AS u_05442 ALTER TABLE t7 MODIFY TTL d + INTERVAL 1 DAY DELETE WHERE aiFilter(x, 'c'); -- { serverError NOT_IMPLEMENTED } +EXECUTE AS u_05442 ALTER TABLE t7 ADD PROJECTION p (SELECT aiGenerate(x) ORDER BY x); -- { serverError NOT_IMPLEMENTED } +EXECUTE AS u_05442 CREATE TABLE t3 (x String) ENGINE = MergeTree PARTITION BY length(aiEmbed(x, 'm')) ORDER BY x; -- { serverError NOT_IMPLEMENTED } +EXECUTE AS u_05442 CREATE TABLE t4 (x String, INDEX i length(aiEmbed(x, 'm')) TYPE minmax) ENGINE = MergeTree ORDER BY x; -- { serverError NOT_IMPLEMENTED } +EXECUTE AS u_05442 ALTER TABLE t7 ADD INDEX i length(aiEmbed(x, 'm')) TYPE minmax; -- { serverError NOT_IMPLEMENTED } +EXECUTE AS u_05442 ALTER TABLE t7 ADD COLUMN y String, MODIFY ORDER BY (x, length(aiEmbed(y, 'm'))); -- { serverError NOT_IMPLEMENTED } + +-- Defaults computed in the background: of a column with a TTL, and added by `ALTER` for existing parts. +EXECUTE AS u_05442 CREATE TABLE t9 (x String, d DateTime, y String DEFAULT aiGenerate(x) TTL d + INTERVAL 1 DAY) ENGINE = MergeTree ORDER BY x; -- { serverError NOT_IMPLEMENTED } +EXECUTE AS u_05442 CREATE TABLE t9 (x String, d DateTime, y String MATERIALIZED aiGenerate(x) TTL d + INTERVAL 1 DAY) ENGINE = MergeTree ORDER BY x; -- { serverError NOT_IMPLEMENTED } +EXECUTE AS u_05442 ALTER TABLE t7 ADD COLUMN y String DEFAULT aiGenerate(x); -- { serverError NOT_IMPLEMENTED } +EXECUTE AS u_05442 ALTER TABLE t7 ADD COLUMN y String MATERIALIZED aiGenerate(x); -- { serverError NOT_IMPLEMENTED } +EXECUTE AS u_05442 ALTER TABLE t7 ADD COLUMN y String, MODIFY COLUMN x String DEFAULT aiGenerate(y); -- { serverError NOT_IMPLEMENTED } +EXECUTE AS u_05442 ALTER TABLE t7 ADD COLUMN y String ALIAS aiGenerate(x); +EXECUTE AS u_05442 CREATE TABLE t9 (x String, y String DEFAULT aiGenerate(x)) ENGINE = MergeTree ORDER BY x; +DROP TABLE t9; +SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = 't7'; + +DROP TABLE t7; +DROP USER u_05442; diff --git a/tests/queries/0_stateless/05443_arrow_null_as_default_non_nullable_field.reference b/tests/queries/0_stateless/05443_arrow_null_as_default_non_nullable_field.reference new file mode 100644 index 0000000000000..8b590edc9e2bb --- /dev/null +++ b/tests/queries/0_stateless/05443_arrow_null_as_default_non_nullable_field.reference @@ -0,0 +1,33 @@ +Arrow +1 a x [1,0] (1) +3 c x [] (0) +42 default none [] (0) +CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN +1 a x Nullable(String) +3 c x Nullable(String) +\N \N \N Nullable(String) +[('x'),(NULL),('x')] +CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN +ArrowStream +1 a x [1,0] (1) +3 c x [] (0) +42 default none [] (0) +CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN +1 a x Nullable(String) +3 c x Nullable(String) +\N \N \N Nullable(String) +[('x'),(NULL),('x')] +CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN +Batches +[0] +[1,2] +[7] +[1,2] +[7] +[NULL] +CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN +Rejected +declared non-nullable but its FieldNode reports 1 nulls +declared non-nullable but its FieldNode reports 1 nulls +declared non-nullable but its FieldNode reports 1 nulls +declared non-nullable but its FieldNode reports 1 nulls diff --git a/tests/queries/0_stateless/05443_arrow_null_as_default_non_nullable_field.sh b/tests/queries/0_stateless/05443_arrow_null_as_default_non_nullable_field.sh new file mode 100755 index 0000000000000..cf8c4ebf9b88b --- /dev/null +++ b/tests/queries/0_stateless/05443_arrow_null_as_default_non_nullable_field.sh @@ -0,0 +1,118 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: needs pyarrow + +# Arrow fields declared non-nullable that still contain nulls are read as nulls: with +# `input_format_null_as_default = 1` they become defaults (including column `DEFAULT` expressions) in non-nullable +# columns; with 0 they are rejected for non-nullable columns and kept as NULL for `Nullable` ones. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +TMP_DIR="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}" +mkdir -p "$TMP_DIR" +trap 'rm -rf "$TMP_DIR"' EXIT + +python3 - "$TMP_DIR" <<'PYEOF' +import struct +import sys +import pyarrow as pa +import pyarrow.ipc as ipc + +out = sys.argv[1] +columns = { + "i": pa.array([1, None, 3], pa.int64()), + "s": pa.array(["a", None, "c"]), + "d": pa.array(["x", None, "x"]).dictionary_encode(), + "arr": pa.array([[1, None], [], None], pa.list_(pa.field("item", pa.int64(), nullable=False))), + "tup": pa.array([{"x": 1}, {"x": None}, None], pa.struct([pa.field("x", pa.int64(), nullable=False)])), +} +schema = pa.schema([pa.field(name, arr.type, nullable=False) for name, arr in columns.items()]) +table = pa.Table.from_arrays(list(columns.values()), schema=schema) +with ipc.new_file(f"{out}/data.arrow", schema) as w: + w.write_table(table) +with ipc.new_stream(f"{out}/data.arrows", schema) as w: + w.write_table(table) + +item = pa.field("item", pa.int64(), nullable=False) +list_type = pa.list_(item) +lists = pa.array([[None], [7]], list_type) + +# The nulls first appear in the second batch. +schema = pa.schema([pa.field("l", list_type, nullable=False)]) +with ipc.new_stream(f"{out}/batches.arrows", schema) as w: + w.write_batch(pa.record_batch([pa.array([[1, 2]], list_type)], schema=schema)) + w.write_batch(pa.record_batch([lists], schema=schema)) + +union_type = pa.sparse_union([pa.field("l", list_type, nullable=False)]) +union = pa.Array.from_buffers(union_type, 2, [None, pa.array([0, 0], pa.int8()).buffers()[1]], children=[lists]) +dict_of_lists = pa.DictionaryArray.from_arrays(pa.array([1, 0], pa.int32()), lists) +for name, arr in (("union", union), ("dictionary", dict_of_lists)): + table = pa.Table.from_arrays([arr], names=["c"]) + with ipc.new_stream(f"{out}/{name}.arrows", table.schema) as w: + w.write_table(table) + +# pyarrow refuses nulls in map entries and keys, so write a list of structs and retag the field as a map. +def table_pos(b, table, slot): + vtable = table - struct.unpack_from("&1 \ + | grep -o -m1 "CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN" + $CLICKHOUSE_LOCAL -q "SELECT i, s, d, toTypeName(d) FROM file('$file', $format, 'i Nullable(Int64), s Nullable(String), d Nullable(String)') ORDER BY i SETTINGS input_format_null_as_default = 0" + $CLICKHOUSE_LOCAL -q "SELECT groupArray(tuple(d)) FROM file('$file', $format, 'd LowCardinality(Nullable(String))') SETTINGS input_format_null_as_default = 0" + $CLICKHOUSE_LOCAL -q "SELECT d FROM file('$file', $format, 'd String') SETTINGS input_format_null_as_default = 0" 2>&1 \ + | grep -o -m1 "CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN" +done + +echo "Batches" +$CLICKHOUSE_LOCAL -q "SELECT l FROM file('$TMP_DIR/batches.arrows', ArrowStream) ORDER BY l" +$CLICKHOUSE_LOCAL -q "SELECT l FROM file('$TMP_DIR/batches.arrows', ArrowStream, 'l Array(Nullable(Int64))') ORDER BY l SETTINGS input_format_null_as_default = 0" +$CLICKHOUSE_LOCAL -q "SELECT l FROM file('$TMP_DIR/batches.arrows', ArrowStream, 'l Array(Int64)') SETTINGS input_format_null_as_default = 0" 2>&1 \ + | grep -o -m1 "CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN" + +# Nulls in non-nullable children of unions and of dictionary values, and in map entries and keys, are rejected. +echo "Rejected" +for name in union dictionary map_entries map_keys; do + $CLICKHOUSE_LOCAL -q "SELECT c FROM file('$TMP_DIR/$name.arrows', ArrowStream)" 2>&1 | grep -o -m1 "declared non-nullable but its FieldNode reports [0-9]* nulls" +done + +$CLICKHOUSE_CLIENT -q "DROP TABLE t_arrow_null_as_default" diff --git a/tests/queries/0_stateless/05443_async_insert_mv_dedup_empty_partition_block.reference b/tests/queries/0_stateless/05443_async_insert_mv_dedup_empty_partition_block.reference new file mode 100644 index 0000000000000..1ee31d8b55207 --- /dev/null +++ b/tests/queries/0_stateless/05443_async_insert_mv_dedup_empty_partition_block.reference @@ -0,0 +1,3 @@ +src 0 1 +src 1 2 +tgt 0 1 diff --git a/tests/queries/0_stateless/05443_async_insert_mv_dedup_empty_partition_block.sql b/tests/queries/0_stateless/05443_async_insert_mv_dedup_empty_partition_block.sql new file mode 100644 index 0000000000000..71c125e9ef63e --- /dev/null +++ b/tests/queries/0_stateless/05443_async_insert_mv_dedup_empty_partition_block.sql @@ -0,0 +1,39 @@ +-- Two async inserts with the same `insert_deduplication_token` but different data are coalesced +-- into one batch. In the materialized view target, the rows of the two tokens land in different +-- partitions. For the partition of the second (duplicate) token, self-deduplication removes all +-- rows, and the recalculated block for that partition is empty. The empty block used to reach +-- `MergeTreeDataWriter::mergeBlock` and trigger the "Required source after the first merge is +-- not 0" logical error with `optimize_on_insert` enabled and a `ReplacingMergeTree` target. + +DROP TABLE IF EXISTS mv_dedup_empty; +DROP TABLE IF EXISTS tgt_dedup_empty; +DROP TABLE IF EXISTS src_dedup_empty; + +CREATE TABLE src_dedup_empty (p UInt64, v UInt64) ENGINE = MergeTree ORDER BY v; +CREATE TABLE tgt_dedup_empty (p UInt64, v UInt64) ENGINE = ReplacingMergeTree PARTITION BY p ORDER BY v + SETTINGS non_replicated_deduplication_window = 100; +CREATE MATERIALIZED VIEW mv_dedup_empty TO tgt_dedup_empty AS SELECT p, v FROM src_dedup_empty; + +SET async_insert = 1; +SET wait_for_async_insert = 0; +SET async_insert_deduplicate = 1; +SET deduplicate_blocks_in_dependent_materialized_views = 1; +SET optimize_on_insert = 1; +-- Prevent an automatic flush between the two inserts; the flush below is explicit. +SET async_insert_busy_timeout_min_ms = 600000; +SET async_insert_busy_timeout_max_ms = 600000; +SET async_insert_use_adaptive_busy_timeout = 0; + +INSERT INTO src_dedup_empty SETTINGS insert_deduplication_token = 'tok' VALUES (0, 1); +INSERT INTO src_dedup_empty SETTINGS insert_deduplication_token = 'tok' VALUES (1, 2); + +SYSTEM FLUSH ASYNC INSERT QUEUE src_dedup_empty; + +-- The source table has no deduplication window, so it keeps both rows. In the target table the +-- second insert is a duplicate of the first by token, so only the first row survives. +SELECT 'src', p, v FROM src_dedup_empty ORDER BY v; +SELECT 'tgt', p, v FROM tgt_dedup_empty ORDER BY v; + +DROP TABLE mv_dedup_empty; +DROP TABLE tgt_dedup_empty; +DROP TABLE src_dedup_empty; diff --git a/tests/queries/0_stateless/05443_join_reorder_converted_outer_join_type_change.reference b/tests/queries/0_stateless/05443_join_reorder_converted_outer_join_type_change.reference new file mode 100644 index 0000000000000..9aab6c3c4ee44 --- /dev/null +++ b/tests/queries/0_stateless/05443_join_reorder_converted_outer_join_type_change.reference @@ -0,0 +1,12 @@ +a1 x1 x1 x Nullable(String) +a3 x1 x1 x Nullable(String) +a1 x1 x1 x +a3 x1 x1 x +a1 x1 x1 x +a3 x1 x1 x +a1 x1 x1 x +a3 x1 x1 x +a1 x1 x1 x +a3 x1 x1 x +a1 x1 x1 x +a3 x1 x1 x diff --git a/tests/queries/0_stateless/05443_join_reorder_converted_outer_join_type_change.sql b/tests/queries/0_stateless/05443_join_reorder_converted_outer_join_type_change.sql new file mode 100644 index 0000000000000..50b4202ef35a0 --- /dev/null +++ b/tests/queries/0_stateless/05443_join_reorder_converted_outer_join_type_change.sql @@ -0,0 +1,71 @@ +-- https://github.com/ClickHouse/ClickHouse/issues/127331: with join_use_nulls = 1, an outer join converted to INNER and +-- reordered so that its null-supplying side is joined first failed with "Cannot fold actions for projection". + +DROP TABLE IF EXISTS a; +DROP TABLE IF EXISTS b; +DROP TABLE IF EXISTS c; +DROP TABLE IF EXISTS d; + +CREATE TABLE a (k String, t String, v String) ENGINE = MergeTree ORDER BY k; +CREATE TABLE b (k String, t String, x String) ENGINE = MergeTree ORDER BY k; +CREATE TABLE c (x String, t String, v String) ENGINE = MergeTree ORDER BY x; +INSERT INTO a VALUES ('k1', 't1', 'a1'), ('k2', 't1', 'a2'), ('k3', 't2', 'a3'); +INSERT INTO b VALUES ('k1', 't1', 'x1'), ('k2', 't1', 'x2'), ('k3', 't2', 'x1'); +INSERT INTO c VALUES ('x1', 't1', 'x'), ('x2', 't1', 'y'), ('x1', 't2', 'x'); + +SELECT a.v, b.x, c.x, c.v, toTypeName(c.x) +FROM a JOIN b ON a.k = b.k AND a.t = b.t LEFT JOIN c ON b.x = c.x AND b.t = c.t +WHERE c.v = 'x' ORDER BY ALL +SETTINGS join_use_nulls = 1, query_plan_convert_outer_join_to_inner_join = 1, enable_join_transitive_predicates = 1, + query_plan_optimize_join_order_limit = 10, use_hash_table_stats_for_join_reordering = 0, + query_plan_optimize_join_order_algorithm = 'greedy', query_plan_optimize_join_order_randomize = 0; + +SELECT a.v, b.x, c.x, c.v +FROM a JOIN b ON a.k = b.k AND a.t = b.t LEFT JOIN c ON b.x = c.x AND b.t = c.t +WHERE c.v = 'x' ORDER BY ALL +SETTINGS join_use_nulls = 1, query_plan_convert_outer_join_to_inner_join = 1, enable_join_transitive_predicates = 1, + query_plan_optimize_join_order_limit = 10, use_hash_table_stats_for_join_reordering = 0, + query_plan_optimize_join_order_algorithm = 'dpsize', query_plan_optimize_join_order_randomize = 0; + +SELECT a.v, b.x, c.x, c.v +FROM a JOIN b ON a.k = b.k AND a.t = b.t LEFT JOIN c ON b.x = c.x AND b.t = c.t +WHERE c.v = 'x' ORDER BY ALL +SETTINGS join_use_nulls = 1, query_plan_convert_outer_join_to_inner_join = 1, enable_join_transitive_predicates = 1, + query_plan_optimize_join_order_limit = 10, use_hash_table_stats_for_join_reordering = 0, + query_plan_optimize_join_order_algorithm = 'dphyp', query_plan_optimize_join_order_randomize = 13; + +SELECT a.v, b.x, c.x, c.v +FROM a JOIN b ON a.k = b.k AND a.t = b.t LEFT JOIN c ON b.x = c.x AND b.t = c.t +WHERE c.v = 'x' ORDER BY ALL +SETTINGS join_use_nulls = 1, query_plan_convert_outer_join_to_inner_join = 1, enable_join_transitive_predicates = 1, + query_plan_optimize_join_order_limit = 10, use_hash_table_stats_for_join_reordering = 0, + query_plan_optimize_join_order_algorithm = 'dpsub', query_plan_optimize_join_order_randomize = 13; + +SELECT ab.v, ab.x, c.x, c.v +FROM c RIGHT JOIN (SELECT a.v AS v, a.t AS t, b.x AS x FROM a JOIN b ON a.k = b.k AND a.t = b.t) AS ab ON ab.x = c.x AND ab.t = c.t +WHERE c.v = 'x' ORDER BY ALL +SETTINGS join_use_nulls = 1, query_plan_convert_outer_join_to_inner_join = 1, enable_join_transitive_predicates = 1, + query_plan_optimize_join_order_limit = 10, use_hash_table_stats_for_join_reordering = 0, + query_plan_optimize_join_order_algorithm = 'greedy', query_plan_optimize_join_order_randomize = 0; + +DROP TABLE a; +DROP TABLE c; +CREATE TABLE a (k String, t String, v String) ENGINE = MergeTree ORDER BY k; +CREATE TABLE c (x String, t String, v String) ENGINE = MergeTree ORDER BY x; +CREATE TABLE d (w String) ENGINE = MergeTree ORDER BY w; +INSERT INTO a VALUES ('x1', 't1', 'a1'), ('x2', 't1', 'a2'), ('x1', 't2', 'a3'); +INSERT INTO c VALUES ('x1', 't1', 'x'), ('x2', 't1', 'y'); +INSERT INTO d VALUES ('x1'), ('x2'); + +-- `c` is joined with `a` before `b` through the equivalence `a.k = d.w = b.x = c.x`. +SELECT a.v, b.x, c.x, c.v +FROM a JOIN d ON a.k = d.w JOIN b ON d.w = b.x AND a.t = b.t LEFT JOIN c ON b.x = c.x +WHERE c.v = 'x' ORDER BY ALL +SETTINGS join_use_nulls = 1, query_plan_convert_outer_join_to_inner_join = 1, enable_join_transitive_predicates = 1, + query_plan_optimize_join_order_limit = 10, use_hash_table_stats_for_join_reordering = 0, + query_plan_optimize_join_order_algorithm = 'dpsize', query_plan_optimize_join_order_randomize = 82; + +DROP TABLE a; +DROP TABLE b; +DROP TABLE c; +DROP TABLE d; diff --git a/tests/queries/0_stateless/05443_parallel_distributed_insert_select_source_access.reference b/tests/queries/0_stateless/05443_parallel_distributed_insert_select_source_access.reference new file mode 100644 index 0000000000000..0d095d7059548 --- /dev/null +++ b/tests/queries/0_stateless/05443_parallel_distributed_insert_select_source_access.reference @@ -0,0 +1,9 @@ +parallel_distributed_insert_select=0 ACCESS_DENIED +parallel_distributed_insert_select=1 ACCESS_DENIED +parallel_distributed_insert_select=2 ACCESS_DENIED +0 +1 +row policy on source ILLEGAL_PREWHERE +additional_table_filters 1 +remote database proxy ACCESS_DENIED +2 diff --git a/tests/queries/0_stateless/05443_parallel_distributed_insert_select_source_access.sh b/tests/queries/0_stateless/05443_parallel_distributed_insert_select_source_access.sh new file mode 100755 index 0000000000000..fe40401030ee3 --- /dev/null +++ b/tests/queries/0_stateless/05443_parallel_distributed_insert_select_source_access.sh @@ -0,0 +1,87 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +user="user_$CLICKHOUSE_DATABASE" + +$CLICKHOUSE_CLIENT -q " + CREATE TABLE secret_local (id UInt64, secret String) ENGINE = MergeTree ORDER BY id; + CREATE TABLE copy_local (id UInt64, secret String) ENGINE = MergeTree ORDER BY id; + INSERT INTO secret_local VALUES (1, 'secret'); + + CREATE TABLE src_dist (id UInt64, secret String) ENGINE = Distributed(test_shard_localhost, currentDatabase(), secret_local, id); + CREATE TABLE dst_dist (id UInt64, secret String) ENGINE = Distributed(test_shard_localhost, currentDatabase(), copy_local, id); + + DROP USER IF EXISTS $user; + CREATE USER $user IDENTIFIED WITH no_password; + GRANT SELECT, INSERT ON $CLICKHOUSE_DATABASE.dst_dist TO $user; +" + +# The cluster has no interserver secret, so the shard runs the forwarded query under the credentials +# from the cluster configuration rather than as the caller. +client=($CLICKHOUSE_CLIENT --user "$user" --prefer_localhost_replica 0) + +for setting in 0 1 2; do + echo -n "parallel_distributed_insert_select=$setting " + "${client[@]}" --parallel_distributed_insert_select "$setting" -q "INSERT INTO dst_dist SELECT * FROM src_dist" |& + grep -o -m1 'ACCESS_DENIED' || echo 'NOT DENIED' +done + +# Read the destination directly: resolving `dst_dist` also authorizes `copy_local` once the shard +# query is planned on the initiator, which depends on the distributed plan settings. +$CLICKHOUSE_CLIENT -q "SELECT count() FROM copy_local" + +# With the grant in place the optimized path must still work. +$CLICKHOUSE_CLIENT -q "GRANT SELECT ON $CLICKHOUSE_DATABASE.src_dist TO $user" +"${client[@]}" --parallel_distributed_insert_select 2 -q "INSERT INTO dst_dist SELECT * FROM src_dist" +$CLICKHOUSE_CLIENT -q "SELECT count() FROM copy_local" + +# A row policy on the source table cannot be applied by the optimized rewrite (it ships the query +# to the shards, where the policy on the Distributed table never applies), so it must fall back to +# the regular path, which rejects a row policy on a Distributed table. +# +# The two fallback cases below pin serialize_query_plan = 0: with plan serialization the initiator +# plans the read of the underlying table itself and requires the caller's own SELECT on it, which +# these cases do not grant. +$CLICKHOUSE_CLIENT -q "CREATE ROW POLICY policy_05443 ON src_dist USING id != 1 TO $user" +echo -n "row policy on source " +"${client[@]}" --parallel_distributed_insert_select 2 --enable_analyzer 1 --enable_parallel_replicas 0 \ + --serialize_query_plan 0 \ + -q "INSERT INTO dst_dist SELECT * FROM src_dist" |& grep -o -m1 'ILLEGAL_PREWHERE' || echo 'NOT REJECTED' +$CLICKHOUSE_CLIENT -q "DROP ROW POLICY policy_05443 ON src_dist" + +# additional_table_filters is keyed by the source table name, which the optimized rewrite would +# drop, so it must fall back to the regular path, which applies the filter: the only row has +# id = 1, nothing is copied and the count stays the same. +echo -n "additional_table_filters " +"${client[@]}" --parallel_distributed_insert_select 2 --enable_parallel_replicas 0 --serialize_query_plan 0 \ + --additional_table_filters "{'src_dist' : 'id != 1'}" -q "INSERT INTO dst_dist SELECT * FROM src_dist" +$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH DISTRIBUTED dst_dist; SELECT count() FROM copy_local" + +# A source from a `Remote` database whose shard points back to this server reads the underlying +# local table, so the caller's own SELECT on it is required (the stored credentials are only for +# genuinely remote servers), like the regular read path enforces. +remote_db="${CLICKHOUSE_DATABASE}_remote" +$CLICKHOUSE_CLIENT -q " + CREATE DATABASE $remote_db ENGINE = Remote('127.0.0.1', '$CLICKHOUSE_DATABASE', 'default', ''); + GRANT SHOW COLUMNS ON $CLICKHOUSE_DATABASE.* TO $user; + GRANT SELECT, INSERT ON $remote_db.* TO $user; + GRANT INSERT ON $CLICKHOUSE_DATABASE.copy_local TO $user; +" +echo -n "remote database proxy " +"${client[@]}" --parallel_distributed_insert_select 2 \ + -q "INSERT INTO $remote_db.copy_local SELECT * FROM $remote_db.secret_local" |& + grep -o -m1 'ACCESS_DENIED' || echo 'NOT DENIED' + +# ...and works once SELECT on the underlying table is granted. +$CLICKHOUSE_CLIENT -q "GRANT SELECT ON $CLICKHOUSE_DATABASE.secret_local TO $user" +"${client[@]}" --parallel_distributed_insert_select 2 \ + -q "INSERT INTO $remote_db.copy_local SELECT * FROM $remote_db.secret_local" +$CLICKHOUSE_CLIENT -q "SELECT count() FROM copy_local" + +$CLICKHOUSE_CLIENT -q " + DROP DATABASE $remote_db; + DROP USER $user; +" diff --git a/tests/queries/0_stateless/05443_parquet_int96_date_time_overflow_behavior.reference b/tests/queries/0_stateless/05443_parquet_int96_date_time_overflow_behavior.reference new file mode 100644 index 0000000000000..f104f80c58911 --- /dev/null +++ b/tests/queries/0_stateless/05443_parquet_int96_date_time_overflow_behavior.reference @@ -0,0 +1,13 @@ +VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE +VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE +0 1677-09-21 00:12:43.145224192 Nullable(DateTime64(9, \'UTC\')) +1 1800-01-01 12:30:00.000000000 Nullable(DateTime64(9, \'UTC\')) +2 2020-05-17 10:20:30.123456000 Nullable(DateTime64(9, \'UTC\')) +3 2262-04-11 23:47:16.854775807 Nullable(DateTime64(9, \'UTC\')) +4 \N Nullable(DateTime64(9, \'UTC\')) +0 1677-09-21 00:12:43.145225 Nullable(DateTime64(6, \'UTC\')) +1 1800-01-01 12:30:00.000000 Nullable(DateTime64(6, \'UTC\')) +2 2020-05-17 10:20:30.123456 Nullable(DateTime64(6, \'UTC\')) +3 2262-04-11 23:47:16.854775 Nullable(DateTime64(6, \'UTC\')) +4 \N Nullable(DateTime64(6, \'UTC\')) +2 diff --git a/tests/queries/0_stateless/05443_parquet_int96_date_time_overflow_behavior.sh b/tests/queries/0_stateless/05443_parquet_int96_date_time_overflow_behavior.sh new file mode 100755 index 0000000000000..184859b094300 --- /dev/null +++ b/tests/queries/0_stateless/05443_parquet_int96_date_time_overflow_behavior.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: the fast test build has no Parquet support + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# INT96 timestamps 0001-01-01, 1800-01-01 12:30, 2020-05-17 10:20:30.123456, 9999-12-31 23:59:59 and NULL, written by +# pyarrow with `use_deprecated_int96_timestamps=True`. The first and the fourth do not fit into Int64 nanoseconds. +DATA_FILE="${CUR_DIR}/data_parquet/05443_int96_out_of_range.parquet" + +${CLICKHOUSE_LOCAL} --query "SELECT * FROM file('${DATA_FILE}', Parquet) ORDER BY id" 2>&1 | grep -o -m1 'VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE' +${CLICKHOUSE_LOCAL} --query "SELECT * FROM file('${DATA_FILE}', Parquet) ORDER BY id SETTINGS date_time_overflow_behavior = 'throw'" 2>&1 | grep -o -m1 'VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE' + +${CLICKHOUSE_LOCAL} --query " + SELECT id, ts, toTypeName(ts) FROM file('${DATA_FILE}', Parquet) ORDER BY id SETTINGS date_time_overflow_behavior = 'saturate'; + SELECT id, ts, toTypeName(ts) FROM file('${DATA_FILE}', Parquet, 'id Int32, ts Nullable(DateTime64(6, \'UTC\'))') ORDER BY id SETTINGS date_time_overflow_behavior = 'saturate'; + SELECT count() FROM file('${DATA_FILE}', Parquet) WHERE ts > '2000-01-01' SETTINGS date_time_overflow_behavior = 'saturate';" diff --git a/tests/queries/0_stateless/05444_ai_functions_no_user.reference b/tests/queries/0_stateless/05444_ai_functions_no_user.reference new file mode 100644 index 0000000000000..aaf616077c107 --- /dev/null +++ b/tests/queries/0_stateless/05444_ai_functions_no_user.reference @@ -0,0 +1,9 @@ +1 +0 +a z +0 +a +b +a +b +a diff --git a/tests/queries/0_stateless/05444_ai_functions_no_user.sql b/tests/queries/0_stateless/05444_ai_functions_no_user.sql new file mode 100644 index 0000000000000..d4d5a104c115b --- /dev/null +++ b/tests/queries/0_stateless/05444_ai_functions_no_user.sql @@ -0,0 +1,98 @@ +-- Tags: no-parallel +-- no-parallel: creates a global named collection and a global user with fixed names + +-- AI functions are not executed with a context without a user (`Buffer` flushes, mutations). +-- No request is made: the check runs before the provider is contacted. + +DROP NAMED COLLECTION IF EXISTS nc_05444; +CREATE NAMED COLLECTION nc_05444 AS + provider = 'openai', + endpoint = 'http://localhost:1/v1/chat/completions', + api_key = 'fake-key'; + +CREATE TABLE t (x String, CONSTRAINT c CHECK aiFilter(x, 'c', map('credentials', 'nc_05444', 'model', 'm'))) ENGINE = MergeTree ORDER BY x; +CREATE TABLE b (x String) ENGINE = Buffer(currentDatabase(), t, 1, 1, 1, 1, 1, 1, 1); +INSERT INTO b VALUES ('a'); -- { serverError ACCESS_DENIED } +CREATE TABLE t_embed (x String, CONSTRAINT c CHECK length(aiEmbed(x, 'm', map('credentials', 'nc_05444'))) > 0) ENGINE = MergeTree ORDER BY x; +CREATE TABLE b_embed (x String) ENGINE = Buffer(currentDatabase(), t_embed, 1, 1, 1, 1, 1, 1, 1); +INSERT INTO b_embed VALUES ('a'); -- { serverError ACCESS_DENIED } +CREATE TABLE t_similarity (x String, CONSTRAINT c CHECK aiSimilarity(x, x, 'm', map('credentials', 'nc_05444')) > 0) ENGINE = MergeTree ORDER BY x; +CREATE TABLE b_similarity (x String) ENGINE = Buffer(currentDatabase(), t_similarity, 1, 1, 1, 1, 1, 1, 1); +INSERT INTO b_similarity VALUES ('a'); -- { serverError ACCESS_DENIED } +-- The failure is reported in `system.warnings`. +SELECT count() > 0 FROM system.warnings WHERE message LIKE 'A background operation%AI function%'; + +-- The user is checked before the credentials: an inaccessible and an absent collection fail with the same error. +DROP USER IF EXISTS u_05444; +CREATE USER u_05444 IDENTIFIED WITH no_password; +GRANT CREATE TABLE, INSERT, SELECT ON * TO u_05444; +GRANT TABLE ENGINE ON MergeTree, TABLE ENGINE ON Buffer TO u_05444; +EXECUTE AS u_05444 CREATE TABLE t_hidden (x String, CONSTRAINT c CHECK aiFilter(x, 'c', map('credentials', 'nc_05444', 'model', 'm'))) ENGINE = MergeTree ORDER BY x; +EXECUTE AS u_05444 CREATE TABLE b_hidden (x String) ENGINE = Buffer(currentDatabase(), t_hidden, 1, 1, 1, 1, 1, 1, 1); +EXECUTE AS u_05444 INSERT INTO b_hidden SELECT 'a'; -- { serverError ACCESS_DENIED } +EXECUTE AS u_05444 CREATE TABLE t_absent (x String, CONSTRAINT c CHECK aiFilter(x, 'c', map('credentials', 'absent_05444', 'model', 'm'))) ENGINE = MergeTree ORDER BY x; +EXECUTE AS u_05444 CREATE TABLE b_absent (x String) ENGINE = Buffer(currentDatabase(), t_absent, 1, 1, 1, 1, 1, 1, 1); +EXECUTE AS u_05444 INSERT INTO b_absent SELECT 'a'; -- { serverError ACCESS_DENIED } + +-- Mutations are executed in the background, so AI functions in them are rejected when submitted. +CREATE TABLE t_update (x String, y String) ENGINE = MergeTree ORDER BY x; +INSERT INTO t_update VALUES ('a', ''); +ALTER TABLE t_update UPDATE y = aiGenerate(x, map('credentials', 'nc_05444', 'model', 'm')) WHERE 1 SETTINGS mutations_sync = 0, allow_nondeterministic_mutations = 1; -- { serverError NOT_IMPLEMENTED } +ALTER TABLE t_update UPDATE y = 'e' WHERE length(aiEmbed(x, 'm', map('credentials', 'nc_05444'))) > 0 SETTINGS mutations_sync = 0; -- { serverError NOT_IMPLEMENTED } +ALTER TABLE t_update DELETE WHERE aiFilter(x, 'c', map('credentials', 'nc_05444', 'model', 'm')) SETTINGS mutations_sync = 0, allow_nondeterministic_mutations = 1; -- { serverError NOT_IMPLEMENTED } +DROP FUNCTION IF EXISTS udf_05444; +CREATE FUNCTION udf_05444 AS x -> aiGenerate(x, map('credentials', 'nc_05444', 'model', 'm')); +ALTER TABLE t_update UPDATE y = udf_05444(x) WHERE 1 SETTINGS mutations_sync = 0, allow_nondeterministic_mutations = 1; -- { serverError NOT_IMPLEMENTED } +DROP FUNCTION udf_05444; +SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = 't_update'; +ALTER TABLE t_update UPDATE y = 'z' WHERE 1 SETTINGS mutations_sync = 1; +SELECT x, y FROM t_update; + +-- `DEFAULT` and `MATERIALIZED` columns are evaluated by the insert, which has a user. +-- The request fails (unreachable endpoint) and returns an empty string instead of throwing. +SET ai_function_throw_on_error = 0, ai_function_max_retries = 0; + +-- Updating a column recomputes a `MATERIALIZED` column that depends on it, which is rejected the same way. +CREATE TABLE t_materialized (k UInt8, x String, m String MATERIALIZED aiGenerate(x, map('credentials', 'nc_05444', 'model', 'm'))) ENGINE = MergeTree ORDER BY k; +INSERT INTO t_materialized (k, x) VALUES (1, 'a'); +ALTER TABLE t_materialized UPDATE x = 'b' WHERE 1 SETTINGS mutations_sync = 0, allow_nondeterministic_mutations = 1; -- { serverError NOT_IMPLEMENTED } +SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = 't_materialized'; + +CREATE TABLE t_default +( + x String, + d String DEFAULT aiGenerate(x, map('credentials', 'nc_05444', 'model', 'm')), + m String MATERIALIZED aiGenerate(x, map('credentials', 'nc_05444', 'model', 'm')) +) +ENGINE = MergeTree ORDER BY x; +INSERT INTO t_default (x) VALUES ('a'); +INSERT INTO t_default (x) VALUES ('b'); +SELECT x, d, m FROM t_default ORDER BY x; + +-- A merge does not evaluate them again. +OPTIMIZE TABLE t_default FINAL; +SELECT x, d, m FROM t_default ORDER BY x; + +-- A lightweight update is evaluated by the query, which has a user. +CREATE TABLE t_lwu (x String, y String) ENGINE = MergeTree ORDER BY x + SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1; +INSERT INTO t_lwu VALUES ('a', 'old'); +UPDATE t_lwu SET y = aiGenerate(x, map('credentials', 'nc_05444', 'model', 'm')) WHERE 1 SETTINGS enable_lightweight_update = 1; +SELECT x, y FROM t_lwu; + +DROP TABLE b; +DROP TABLE t; +DROP TABLE b_embed; +DROP TABLE t_embed; +DROP TABLE b_similarity; +DROP TABLE t_similarity; +DROP TABLE t_update; +DROP TABLE t_default; +DROP TABLE t_lwu; +DROP TABLE t_materialized; +DROP TABLE b_hidden; +DROP TABLE t_hidden; +DROP TABLE b_absent; +DROP TABLE t_absent; +DROP USER u_05444; +DROP NAMED COLLECTION nc_05444; diff --git a/tests/queries/0_stateless/05444_parquet_int96_lower_boundary.reference b/tests/queries/0_stateless/05444_parquet_int96_lower_boundary.reference new file mode 100644 index 0000000000000..4d2fdda745397 --- /dev/null +++ b/tests/queries/0_stateless/05444_parquet_int96_lower_boundary.reference @@ -0,0 +1,2 @@ +0 1677-09-21 12:00:00.000000000 -9223329600000000000 +0 1677-09-21 12:00:00.000000000 -9223329600000000000 diff --git a/tests/queries/0_stateless/05444_parquet_int96_lower_boundary.sh b/tests/queries/0_stateless/05444_parquet_int96_lower_boundary.sh new file mode 100755 index 0000000000000..39e02424f38a4 --- /dev/null +++ b/tests/queries/0_stateless/05444_parquet_int96_lower_boundary.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: the fast test build has no Parquet support + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A single INT96 timestamp 1677-09-21 12:00:00 (julian day 2333836, time of day 43200000000000 ns), written by pyarrow +# with `use_deprecated_int96_timestamps=True`. Midnight of that day does not fit into Int64 nanoseconds, but the full value +# -9223329600000000000 does, so it must be read exactly instead of being saturated to the Int64 minimum or rejected. +DATA_FILE="${CUR_DIR}/data_parquet/05444_int96_lower_boundary.parquet" + +${CLICKHOUSE_LOCAL} --query " + SELECT id, ts, toUnixTimestamp64Nano(ts) FROM file('${DATA_FILE}', Parquet) SETTINGS date_time_overflow_behavior = 'saturate'; + SELECT id, ts, toUnixTimestamp64Nano(ts) FROM file('${DATA_FILE}', Parquet) SETTINGS date_time_overflow_behavior = 'throw';" diff --git a/tests/queries/0_stateless/05445_distributed_async_insert_batch_per_user.reference b/tests/queries/0_stateless/05445_distributed_async_insert_batch_per_user.reference new file mode 100644 index 0000000000000..aa20b4a608862 --- /dev/null +++ b/tests/queries/0_stateless/05445_distributed_async_insert_batch_per_user.reference @@ -0,0 +1 @@ +ACCESS_DENIED diff --git a/tests/queries/0_stateless/05445_distributed_async_insert_batch_per_user.sh b/tests/queries/0_stateless/05445_distributed_async_insert_batch_per_user.sh new file mode 100755 index 0000000000000..71282f1c84378 --- /dev/null +++ b/tests/queries/0_stateless/05445_distributed_async_insert_batch_per_user.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Tags: distributed, no-fasttest +# Tag no-fasttest: the interserver `` requires SSL, which the fast test build does not have. + +# Queued `Distributed` inserts of different users must not be sent in one batch: the batch is sent with +# the `ClientInfo` of its first file, and over a cluster with an interserver `` the shard +# authenticates the whole batch as that file's `initial_user`. Before the fix, rows of a user without +# the remote `INSERT` grant were delivered under the identity of a user that had it. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Flushing the rejected file (also when the table is dropped) logs the error on the server. +CLICKHOUSE_CLIENT=$(echo "$CLICKHOUSE_CLIENT" | sed "s/--send_logs_level=${CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL:-none}/--send_logs_level=none/g") + +WRITER="writer_${CLICKHOUSE_DATABASE}" +TENANT="tenant_${CLICKHOUSE_DATABASE}" + +${CLICKHOUSE_CLIENT} -m -q " + DROP USER IF EXISTS ${WRITER}, ${TENANT}; + CREATE TABLE target (key Int) ENGINE = MergeTree ORDER BY key; + CREATE TABLE dist (key Int) ENGINE = Distributed(test_cluster_interserver_secret, currentDatabase(), target, 0) + SETTINGS background_insert_batch = 1; + SYSTEM STOP DISTRIBUTED SENDS dist; + CREATE USER ${WRITER} IDENTIFIED WITH no_password; + CREATE USER ${TENANT} IDENTIFIED WITH no_password; + GRANT INSERT ON ${CLICKHOUSE_DATABASE}.dist TO ${WRITER}, ${TENANT}; + GRANT INSERT ON ${CLICKHOUSE_DATABASE}.target TO ${WRITER}; +" + +${CLICKHOUSE_CLIENT} --user "${WRITER}" --distributed_foreground_insert 0 --prefer_localhost_replica 0 -q "INSERT INTO dist VALUES (1)" +${CLICKHOUSE_CLIENT} --user "${TENANT}" --distributed_foreground_insert 0 --prefer_localhost_replica 0 -q "INSERT INTO dist VALUES (2)" + +${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH DISTRIBUTED dist" 2>&1 | grep -o -m1 "ACCESS_DENIED" +${CLICKHOUSE_CLIENT} -q "SELECT key FROM target WHERE key = 2" + +${CLICKHOUSE_CLIENT} -m -q " + DROP TABLE dist; + DROP USER ${WRITER}, ${TENANT}; +" diff --git a/tests/queries/0_stateless/05445_webterminal_concurrent_sessions_input.reference b/tests/queries/0_stateless/05445_webterminal_concurrent_sessions_input.reference new file mode 100644 index 0000000000000..e6c10cc871478 --- /dev/null +++ b/tests/queries/0_stateless/05445_webterminal_concurrent_sessions_input.reference @@ -0,0 +1,2 @@ +session 0: ok +session 1: ok diff --git a/tests/queries/0_stateless/05445_webterminal_concurrent_sessions_input.sh b/tests/queries/0_stateless/05445_webterminal_concurrent_sessions_input.sh new file mode 100755 index 0000000000000..71c889eb14248 --- /dev/null +++ b/tests/queries/0_stateless/05445_webterminal_concurrent_sessions_input.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Concurrent web terminal sessions run in the same server process and must not share +# the state of the line editor: the input of each session is sent byte by byte, +# interleaved with the other session, with multi-byte UTF-8 characters split +# between the sessions, and each session must decode its own query correctly. + +python3 - "${CLICKHOUSE_HOST}" "${CLICKHOUSE_PORT_HTTP}" <<'PYEOF' +import base64 +import json +import os +import socket +import struct +import sys +import time + +host, port = sys.argv[1], int(sys.argv[2]) + + +def recv_exact(sock, n): + buf = b"" + while len(buf) < n: + chunk = sock.recv(n - len(buf)) + if not chunk: + raise EOFError("connection closed") + buf += chunk + return buf + + +def send_frame(sock, opcode, payload): + mask = os.urandom(4) + masked = bytes(b ^ mask[i % 4] for i, b in enumerate(payload)) + if len(payload) < 126: + header = bytes([0x80 | opcode, 0x80 | len(payload)]) + else: + header = bytes([0x80 | opcode, 0x80 | 126]) + struct.pack(">H", len(payload)) + sock.sendall(header + mask + masked) + + +def read_frame(sock): + header = recv_exact(sock, 2) + opcode = header[0] & 0x0F + length = header[1] & 0x7F + if length == 126: + length = struct.unpack(">H", recv_exact(sock, 2))[0] + elif length == 127: + length = struct.unpack(">Q", recv_exact(sock, 8))[0] + return opcode, recv_exact(sock, length) + + +def read_until(sock, needle, timeout): + """Accumulate the terminal output until it contains `needle`.""" + deadline = time.monotonic() + timeout + output = b"" + while needle not in output: + if time.monotonic() > deadline: + raise TimeoutError(f"{needle!r} not found in {output[-500:]!r}") + opcode, payload = read_frame(sock) + if opcode == 0x08: + raise EOFError(f"closed by the server: {payload!r}") + if opcode == 0x09: + send_frame(sock, 0x0A, payload) + elif opcode == 0x02: + output += payload + return output + + +def connect(): + sock = socket.create_connection((host, port), timeout=60) + key = base64.b64encode(os.urandom(16)).decode() + request = ( + f"GET /webterminal HTTP/1.1\r\n" + f"Host: {host}:{port}\r\n" + f"Origin: http://{host}:{port}\r\n" + f"Upgrade: websocket\r\n" + f"Connection: Upgrade\r\n" + f"Sec-WebSocket-Key: {key}\r\n" + f"Sec-WebSocket-Version: 13\r\n\r\n" + ) + sock.sendall(request.encode()) + response = b"" + while b"\r\n\r\n" not in response: + response += recv_exact(sock, 1) + assert response.startswith(b"HTTP/1.1 101"), response + send_frame(sock, 0x01, json.dumps({"type": "auth", "user": "default", "password": ""}).encode()) + read_until(sock, b":) ", 60) + return sock + + +# Different multi-byte characters in each session, so that a mixed-up decoder state +# produces a different string. +texts = ["привет", "€😀日本"] +sessions = [connect() for _ in texts] +queries = [f"SELECT hex('{text}') AS h_{i}\r".encode() for i, text in enumerate(texts)] + +for pos in range(max(len(q) for q in queries)): + for sock, query in zip(sessions, queries): + if pos < len(query): + send_frame(sock, 0x02, query[pos:pos + 1]) + +for i, (sock, text) in enumerate(zip(sessions, texts)): + expected = text.encode().hex().upper().encode() + read_until(sock, expected, 60) + print(f"session {i}: ok") + sock.close() +PYEOF diff --git a/tests/queries/0_stateless/05446_insert_select_cluster_function_column_defaults.reference b/tests/queries/0_stateless/05446_insert_select_cluster_function_column_defaults.reference new file mode 100644 index 0000000000000..3f826b9aceefd --- /dev/null +++ b/tests/queries/0_stateless/05446_insert_select_cluster_function_column_defaults.reference @@ -0,0 +1,18 @@ +--- s3Cluster, column list --- +1 2000-01-01 7 unknown 10 +2 2001-02-03 5 x 6 +--- s3Cluster, asterisk --- +1 2000-01-01 7 unknown 10 +2 2001-02-03 5 x 6 +--- urlCluster --- +1 2000-01-01 7 unknown 10 +2 2001-02-03 5 x 6 +--- s3 with parallel replicas --- +1 2000-01-01 7 unknown 10 +2 2001-02-03 5 x 6 +--- s3Cluster, explicit structure --- +1 1.50 +--- s3Cluster, explicit structure as key-value argument --- +1 1.50 +--- ClickPipes shape, DEFAULT now64(3) with TTL --- +3 1 diff --git a/tests/queries/0_stateless/05446_insert_select_cluster_function_column_defaults.sh b/tests/queries/0_stateless/05446_insert_select_cluster_function_column_defaults.sh new file mode 100755 index 0000000000000..676f062b7e386 --- /dev/null +++ b/tests/queries/0_stateless/05446_insert_select_cluster_function_column_defaults.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, zookeeper +# Tag no-fasttest: needs MinIO and a Replicated table + +# A distributed `INSERT ... SELECT FROM ` must fill the fields missing in the files with the +# `DEFAULT` expressions of the destination table, the same as a plain `INSERT ... SELECT FROM s3(...)`. +# +# The forwarded query used to carry the structure of the source storage, which has names and types only, in +# place of `'auto'`. The nodes then did not take the structure from the insertion table and filled the missing +# fields with the default values of their types (0, '', 1970-01-01). +# +# All nodes of the clusters below are this server. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +set -eu + +S3_DIR="http://localhost:11111/test/${CLICKHOUSE_DATABASE}/05446" + +# The first file has none of the defaulted fields, the second one has all of them. +$CLICKHOUSE_CLIENT -q " + INSERT INTO FUNCTION s3('${S3_DIR}/part_1.jsonl', 'JSONEachRow', 'id UInt32') VALUES (1)" +$CLICKHOUSE_CLIENT -q " + INSERT INTO FUNCTION s3('${S3_DIR}/part_2.jsonl', 'JSONEachRow', 'id UInt32, d Date, q UInt32, c String, e UInt32') + VALUES (2, '2001-02-03', 5, 'x', 6)" + +COLUMNS="id UInt32, d Date DEFAULT '2000-01-01', q UInt32 DEFAULT 7, c String DEFAULT 'unknown', e UInt32 DEFAULT id * 10" + +$CLICKHOUSE_CLIENT -q " + DROP TABLE IF EXISTS dst_05446 SYNC; + CREATE TABLE dst_05446 (${COLUMNS}) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/dst_05446', 'r1') ORDER BY id; +" + +S3_GLOB="${S3_DIR}/part_{1,2}.jsonl" +CLUSTER="test_cluster_one_shard_three_replicas_localhost" +PARALLEL_REPLICAS="enable_parallel_replicas = 1, automatic_parallel_replicas_mode = 0, max_parallel_replicas = 3, + cluster_for_parallel_replicas = '${CLUSTER}', parallel_replicas_for_cluster_engines = 1" + +function check() +{ + local table=$1 + echo "--- $2 ---" + $CLICKHOUSE_CLIENT -q "SELECT * FROM ${table} ORDER BY id" + $CLICKHOUSE_CLIENT -q "TRUNCATE TABLE ${table}" +} + +# The shape of the queries of object storage ClickPipes. +$CLICKHOUSE_CLIENT -q " + INSERT INTO dst_05446 (id, d, q, c, e) SELECT id, d, q, c, e FROM s3Cluster('${CLUSTER}', '${S3_GLOB}', 'JSONEachRow') + SETTINGS parallel_distributed_insert_select = 2" +check dst_05446 "s3Cluster, column list" + +$CLICKHOUSE_CLIENT -q " + INSERT INTO dst_05446 SELECT * FROM s3Cluster('${CLUSTER}', '${S3_GLOB}', 'JSONEachRow') + SETTINGS parallel_distributed_insert_select = 2" +check dst_05446 "s3Cluster, asterisk" + +$CLICKHOUSE_CLIENT -q " + INSERT INTO dst_05446 SELECT * FROM urlCluster('${CLUSTER}', '${S3_GLOB}', 'JSONEachRow') + SETTINGS parallel_distributed_insert_select = 2" +check dst_05446 "urlCluster" + +$CLICKHOUSE_CLIENT -q " + INSERT INTO dst_05446 SELECT * FROM s3('${S3_GLOB}', 'JSONEachRow') + SETTINGS parallel_distributed_insert_select = 2, ${PARALLEL_REPLICAS}" +check dst_05446 "s3 with parallel replicas" + +# An explicit structure must reach the nodes as written, positional or `key = value`. The `SELECT` list has an +# expression, so the nodes cannot take the structure from the insertion table: with `'auto'` they would infer +# `v` as `Float64` and insert '1.5' instead of '1.50'. +$CLICKHOUSE_CLIENT -q "INSERT INTO FUNCTION s3('${S3_DIR}/typed.jsonl', 'LineAsString', 'l String') VALUES ('{\"id\":1,\"v\":1.50}')" +$CLICKHOUSE_CLIENT -q " + DROP TABLE IF EXISTS typed_05446 SYNC; + CREATE TABLE typed_05446 (id UInt32, v String) ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/typed_05446', 'r1') ORDER BY id; +" + +$CLICKHOUSE_CLIENT -q " + INSERT INTO typed_05446 SELECT id, toString(v) FROM s3Cluster('${CLUSTER}', '${S3_DIR}/typed.jsonl', 'JSONEachRow', 'id UInt32, v String') + SETTINGS parallel_distributed_insert_select = 2" +check typed_05446 "s3Cluster, explicit structure" + +$CLICKHOUSE_CLIENT -q " + INSERT INTO typed_05446 SELECT id, toString(v) FROM s3Cluster('${CLUSTER}', '${S3_DIR}/typed.jsonl', format = 'JSONEachRow', structure = 'id UInt32, v String') + SETTINGS parallel_distributed_insert_select = 2" +check typed_05446 "s3Cluster, explicit structure as key-value argument" + +# The shape of the queries of object storage ClickPipes, with a `DEFAULT now64(3)` column missing in the files and +# a TTL on it: with the default values of the type the rows got `1970-01-01` and the TTL deleted them. +$CLICKHOUSE_CLIENT -q "INSERT INTO FUNCTION s3('${S3_DIR}/clickpipes_a.json', 'LineAsString', 'l String') VALUES ('{\"document\":\"d1\"}'), ('{\"document\":\"d2\"}')" +$CLICKHOUSE_CLIENT -q "INSERT INTO FUNCTION s3('${S3_DIR}/clickpipes_b.json', 'LineAsString', 'l String') VALUES ('{\"document\":\"d3\"}')" +$CLICKHOUSE_CLIENT -q " + DROP TABLE IF EXISTS clickpipes_05446 SYNC; + CREATE TABLE clickpipes_05446 (document String, _synced_at DateTime64(3, 'UTC') DEFAULT now64(3)) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{database}/clickpipes_05446', 'r1') ORDER BY document + TTL toDateTime(_synced_at) + INTERVAL 30 DAY; +" + +$CLICKHOUSE_CLIENT --param_url="${S3_DIR}/clickpipes_{a,b}.json" --param_format=JSONEachRow -q " + INSERT INTO \`${CLICKHOUSE_DATABASE}\`.\`clickpipes_05446\` (\`_synced_at\`, \`document\`) + SELECT \`_synced_at\` AS \`_synced_at\`, \`document\` AS \`document\` + FROM s3Cluster('${CLUSTER}', {url:String}, NOSIGN, {format:String}) + SETTINGS parallel_distributed_insert_select = 2, insert_deduplicate = 0, optimize_on_insert = 0" +$CLICKHOUSE_CLIENT -q "OPTIMIZE TABLE clickpipes_05446 FINAL" +echo "--- ClickPipes shape, DEFAULT now64(3) with TTL ---" +$CLICKHOUSE_CLIENT -q "SELECT count(), min(_synced_at) > now64(3) - INTERVAL 1 HOUR FROM clickpipes_05446" + +$CLICKHOUSE_CLIENT -q " + DROP TABLE dst_05446 SYNC; + DROP TABLE typed_05446 SYNC; + DROP TABLE clickpipes_05446 SYNC; +" diff --git a/tests/queries/0_stateless/05447_azure_sas_url_hidden_in_table_function.reference b/tests/queries/0_stateless/05447_azure_sas_url_hidden_in_table_function.reference new file mode 100644 index 0000000000000..65bc13dc12ac7 --- /dev/null +++ b/tests/queries/0_stateless/05447_azure_sas_url_hidden_in_table_function.reference @@ -0,0 +1,13 @@ +SELECT * FROM azureBlobStorage('[HIDDEN]', 'container', 'blob') +SELECT * FROM azureBlobStorage('[HIDDEN]', 'container', 'blob') +SELECT * FROM azureBlobStorage('[HIDDEN]', 'container', 'blob') +SELECT * FROM azureBlobStorageCluster('cluster', '[HIDDEN]', 'container', 'blob') +SELECT * FROM icebergAzure('[HIDDEN]', 'container', 'path') +SELECT * FROM azureBlobStorage(azure_conf, storage_account_url = '[HIDDEN]', container = 'container', blob_path = 'blob') +SELECT * FROM azureBlobStorageCluster('cluster', azure_conf, connection_string = '[HIDDEN]', container = 'container') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', 'CSV') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', 'account', '[HIDDEN]') +SELECT * FROM azureBlobStorage(azure_conf, storage_account_url = 'https://account.blob.core.windows.net', container = 'container') +HIDDEN +0 1 +0 1 diff --git a/tests/queries/0_stateless/05447_azure_sas_url_hidden_in_table_function.sh b/tests/queries/0_stateless/05447_azure_sas_url_hidden_in_table_function.sh new file mode 100755 index 0000000000000..8ef143807a791 --- /dev/null +++ b/tests/queries/0_stateless/05447_azure_sas_url_hidden_in_table_function.sh @@ -0,0 +1,43 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: the fast test build has no `azureBlobStorage` table function + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A storage account url carrying a credential of its own (a SAS `?sig=...`, `user:key@`), or a value that is +# not a plain string literal, is hidden whole in the Azure table functions, positional or as a named-collection +# override, as the `AzureBlobStorage` engine and the backup destination already do. +while read -r query; do + $CLICKHOUSE_FORMAT --oneline --query "$query" +done <<'EOF' +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/?sp=r&sig=plain_sas_signature', 'container', 'blob') +SELECT * FROM azureBlobStorage('https://user:plain_password@account.blob.core.windows.net', 'container', 'blob') +SELECT * FROM azureBlobStorage(concat('https://account.blob.core.windows.net/?sig=', 'plain_sas_signature'), 'container', 'blob') +SELECT * FROM azureBlobStorageCluster('cluster', 'https://account.blob.core.windows.net/?sp=r&sig=plain_sas_signature', 'container', 'blob') +SELECT * FROM icebergAzure('https://account.blob.core.windows.net/?sp=r&sig=plain_sas_signature', 'container', 'path') +SELECT * FROM azureBlobStorage(azure_conf, storage_account_url = 'https://account.blob.core.windows.net/?sp=r&sig=plain_sas_signature', container = 'container', blob_path = 'blob') +SELECT * FROM azureBlobStorageCluster('cluster', azure_conf, connection_string = 'https://account.blob.core.windows.net/?sp=r&sig=plain_sas_signature', container = 'container') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', 'CSV') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', 'account', 'plain_account_key') +SELECT * FROM azureBlobStorage(azure_conf, storage_account_url = 'https://account.blob.core.windows.net', container = 'container') +EOF + +# `SELECT` on a view implies `SHOW COLUMNS`, so a user granted only that sees the view definition through +# `SHOW CREATE TABLE` and `system.tables`, but not the SAS in it. +user="user_${CLICKHOUSE_TEST_UNIQUE_NAME}" +$CLICKHOUSE_CLIENT -q "DROP USER IF EXISTS ${user}" +$CLICKHOUSE_CLIENT -q "CREATE VIEW ${CLICKHOUSE_DATABASE}.v_sas AS SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/?sp=r&sig=plain_sas_signature', 'container', 'file.csv', 'CSV', 'none', 'id UInt8')" +$CLICKHOUSE_CLIENT -q "CREATE USER ${user} IDENTIFIED WITH plaintext_password BY 'hello'" +$CLICKHOUSE_CLIENT -q "GRANT SELECT ON ${CLICKHOUSE_DATABASE}.v_sas TO ${user}" + +$CLICKHOUSE_CLIENT --user "${user}" --password hello -q "SHOW CREATE TABLE ${CLICKHOUSE_DATABASE}.v_sas" | grep -o -e 'plain_sas_signature' -e 'HIDDEN' +$CLICKHOUSE_CLIENT --user "${user}" --password hello -q "SELECT countIf(create_table_query LIKE '%plain_sas_signature%'), countIf(create_table_query LIKE '%[HIDDEN]%') FROM system.tables WHERE database = '${CLICKHOUSE_DATABASE}' AND name = 'v_sas'" + +# `EXPLAIN AST optimize = 1` expands the view, which executes the table function and needs more than `SELECT` +# on the view, so it is run as the default user, who sees no secrets either without +# `format_display_secrets_in_show_and_select`. +$CLICKHOUSE_CLIENT --format_display_secrets_in_show_and_select 0 -q "SELECT countIf(explain LIKE '%plain_sas_signature%'), countIf(explain LIKE '%[HIDDEN]%') FROM (EXPLAIN AST optimize = 1 SELECT * FROM ${CLICKHOUSE_DATABASE}.v_sas)" + +$CLICKHOUSE_CLIENT -q "DROP USER ${user}" diff --git a/tests/queries/0_stateless/05447_orc_native_reader_writer.reference b/tests/queries/0_stateless/05447_orc_native_reader_writer.reference new file mode 100644 index 0000000000000..a46fd04699683 --- /dev/null +++ b/tests/queries/0_stateless/05447_orc_native_reader_writer.reference @@ -0,0 +1,14 @@ +1900-06-01 00:01:41.000007000 -2195942298999993 +1900-06-01 00:01:41.000000000 -2195942299000000 +1960-06-01 00:01:41.000007000 -302486298999993 +2000-06-01 00:01:41.000007000 959817701000007 +none 25000 609103159500350648 +zstd 25000 609103159500350648 +lz4 25000 609103159500350648 +snappy 25000 609103159500350648 +zlib 25000 609103159500350648 +expected 25000 609103159500350648 +1001 4500 5500 +1923 +['a','b','a','b'] +['a','b','a','b'] Array(LowCardinality(String)) diff --git a/tests/queries/0_stateless/05447_orc_native_reader_writer.sh b/tests/queries/0_stateless/05447_orc_native_reader_writer.sh new file mode 100755 index 0000000000000..bee99fd058401 --- /dev/null +++ b/tests/queries/0_stateless/05447_orc_native_reader_writer.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The Arrow ORC adapter stores negative nanoseconds for the timestamps before the epoch, +# and the reader must decode them as the reference implementation does. +$CLICKHOUSE_LOCAL -q "SELECT t, toUnixTimestamp64Micro(t) FROM file('$CUR_DIR/data_orc/timestamps_before_epoch_arrow.orc', ORC) SETTINGS session_timezone = 'UTC'" + +DATA_FILE="$CLICKHOUSE_TMP/05447_${CLICKHOUSE_DATABASE}.orc" + +# Dictionary encoding of strings (with the row index and several row groups), a non-GMT writer time +# zone, and the values that need all integer encodings: repeats, deltas, outliers. +QUERY="SELECT number AS n, + toString(number % 13) AS low_cardinality, + if(number % 5 = 0, NULL, concat('value_', toString(number))) AS high_cardinality, + toDateTime64(1700000000 + number * 37, 3, 'UTC') AS ts, + if(number % 1000 = 0, 1000000000000, number % 7) AS outliers, + intDiv(number, 100) AS runs, + toDecimal128(number, 10) * -3 AS dec +FROM numbers(25000)" + +for compression in none zstd lz4 snappy zlib +do + $CLICKHOUSE_LOCAL -q "$QUERY FORMAT ORC SETTINGS output_format_orc_compression_method = '$compression', + output_format_orc_dictionary_key_size_threshold = 0.5, output_format_orc_row_index_stride = 1000, + output_format_orc_writer_time_zone_name = 'America/Los_Angeles'" > "$DATA_FILE" + $CLICKHOUSE_LOCAL -q "SELECT '$compression', count(), sum(cityHash64(*)) FROM file('$DATA_FILE', ORC, + 'n UInt64, low_cardinality String, high_cardinality Nullable(String), ts DateTime64(3, \'UTC\'), outliers UInt64, runs UInt64, dec Decimal(38, 10)') + SETTINGS input_format_orc_reader_time_zone_name = 'America/Los_Angeles'" +done +$CLICKHOUSE_LOCAL -q "SELECT 'expected', count(), sum(cityHash64(*)) FROM ($QUERY)" + +# Filter pushdown skips the row groups by the statistics in the row index. +$CLICKHOUSE_LOCAL -q "SELECT count(), min(n), max(n) FROM file('$DATA_FILE', ORC) WHERE n BETWEEN 4500 AND 5500 SETTINGS input_format_orc_filter_push_down = 1" +$CLICKHOUSE_LOCAL -q "SELECT count() FROM file('$DATA_FILE', ORC) WHERE low_cardinality = '12' SETTINGS input_format_orc_filter_push_down = 1" + +# The dictionary of a string column nested in an array can have more entries than the stripe has rows. +$CLICKHOUSE_LOCAL -q "SELECT ['a', 'b', 'a', 'b'] AS arr FORMAT ORC SETTINGS output_format_orc_dictionary_key_size_threshold = 0.5" > "$DATA_FILE" +$CLICKHOUSE_LOCAL -q "SELECT arr FROM file('$DATA_FILE', ORC, 'arr Array(String)')" +$CLICKHOUSE_LOCAL -q "SELECT arr, toTypeName(arr) FROM file('$DATA_FILE', ORC, 'arr Array(LowCardinality(String))')" + +rm "$DATA_FILE" diff --git a/tests/queries/0_stateless/05448_orc_date_after_2059.reference b/tests/queries/0_stateless/05448_orc_date_after_2059.reference new file mode 100644 index 0000000000000..62cea6b447e40 --- /dev/null +++ b/tests/queries/0_stateless/05448_orc_date_after_2059.reference @@ -0,0 +1,12 @@ +0 1970-01-01 Nullable(Date32) +1 2059-09-17 Nullable(Date32) +2 2059-09-18 Nullable(Date32) +3 2100-01-01 Nullable(Date32) +4 2149-06-06 Nullable(Date32) +0 1970-01-01 +1 2059-09-17 +2 2059-09-18 +3 2100-01-01 +4 2149-06-06 +3 +4 diff --git a/tests/queries/0_stateless/05448_orc_date_after_2059.sh b/tests/queries/0_stateless/05448_orc_date_after_2059.sh new file mode 100755 index 0000000000000..8cc480ff4b24c --- /dev/null +++ b/tests/queries/0_stateless/05448_orc_date_after_2059.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `Date` is the unsigned number of days, and the dates after 2059-09-18 (more than 32767 days) used +# to be written to ORC as negative numbers, which read back as the dates before 1970. +DATA_FILE="$CLICKHOUSE_TMP/05448_${CLICKHOUSE_DATABASE}.orc" + +$CLICKHOUSE_LOCAL -q "SELECT number AS n, toDate(d) AS d FROM values('number UInt64, d String', (0, '1970-01-01'), (1, '2059-09-17'), (2, '2059-09-18'), (3, '2100-01-01'), (4, '2149-06-06')) FORMAT ORC" > "$DATA_FILE" + +$CLICKHOUSE_LOCAL -q "SELECT n, d, toTypeName(d) FROM file('$DATA_FILE', ORC) ORDER BY n" +$CLICKHOUSE_LOCAL -q "SELECT n, d FROM file('$DATA_FILE', ORC, 'n UInt64, d Date') ORDER BY n" + +# The statistics are correct as well, so the filter pushdown keeps the rows. +$CLICKHOUSE_LOCAL -q "SELECT n FROM file('$DATA_FILE', ORC, 'n UInt64, d Date') WHERE d > '2060-01-01' ORDER BY n SETTINGS input_format_orc_filter_push_down = 1" + +rm "$DATA_FILE" diff --git a/tests/queries/0_stateless/05448_text_formats_read_padding_object_storage.reference b/tests/queries/0_stateless/05448_text_formats_read_padding_object_storage.reference new file mode 100644 index 0000000000000..f40a36898a891 --- /dev/null +++ b/tests/queries/0_stateless/05448_text_formats_read_padding_object_storage.reference @@ -0,0 +1,18 @@ +-- page cache, direct read +0 0 +1 10 +2 20 +1 +-- page cache, prefetching read +0 0 +1 10 +2 20 +1 +-- page cache, reader executor, read twice +0 0 +1 10 +2 20 +0 0 +1 10 +2 20 +1 diff --git a/tests/queries/0_stateless/05448_text_formats_read_padding_object_storage.sh b/tests/queries/0_stateless/05448_text_formats_read_padding_object_storage.sh new file mode 100755 index 0000000000000..6d3c2b25435db --- /dev/null +++ b/tests/queries/0_stateless/05448_text_formats_read_padding_object_storage.sh @@ -0,0 +1,46 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: AzureBlobStorage (Azurite) is not available in the fast test. + +# Parsing a short CSV object through the userspace page cache must not read past the end of the page cache cell that +# holds it: with a direct read, a prefetching read and the reader executor (read twice, so that the second read is +# served from the cache). A sanitizer build aborts on such a read. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +AZURE_CONN="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;" +# Azure container names are limited to 63 characters and must be lowercase alphanumeric, so hash the unique name. +AZURE_CONT="cont$(echo "${CLICKHOUSE_TEST_UNIQUE_NAME}" | md5sum | cut -c1-24)" + +CONFIG_FILE="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}_page_cache.yaml" +echo "page_cache_max_size: 134217728" > "${CONFIG_FILE}" + +# A 14-byte object: "0,0\n1,10\n2,20\n". +${CLICKHOUSE_CLIENT} --query " + INSERT INTO FUNCTION azureBlobStorage('${AZURE_CONN}', '${AZURE_CONT}', 'data.csv', 'CSV', 'auto', 'a UInt32, b UInt32') + SELECT number, number * 10 FROM numbers(3) SETTINGS azure_truncate_on_insert = 1" + +SELECT_QUERY="SELECT * FROM azureBlobStorage('${AZURE_CONN}', '${AZURE_CONT}', 'data.csv', 'CSV', 'auto', 'a String, b UInt32') + ORDER BY b SETTINGS input_format_parallel_parsing = 0" +PAGE_CACHE_USED="SELECT value > 0 FROM system.metrics WHERE metric = 'PageCacheBytes'" +PAGE_CACHE_HIT="SELECT sum(value) > 0 FROM system.events WHERE event = 'PageCacheReadBytes'" + +echo "-- page cache, direct read" +${CLICKHOUSE_LOCAL} --config-file "${CONFIG_FILE}" --query "${SELECT_QUERY}, + use_page_cache_for_object_storage = 1, remote_filesystem_read_method = 'read', use_reader_executor = 0; + ${PAGE_CACHE_USED}" + +echo "-- page cache, prefetching read" +${CLICKHOUSE_LOCAL} --config-file "${CONFIG_FILE}" --query "${SELECT_QUERY}, + use_page_cache_for_object_storage = 1, remote_filesystem_read_method = 'threadpool', remote_filesystem_read_prefetch = 1, + use_reader_executor = 0; + ${PAGE_CACHE_USED}" + +echo "-- page cache, reader executor, read twice" +READER_EXECUTOR_QUERY="${SELECT_QUERY}, + use_page_cache_for_object_storage = 1, remote_filesystem_read_method = 'read', use_reader_executor = 1" +${CLICKHOUSE_LOCAL} --config-file "${CONFIG_FILE}" --query "${READER_EXECUTOR_QUERY}; ${READER_EXECUTOR_QUERY}; ${PAGE_CACHE_HIT}" + +rm "${CONFIG_FILE}" diff --git a/tests/queries/0_stateless/05453_orc_bad_bloom_filter.reference b/tests/queries/0_stateless/05453_orc_bad_bloom_filter.reference new file mode 100644 index 0000000000000..a4a9c43f01b26 --- /dev/null +++ b/tests/queries/0_stateless/05453_orc_bad_bloom_filter.reference @@ -0,0 +1,8 @@ +1.6.0 0 18000000000 Mike +1.6.0 0 1 +1.6.0 1 18000000000 Mike +1.6.0 1 1 +1.6.11 0 18000000000 Mike +1.6.11 0 1 +1.6.11 1 18000000000 Mike +1.6.11 1 1 diff --git a/tests/queries/0_stateless/05453_orc_bad_bloom_filter.sh b/tests/queries/0_stateless/05453_orc_bad_bloom_filter.sh new file mode 100755 index 0000000000000..30badff1fa0dc --- /dev/null +++ b/tests/queries/0_stateless/05453_orc_bad_bloom_filter.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + + +# The bloom filters written by the C++ writer of ORC 1.6.0 - 1.6.11 and 1.7.0 use incorrect hashing (ORC-1024) +# and must be ignored, otherwise the filter pushdown skips the matching rows. +for version in 1.6.0 1.6.11 +do + for push_down in 0 1 + do + $CLICKHOUSE_LOCAL -q " + SELECT '$version', $push_down, id, name + FROM file('$CUR_DIR/data_orc/bad_bloom_filter_$version.orc') + WHERE id = 18000000000 + SETTINGS input_format_orc_filter_push_down = $push_down" + $CLICKHOUSE_LOCAL -q " + SELECT '$version', $push_down, count() + FROM file('$CUR_DIR/data_orc/bad_bloom_filter_$version.orc') + WHERE id IN (18000000000, 18000000001) + SETTINGS input_format_orc_filter_push_down = $push_down" + done +done diff --git a/tests/queries/0_stateless/05457_dictionary_remote_clickhouse_invalidate_query.reference b/tests/queries/0_stateless/05457_dictionary_remote_clickhouse_invalidate_query.reference new file mode 100644 index 0000000000000..e443bdf2664a0 --- /dev/null +++ b/tests/queries/0_stateless/05457_dictionary_remote_clickhouse_invalidate_query.reference @@ -0,0 +1,4 @@ +a +0 +b c + diff --git a/tests/queries/0_stateless/05457_dictionary_remote_clickhouse_invalidate_query.sh b/tests/queries/0_stateless/05457_dictionary_remote_clickhouse_invalidate_query.sh new file mode 100755 index 0000000000000..9da82e9a67558 --- /dev/null +++ b/tests/queries/0_stateless/05457_dictionary_remote_clickhouse_invalidate_query.sh @@ -0,0 +1,52 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# 127.0.0.2 is not a local address for the dictionary source, so the invalidate query is sent through +# RemoteQueryExecutor, like for a source on another server. Its result structure is unknown in advance. + +$CLICKHOUSE_CLIENT --query " +CREATE TABLE dict_src (id UInt64, val String, upd DateTime) ENGINE = Memory; +INSERT INTO dict_src VALUES (1, 'a', '2020-01-01 00:00:00'); + +CREATE DICTIONARY dict_remote (id UInt64, val String) +PRIMARY KEY id +SOURCE(CLICKHOUSE(HOST '127.0.0.2' PORT tcpPort() USER 'default' DB currentDatabase() TABLE 'dict_src' INVALIDATE_QUERY 'SELECT max(upd) FROM dict_src')) +LAYOUT(FLAT()) +LIFETIME(1); + +SELECT dictGet('dict_remote', 'val', toUInt64(1)); +" + +function wait_until() +{ + local deadline=$((SECONDS + 60)) + while [ "$($CLICKHOUSE_CLIENT --query "$1")" != "1" ] && [ $SECONDS -lt $deadline ] + do + sleep 1 + done +} + +DICT_INFO="FROM system.dictionaries WHERE database = currentDatabase() AND name = 'dict_remote'" +# Invalidate queries seen by the "remote" server. +CHECKS="FROM system.query_log WHERE current_database = currentDatabase() AND type != 'QueryStart' AND query LIKE 'SELECT max(upd)%'" + +# The first periodic check always reloads: there is no previous invalidate query response to compare with. +# The reloaded dictionary is a new object, so its `query_count` is 0 again (the `dictGet` above made it 1). +wait_until "SELECT query_count = 0 $DICT_INFO" + +# A new row that does not change the invalidate query result. +$CLICKHOUSE_CLIENT --query "INSERT INTO dict_src VALUES (2, 'b', '2019-01-01 00:00:00')" +c0=$($CLICKHOUSE_CLIENT --query "SYSTEM FLUSH LOGS query_log; SELECT count() $CHECKS") +wait_until "SYSTEM FLUSH LOGS query_log; SELECT count() >= $c0 + 2 $CHECKS" +# Two checks have passed since the insert, the dictionary must not have been reloaded. +$CLICKHOUSE_CLIENT --query "SELECT dictHas('dict_remote', toUInt64(2))" + +# A new row that changes the invalidate query result. +$CLICKHOUSE_CLIENT --query "INSERT INTO dict_src VALUES (3, 'c', '2021-01-01 00:00:00')" +wait_until "SELECT dictHas('dict_remote', toUInt64(3))" +$CLICKHOUSE_CLIENT --query "SELECT dictGet('dict_remote', 'val', toUInt64(2)), dictGet('dict_remote', 'val', toUInt64(3))" + +$CLICKHOUSE_CLIENT --query "SELECT last_exception $DICT_INFO" diff --git a/tests/queries/0_stateless/05457_json_extract_datetime_compatibility.reference b/tests/queries/0_stateless/05457_json_extract_datetime_compatibility.reference new file mode 100644 index 0000000000000..414d8128901e7 --- /dev/null +++ b/tests/queries/0_stateless/05457_json_extract_datetime_compatibility.reference @@ -0,0 +1,5 @@ +1970-01-01 00:00:00.000 +2026-10-19 07:30:26.743 +2026-10-19 07:30:26 +2026-10-19 07:30:26.743 +1970-01-01 00:00:00.000 diff --git a/tests/queries/0_stateless/05457_json_extract_datetime_compatibility.sql b/tests/queries/0_stateless/05457_json_extract_datetime_compatibility.sql new file mode 100644 index 0000000000000..0d3234ea3e1b4 --- /dev/null +++ b/tests/queries/0_stateless/05457_json_extract_datetime_compatibility.sql @@ -0,0 +1,19 @@ +-- `compatibility` with versions before 26.8 must restore the old behavior of `JSONExtract`, which parsed +-- `DateTime` values according to `date_time_input_format` rather than `cast_string_to_date_time_mode`. + +SET date_time_input_format = 'best_effort', cast_string_to_date_time_mode = 'basic'; + +SET json_extract_use_cast_string_to_date_time_mode = 1; +SELECT JSONExtract('{"createdAt":"2026-10-19T07:30:26.743Z"}', 'createdAt', 'DateTime64(3, \'UTC\')'); + +SET json_extract_use_cast_string_to_date_time_mode = 0; +SELECT JSONExtract('{"createdAt":"2026-10-19T07:30:26.743Z"}', 'createdAt', 'DateTime64(3, \'UTC\')'); +SELECT JSONExtract('{"createdAt":"2026-10-19T07:30:26Z"}', 'createdAt', 'DateTime(\'UTC\')'); + +-- `compatibility` resets the setting, but does not touch `date_time_input_format` set explicitly. +SET json_extract_use_cast_string_to_date_time_mode = DEFAULT; +SET compatibility = '23.9', date_time_input_format = 'best_effort'; +SELECT JSONExtract('{"createdAt":"2026-10-19T07:30:26.743Z"}', 'createdAt', 'DateTime64(3, \'UTC\')'); + +SET compatibility = '26.8', cast_string_to_date_time_mode = 'basic'; +SELECT JSONExtract('{"createdAt":"2026-10-19T07:30:26.743Z"}', 'createdAt', 'DateTime64(3, \'UTC\')'); diff --git a/tests/queries/0_stateless/05457_part_check_ignores_query_cancellation.reference b/tests/queries/0_stateless/05457_part_check_ignores_query_cancellation.reference new file mode 100644 index 0000000000000..a50066ed3c6a0 --- /dev/null +++ b/tests/queries/0_stateless/05457_part_check_ignores_query_cancellation.reference @@ -0,0 +1,4 @@ +1 +499999500000 +waiter timed out inside the cache read +spurious part checks: 0 diff --git a/tests/queries/0_stateless/05457_part_check_ignores_query_cancellation.sh b/tests/queries/0_stateless/05457_part_check_ignores_query_cancellation.sh new file mode 100755 index 0000000000000..c8c64a04ad583 --- /dev/null +++ b/tests/queries/0_stateless/05457_part_check_ignores_query_cancellation.sh @@ -0,0 +1,144 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-parallel, no-random-settings, no-random-merge-tree-settings, no-replicated-database, no-object-storage, no-parallel-replicas, zookeeper, no-shared-merge-tree +# no-parallel: enables a global pauseable failpoint which pauses every filesystem cache write on the server. +# zookeeper: a part check is requested through the broken-part callback, which only `ReplicatedMergeTree` installs. +# no-shared-merge-tree: the table pins a private cache disk, and SharedMergeTree requires a keeper-metadata disk. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A query that waits in `FileSegment::wait` for another query's cache download, and is cancelled while it +# waits, throws `TIMEOUT_EXCEEDED` or `QUERY_WAS_CANCELLED` from inside `MergeTreeReaderWide::readRows`. +# Neither code is a retryable exception, so the reader reported the part as broken and a healthy part was +# enqueued for a full data check. On object storage such a check reads the whole part, and a replica +# restart (`partialShutdown` -> `ReplicatedMergeTreePartCheckThread::stop`) blocks until it ends. +# +# Here the cancellation comes from `max_execution_time`: the cancellation checker kills the query while +# its only thread waits for the paused download, and the wait loop observes the kill and throws. +# A killed or timed-out query says nothing about the data, so no check must be requested. + +# A private cache keeps the test independent from other cache users on the server. The whole data file +# must fit in one file segment, so that the waiter below never becomes a downloader itself (it would +# pause on its own cache write, because the failpoint is disabled only after it returns). +cache_name="cache_05457_${CLICKHOUSE_DATABASE}" +table="t_part_check_cancel" + +$CLICKHOUSE_CLIENT --query " + DROP TABLE IF EXISTS $table SYNC; + CREATE TABLE $table (a UInt64) + ENGINE = ReplicatedMergeTree('/clickhouse/tables/$CLICKHOUSE_TEST_ZOOKEEPER_PREFIX/$table', 'r1') + ORDER BY a + SETTINGS + min_bytes_for_wide_part = 0, + disk = disk( + type = cache, + name = '$cache_name', + path = '$cache_name/', + max_size = '1Gi', + max_file_segment_size = '256Mi', + cache_on_write_operations = 0, + load_metadata_asynchronously = 0, + disk = 'local_disk'); + INSERT INTO $table SELECT number FROM numbers(1000000); +" + +# Warm up the in-memory mark cache, so that both queries below take marks from it and their only +# filesystem cache accesses are for the data file. Otherwise the paused downloader would hold the +# mark cache load token and the waiter would block on it instead of `FileSegment::wait`. +# Strictly synchronous, no prefetches and no background download: a cache write still in flight +# after this query would pause on the failpoint while holding a segment the queries below need. +$CLICKHOUSE_CLIENT --max_threads 1 \ + --remote_filesystem_read_prefetch 0 \ + --allow_prefetched_read_pool_for_remote_filesystem 0 \ + --filesystem_cache_allow_background_download 0 \ + --query "SELECT a FROM $table LIMIT 1 FORMAT Null" + +$CLICKHOUSE_CLIENT --query " + SYSTEM DROP FILESYSTEM CACHE '$cache_name'; + SYSTEM ENABLE FAILPOINT file_segment_pause_before_write; +" + +# The failpoint is server-global, so it must be disabled even if the script aborts before reaching +# the explicit disable below. Otherwise every filesystem cache write on the server stays paused and +# the following tests hang behind it, masking the failure that happened here. +trap '$CLICKHOUSE_CLIENT --query "SYSTEM DISABLE FAILPOINT file_segment_pause_before_write" > /dev/null 2>&1 || true' EXIT + +# The downloader pauses inside its first cache write, holding the file segment in DOWNLOADING state. +$CLICKHOUSE_CLIENT --max_threads 1 \ + --enable_filesystem_cache 1 \ + --read_from_filesystem_cache_if_exists_otherwise_bypass_cache 0 \ + --remote_filesystem_read_prefetch 0 \ + --allow_prefetched_read_pool_for_remote_filesystem 0 \ + --filesystem_cache_allow_background_download 0 \ + --query "SELECT sum(a) FROM $table" & +downloader_pid=$! + +# The failpoint pauses every cache write on the server, so the first pause is not necessarily the +# downloader's: additionally wait until the downloader holds the segment of this test's cache. +# Bounded, so that a downloader that never writes to the cache is reported instead of blocking +# the test until its timeout. +if ! timeout 120 $CLICKHOUSE_CLIENT --query "SYSTEM WAIT FAILPOINT file_segment_pause_before_write PAUSE" > /dev/null 2>&1; then + echo "FAIL: no cache write paused on the failpoint" +fi +downloading=0 +for _ in {1..600}; do + downloading=$($CLICKHOUSE_CLIENT --query " + SELECT count() FROM system.filesystem_cache WHERE cache_name = '$cache_name' AND state = 'DOWNLOADING'") + [[ "$downloading" == "1" ]] && break + sleep 0.1 +done +echo "$downloading" + +# The waiter needs the paused segment. Its cache wait is long enough to outlive `max_execution_time`, +# so the cancellation checker kills it while it waits, and the wait loop throws `TIMEOUT_EXCEEDED` +# from inside the read. `Cache info:` in the error proves the throw happened inside the cache read +# buffer, i.e. under the reader's catch, and not at a cancellation point of the pipeline. +# Backgrounded with a bounded wait, so that a waiter which unexpectedly pauses on the failpoint +# itself is reported instead of blocking the test until its timeout. +waiter_out="${CLICKHOUSE_TMP:-/tmp}/05457_waiter_${CLICKHOUSE_DATABASE}.out" +rm -f "$waiter_out" +waiter_query_id="05457_waiter_$(random_str 10)" +$CLICKHOUSE_CLIENT --query_id "$waiter_query_id" --max_threads 1 \ + --enable_filesystem_cache 1 \ + --read_from_filesystem_cache_if_exists_otherwise_bypass_cache 0 \ + --remote_filesystem_read_prefetch 0 \ + --allow_prefetched_read_pool_for_remote_filesystem 0 \ + --filesystem_cache_allow_background_download 0 \ + --use_uncompressed_cache 0 \ + --filesystem_cache_wait_for_concurrent_download_timeout_milliseconds 600000 \ + --max_execution_time 3 \ + --query "SELECT sum(a) FROM $table" > "$waiter_out" 2>&1 & +waiter_pid=$! + +waiter_finished=0 +for _ in {1..600}; do + if ! kill -0 "$waiter_pid" 2>/dev/null; then + waiter_finished=1 + break + fi + sleep 0.1 +done + +$CLICKHOUSE_CLIENT --query "SYSTEM DISABLE FAILPOINT file_segment_pause_before_write" +wait "$downloader_pid" +wait "$waiter_pid" 2>/dev/null || true + +waiter_result=$(cat "$waiter_out") +rm -f "$waiter_out" +if [[ "$waiter_finished" != "1" ]]; then + echo "FAIL: the waiter did not finish while the failpoint was armed: $waiter_result" +elif [[ "$waiter_result" == *"Timeout exceeded"*"Cache info:"* ]]; then + echo "waiter timed out inside the cache read" +else + echo "FAIL: unexpected waiter result: $waiter_result" +fi + +$CLICKHOUSE_CLIENT --query "SYSTEM FLUSH LOGS text_log" +checks=$($CLICKHOUSE_CLIENT --query " + SELECT count() FROM system.text_log + WHERE event_date >= yesterday() AND query_id = '$waiter_query_id' + AND message LIKE 'Enqueueing % for check after %'") +echo "spurious part checks: $checks" + +$CLICKHOUSE_CLIENT --query "DROP TABLE $table SYNC" diff --git a/tests/queries/0_stateless/05458_azure_credentials_hidden_in_logged_queries.reference b/tests/queries/0_stateless/05458_azure_credentials_hidden_in_logged_queries.reference new file mode 100644 index 0000000000000..cf669808c66d3 --- /dev/null +++ b/tests/queries/0_stateless/05458_azure_credentials_hidden_in_logged_queries.reference @@ -0,0 +1,39 @@ +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', '[HIDDEN]', 'account', '[HIDDEN]') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', '[HIDDEN]') +SELECT * FROM azureBlobStorageCluster('cluster', 'https://account.blob.core.windows.net/container/blob.csv', '[HIDDEN]') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', '[HIDDEN]', SETTINGS input_format_csv_skip_first_lines = 0) +SELECT * FROM azureBlobStorageCluster('cluster', 'https://account.blob.core.windows.net/container/blob.csv', '[HIDDEN]', SETTINGS input_format_csv_skip_first_lines = 0) +SELECT * FROM icebergAzure('https://account.blob.core.windows.net/container/blob.csv', '[HIDDEN]', SETTINGS input_format_csv_skip_first_lines = 0) +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', '[HIDDEN]', partition_strategy = 'none') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', '[HIDDEN]') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', '[HIDDEN]', '[HIDDEN]') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', account_key = '[HIDDEN]') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', partition_strategy = 'none', '[HIDDEN]') +SELECT * FROM azureBlobStorage('[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]', '[HIDDEN]') +SELECT * FROM azureBlobStorageCluster('[HIDDEN]', 'https://account.blob.core.windows.net/container/blob.csv', '[HIDDEN]') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', '[HIDDEN]') +SELECT * FROM azureBlobStorage('AccountName=account;AccountKey=[HIDDEN];BlobEndpoint=https://account.blob.core.windows.net', 'container', 'blob', 'account', '[HIDDEN]') +SELECT * FROM azureBlobStorage('AccountName=account;AccountKey=[HIDDEN];SharedAccessSignature=[HIDDEN]', 'container', 'blob') +SELECT * FROM azureBlobStorage('AccountName=account;AccountKey=[HIDDEN];AccountKey=[HIDDEN]', 'container', 'blob') +SELECT * FROM azureBlobStorage('AccountName=account;BlobEndpoint=[HIDDEN]', 'container', 'blob') +SELECT * FROM azureBlobStorage('[HIDDEN]', 'container', 'blob') +SELECT * FROM azureBlobStorage('[HIDDEN]', 'container', 'blob') +SELECT * FROM azureBlobStorage(azure_conf, connection_string = 'AccountName=account;AccountKey=[HIDDEN]', account_key = '[HIDDEN]') +SELECT * FROM azureBlobStorage(azure_conf, '[HIDDEN]', blob_path = 'blob') +SELECT * FROM azureBlobStorage(azure_conf, container = '[HIDDEN]', blob_path = 'blob') +SELECT * FROM azureBlobStorage(azure_conf, connection_string = '[HIDDEN]', storage_account_url = '[HIDDEN]') +SELECT * FROM azureBlobStorageCluster('cluster', azure_conf, connection_string = '[HIDDEN]', connection_string = '[HIDDEN]') +CREATE TABLE test_azure (`key` UInt64) ENGINE = AzureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', '[HIDDEN]', partition_strategy = 'none') +CREATE TABLE test_azure (`key` UInt64) ENGINE = AzureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', partition_strategy = 'none', '[HIDDEN]') +CREATE TABLE test_azure (`key` UInt64) ENGINE = AzureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', 'CSV', '[HIDDEN]') +CREATE TABLE test_azure (`key` UInt64) ENGINE = AzureBlobStorage('[HIDDEN]', '[HIDDEN]') +CREATE TABLE test_azure (`key` UInt64) ENGINE = AzureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', '[HIDDEN]') +CREATE TABLE test_azure_queue (`key` UInt64) ENGINE = AzureQueue('https://account.blob.core.windows.net', 'container', '*', 'account', '[HIDDEN]') +CREATE TABLE test_azure_queue (`key` UInt64) ENGINE = AzureQueue('https://account.blob.core.windows.net', 'container', '*', 'CSV') SETTINGS after_processing_move_connection_string = '[HIDDEN]' +CREATE TABLE test_azure_queue (`key` UInt64) ENGINE = AzureQueue('https://account.blob.core.windows.net', 'container', '*', 'CSV') SETTINGS after_processing_move_connection_string = '[HIDDEN]' +CREATE TABLE test_azure_queue (`key` UInt64) ENGINE = AzureQueue('https://account.blob.core.windows.net', 'container', '*', 'CSV') SETTINGS after_processing_move_connection_string = 'AccountName=account;AccountKey=[HIDDEN];AccountKey=[HIDDEN]' +0 1 +0 1 +0 1 +0 1 +0 1 diff --git a/tests/queries/0_stateless/05458_azure_credentials_hidden_in_logged_queries.sh b/tests/queries/0_stateless/05458_azure_credentials_hidden_in_logged_queries.sh new file mode 100755 index 0000000000000..6246ea1d8cb32 --- /dev/null +++ b/tests/queries/0_stateless/05458_azure_credentials_hidden_in_logged_queries.sh @@ -0,0 +1,62 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: the fast test build has no Azure table functions and engines, whose arguments it then hides whole + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `clickhouse-format` hides the credentials in the arguments of the Azure table functions and engines and in the +# `SETTINGS` clause of `AzureQueue`, wherever the parser reads them from. +while read -r query; do + $CLICKHOUSE_FORMAT --oneline --query "$query" +done <<'EOF' +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', extra_credentials(client_id = 'id', tenant_id = 'tid'), 'account', 'plain_account_key') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', 'sp=r&sig=plain_sas_signature') +SELECT * FROM azureBlobStorageCluster('cluster', 'https://account.blob.core.windows.net/container/blob.csv', 'sp=r&sig=plain_sas_signature') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', 'sp=r&sig=plain_sas_signature', SETTINGS input_format_csv_skip_first_lines = 0) +SELECT * FROM azureBlobStorageCluster('cluster', 'https://account.blob.core.windows.net/container/blob.csv', 'sp=r&sig=plain_sas_signature', SETTINGS input_format_csv_skip_first_lines = 0) +SELECT * FROM icebergAzure('https://account.blob.core.windows.net/container/blob.csv', 'sp=r&sig=plain_sas_signature', SETTINGS input_format_csv_skip_first_lines = 0) +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', 'sp=r&sig=plain_sas_signature', partition_strategy = 'none') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', 'sp=r&sig=plain_sas_signature' = 'x') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', 'sp=r&sig=plain_sas_signature', extra_credentials(client_id = 'id', client_secret = 'plain_client_secret', role_arn = 'plain_role_arn')) +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', account_key = 'plain_account_key') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', partition_strategy = 'none', 'plain_account_key') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', extra_credentials(client_id = 'id'), extra_credentials(tenant_id = 'tid'), 'plain_account_key', 'CSV') +SELECT * FROM azureBlobStorageCluster(extra_credentials(client_id = 'id'), 'https://account.blob.core.windows.net/container/blob.csv', 'sp=r&sig=plain_sas_signature') +SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', extra_credentials('plain_client_secret' = 'x', client_id = 'plain_client_id')) +SELECT * FROM azureBlobStorage('AccountName=account;AccountKey=plain_account_key;BlobEndpoint=https://account.blob.core.windows.net', 'container', 'blob', 'account', 'plain_account_key') +SELECT * FROM azureBlobStorage('AccountName=account;AccountKey=plain_account_key;SharedAccessSignature=sp=r&sig=plain_sas_signature', 'container', 'blob') +SELECT * FROM azureBlobStorage('AccountName=account;AccountKey=plain_account_key;AccountKey=plain_account_key_2', 'container', 'blob') +SELECT * FROM azureBlobStorage('AccountName=account;BlobEndpoint=https://account.blob.core.windows.net/?sig=plain_sas_signature', 'container', 'blob') +SELECT * FROM azureBlobStorage('account.blob.core.windows.net/?sig=plain_sas_signature', 'container', 'blob') +SELECT * FROM azureBlobStorage('HTTPS://user:plain_password=x@account.blob.core.windows.net/', 'container', 'blob') +SELECT * FROM azureBlobStorage(azure_conf, connection_string = 'AccountName=account;AccountKey=plain_account_key', account_key = 'plain_account_key') +SELECT * FROM azureBlobStorage(azure_conf, client_secret = 'plain_client_secret', blob_path = 'blob') +SELECT * FROM azureBlobStorage(azure_conf, container = headers('Authorization' = 'plain_header_secret'), blob_path = 'blob') +SELECT * FROM azureBlobStorage(azure_conf, connection_string = 'AccountName=account;AccountKey=plain_account_key', storage_account_url = 'AccountName=account;AccountKey=plain_account_key_2') +SELECT * FROM azureBlobStorageCluster('cluster', azure_conf, connection_string = concat('AccountName=account;AccountKey=', 'plain_account_key'), connection_string = 'AccountName=account;AccountKey=plain_account_key_2') +CREATE TABLE test_azure (key UInt64) ENGINE = AzureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', 'sp=r&sig=plain_sas_signature', partition_strategy = 'none') +CREATE TABLE test_azure (key UInt64) ENGINE = AzureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', partition_strategy = 'none', 'plain_account_key') +CREATE TABLE test_azure (key UInt64) ENGINE = AzureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', 'CSV', extra_credentials(client_id = 'id', client_secret = 'plain_client_secret', role_arn = 'plain_role_arn')) +CREATE TABLE test_azure (key UInt64) ENGINE = AzureBlobStorage(azure_conf, sas_token = 'plain_sas_signature') +CREATE TABLE test_azure (key UInt64) ENGINE = AzureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', 'sp=r&sig=plain_sas_signature' = 'x') +CREATE TABLE test_azure_queue (key UInt64) ENGINE = AzureQueue('https://account.blob.core.windows.net', 'container', '*', 'account', 'plain_account_key' = 'x') +CREATE TABLE test_azure_queue (key UInt64) ENGINE = AzureQueue('https://account.blob.core.windows.net', 'container', '*', 'CSV') SETTINGS after_processing_move_connection_string = 'HTTPS://account.blob.core.windows.net/?sig=plain_sas_signature' +CREATE TABLE test_azure_queue (key UInt64) ENGINE = AzureQueue('https://account.blob.core.windows.net', 'container', '*', 'CSV') SETTINGS after_processing_move_connection_string = 'https://user:plain_password=x@account.blob.core.windows.net/' +CREATE TABLE test_azure_queue (key UInt64) ENGINE = AzureQueue('https://account.blob.core.windows.net', 'container', '*', 'CSV') SETTINGS after_processing_move_connection_string = 'AccountName=account;AccountKey=plain_account_key;AccountKey=plain_account_key_2' +EOF + +# The query tree hides an argument masked whole, not only the right side of a comparison. +$CLICKHOUSE_CLIENT -q "SELECT countIf(explain LIKE '%plain_sas_signature%'), countIf(explain LIKE '%[HIDDEN]%') FROM (EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', 'sp=r&sig=plain_sas_signature' = 'x'))" +$CLICKHOUSE_CLIENT -q "SELECT countIf(explain LIKE '%plain_sas_signature%'), countIf(explain LIKE '%[HIDDEN]%') FROM (EXPLAIN QUERY TREE run_passes = 0 SELECT * FROM azureBlobStorage(azure_conf, 'sp=r&sig=plain_sas_signature' = 'x'))" + +# Nothing inside an Azure `extra_credentials(...)` is dumped, keys included. +for explain in "AST" "QUERY TREE run_passes = 0"; do + $CLICKHOUSE_CLIENT -q "SELECT countIf(explain LIKE '%plain_client_%'), countIf(explain LIKE '%[HIDDEN]%') FROM (EXPLAIN $explain SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net', 'container', 'blob', extra_credentials('plain_client_secret' = 'x', client_id = 'plain_client_id')))" +done + +# The statement is logged with the credential hidden. +$CLICKHOUSE_CLIENT -q "EXPLAIN AST SELECT * FROM azureBlobStorage('https://account.blob.core.windows.net/container/blob.csv', 'sp=r&sig=plain_sas_logged')" > /dev/null +$CLICKHOUSE_CLIENT -q "SYSTEM FLUSH LOGS query_log" +$CLICKHOUSE_CLIENT -q "SELECT countIf(query LIKE '%plain_sas_logged%'), countIf(query LIKE '%[HIDDEN]%') FROM system.query_log WHERE current_database = currentDatabase() AND query LIKE 'EXPLAIN AST%azureBlobStorage%' AND type = 'QueryFinish' AND is_initial_query" diff --git a/tests/queries/0_stateless/05459_ai_functions_background_defaults.reference b/tests/queries/0_stateless/05459_ai_functions_background_defaults.reference new file mode 100644 index 0000000000000..7b06fcf9e8f7e --- /dev/null +++ b/tests/queries/0_stateless/05459_ai_functions_background_defaults.reference @@ -0,0 +1,3 @@ +x +y DEFAULT +z ALIAS diff --git a/tests/queries/0_stateless/05459_ai_functions_background_defaults.sql b/tests/queries/0_stateless/05459_ai_functions_background_defaults.sql new file mode 100644 index 0000000000000..a625113c2e5a7 --- /dev/null +++ b/tests/queries/0_stateless/05459_ai_functions_background_defaults.sql @@ -0,0 +1,28 @@ +-- AI functions are rejected in `DEFAULT` and `MATERIALIZED` expressions computed in the background: +-- of a column with a TTL, and of a column added or modified by `ALTER`. + +CREATE TABLE t1 (x String, d DateTime, y String DEFAULT aiGenerate(x) TTL d + INTERVAL 1 DAY) ENGINE = MergeTree ORDER BY x; -- { serverError NOT_IMPLEMENTED } +CREATE TABLE t1 (x String, d DateTime, y String MATERIALIZED aiGenerate(x) TTL d + INTERVAL 1 DAY) ENGINE = MergeTree ORDER BY x; -- { serverError NOT_IMPLEMENTED } + +CREATE TABLE t2 (x String, y String DEFAULT aiGenerate(x)) ENGINE = MergeTree ORDER BY x; +ALTER TABLE t2 ADD COLUMN z String DEFAULT aiGenerate(x); -- { serverError NOT_IMPLEMENTED } +ALTER TABLE t2 ADD COLUMN z String MATERIALIZED aiGenerate(x); -- { serverError NOT_IMPLEMENTED } +ALTER TABLE t2 MODIFY COLUMN x String DEFAULT aiGenerate(y); -- { serverError NOT_IMPLEMENTED } +ALTER TABLE t2 ADD COLUMN z String ALIAS aiGenerate(x); +ALTER TABLE t2 MATERIALIZE COLUMN y; -- { serverError NOT_IMPLEMENTED } +SELECT name, default_kind FROM system.columns WHERE database = currentDatabase() AND table = 't2' ORDER BY position; + +DROP TABLE t2; + +-- Column TTL added by `ALTER`. +CREATE TABLE t3 (x String, d DateTime, y String, w String DEFAULT aiGenerate(x)) ENGINE = MergeTree ORDER BY x; +ALTER TABLE t3 MODIFY COLUMN y TTL d + toIntervalDay(length(aiGenerate(x))); -- { serverError NOT_IMPLEMENTED } +ALTER TABLE t3 MODIFY COLUMN w TTL d; -- { serverError NOT_IMPLEMENTED } +DROP TABLE t3; + +-- A lightweight `DELETE` executed as a mutation is rejected. Executed as a lightweight update, it runs in the query +-- and fails on the missing credentials. +CREATE TABLE t4 (x String) ENGINE = MergeTree ORDER BY x SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1; +DELETE FROM t4 WHERE aiFilter(x, 'c') SETTINGS lightweight_delete_mode = 'alter_update'; -- { serverError NOT_IMPLEMENTED } +DELETE FROM t4 WHERE aiFilter(x, 'c') SETTINGS lightweight_delete_mode = 'lightweight_update_force', enable_lightweight_update = 1; -- { serverError BAD_ARGUMENTS } +DROP TABLE t4; diff --git a/tests/queries/0_stateless/05473_orc_footer_content_length.reference b/tests/queries/0_stateless/05473_orc_footer_content_length.reference new file mode 100644 index 0000000000000..ae5399ce4997a --- /dev/null +++ b/tests/queries/0_stateless/05473_orc_footer_content_length.reference @@ -0,0 +1,2 @@ +1 True True +1 True True diff --git a/tests/queries/0_stateless/05473_orc_footer_content_length.sh b/tests/queries/0_stateless/05473_orc_footer_content_length.sh new file mode 100755 index 0000000000000..78921273f950c --- /dev/null +++ b/tests/queries/0_stateless/05473_orc_footer_content_length.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest. +# The `content_length` in the ORC footer covers only the stripes, not the metadata section with the stripe statistics. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +check() +{ + $CLICKHOUSE_LOCAL --query "$1" | python3 -c ' +import sys + +data = sys.stdin.buffer.read() + +def varint(buf, pos): + result = shift = 0 + while True: + byte = buf[pos] + pos += 1 + result |= (byte & 0x7f) << shift + shift += 7 + if byte < 0x80: + return result, pos + +def fields(buf): + pos = 0 + while pos < len(buf): + key, pos = varint(buf, pos) + if key & 7 == 0: + value, pos = varint(buf, pos) + elif key & 7 == 2: + length, pos = varint(buf, pos) + value = buf[pos:pos + length] + pos += length + else: + raise Exception("unexpected wire type") + yield key >> 3, value + +postscript_length = data[-1] +postscript = dict(fields(data[-1 - postscript_length:-1])) +footer_length = postscript[1] +footer = list(fields(data[-1 - postscript_length - footer_length:-1 - postscript_length])) +content_length = next(v for f, v in footer if f == 2) +stripes = [dict(fields(v)) for f, v in footer if f == 3] +stripes_length = sum(s.get(2, 0) + s.get(3, 0) + s.get(4, 0) for s in stripes) +print(len(stripes), content_length == stripes_length, postscript[5] > 0) +' +} + +check "SELECT number FROM numbers(3) FORMAT ORC SETTINGS output_format_orc_compression_method = 'none'" +check "SELECT number, toString(number) FROM numbers(100000) FORMAT ORC SETTINGS output_format_orc_compression_method = 'none'" diff --git a/tests/queries/0_stateless/05474_orc_decimal64_v2.reference b/tests/queries/0_stateless/05474_orc_decimal64_v2.reference new file mode 100644 index 0000000000000..754cd5dae7684 --- /dev/null +++ b/tests/queries/0_stateless/05474_orc_decimal64_v2.reference @@ -0,0 +1,20 @@ +a Nullable(Int64) +b Nullable(Decimal(12, 0)) +c Nullable(Decimal(20, 2)) +d Nullable(Decimal(12, 2)) +e Nullable(Decimal(2, 2)) +17292380420 24 36164.16 0.03 0.01 +17292380421 38 63351.7 0.08 0.01 +17292380421 28 42673.96 0.09 0.06 +17292380421 40 76677.6 0.05 0.04 +17292380421 2 2096.48 0.07 0.07 +17292380421 42 45284.82 0.07 0.05 +17292380421 10 18572.9 0.01 0.08 +17292380422 12 14836.8 0.09 0.06 +17292380422 41 82152.52 0.07 0.02 +17292380422 38 47240.84 0.1 0 +38 0.08 +40 0.05 +42 0.07 +41 0.07 +38 0.1 diff --git a/tests/queries/0_stateless/05474_orc_decimal64_v2.sh b/tests/queries/0_stateless/05474_orc_decimal64_v2.sh new file mode 100755 index 0000000000000..94e31bf561544 --- /dev/null +++ b/tests/queries/0_stateless/05474_orc_decimal64_v2.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `decimal64_v2.orc` from the examples of apache/orc has the format version 1.9999 (`UNSTABLE_PRE_2_0`), +# where the DECIMAL columns with the precision up to 18 are signed RLEv2 integers without the SECONDARY stream. +$CLICKHOUSE_LOCAL -q "DESCRIBE file('$CUR_DIR/data_orc/decimal64_v2.orc', ORC)" +$CLICKHOUSE_LOCAL -q "SELECT * FROM file('$CUR_DIR/data_orc/decimal64_v2.orc', ORC)" +$CLICKHOUSE_LOCAL -q "SELECT b, d FROM file('$CUR_DIR/data_orc/decimal64_v2.orc', ORC) WHERE b > 30 SETTINGS input_format_orc_filter_push_down = 1" diff --git a/tests/queries/0_stateless/05480_iceberg_add_column_too_small_last_column_id_128455.reference b/tests/queries/0_stateless/05480_iceberg_add_column_too_small_last_column_id_128455.reference new file mode 100644 index 0000000000000..1351103cad4e9 --- /dev/null +++ b/tests/queries/0_stateless/05480_iceberg_add_column_too_small_last_column_id_128455.reference @@ -0,0 +1,10 @@ +1 [1,2] \N +1 [1,2] \N +2 [3] x +id=1 arr=2 arr.element=3 extra=4 last-column-id=4 +1 {'k':[1,2]} \N +id=1 m=2 m.key=3 m.value=4 m.value.element=5 extra=6 last-column-id=6 +1 +1 \N \N +id=1 c=5 d=6 last-column-id=6 +a=1 a.b=2 a.b=3 c=4 last-column-id=4 diff --git a/tests/queries/0_stateless/05480_iceberg_add_column_too_small_last_column_id_128455.sh b/tests/queries/0_stateless/05480_iceberg_add_column_too_small_last_column_id_128455.sh new file mode 100755 index 0000000000000..37cc7e4c38a7e --- /dev/null +++ b/tests/queries/0_stateless/05480_iceberg_add_column_too_small_last_column_id_128455.sh @@ -0,0 +1,130 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# - no-fasttest: requires IcebergLocal (USE_AVRO build option) + +# Regression test for https://github.com/ClickHouse/ClickHouse/issues/128455 +# Tables created by 26.6 and older record a `last-column-id` that does not count nested field ids. ADD COLUMN must +# not reuse such an id: new ids start after the largest id of any schema (the current one and those of dropped columns). + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +TABLE_A="t_${CLICKHOUSE_DATABASE}_${RANDOM}_a" +TABLE_A_DIR="${USER_FILES_PATH}/${TABLE_A}" +TABLE_B="t_${CLICKHOUSE_DATABASE}_${RANDOM}_b" +TABLE_B_DIR="${USER_FILES_PATH}/${TABLE_B}" +TABLE_C="t_${CLICKHOUSE_DATABASE}_${RANDOM}_c" +TABLE_C_DIR="${USER_FILES_PATH}/${TABLE_C}" +TABLE_D="t_${CLICKHOUSE_DATABASE}_${RANDOM}_d" +TABLE_D_DIR="${USER_FILES_PATH}/${TABLE_D}" +rm -rf "${TABLE_A_DIR}" "${TABLE_B_DIR}" "${TABLE_C_DIR}" "${TABLE_D_DIR}" +cleanup() { + rm -rf "${TABLE_A_DIR}" "${TABLE_B_DIR}" "${TABLE_C_DIR}" "${TABLE_D_DIR}" + for table in ${TABLE_A} ${TABLE_B} ${TABLE_C} ${TABLE_D}; do + ${CLICKHOUSE_CLIENT} --query "DROP TABLE IF EXISTS ${table}; DROP TABLE IF EXISTS ${table}_reopened" + done +} +trap cleanup EXIT + +# Publishes a new metadata version that differs from the latest one only by `last-column-id`, as 26.6 wrote it. +# A new file rather than an in-place edit, because the server caches parsed metadata files by path. +lower_last_column_id() { + python3 - "$1/metadata" "$2" <<'PY' +import json, os, re, sys +metadata_dir, value = sys.argv[1], int(sys.argv[2]) +latest = max(int(m.group(1)) for f in os.listdir(metadata_dir) if (m := re.fullmatch(r"v(\d+)\.metadata\.json", f))) +with open(os.path.join(metadata_dir, f"v{latest}.metadata.json")) as f: + metadata = json.load(f) +metadata["last-column-id"] = value +metadata["last-updated-ms"] += 1 +tmp = os.path.join(metadata_dir, ".tmp_next") +with open(tmp, "w") as f: + json.dump(metadata, f) +os.rename(tmp, os.path.join(metadata_dir, f"v{latest + 1}.metadata.json")) +PY +} + +# Field ids of the current schema, nested ones included, and `last-column-id` of the latest metadata version. +field_ids() { + python3 - "$1/metadata" <<'PY' +import json, os, re, sys +metadata_dir = sys.argv[1] +latest = max(int(m.group(1)) for f in os.listdir(metadata_dir) if (m := re.fullmatch(r"v(\d+)\.metadata\.json", f))) +with open(os.path.join(metadata_dir, f"v{latest}.metadata.json")) as f: + metadata = json.load(f) +schema = next(s for s in metadata["schemas"] if s["schema-id"] == metadata["current-schema-id"]) +ids = [] +def walk(node, prefix): + if not isinstance(node, dict): + return + if node["type"] == "struct": + for field in node["fields"]: + ids.append((field["id"], prefix + field["name"])) + walk(field["type"], prefix + field["name"] + ".") + elif node["type"] == "list": + ids.append((node["element-id"], prefix + "element")) + walk(node["element"], prefix + "element.") + elif node["type"] == "map": + ids.append((node["key-id"], prefix + "key")) + ids.append((node["value-id"], prefix + "value")) + walk(node["key"], prefix + "key.") + walk(node["value"], prefix + "value.") +walk(schema, "") +print(" ".join(f"{path}={field_id}" for field_id, path in sorted(ids)), f"last-column-id={metadata['last-column-id']}") +PY +} + +# The reproducer from the issue: ids id=1 arr=2 arr.element=3, and 26.6 records last-column-id=2. +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --multiquery -q " +CREATE TABLE ${TABLE_A} (id Int64, arr Array(Int64)) ENGINE = IcebergLocal('${TABLE_A_DIR}/'); +INSERT INTO ${TABLE_A} VALUES (1, [1, 2]); +" +lower_last_column_id "${TABLE_A_DIR}" 2 +# A second table over the same path starts from the lowered value, as a newer server opening an old table does. +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --iceberg_metadata_staleness_ms=0 --multiquery -q " +CREATE TABLE ${TABLE_A}_reopened ENGINE = IcebergLocal('${TABLE_A_DIR}/'); +ALTER TABLE ${TABLE_A}_reopened ADD COLUMN extra Nullable(String); +SELECT * FROM ${TABLE_A}_reopened ORDER BY id; +INSERT INTO ${TABLE_A}_reopened VALUES (2, [3], 'x'); +SELECT * FROM ${TABLE_A}_reopened ORDER BY id; +" +field_ids "${TABLE_A_DIR}" + +# The same with a `Map`: ids id=1 m=2 m.key=3 m.value=4 m.value.element=5. +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --multiquery -q " +CREATE TABLE ${TABLE_D} (id Int64, m Map(String, Array(Int64))) ENGINE = IcebergLocal('${TABLE_D_DIR}/'); +INSERT INTO ${TABLE_D} VALUES (1, map('k', [1, 2])); +" +lower_last_column_id "${TABLE_D_DIR}" 2 +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --iceberg_metadata_staleness_ms=0 --multiquery -q " +CREATE TABLE ${TABLE_D}_reopened ENGINE = IcebergLocal('${TABLE_D_DIR}/'); +ALTER TABLE ${TABLE_D}_reopened ADD COLUMN extra Nullable(String); +SELECT * FROM ${TABLE_D}_reopened ORDER BY id; +" +field_ids "${TABLE_D_DIR}" + +# A dropped column keeps its ids 2, 3, 4 in the first schema and in the data file, so they must not be reused either. +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --multiquery -q " +CREATE TABLE ${TABLE_B} (id Int64, t Tuple(a Int64, b Int64)) ENGINE = IcebergLocal('${TABLE_B_DIR}/'); +INSERT INTO ${TABLE_B} VALUES (1, (10, 20)); +ALTER TABLE ${TABLE_B} DROP COLUMN t; +" +lower_last_column_id "${TABLE_B_DIR}" 2 +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --iceberg_metadata_staleness_ms=0 --multiquery -q " +CREATE TABLE ${TABLE_B}_reopened ENGINE = IcebergLocal('${TABLE_B_DIR}/'); +SELECT * FROM ${TABLE_B}_reopened ORDER BY id; +ALTER TABLE ${TABLE_B}_reopened ADD COLUMN c Nullable(Int64); +ALTER TABLE ${TABLE_B}_reopened ADD COLUMN d Nullable(Int64); +SELECT * FROM ${TABLE_B}_reopened ORDER BY id; +" +field_ids "${TABLE_B_DIR}" + +# A top-level column named `a.b` (id 2) and the field `b` of the tuple `a` (id 3) have the same dotted path. +${CLICKHOUSE_CLIENT} --query "CREATE TABLE ${TABLE_C} (a Tuple(b Int64), \`a.b\` Int64) ENGINE = IcebergLocal('${TABLE_C_DIR}/')" +lower_last_column_id "${TABLE_C_DIR}" 2 +${CLICKHOUSE_CLIENT} --allow_insert_into_iceberg=1 --iceberg_metadata_staleness_ms=0 --multiquery -q " +CREATE TABLE ${TABLE_C}_reopened ENGINE = IcebergLocal('${TABLE_C_DIR}/'); +ALTER TABLE ${TABLE_C}_reopened ADD COLUMN c Nullable(Int64); +" +field_ids "${TABLE_C_DIR}" diff --git a/tests/queries/0_stateless/05480_snappy_framed_read_padding.reference b/tests/queries/0_stateless/05480_snappy_framed_read_padding.reference new file mode 100644 index 0000000000000..1daaa760f7f48 --- /dev/null +++ b/tests/queries/0_stateless/05480_snappy_framed_read_padding.reference @@ -0,0 +1,4 @@ +01 +90 50 +00 +129 20 diff --git a/tests/queries/0_stateless/05480_snappy_framed_read_padding.sh b/tests/queries/0_stateless/05480_snappy_framed_read_padding.sh new file mode 100755 index 0000000000000..510f5a8accfff --- /dev/null +++ b/tests/queries/0_stateless/05480_snappy_framed_read_padding.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Tags: no-fasttest + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Reading a framed-snappy text file without parallel parsing must not read past the decompressed chunk. +# Each case prints the type of the first chunk (00 = compressed, 01 = uncompressed) and the read result. + +DIR="${USER_FILES_PATH}/${CLICKHOUSE_TEST_UNIQUE_NAME}" +mkdir -p "${DIR}" +chmod 777 "${DIR}" +trap 'rm -rf "${DIR}"' EXIT + +function check() +{ + local file="${DIR}/$1.snappy" + ${CLICKHOUSE_CLIENT} -q " + INSERT INTO TABLE FUNCTION file('${file}', 'LineAsString', 'line String', 'snappy') + $2 + SETTINGS snappy_mode = 'framed'; + " + od -A n -t x1 -j 10 -N 1 "${file}" | tr -d ' ' + ${CLICKHOUSE_CLIENT} -q " + SELECT sum(length(line)), count() FROM file('${file}', 'LineAsString', 'line String', 'snappy') + SETTINGS snappy_mode = 'framed', input_format_parallel_parsing = 0; + " +} + +check uncompressed "SELECT toString(number) FROM numbers(50)" +check compressed "SELECT repeat('a', number % 13) || toString(number) FROM numbers(20)" diff --git a/tests/queries/0_stateless/05481_object_storage_parquet_footer_prefetch.reference b/tests/queries/0_stateless/05481_object_storage_parquet_footer_prefetch.reference new file mode 100644 index 0000000000000..8588a3ce6bb67 --- /dev/null +++ b/tests/queries/0_stateless/05481_object_storage_parquet_footer_prefetch.reference @@ -0,0 +1,4 @@ +400 79800 0 399 +58 11571 +INCORRECT_DATA +1 diff --git a/tests/queries/0_stateless/05481_object_storage_parquet_footer_prefetch.sh b/tests/queries/0_stateless/05481_object_storage_parquet_footer_prefetch.sh new file mode 100755 index 0000000000000..aae1c258a604e --- /dev/null +++ b/tests/queries/0_stateless/05481_object_storage_parquet_footer_prefetch.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: needs MinIO + +# A source of an object storage reads the Parquet footer of its next file in the background, while it reads the current one. +# One stream (`max_threads = 1`) reads all files, so every file but the first one is read through a prefetched footer. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +DIR="${CLICKHOUSE_TEST_UNIQUE_NAME}" +Q="${CLICKHOUSE_TEST_UNIQUE_NAME}_" + +for i in 0 1 2 3; do + ${CLICKHOUSE_CLIENT} --query "INSERT INTO FUNCTION s3(s3_conn, filename = '${DIR}/good/${i}.parquet', format = Parquet) SELECT number + ${i} * 100 AS x FROM numbers(100)" +done +${CLICKHOUSE_CLIENT} --query "INSERT INTO FUNCTION s3(s3_conn, filename = '${DIR}/bad/0.parquet', format = Parquet) SELECT number AS x FROM numbers(100)" +${CLICKHOUSE_CLIENT} --query "INSERT INTO FUNCTION s3(s3_conn, filename = '${DIR}/bad/1.parquet', format = RawBLOB) SELECT 'not a parquet file'" + +SETTINGS="--max_threads 1 --use_parquet_metadata_cache 1" +${CLICKHOUSE_CLIENT} $SETTINGS --query_id "${Q}1" --query "SELECT count(x), sum(x), min(x), max(x) FROM s3(s3_conn, filename = '${DIR}/good/*.parquet')" +${CLICKHOUSE_CLIENT} $SETTINGS --query_id "${Q}2" --query "SELECT count(x), sum(x) FROM s3(s3_conn, filename = '${DIR}/good/*.parquet') WHERE x % 7 = 0" + +# The file that is not Parquet fails the query as it did before the footer was read in advance. +${CLICKHOUSE_CLIENT} $SETTINGS --query "SELECT sum(x) FROM s3(s3_conn, filename = '${DIR}/bad/*.parquet')" 2>&1 | grep -o -m1 "INCORRECT_DATA" + +${CLICKHOUSE_CLIENT} --query "SYSTEM FLUSH LOGS query_log" + +# The second query finds all footers in the Parquet metadata cache. +# With parallel replicas, `s3` is rewritten to `s3Cluster` and the files are read by secondary queries, +# which run with `current_database = 'default'`, so the hits are summed over `initial_query_id`. +${CLICKHOUSE_CLIENT} --query " + SELECT sum(ProfileEvents['ParquetMetadataCacheHits']) >= 4 + FROM system.query_log + WHERE initial_query_id = '${Q}2' AND type = 'QueryFinish' + AND (current_database = currentDatabase() OR NOT is_initial_query) +" diff --git a/tests/queries/0_stateless/05481_view_with_two_array_joins.reference b/tests/queries/0_stateless/05481_view_with_two_array_joins.reference new file mode 100644 index 0000000000000..aa8e3286a4af2 --- /dev/null +++ b/tests/queries/0_stateless/05481_view_with_two_array_joins.reference @@ -0,0 +1,38 @@ +v 1 1 +v 1 2 +v 1 3 +v_definer 1 1 +v_definer 1 2 +v_definer 1 3 +v_definer, no pushdown 1 1 +v_definer, no pushdown 1 2 +v_definer, no pushdown 1 3 +v_dist 1 1 +v_dist 1 2 +v_dist 1 3 +v_full 1 1 1 +v_full 1 2 1 +v_full 1 3 1 +v_paste 1 1 0 +v_paste 1 2 0 +v_paste 1 3 1 +v_asof 1 1 1 +v_asof 1 2 1 +v_asof 1 3 1 +v_cross 1 1 1 +v_cross 1 1 2 +v_cross 1 2 1 +v_cross 1 2 2 +v_cross 1 3 1 +v_cross 1 3 2 +v_left_semi 1 1 1 +v_left_semi 1 2 1 +v_left_semi 1 3 1 +v_right_semi 1 1 1 +v_right_semi 1 2 1 +v_right_semi 1 3 1 +mv 1 1 +mv 1 2 +mv 1 3 +v_dist_plain pushdown 1 +v_dist no pushdown 1 diff --git a/tests/queries/0_stateless/05481_view_with_two_array_joins.sql b/tests/queries/0_stateless/05481_view_with_two_array_joins.sql new file mode 100644 index 0000000000000..5b7100c3f32aa --- /dev/null +++ b/tests/queries/0_stateless/05481_view_with_two_array_joins.sql @@ -0,0 +1,47 @@ +-- Tags: distributed +-- A view whose query has more than one ARRAY JOIN must be readable (issue #128410). +CREATE TABLE t (id UInt32, a Array(Array(UInt32))) ENGINE = MergeTree ORDER BY id; +INSERT INTO t VALUES (1, [[1, 2], [3]]); +CREATE TABLE t_dist AS t ENGINE = Distributed(test_shard_localhost, currentDatabase(), t); + +CREATE VIEW v AS SELECT id, y FROM t ARRAY JOIN a AS x ARRAY JOIN x AS y; +CREATE VIEW v_definer SQL SECURITY DEFINER AS SELECT id, y FROM t ARRAY JOIN a AS x ARRAY JOIN x AS y; +CREATE VIEW v_dist AS SELECT id, y FROM t_dist ARRAY JOIN a AS x ARRAY JOIN x AS y; + +SELECT 'v', * FROM v ORDER BY y SETTINGS optimize_trivial_view_pushdown_to_distributed = 1; +SELECT 'v_definer', * FROM v_definer ORDER BY y SETTINGS optimize_trivial_view_pushdown_to_distributed = 1; +SELECT 'v_definer, no pushdown', * FROM v_definer ORDER BY y SETTINGS optimize_trivial_view_pushdown_to_distributed = 0; +SELECT 'v_dist', * FROM v_dist ORDER BY y SETTINGS optimize_trivial_view_pushdown_to_distributed = 1; + +-- Other JOIN kinds between or around the ARRAY JOINs. +CREATE TABLE r (id UInt32, k UInt32) ENGINE = MergeTree ORDER BY id; +INSERT INTO r VALUES (1, 1), (2, 2); + +CREATE VIEW v_full AS SELECT id, y, k FROM t ARRAY JOIN a AS x FULL OUTER JOIN r ON t.id = r.id ARRAY JOIN x AS y; +CREATE VIEW v_paste AS SELECT id, y, n FROM t ARRAY JOIN a AS x PASTE JOIN (SELECT number AS n FROM numbers(2)) AS p ARRAY JOIN x AS y; +CREATE VIEW v_asof AS SELECT id, y, k FROM t ARRAY JOIN a AS x ASOF JOIN r ON t.id = r.id AND length(x) >= r.k ARRAY JOIN x AS y; +CREATE VIEW v_cross AS SELECT id, y, k FROM t ARRAY JOIN a AS x CROSS JOIN r ARRAY JOIN x AS y; +CREATE VIEW v_left_semi AS SELECT id, y, k FROM t ARRAY JOIN a AS x LEFT SEMI JOIN r ON t.id = r.id ARRAY JOIN x AS y; +CREATE VIEW v_right_semi AS SELECT id, y, k FROM t RIGHT SEMI JOIN r ON t.id = r.id ARRAY JOIN a AS x ARRAY JOIN x AS y; + +SELECT 'v_full', * FROM v_full ORDER BY ALL SETTINGS optimize_trivial_view_pushdown_to_distributed = 1; +SELECT 'v_paste', * FROM v_paste ORDER BY ALL SETTINGS optimize_trivial_view_pushdown_to_distributed = 1; +SELECT 'v_asof', * FROM v_asof ORDER BY ALL SETTINGS optimize_trivial_view_pushdown_to_distributed = 1; +SELECT 'v_cross', * FROM v_cross ORDER BY ALL SETTINGS optimize_trivial_view_pushdown_to_distributed = 1; +SELECT 'v_left_semi', * FROM v_left_semi ORDER BY ALL SETTINGS optimize_trivial_view_pushdown_to_distributed = 1; +SELECT 'v_right_semi', * FROM v_right_semi ORDER BY ALL SETTINGS optimize_trivial_view_pushdown_to_distributed = 1; + +-- A refreshable materialized view reading the view. +CREATE TABLE dst (id UInt32, y UInt32) ENGINE = MergeTree ORDER BY (id, y); +CREATE MATERIALIZED VIEW mv REFRESH AFTER 1 YEAR SETTINGS all_replicas = 1 APPEND TO dst EMPTY AS SELECT id, y FROM v; +SYSTEM REFRESH VIEW mv; +SYSTEM WAIT VIEW mv; +SELECT 'mv', * FROM dst ORDER BY y; + +-- The view over `Distributed` is not trivial, so the pushdown does not fire for it; a plain view is the positive control. +-- The default EXPLAIN format does not print the `VIEW subquery` step. +SET explain_query_plan_default = 'legacy'; +SET prefer_localhost_replica = 0; +CREATE VIEW v_dist_plain AS SELECT id FROM t_dist; +SELECT 'v_dist_plain pushdown', countIf(explain LIKE '%VIEW subquery%') = 0 FROM (EXPLAIN SELECT id FROM v_dist_plain SETTINGS optimize_trivial_view_pushdown_to_distributed = 1); +SELECT 'v_dist no pushdown', countIf(explain LIKE '%VIEW subquery%') > 0 FROM (EXPLAIN SELECT id, y FROM v_dist SETTINGS optimize_trivial_view_pushdown_to_distributed = 1); diff --git a/tests/queries/0_stateless/05483_arrow_record_batch_split_many_batches.reference b/tests/queries/0_stateless/05483_arrow_record_batch_split_many_batches.reference new file mode 100644 index 0000000000000..62464cbf5e5c0 --- /dev/null +++ b/tests/queries/0_stateless/05483_arrow_record_batch_split_many_batches.reference @@ -0,0 +1,2 @@ +batches: 4096 row counts: [64] +262144 34359607296 1000 16777216 diff --git a/tests/queries/0_stateless/05483_arrow_record_batch_split_many_batches.sh b/tests/queries/0_stateless/05483_arrow_record_batch_split_many_batches.sh new file mode 100755 index 0000000000000..818ccdf9b491c --- /dev/null +++ b/tests/queries/0_stateless/05483_arrow_record_batch_split_many_batches.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# ^ pyarrow is not available in the fast test + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# One large block split by a small `output_format_arrow_max_record_batch_size_bytes` into thousands of +# record batches. The `Variant` and `LowCardinality` columns never need a split themselves, but measuring +# them is linear in the rows given, so this is the shape where measuring the whole remainder of the block +# for every batch would be quadratic. Every batch is bounded by the `String` column: 64 rows of 64 bytes. + +FILE="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_many_batches.arrows" +trap 'rm -f "$FILE"' EXIT + +$CLICKHOUSE_LOCAL --query " + SELECT + number::Variant(UInt64, String) AS v, + toLowCardinality(toString(number % 1000)) AS lc, + repeat('x', 64) AS s + FROM numbers(262144) + SETTINGS + max_block_size = 262144, + output_format_arrow_max_record_batch_size_bytes = 4096, + output_format_arrow_compression_method = 'none' + FORMAT ArrowStream" > "$FILE" + +python3 - "$FILE" <<'PY' +import sys +import pyarrow as pa +import pyarrow.ipc as ipc + +with pa.OSFile(sys.argv[1], "rb") as src: + batches = list(ipc.open_stream(src)) +print("batches:", len(batches), "row counts:", sorted({batch.num_rows for batch in batches})) +PY + +$CLICKHOUSE_LOCAL --query " + SELECT count(), sum(variantElement(v, 'UInt64')), uniqExact(lc), sum(length(s)) + FROM file('$FILE', ArrowStream)" diff --git a/tests/queries/0_stateless/05483_replicated_database_ddl_during_restore_replica.reference b/tests/queries/0_stateless/05483_replicated_database_ddl_during_restore_replica.reference new file mode 100644 index 0000000000000..d00491fd7e5bb --- /dev/null +++ b/tests/queries/0_stateless/05483_replicated_database_ddl_during_restore_replica.reference @@ -0,0 +1 @@ +1 diff --git a/tests/queries/0_stateless/05483_replicated_database_ddl_during_restore_replica.sh b/tests/queries/0_stateless/05483_replicated_database_ddl_during_restore_replica.sh new file mode 100755 index 0000000000000..eec443dcca491 --- /dev/null +++ b/tests/queries/0_stateless/05483_replicated_database_ddl_during_restore_replica.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +# Tags: zookeeper + +# `SYSTEM RESTORE DATABASE REPLICA` stops and replaces the DDL worker of a `Replicated` database. +# A DDL query that was concurrently waiting in the worker used to keep a raw pointer to it +# and dereference the destroyed worker (segmentation fault). Now such a query either succeeds +# or fails with an exception, and the server stays alive. + +# The racing queries fail with various errors by design; they are not interesting. +CLICKHOUSE_CLIENT_SERVER_LOGS_LEVEL=none + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +DB="rdb_${CLICKHOUSE_TEST_UNIQUE_NAME}" +ZK_PATH="/test/${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}/${CLICKHOUSE_TEST_UNIQUE_NAME}" + +$CLICKHOUSE_CLIENT --query "CREATE DATABASE ${DB} ENGINE = Replicated('${ZK_PATH}', 'shard1', 'replica1')" + +TIMEOUT=10 + +function ddl_thread() +{ + local i=0 + local end=$((SECONDS + TIMEOUT)) + while [ $SECONDS -lt $end ]; do + i=$((i + 1)) + $CLICKHOUSE_CLIENT --query "CREATE TABLE ${DB}.t_${1}_${i} (x UInt64) ENGINE = MergeTree ORDER BY x" >/dev/null 2>&1 + $CLICKHOUSE_CLIENT --query "DROP TABLE IF EXISTS ${DB}.t_${1}_${i} SYNC" >/dev/null 2>&1 + done +} + +function sync_thread() +{ + local end=$((SECONDS + TIMEOUT)) + while [ $SECONDS -lt $end ]; do + $CLICKHOUSE_CLIENT --query "SYSTEM SYNC DATABASE REPLICA ${DB}" >/dev/null 2>&1 + done +} + +function restore_thread() +{ + local end=$((SECONDS + TIMEOUT)) + while [ $SECONDS -lt $end ]; do + $CLICKHOUSE_CLIENT --query "SYSTEM RESTORE DATABASE REPLICA ${DB}" >/dev/null 2>&1 + done +} + +ddl_thread 1 & +ddl_thread 2 & +ddl_thread 3 & +sync_thread & +restore_thread & +restore_thread & +wait + +# The server is alive and the database still accepts DDL. +for _ in $(seq 1 60); do + $CLICKHOUSE_CLIENT --query "CREATE TABLE ${DB}.t_final (x UInt64) ENGINE = MergeTree ORDER BY x" >/dev/null 2>&1 && break + sleep 1 +done +$CLICKHOUSE_CLIENT --query "EXISTS TABLE ${DB}.t_final" + +$CLICKHOUSE_CLIENT --query "DROP DATABASE ${DB} SYNC" diff --git a/tests/queries/0_stateless/05484_replicated_database_restore_replica_waits_for_initial_query.reference b/tests/queries/0_stateless/05484_replicated_database_restore_replica_waits_for_initial_query.reference new file mode 100644 index 0000000000000..587aee15552be --- /dev/null +++ b/tests/queries/0_stateless/05484_replicated_database_restore_replica_waits_for_initial_query.reference @@ -0,0 +1,5 @@ +restore is waiting 1 +create succeeded +restore finished +1 +0 diff --git a/tests/queries/0_stateless/05484_replicated_database_restore_replica_waits_for_initial_query.sh b/tests/queries/0_stateless/05484_replicated_database_restore_replica_waits_for_initial_query.sh new file mode 100755 index 0000000000000..6ef139d50d2d2 --- /dev/null +++ b/tests/queries/0_stateless/05484_replicated_database_restore_replica_waits_for_initial_query.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Tags: zookeeper, no-fasttest, no-parallel +# no-parallel: the `database_replicated_pause_before_initial_query_execution` failpoint fires globally, +# it would pause (or be consumed by) a DDL query of another `Replicated` database. + +# An initial DDL query of a `Replicated` database executes and commits on the submitting thread. +# `SYSTEM RESTORE DATABASE REPLICA` must wait for such a query that was already admitted by the old +# DDL worker before the new worker recovers the replica. Otherwise the query could commit its metadata +# while the recovery compares the Keeper snapshot with the local tables, and the recovery would move +# the just created table to the `_broken_replicated_tables` database. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +DB="rdb_${CLICKHOUSE_TEST_UNIQUE_NAME}" +ZK_PATH="/test/${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}/${CLICKHOUSE_TEST_UNIQUE_NAME}" +FAILPOINT="database_replicated_pause_before_initial_query_execution" +RESTORE_QUERY_ID="restore_${CLICKHOUSE_TEST_UNIQUE_NAME}" + +function cleanup() +{ + $CLICKHOUSE_CLIENT --query "SYSTEM DISABLE FAILPOINT $FAILPOINT" >/dev/null 2>&1 ||: + $CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS ${DB} SYNC" >/dev/null 2>&1 ||: + $CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS ${DB}_broken_replicated_tables SYNC" >/dev/null 2>&1 ||: +} +trap cleanup EXIT + +$CLICKHOUSE_CLIENT --query "CREATE DATABASE ${DB} ENGINE = Replicated('${ZK_PATH}', 'shard1', 'replica1')" + +# Hold the initial `CREATE TABLE` after the DDL worker has admitted it, before it executes and commits. +$CLICKHOUSE_CLIENT --query "SYSTEM ENABLE FAILPOINT $FAILPOINT" +$CLICKHOUSE_CLIENT --distributed_ddl_output_mode=none --query "CREATE TABLE ${DB}.t (x UInt64) ENGINE = MergeTree ORDER BY x" & +create_pid=$! +$CLICKHOUSE_CLIENT --query "SYSTEM WAIT FAILPOINT $FAILPOINT PAUSE" + +$CLICKHOUSE_CLIENT --query_id "$RESTORE_QUERY_ID" --query "SYSTEM RESTORE DATABASE REPLICA ${DB}" >/dev/null 2>&1 & +restore_pid=$! + +# The restore stops the old DDL worker and then has to wait for the paused `CREATE TABLE`. +for _ in $(seq 1 600); do + [ "$($CLICKHOUSE_CLIENT --query "SELECT count() FROM system.processes WHERE query_id = '$RESTORE_QUERY_ID'")" = 1 ] && break + sleep 0.1 +done +sleep 2 +$CLICKHOUSE_CLIENT --query "SELECT 'restore is waiting', count() FROM system.processes WHERE query_id = '$RESTORE_QUERY_ID'" + +$CLICKHOUSE_CLIENT --query "SYSTEM NOTIFY FAILPOINT $FAILPOINT" +wait $create_pid && echo "create succeeded" +# The restore proceeds only after the query has committed. The replica is not lost then (its digest is not empty), +# so the restore may refuse with `REPLICA_ALREADY_EXISTS`; either way the DDL worker is re-created. +wait $restore_pid ||: +echo "restore finished" + +# The table created by the successful query is still there after the recovery of the replica. +$CLICKHOUSE_CLIENT --query "SYSTEM SYNC DATABASE REPLICA ${DB}" +$CLICKHOUSE_CLIENT --query "EXISTS TABLE ${DB}.t" +$CLICKHOUSE_CLIENT --query "SELECT count() FROM system.tables WHERE database = '${DB}_broken_replicated_tables'" diff --git a/tests/queries/0_stateless/05488_ssd_cache_dictionary_cache_file_cannot_grow.reference b/tests/queries/0_stateless/05488_ssd_cache_dictionary_cache_file_cannot_grow.reference new file mode 100644 index 0000000000000..799f581f2dea1 --- /dev/null +++ b/tests/queries/0_stateless/05488_ssd_cache_dictionary_cache_file_cannot_grow.reference @@ -0,0 +1,6 @@ +d_complex LOADED +d_simple LOADED +1 1 +alive +4 +0 diff --git a/tests/queries/0_stateless/05488_ssd_cache_dictionary_cache_file_cannot_grow.sh b/tests/queries/0_stateless/05488_ssd_cache_dictionary_cache_file_cannot_grow.sh new file mode 100755 index 0000000000000..aa8ddefa83721 --- /dev/null +++ b/tests/queries/0_stateless/05488_ssd_cache_dictionary_cache_file_cannot_grow.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-llvm-coverage +# no-fasttest: waits 7 s for the update backoff; no-llvm-coverage: the file size limit below would also cut the coverage profile. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# `ssd_cache` dictionaries whose cache file cannot grow for the next partition stay usable after the failed update, +# and the next update fails the same way. The file size limit lets the first 64 KiB of each cache file be allocated, +# but not the next 64 KiB, while 100 values of 1.2 KB need more than that. + +dir="${CLICKHOUSE_USER_FILES_UNIQUE:?}" +rm -rf "${dir:?}" +mkdir -p "$dir" +trap 'rm -rf "${dir:?}"' EXIT + +query=" +CREATE TABLE src (k UInt64, v String) ENGINE = Memory; +INSERT INTO src SELECT number, toString(number) || ':' || repeat('x', 1200) FROM numbers(100); + +CREATE DICTIONARY d_simple (k UInt64, v String DEFAULT '') PRIMARY KEY k +SOURCE(CLICKHOUSE(TABLE 'src')) LIFETIME(MIN 1000 MAX 1000) +LAYOUT(SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 65536 WRITE_BUFFER_SIZE 4096 MAX_PARTITIONS_COUNT 4 PATH '$dir/d_simple')); + +CREATE DICTIONARY d_complex (k UInt64, v String DEFAULT '') PRIMARY KEY k +SOURCE(CLICKHOUSE(TABLE 'src')) LIFETIME(MIN 1000 MAX 1000) +LAYOUT(COMPLEX_KEY_SSD_CACHE(BLOCK_SIZE 4096 FILE_SIZE 65536 WRITE_BUFFER_SIZE 4096 MAX_PARTITIONS_COUNT 4 PATH '$dir/d_complex')); + +-- Both updates fail, because the cache files cannot grow. +SELECT dictGet('d_simple', 'v', number) FROM numbers(100) FORMAT Null; +SELECT dictGet('d_complex', 'v', tuple(number)) FROM numbers(100) FORMAT Null; + +-- Wait for the 5-6 s backoff after a failed update, so that the next lookups update the dictionaries again. +SELECT sleep(3) FORMAT Null; +SELECT sleep(3) FORMAT Null; +SELECT sleep(1) FORMAT Null; + +SELECT dictGet('d_simple', 'v', number) FROM numbers(100) FORMAT Null; +SELECT dictGet('d_complex', 'v', tuple(number)) FROM numbers(100) FORMAT Null; + +SELECT name, status FROM system.dictionaries ORDER BY name; +SELECT dictGet('d_simple', 'v', toUInt64(0)) = '0:' || repeat('x', 1200), dictGet('d_complex', 'v', tuple(toUInt64(0))) = '0:' || repeat('x', 1200); +SELECT 'alive'; +" + +( + cd "$dir" || exit 1 + ulimit -c 0 + trap '' XFSZ + ulimit -f 100 + $CLICKHOUSE_LOCAL --ignore-error --send_logs_level=error --query "$query" 2>"$dir/stderr" +) + +# Each of the four failed updates reports the failed allocation, and none of them was refused by the backoff. +grep -o -F 'Cannot preallocate space' "$dir/stderr" | wc -l +grep -o -F 'nearest update is scheduled' "$dir/stderr" | wc -l diff --git a/tests/queries/0_stateless/05493_describe_table_function_create_temporary_table_access.reference b/tests/queries/0_stateless/05493_describe_table_function_create_temporary_table_access.reference new file mode 100644 index 0000000000000..8a264f0dc70c0 --- /dev/null +++ b/tests/queries/0_stateless/05493_describe_table_function_create_temporary_table_access.reference @@ -0,0 +1,31 @@ +--- readonly_1 +READONLY +READONLY +READONLY +READONLY +READONLY +READONLY +READONLY +OK +--- readonly_2 +OK +OK +OK +OK +OK +OK +--- readonly_0 +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +ACCESS_DENIED +OK +--- requests to the endpoint +readonly_2_cluster 1 +readonly_2_describe 1 +readonly_2_remote 1 +readonly_2_select 1 +readonly_2_view_if_permitted 1 diff --git a/tests/queries/0_stateless/05493_describe_table_function_create_temporary_table_access.sh b/tests/queries/0_stateless/05493_describe_table_function_create_temporary_table_access.sh new file mode 100755 index 0000000000000..c8437fd1f3aea --- /dev/null +++ b/tests/queries/0_stateless/05493_describe_table_function_create_temporary_table_access.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Resolving the structure of a table function (`DESCRIBE TABLE url(...)`, or a table function nested in +# `cluster` or `viewIfPermitted`) must require the same `CREATE TEMPORARY TABLE` grant as reading from it, and must be refused +# before the source is contacted. Otherwise a user who cannot run `url` (e.g. under `readonly = 1`) can make +# the server request an arbitrary endpoint and read the response from the schema inference error. + +USER_READONLY_1="readonly_1_${CLICKHOUSE_TEST_UNIQUE_NAME}" +USER_READONLY_2="readonly_2_${CLICKHOUSE_TEST_UNIQUE_NAME}" +USER_READONLY_0="readonly_0_${CLICKHOUSE_TEST_UNIQUE_NAME}" +TEMP_CREDENTIALS="unknown" + +# Create users with different readonly settings. +function create_users() { + ${CLICKHOUSE_CLIENT} -m -q " + DROP USER IF EXISTS ${USER_READONLY_1}, ${USER_READONLY_2}, ${USER_READONLY_0}; + CREATE USER ${USER_READONLY_1} IDENTIFIED WITH PLAINTEXT_PASSWORD BY '${TEMP_CREDENTIALS}' SETTINGS readonly = 1; + CREATE USER ${USER_READONLY_2} IDENTIFIED WITH PLAINTEXT_PASSWORD BY '${TEMP_CREDENTIALS}' SETTINGS readonly = 2; + -- \`readonly = 0\`, but without the \`CREATE TEMPORARY TABLE\` grant. + CREATE USER ${USER_READONLY_0} IDENTIFIED WITH PLAINTEXT_PASSWORD BY '${TEMP_CREDENTIALS}'; + GRANT SELECT ON *.*, READ ON URL, READ ON REMOTE TO ${USER_READONLY_1}, ${USER_READONLY_2}, ${USER_READONLY_0}; + GRANT CREATE TEMPORARY TABLE ON *.* TO ${USER_READONLY_1}, ${USER_READONLY_2}; + " +} + +function drop_users() { + ${CLICKHOUSE_CLIENT} -q "DROP USER ${USER_READONLY_1}, ${USER_READONLY_2}, ${USER_READONLY_0}" +} + +# Instead of a mock server, use this server's HTTP interface: every request to it is logged in `system.query_log` +# with the marker given as the argument, so `check_endpoint_requests` can tell whether the source was contacted. +# It always responds in `JSONEachRow`, like the JSON metadata service in the original report: the format passed +# to `url` is how the response is parsed, so `Values` fails to parse it and would echo the body in the error. +function endpoint() { + local marker="$1" + echo "${CLICKHOUSE_PORT_HTTP_PROTO}://${CLICKHOUSE_HOST}:${CLICKHOUSE_PORT_HTTP}/?database=${CLICKHOUSE_DATABASE}&query=SELECT+1+AS+x,+%27marker_${marker}%27+AS+m+FORMAT+JSONEachRow" +} + +# Run a query as the given user and print `READONLY`, `ACCESS_DENIED`, `OK`, or the full unexpected error. +# `CLICKHOUSE_CLIENT` passes randomized settings, which a `readonly = 1` user is not allowed to change. +function run() { + local user="$1" + local query="$2" + local output + output=$(${CLICKHOUSE_CLIENT_BINARY} --host "${CLICKHOUSE_HOST}" --port "${CLICKHOUSE_PORT_TCP}" \ + --user "${user}" --password "${TEMP_CREDENTIALS}" -q "${query}" 2>&1) + if [[ "${output}" =~ \((READONLY|ACCESS_DENIED)\) ]]; then + echo "${BASH_REMATCH[1]}" + elif [[ "${output}" == *Exception* ]]; then + echo "${output}" + else + echo "OK" + fi +} + +# Run all the queries as one user. The user name without the unique suffix (e.g. `readonly_1`) +# labels the output and the endpoint markers. +function check_user() { + local user="$1" + local label="${user%_"${CLICKHOUSE_TEST_UNIQUE_NAME}"}" + echo "--- ${label}" + run "${user}" "SELECT * FROM url('$(endpoint "${label}_select")', 'JSONEachRow')" + run "${user}" "DESCRIBE TABLE url('$(endpoint "${label}_describe")', 'JSONEachRow')" + # With a format that fails to parse the response, the schema inference error would contain the response body. + # It is skipped for `readonly_2`, which is allowed to read the URL: its error text depends on the server version. + if [[ "${label}" != "readonly_2" ]]; then + run "${user}" "DESCRIBE TABLE url('$(endpoint "${label}_values")', 'Values')" + fi + + # `remote` is not allowed in readonly mode, so it is refused before the nested `url` is resolved. + run "${user}" "DESCRIBE TABLE remote('127.0.0.1', url('$(endpoint "${label}_remote")', 'JSONEachRow'))" + # `cluster` is allowed in readonly mode, but it resolves the structure of the nested `url` on the local shard. + run "${user}" "DESCRIBE TABLE cluster('test_shard_localhost', url('$(endpoint "${label}_cluster")', 'JSONEachRow'))" + + # `viewIfPermitted` is allowed in readonly mode, but it resolves the structure of its `ELSE` table function. + run "${user}" "DESCRIBE TABLE viewIfPermitted(SELECT 1 ELSE url('$(endpoint "${label}_view_if_permitted")', 'JSONEachRow'))" + # `SELECT` from `viewIfPermitted` resolves the structure of its `ELSE` table function too, before choosing a branch. + # It is skipped for `readonly_2`: `SELECT 1` is permitted for it and does not match the structure of the `ELSE` branch. + if [[ "${label}" != "readonly_2" ]]; then + run "${user}" "SELECT * FROM viewIfPermitted(SELECT 1 ELSE url('$(endpoint "${label}_select_view_if_permitted")', 'JSONEachRow'))" + fi + + # Table functions allowed in readonly mode stay available without the grant. + run "${user}" "DESCRIBE TABLE numbers(1)" +} + +# Print the markers of the requests that reached the endpoint, i.e. which queries contacted the source. +# Only the queries of the `readonly = 2` user may appear: a marker without requests has no row. +function check_endpoint_requests() { + echo "--- requests to the endpoint" + ${CLICKHOUSE_CLIENT} -m -q " + SYSTEM FLUSH LOGS query_log; + SELECT extract(query, 'marker_([a-z0-9_]+)') AS marker, count() > 0 + FROM system.query_log + WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND query LIKE 'SELECT 1 AS x, %marker_%' + GROUP BY marker + ORDER BY marker; + " +} + +create_users +for user in "${USER_READONLY_1}" "${USER_READONLY_2}" "${USER_READONLY_0}"; do + check_user "${user}" +done +check_endpoint_requests +drop_users diff --git a/tests/queries/0_stateless/05493_msgpack_async_insert_after_invalid_entry.reference b/tests/queries/0_stateless/05493_msgpack_async_insert_after_invalid_entry.reference new file mode 100644 index 0000000000000..2e0aea006e630 --- /dev/null +++ b/tests/queries/0_stateless/05493_msgpack_async_insert_after_invalid_entry.reference @@ -0,0 +1,6 @@ +1 +2 +3 +4 +1 [1,2,3] +1 [1,2,3] diff --git a/tests/queries/0_stateless/05493_msgpack_async_insert_after_invalid_entry.sh b/tests/queries/0_stateless/05493_msgpack_async_insert_after_invalid_entry.sh new file mode 100755 index 0000000000000..19aeaf05141fd --- /dev/null +++ b/tests/queries/0_stateless/05493_msgpack_async_insert_after_invalid_entry.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: MsgPack format is not available in the fasttest environment + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# One async insert batch reuses one MsgPack input format for all its entries. Each case queues an +# invalid entry followed by a valid one: the valid entry must be parsed from a clean state. + +url="${CLICKHOUSE_URL}&async_insert=1&wait_for_async_insert=0&async_insert_use_adaptive_busy_timeout=0&async_insert_busy_timeout_min_ms=100000&async_insert_busy_timeout_max_ms=100000" + +function insert_pair() +{ + local table=$1 bad=$2 good=$3 + printf '%b' "$bad" | ${CLICKHOUSE_CURL} -sS "${url}&query=INSERT+INTO+${table}+FORMAT+MsgPack" --data-binary @- + printf '%b' "$good" | ${CLICKHOUSE_CURL} -sS "${url}&query=INSERT+INTO+${table}+FORMAT+MsgPack" --data-binary @- + ${CLICKHOUSE_CLIENT} -q "SYSTEM FLUSH ASYNC INSERT QUEUE ${table}" + ${CLICKHOUSE_CLIENT} -q "SELECT * FROM ${table} ORDER BY ALL" +} + +${CLICKHOUSE_CLIENT} -q " + CREATE TABLE t_scalar (x UInt64) ENGINE = MergeTree ORDER BY x; + CREATE TABLE t_exception (id UInt64, arr Array(UInt8)) ENGINE = MergeTree ORDER BY id; + CREATE TABLE t_array (id UInt64, arr Array(UInt8)) ENGINE = MergeTree ORDER BY id; +" + +# A uint32 with 1 of its 4 bytes; the valid entry is four rows 1, 2, 3, 4. +insert_pair t_scalar '\xce\x00' '\x01\x02\x03\x04' +# A string inside Array(UInt8) is rejected in the middle of the array. +insert_pair t_exception '\x07\x92\x01\xa1x' '\x01\x93\x01\x02\x03' +# An array of 3 elements with only 1 present. +insert_pair t_array '\x07\x93\x01' '\x01\x93\x01\x02\x03' + +${CLICKHOUSE_CLIENT} -q "DROP TABLE t_scalar; DROP TABLE t_exception; DROP TABLE t_array" diff --git a/tests/queries/0_stateless/05493_mysql_host_connector_syntax.reference b/tests/queries/0_stateless/05493_mysql_host_connector_syntax.reference new file mode 100644 index 0000000000000..60ef3fadf28cc --- /dev/null +++ b/tests/queries/0_stateless/05493_mysql_host_connector_syntax.reference @@ -0,0 +1,13 @@ +which the MySQL client library would read as a connection string +which the MySQL client library would read as a connection string +which the MySQL client library would read as a list of hosts +which the MySQL client library would read as a connection plugin URL +which the MySQL client library would read as the end of the host +connection failed +port 0: rejected +dictionary 127.0.0.1 port 0: rejected +dictionary localhost port 0: unix socket +dictionary localhost port 0: rejected +localhost: tcp +which the MySQL client library would read as a connection string +which the MySQL client library would read as a connection string diff --git a/tests/queries/0_stateless/05493_mysql_host_connector_syntax.sh b/tests/queries/0_stateless/05493_mysql_host_connector_syntax.sh new file mode 100755 index 0000000000000..bc24f80c96227 --- /dev/null +++ b/tests/queries/0_stateless/05493_mysql_host_connector_syntax.sh @@ -0,0 +1,77 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: the MySQL client library is not built in the fast test + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The MySQL client library reinterprets some host strings: a connection string (which can set a socket, a plugin +# directory, TLS files), a connection plugin URL, a list of hosts, or a host cut at a NUL. They must be rejected +# before any connection attempt. + +# Prints how the host was rejected, or "connection failed" for an ordinary connection failure. +function check() +{ + local output + output=$($CLICKHOUSE_CLIENT --send_logs_level=fatal -q "$1" 2>&1 | tr -d '\000') + if grep -q -F "which the MySQL client library would read as" <<< "$output"; then + grep -o -m1 "which the MySQL client library would read as [a-zA-Z ]*" <<< "$output" + elif grep -q -F "ALL_CONNECTION_TRIES_FAILED" <<< "$output"; then + echo "connection failed" + else + echo "unexpected output: $output" + fi +} + +check "SELECT * FROM mysql('socket=/nonexistent/mysqld.sock', 'db', 't', 'u', 'p')" +check "SELECT * FROM mysql('host=localhost;socket=/nonexistent/mysqld.sock', 'db', 't', 'u', 'p')" +check "SELECT * FROM mysql('127.0.0.1,localhost:1', 'db', 't', 'u', 'p')" +check "SELECT * FROM mysql('[replication://127.0.0.1]:1', 'db', 't', 'u', 'p')" +check "SELECT * FROM mysql('127.0.0.1\\0.example.com:1', 'db', 't', 'u', 'p')" + +# A plain host is passed through and fails to connect as usual. +check "SELECT * FROM mysql('127.0.0.1:1', 'db', 't', 'u', 'p')" + +# Port 0 would be replaced by the client library's default port. +$CLICKHOUSE_CLIENT --send_logs_level=fatal -q "SELECT * FROM mysql('127.0.0.1:0', 'db', 't', 'u', 'p')" 2>&1 \ + | grep -q -F "MySQL port 0" && echo "port 0: rejected" || echo "port 0: connection failed" + +# A dictionary socket is used only for `localhost`; for another host it is ignored and port 0 is rejected. +function check_dictionary() +{ + $CLICKHOUSE_CLIENT --send_logs_level=fatal -q " + CREATE DICTIONARY ${CLICKHOUSE_DATABASE}.d_mysql (x UInt64, y String) PRIMARY KEY x + SOURCE(MYSQL(HOST '$1' PORT 0 SOCKET '$2' USER 'u' PASSWORD 'p' DB 'db' TABLE 't')) + LAYOUT(FLAT()) LIFETIME(0)" + local output + output=$($CLICKHOUSE_CLIENT --send_logs_level=fatal -q "SELECT * FROM ${CLICKHOUSE_DATABASE}.d_mysql" 2>&1) + if grep -q -E "MySQL port 0|MySQL socket" <<< "$output"; then + echo "dictionary $1 port 0: rejected" + elif grep -q -F "through socket" <<< "$output"; then + echo "dictionary $1 port 0: unix socket" + else + echo "unexpected output: $output" + fi + $CLICKHOUSE_CLIENT -q "DROP DICTIONARY IF EXISTS ${CLICKHOUSE_DATABASE}.d_mysql" +} +check_dictionary "127.0.0.1" "/nonexistent/mysqld.sock" +check_dictionary "localhost" "/nonexistent/mysqld.sock" +# The client library would read a socket starting with NUL as no socket and connect over TCP. +check_dictionary "localhost" "\\0" + +# `localhost` uses TCP to the given port, not the client library's default Unix socket. +output=$($CLICKHOUSE_CLIENT --send_logs_level=fatal -q "SELECT * FROM mysql('localhost:1', 'db', 't', 'u', 'p')" 2>&1) +if grep -q -F "Can't connect to server on 'localhost'" <<< "$output"; then + echo "localhost: tcp" +elif grep -q -F "through socket" <<< "$output"; then + echo "localhost: unix socket" +else + echo "unexpected output: $output" +fi + +# The `MySQL` database engine and table engine use the same connection pool. +check "CREATE DATABASE ${CLICKHOUSE_DATABASE}_mysql ENGINE = MySQL('socket=/nonexistent/mysqld.sock', 'db', 'u', 'p')" +check "CREATE TABLE ${CLICKHOUSE_DATABASE}.t_mysql (x UInt32) ENGINE = MySQL('socket=/nonexistent/mysqld.sock', 'db', 't', 'u', 'p'); SELECT * FROM ${CLICKHOUSE_DATABASE}.t_mysql" + +$CLICKHOUSE_CLIENT -q "DROP DATABASE IF EXISTS ${CLICKHOUSE_DATABASE}_mysql; DROP TABLE IF EXISTS ${CLICKHOUSE_DATABASE}.t_mysql; DROP DICTIONARY IF EXISTS ${CLICKHOUSE_DATABASE}.d_mysql" diff --git a/tests/queries/0_stateless/05493_s3queue_settings_system_table_hide_secrets.reference b/tests/queries/0_stateless/05493_s3queue_settings_system_table_hide_secrets.reference new file mode 100644 index 0000000000000..6ccddfe23c735 --- /dev/null +++ b/tests/queries/0_stateless/05493_s3queue_settings_system_table_hide_secrets.reference @@ -0,0 +1,4 @@ +after_processing_move_access_key_id key_id_05493 +after_processing_move_connection_string [HIDDEN] +after_processing_move_secret_access_key [HIDDEN] +mode unordered diff --git a/tests/queries/0_stateless/05493_s3queue_settings_system_table_hide_secrets.sql b/tests/queries/0_stateless/05493_s3queue_settings_system_table_hide_secrets.sql new file mode 100644 index 0000000000000..63571197469ef --- /dev/null +++ b/tests/queries/0_stateless/05493_s3queue_settings_system_table_hide_secrets.sql @@ -0,0 +1,22 @@ +-- Tags: no-fasttest, zookeeper +-- no-fasttest: the `S3Queue` engine is not built in the fast test. + +-- `system.s3_queue_settings` hides the secrets that `SHOW CREATE TABLE` hides. + +DROP TABLE IF EXISTS t_secrets; + +CREATE TABLE t_secrets (a String) +ENGINE = S3Queue('http://whatever-we-dont-care:9001/root/t_secrets/*', NOSIGN, 'CSV') +SETTINGS mode = 'unordered', + after_processing_move_access_key_id = 'key_id_05493', + after_processing_move_secret_access_key = 'SEKRIT_05493_1', + after_processing_move_connection_string = 'DefaultEndpointsProtocol=https;AccountName=a;AccountKey=SEKRIT_05493_2;'; + +SELECT name, value +FROM system.s3_queue_settings +WHERE database = currentDatabase() AND table = 't_secrets' + AND name IN ('after_processing_move_access_key_id', 'after_processing_move_secret_access_key', + 'after_processing_move_connection_string', 'mode') +ORDER BY name; + +DROP TABLE t_secrets; diff --git a/tests/queries/0_stateless/05493_skip_index_archive_newer_format_version.reference b/tests/queries/0_stateless/05493_skip_index_archive_newer_format_version.reference new file mode 100644 index 0000000000000..8186bcc258c48 --- /dev/null +++ b/tests/queries/0_stateless/05493_skip_index_archive_newer_format_version.reference @@ -0,0 +1,7 @@ +read and merge +1000 +500 +1 +mutation, then upgrade +1 +1 diff --git a/tests/queries/0_stateless/05493_skip_index_archive_newer_format_version.sh b/tests/queries/0_stateless/05493_skip_index_archive_newer_format_version.sh new file mode 100755 index 0000000000000..c8f25f150af50 --- /dev/null +++ b/tests/queries/0_stateless/05493_skip_index_archive_newer_format_version.sh @@ -0,0 +1,53 @@ +#!/usr/bin/env bash + +# A part whose skip index archive skp_idx.packed was written in a newer, unknown format version +# (as after a rollback to an older server) stays readable, mergeable and mutable; only its skip indices are not used. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +data_path="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_newer_archive" +rm -rf "$data_path" +trap 'rm -rf "$data_path"' EXIT + +# Creates a table with one part that has skp_idx.packed and replaces the archive's format version (its first byte) with 0xff. +function create_part_with_newer_archive() +{ + rm -rf "$data_path" + ${CLICKHOUSE_LOCAL} --path "$data_path" --query " + CREATE DATABASE d ENGINE = Atomic; + CREATE TABLE d.t (id UInt64, v UInt64, INDEX i v TYPE minmax GRANULARITY 1) + ENGINE = MergeTree ORDER BY id + SETTINGS packed_skip_index_max_bytes = '1M', index_granularity = 100, $1 + min_bytes_for_full_part_storage = 0, min_rows_for_full_part_storage = 0, + min_level_for_full_part_storage = 0; + INSERT INTO d.t SELECT number, number FROM numbers(1000); + " + archives=("$data_path"/store/*/*/all_1_1_0/skp_idx.packed) + test -f "${archives[0]}" || { echo "no skp_idx.packed"; exit 1; } + original_version=$(head -c 1 "${archives[0]}" | od -An -tx1 | tr -d ' \n') + printf '\xff' | dd of="${archives[0]}" bs=1 count=1 conv=notrunc 2>/dev/null +} + +echo "read and merge" +create_part_with_newer_archive "" +${CLICKHOUSE_LOCAL} --path "$data_path" --query " + SELECT count() FROM d.t; + SELECT id FROM d.t WHERE v = 500; + INSERT INTO d.t SELECT number, number FROM numbers(1000, 10); + OPTIMIZE TABLE d.t FINAL; + SELECT count() FROM d.t WHERE v = 500 SETTINGS force_data_skipping_indices = 'i'; +" + +# The mutated part must not keep the old archive: after an upgrade it would be readable again but describe the old values. +echo "mutation, then upgrade" +create_part_with_newer_archive "min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0," +${CLICKHOUSE_LOCAL} --path "$data_path" --query "ALTER TABLE d.t UPDATE v = 1000000 WHERE id = 500 SETTINGS mutations_sync = 2" +for archive in "$data_path"/store/*/*/all_*/skp_idx.packed; do + test -f "$archive" && printf "\\x${original_version}" | dd of="$archive" bs=1 count=1 conv=notrunc 2>/dev/null +done +${CLICKHOUSE_LOCAL} --path "$data_path" --query " + SELECT count() FROM d.t WHERE v = 1000000; + SELECT count() FROM d.t WHERE v = 1000000 SETTINGS use_skip_indexes = 0; +" diff --git a/tests/queries/0_stateless/05494_custom_disk_identity_without_credentials_marker.reference b/tests/queries/0_stateless/05494_custom_disk_identity_without_credentials_marker.reference new file mode 100644 index 0000000000000..d6f64353077fd --- /dev/null +++ b/tests/queries/0_stateless/05494_custom_disk_identity_without_credentials_marker.reference @@ -0,0 +1,25 @@ +--- created_before_26_7 +reuse after upgrade +named_ok +unnamed_ok +named_disk +unnamed +the new tables store the definition of the existing tables +a2 same as a +b2 same as b +after restart +1 +1 +1 +1 +named_disk +unnamed +--- permanently_detached +1 +attach +2 +after restart +2 +--- mixed +wrapper_ok +w2 same as w diff --git a/tests/queries/0_stateless/05494_custom_disk_identity_without_credentials_marker.sh b/tests/queries/0_stateless/05494_custom_disk_identity_without_credentials_marker.sh new file mode 100755 index 0000000000000..c62bc4af79251 --- /dev/null +++ b/tests/queries/0_stateless/05494_custom_disk_identity_without_credentials_marker.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A disk definition that uses `from_env` is stored with the `_server_credentials_allowed` marker. Tables created by +# versions before 26.7 store it without the marker. After an upgrade, declaring the same disk again in a new table must +# reuse the existing disk, and the new table must load after a restart. +# `type = 'from_env T'` with `T=local` counts as a disk that may use server credentials, without needing S3. + +export T=local + +run_case() +{ + local name=$1 strip=$2 type="'from_env T'" + local dir="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_${name}" + rm -rf "${dir}" + local named="disk(name = 'named_disk', type = ${type}, path = '${dir}/disk_named/')" + local unnamed="disk(type = ${type}, path = '${dir}/disk_unnamed/')" + + run() { ${CLICKHOUSE_LOCAL} --path "${dir}/data" --dynamic_disk_allow_from_env=1 -mn -q "$1" -- --custom_local_disks_base_directory="${dir}/" 2>&1 | grep -oE '^[0-9]+$|^[a-z0-9_]+$|\([A-Z_]+\)'; } + disks="SELECT if(name = 'named_disk', name, 'unnamed') FROM system.disks WHERE name = 'named_disk' OR name LIKE '__tmp_internal_%' ORDER BY name DESC" + + echo "--- ${name}" + run "CREATE TABLE a (x UInt8) ENGINE = MergeTree ORDER BY x SETTINGS disk = ${named}; INSERT INTO a VALUES (1); + CREATE TABLE b (x UInt8) ENGINE = MergeTree ORDER BY x SETTINGS disk = ${unnamed}; INSERT INTO b VALUES (1);" + # Make the stored definitions look as if an older version had written them. + sed -i "${strip}" "${dir}"/data/store/*/*/{a,b}.sql + + echo "reuse after upgrade" + run "CREATE TABLE a2 (x UInt8) ENGINE = MergeTree ORDER BY x SETTINGS disk = ${named}; SELECT 'named_ok'" + run "CREATE TABLE b2 (x UInt8) ENGINE = MergeTree ORDER BY x SETTINGS disk = ${unnamed}; SELECT 'unnamed_ok'" + run "INSERT INTO a2 VALUES (2); INSERT INTO b2 VALUES (2); ${disks}" + + echo "the new tables store the definition of the existing tables" + for t in a b; do + cmp -s <(grep -o "disk = disk(.*)" "${dir}"/data/store/*/*/${t}.sql) <(grep -o "disk = disk(.*)" "${dir}"/data/store/*/*/${t}2.sql) && echo "${t}2 same as ${t}" + done + + echo "after restart" + run "SELECT count() FROM a; SELECT count() FROM a2; SELECT count() FROM b; SELECT count() FROM b2; ${disks}" + rm -rf "${dir}" +} + +run_case created_before_26_7 's/, equals(_server_credentials_allowed, 1)//' + +# A table detached permanently is not loaded at startup, so its disk is not registered when it is attached again. The +# short `ATTACH` must still find the disk by the definition stored by the older version, not create an empty one. +# `local_blob_storage` keeps its metadata in a directory named after the disk. +export T2=local_blob_storage +dir="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_permanently_detached" +rm -rf "${dir}" +run() { ${CLICKHOUSE_LOCAL} --path "${dir}/data" --dynamic_disk_allow_from_env=1 -mn -q "$1" -- --custom_local_disks_base_directory="${dir}/" 2>&1 | grep -oE '^[0-9]+$|\([A-Z_]+\)'; } +echo "--- permanently_detached" +run "CREATE DATABASE d ENGINE = Atomic; CREATE TABLE d.p (x UInt8) ENGINE = MergeTree ORDER BY x SETTINGS disk = disk(type = 'from_env T2', path = '${dir}/blobs/'); INSERT INTO d.p VALUES (7)" +old_disk=$(ls "${dir}/data/disks" | grep __tmp_internal_) +sed -i 's/, equals(_server_credentials_allowed, 1)//' "${dir}"/data/store/*/*/p.sql +# Load the definition once to learn the name an older version gave the disk, and move the metadata there. +run "SELECT 1" > /dev/null +legacy_disk=$(ls "${dir}/data/disks" | grep __tmp_internal_ | grep -v "${old_disk}") +rm -rf "${dir}/data/disks/${legacy_disk}" +mv "${dir}/data/disks/${old_disk}" "${dir}/data/disks/${legacy_disk}" +run "SELECT count() FROM d.p" +run "DETACH TABLE d.p PERMANENTLY" +echo "attach" +run "ATTACH TABLE d.p; INSERT INTO d.p VALUES (8); SELECT count() FROM d.p" +echo "after restart" +run "SELECT count() FROM d.p" +rm -rf "${dir}" + +# A wrapper declared by an older version names the disk of the older form below it, while a newer table can have +# registered the disk of the newer form too. Declaring the same wrapper again must find the wrapper. +dir="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_mixed" +rm -rf "${dir}" +run() { ${CLICKHOUSE_LOCAL} --path "${dir}/data" --dynamic_disk_allow_from_env=1 -mn -q "$1" -- --custom_local_disks_base_directory="${dir}/" --custom_cached_disks_base_directory="${dir}/caches/" 2>&1 | grep -oE '^[0-9a-z_]+$|\([A-Z_]+\)'; } +leaf="disk(type = 'from_env T2', path = '${dir}/blobs/')" +wrapper="disk(name = 'wrapper_disk', type = cache, path = '${dir}/caches/w/', max_size = '1Gi', disk = ${leaf})" +echo "--- mixed" +run "CREATE TABLE w (x UInt8) ENGINE = MergeTree ORDER BY x SETTINGS disk = ${wrapper}; CREATE TABLE l (x UInt8) ENGINE = MergeTree ORDER BY x SETTINGS disk = ${leaf}" +sed -i 's/, equals(_server_credentials_allowed, 1)//' "${dir}"/data/store/*/*/w.sql +run "CREATE TABLE w2 (x UInt8) ENGINE = MergeTree ORDER BY x SETTINGS disk = ${wrapper}; SELECT 'wrapper_ok'" +w2_definition=$(grep -o "disk = disk(.*)" "${dir}"/data/store/*/*/w2.sql) +[ -n "${w2_definition}" ] && [ "${w2_definition}" = "$(grep -o "disk = disk(.*)" "${dir}"/data/store/*/*/w.sql)" ] && echo "w2 same as w" +rm -rf "${dir}" diff --git a/tests/queries/0_stateless/05496_dynamic_disk_wrapper_of_server_credentials_disk.reference b/tests/queries/0_stateless/05496_dynamic_disk_wrapper_of_server_credentials_disk.reference new file mode 100644 index 0000000000000..c3d951aa50dcc --- /dev/null +++ b/tests/queries/0_stateless/05496_dynamic_disk_wrapper_of_server_credentials_disk.reference @@ -0,0 +1,21 @@ +with the opt-in: create a disk with server credentials: OK +with the opt-in: wrap it: OK +with the opt-in: create a disk without server credentials: OK +without the opt-in: wrap the disk with server credentials: (ACCESS_DENIED) +without the opt-in: wrap its wrapper: (ACCESS_DENIED) +without the opt-in: wrap the disk without server credentials: OK +with the opt-in: wrap the disk with server credentials: OK +with the opt-in: wrap it with from_zk: OK +without the opt-in: the same definition after the node changed: (ACCESS_DENIED) +without the opt-in: the storage policy of the disk with server credentials: (ACCESS_DENIED) +without the opt-in: the storage policy of the disk without server credentials: OK +without the opt-in: change to the storage policy of the disk with server credentials: (ACCESS_DENIED) +with the opt-in: the storage policy of the disk with server credentials: OK +without the opt-in: alter that table otherwise: OK +without the opt-in: change another setting of that table: OK +without the opt-in: Log on the disk with server credentials: (ACCESS_DENIED) +without the opt-in: StripeLog on its storage policy: (ACCESS_DENIED) +without the opt-in: Set on the disk: (ACCESS_DENIED) +without the opt-in: Join on the disk: (ACCESS_DENIED) +without the opt-in: Distributed on its storage policy: (ACCESS_DENIED) +without the opt-in: Log on the disk without server credentials: OK diff --git a/tests/queries/0_stateless/05496_dynamic_disk_wrapper_of_server_credentials_disk.sh b/tests/queries/0_stateless/05496_dynamic_disk_wrapper_of_server_credentials_disk.sh new file mode 100755 index 0000000000000..ad47a415bdc3f --- /dev/null +++ b/tests/queries/0_stateless/05496_dynamic_disk_wrapper_of_server_credentials_disk.sh @@ -0,0 +1,74 @@ +#!/usr/bin/env bash +# Tags: zookeeper, no-fasttest +# Tag zookeeper: the disk type is read with `from_zk`. +# Tag no-fasttest: needs the `encrypted` disk, which requires SSL. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A session without `s3_allow_server_credentials_in_user_queries` may not wrap, by name, a disk that a session with it +# created with the server's own credentials, nor a wrapper of such a disk. A `type` read with `from_zk` counts as a +# disk that may use server credentials; here it resolves to an `encrypted` disk, so no object storage is needed. +# The `encrypted` disks wrap local disks under `custom_local_disks_base_directory`. A disk stays registered when its +# table is dropped, so every table is dropped right away: a table naming a disk of another table could not be loaded +# after a restart. + +ZK_PATH="/clickhouse/${CLICKHOUSE_TEST_UNIQUE_NAME}" +KEY="1234567812345678" +SERVER_DISK="server_${CLICKHOUSE_DATABASE}" +WRAPPER_DISK="wrapper_${CLICKHOUSE_DATABASE}" +PLAIN_DISK="plain_${CLICKHOUSE_DATABASE}" + +run() +{ + local out + if out=$(${CLICKHOUSE_CLIENT} --dynamic_disk_allow_from_zk=1 "$@" 2>&1); then echo "OK"; else echo "$out" | grep -oE '\([A-Z_]+\)' | head -1; fi + ${CLICKHOUSE_CLIENT} -q "DROP TABLE IF EXISTS created SYNC" +} +create() { echo "CREATE TABLE $1 (x UInt8) ENGINE = MergeTree ORDER BY x SETTINGS disk = $2"; } +wrap() { echo "disk(type = encrypted, disk = '$1', path = '$2/', key = '${KEY}')"; } + +OPT_IN="--s3_allow_server_credentials_in_user_queries=1" +local_disk() { echo "disk(type = local, path = '${CLICKHOUSE_DISKS_FILES}/${CLICKHOUSE_TEST_UNIQUE_NAME}_$1/')"; } + +$CLICKHOUSE_KEEPER_CLIENT -q "create '${ZK_PATH}' 'disk'" +$CLICKHOUSE_KEEPER_CLIENT -q "create '${ZK_PATH}/type' 'encrypted'" + +echo "with the opt-in: create a disk with server credentials: $(run ${OPT_IN} -q "$(create created "disk(name = '${SERVER_DISK}', type = 'from_zk ${ZK_PATH}/type', disk = $(local_disk server), path = 'encrypted/', key = '${KEY}')")")" +echo "with the opt-in: wrap it: $(run ${OPT_IN} -q "$(create created "disk(name = '${WRAPPER_DISK}', type = encrypted, disk = '${SERVER_DISK}', path = 'wrapper/', key = '${KEY}')")")" +echo "with the opt-in: create a disk without server credentials: $(run ${OPT_IN} -q "$(create created "disk(name = '${PLAIN_DISK}', type = encrypted, disk = $(local_disk plain), path = 'encrypted/', key = '${KEY}')")")" + +echo "without the opt-in: wrap the disk with server credentials: $(run -q "$(create created "$(wrap "${SERVER_DISK}" t1)")")" +echo "without the opt-in: wrap its wrapper: $(run -q "$(create created "$(wrap "${WRAPPER_DISK}" t2)")")" +echo "without the opt-in: wrap the disk without server credentials: $(run -q "$(create created "$(wrap "${PLAIN_DISK}" t3)")")" +echo "with the opt-in: wrap the disk with server credentials: $(run ${OPT_IN} -q "$(create created "$(wrap "${SERVER_DISK}" t4)")")" + +# The wrapped disk is read with `from_zk`: after the node changes, the same definition no longer resolves to the disk with +# server credentials, but it still names the disk created from it. +$CLICKHOUSE_KEEPER_CLIENT -q "create '${ZK_PATH}/wrapped' '${SERVER_DISK}'" +INDIRECT_WRAPPER="disk(type = encrypted, disk = 'from_zk ${ZK_PATH}/wrapped', path = 'indirect/', key = '${KEY}')" +echo "with the opt-in: wrap it with from_zk: $(run ${OPT_IN} -q "$(create created "${INDIRECT_WRAPPER}")")" +$CLICKHOUSE_KEEPER_CLIENT -q "set '${ZK_PATH}/wrapped' '${PLAIN_DISK}'" +echo "without the opt-in: the same definition after the node changed: $(run -q "$(create created "${INDIRECT_WRAPPER}")")" + +# A table on a disk gets the storage policy `__`, which names the disk as well. +echo "without the opt-in: the storage policy of the disk with server credentials: $(run -q "CREATE TABLE created (x UInt8) ENGINE = MergeTree ORDER BY x SETTINGS storage_policy = '__${SERVER_DISK}'")" +echo "without the opt-in: the storage policy of the disk without server credentials: $(run -q "CREATE TABLE created (x UInt8) ENGINE = MergeTree ORDER BY x SETTINGS storage_policy = '__${PLAIN_DISK}'")" +echo "without the opt-in: change to the storage policy of the disk with server credentials: $(run -q "CREATE TABLE created (x UInt8) ENGINE = MergeTree ORDER BY x SETTINGS storage_policy = '__${PLAIN_DISK}'; ALTER TABLE created MODIFY SETTING storage_policy = '__${SERVER_DISK}'")" + +# A table that already has the policy can still be altered otherwise. +echo "with the opt-in: the storage policy of the disk with server credentials: $(run ${OPT_IN} -q "CREATE TABLE t_policy (x UInt8) ENGINE = MergeTree ORDER BY x SETTINGS storage_policy = '__${SERVER_DISK}'")" +echo "without the opt-in: alter that table otherwise: $(run -q "ALTER TABLE t_policy ADD COLUMN y UInt8")" +echo "without the opt-in: change another setting of that table: $(run -q "ALTER TABLE t_policy MODIFY SETTING parts_to_throw_insert = 500")" +${CLICKHOUSE_CLIENT} -q "DROP TABLE t_policy SYNC" + +# Other engines name a disk or a storage policy as well. +echo "without the opt-in: Log on the disk with server credentials: $(run -q "CREATE TABLE created (x UInt8) ENGINE = Log SETTINGS disk = '${SERVER_DISK}'")" +echo "without the opt-in: StripeLog on its storage policy: $(run -q "CREATE TABLE created (x UInt8) ENGINE = StripeLog SETTINGS storage_policy = '__${SERVER_DISK}'")" +echo "without the opt-in: Set on the disk: $(run -q "CREATE TABLE created (x UInt8) ENGINE = Set SETTINGS disk = '${SERVER_DISK}'")" +echo "without the opt-in: Join on the disk: $(run -q "CREATE TABLE created (x UInt8) ENGINE = Join(ANY, LEFT, x) SETTINGS disk = '${SERVER_DISK}'")" +echo "without the opt-in: Distributed on its storage policy: $(run -q "CREATE TABLE created (x UInt8) ENGINE = Distributed('test_shard_localhost', currentDatabase(), t_plain, rand(), '__${SERVER_DISK}')")" +echo "without the opt-in: Log on the disk without server credentials: $(run -q "CREATE TABLE created (x UInt8) ENGINE = Log SETTINGS disk = '${PLAIN_DISK}'")" + +$CLICKHOUSE_KEEPER_CLIENT -q "rmr '${ZK_PATH}'" diff --git a/tests/queries/0_stateless/05498_group_by_two_level_few_keys_prefetch.reference b/tests/queries/0_stateless/05498_group_by_two_level_few_keys_prefetch.reference new file mode 100644 index 0000000000000..10172e5b7749e --- /dev/null +++ b/tests/queries/0_stateless/05498_group_by_two_level_few_keys_prefetch.reference @@ -0,0 +1,115 @@ +LowCardinality(Nullable(String)), prefetch on +category_value_01_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 120928 +category_value_02_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 88992 +category_value_03_cccccccccccccccccccccccccccccccccccccccc 74464 +category_value_10_dddddddddddddddddddddddddddddddddddddddd 1029 +category_value_11_dddddddddddddddddddddddddddddddddddddddd 1063 +category_value_12_dddddddddddddddddddddddddddddddddddddddd 1025 +category_value_4_dddddddddddddddddddddddddddddddddddddddd 959 +category_value_5_dddddddddddddddddddddddddddddddddddddddd 1012 +category_value_6_dddddddddddddddddddddddddddddddddddddddd 1037 +category_value_7_dddddddddddddddddddddddddddddddddddddddd 1043 +category_value_8_dddddddddddddddddddddddddddddddddddddddd 982 +category_value_9_dddddddddddddddddddddddddddddddddddddddd 1082 +\N 6384 +LowCardinality(Nullable(String)), prefetch off +category_value_01_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 120928 +category_value_02_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 88992 +category_value_03_cccccccccccccccccccccccccccccccccccccccc 74464 +category_value_10_dddddddddddddddddddddddddddddddddddddddd 1029 +category_value_11_dddddddddddddddddddddddddddddddddddddddd 1063 +category_value_12_dddddddddddddddddddddddddddddddddddddddd 1025 +category_value_4_dddddddddddddddddddddddddddddddddddddddd 959 +category_value_5_dddddddddddddddddddddddddddddddddddddddd 1012 +category_value_6_dddddddddddddddddddddddddddddddddddddddd 1037 +category_value_7_dddddddddddddddddddddddddddddddddddddddd 1043 +category_value_8_dddddddddddddddddddddddddddddddddddddddd 982 +category_value_9_dddddddddddddddddddddddddddddddddddddddd 1082 +\N 6384 +Nullable(String), prefetch on +category_value_01_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 120928 +category_value_02_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 88992 +category_value_03_cccccccccccccccccccccccccccccccccccccccc 74464 +category_value_10_dddddddddddddddddddddddddddddddddddddddd 1029 +category_value_11_dddddddddddddddddddddddddddddddddddddddd 1063 +category_value_12_dddddddddddddddddddddddddddddddddddddddd 1025 +category_value_4_dddddddddddddddddddddddddddddddddddddddd 959 +category_value_5_dddddddddddddddddddddddddddddddddddddddd 1012 +category_value_6_dddddddddddddddddddddddddddddddddddddddd 1037 +category_value_7_dddddddddddddddddddddddddddddddddddddddd 1043 +category_value_8_dddddddddddddddddddddddddddddddddddddddd 982 +category_value_9_dddddddddddddddddddddddddddddddddddddddd 1082 +\N 6384 +Nullable(String), prefetch off +category_value_01_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 120928 +category_value_02_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 88992 +category_value_03_cccccccccccccccccccccccccccccccccccccccc 74464 +category_value_10_dddddddddddddddddddddddddddddddddddddddd 1029 +category_value_11_dddddddddddddddddddddddddddddddddddddddd 1063 +category_value_12_dddddddddddddddddddddddddddddddddddddddd 1025 +category_value_4_dddddddddddddddddddddddddddddddddddddddd 959 +category_value_5_dddddddddddddddddddddddddddddddddddddddd 1012 +category_value_6_dddddddddddddddddddddddddddddddddddddddd 1037 +category_value_7_dddddddddddddddddddddddddddddddddddddddd 1043 +category_value_8_dddddddddddddddddddddddddddddddddddddddd 982 +category_value_9_dddddddddddddddddddddddddddddddddddddddd 1082 +\N 6384 +keys256, prefetch on +0 7 11 23584 +1 7 11 22480 +2 7 11 23504 +3 7 11 23424 +4 7 11 22992 +5 7 11 23248 +6 7 11 22848 +7 7 11 23168 +8 7 11 21952 +9 7 11 23200 +10 7 11 23456 +11 7 11 23392 +12 7 11 22752 +keys256, prefetch off +0 7 11 23584 +1 7 11 22480 +2 7 11 23504 +3 7 11 23424 +4 7 11 22992 +5 7 11 23248 +6 7 11 22848 +7 7 11 23168 +8 7 11 21952 +9 7 11 23200 +10 7 11 23456 +11 7 11 23392 +12 7 11 22752 +serialized, prefetch on + 7 6384 +category_value_01_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 7 120928 +category_value_02_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 7 88992 +category_value_03_cccccccccccccccccccccccccccccccccccccccc 7 74464 +category_value_10_dddddddddddddddddddddddddddddddddddddddd 7 1029 +category_value_11_dddddddddddddddddddddddddddddddddddddddd 7 1063 +category_value_12_dddddddddddddddddddddddddddddddddddddddd 7 1025 +category_value_4_dddddddddddddddddddddddddddddddddddddddd 7 959 +category_value_5_dddddddddddddddddddddddddddddddddddddddd 7 1012 +category_value_6_dddddddddddddddddddddddddddddddddddddddd 7 1037 +category_value_7_dddddddddddddddddddddddddddddddddddddddd 7 1043 +category_value_8_dddddddddddddddddddddddddddddddddddddddd 7 982 +category_value_9_dddddddddddddddddddddddddddddddddddddddd 7 1082 +serialized, prefetch off + 7 6384 +category_value_01_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa 7 120928 +category_value_02_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb 7 88992 +category_value_03_cccccccccccccccccccccccccccccccccccccccc 7 74464 +category_value_10_dddddddddddddddddddddddddddddddddddddddd 7 1029 +category_value_11_dddddddddddddddddddddddddddddddddddddddd 7 1063 +category_value_12_dddddddddddddddddddddddddddddddddddddddd 7 1025 +category_value_4_dddddddddddddddddddddddddddddddddddddddd 7 959 +category_value_5_dddddddddddddddddddddddddddddddddddddddd 7 1012 +category_value_6_dddddddddddddddddddddddddddddddddddddddd 7 1037 +category_value_7_dddddddddddddddddddddddddddddddddddddddd 7 1043 +category_value_8_dddddddddddddddddddddddddddddddddddddddd 7 982 +category_value_9_dddddddddddddddddddddddddddddddddddddddd 7 1082 +grown two-level, prefetch on and off +100000 300000 14999850000 +100000 300000 14999850000 diff --git a/tests/queries/0_stateless/05498_group_by_two_level_few_keys_prefetch.sql b/tests/queries/0_stateless/05498_group_by_two_level_few_keys_prefetch.sql new file mode 100644 index 0000000000000..231b76d6e9d6a --- /dev/null +++ b/tests/queries/0_stateless/05498_group_by_two_level_few_keys_prefetch.sql @@ -0,0 +1,62 @@ +-- A two-level `GROUP BY` table that holds few keys decides the software prefetch by its keys, not by the preallocated +-- buckets (see `bytesForPrefetch`). The decision never changes the result, so the test only checks that every two-level +-- table shape behind that gate returns the same rows with the prefetch on and off: the issue's +-- `LowCardinality(Nullable(String))` key, `Nullable(String)` (`TwoLevelStringHashMap`), `keys256`, serialized keys +-- (`HashMethodSerialized` has a gate of its own), and a two-level table that has outgrown its initial allocation. + +DROP TABLE IF EXISTS t_two_level_few_keys; + +CREATE TABLE t_two_level_few_keys +( + lc LowCardinality(Nullable(String)), + s Nullable(String), + str String, + a UInt64, + b UInt64, + c UInt64, + many UInt64 +) +ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO t_two_level_few_keys +SELECT v, v, ifNull(v, ''), intHash32(intDiv(number, 16)) % 13, 7, 11, number % 100000 +FROM +( + SELECT + number, + multiIf( + intHash32(intDiv(number, 16)) % 100 < 40, 'category_value_01_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', + intHash32(intDiv(number, 16)) % 100 < 70, 'category_value_02_bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', + intHash32(intDiv(number, 16)) % 100 < 95, 'category_value_03_cccccccccccccccccccccccccccccccccccccccc', + intHash32(intDiv(number, 16)) % 100 < 97, NULL, + concat('category_value_', toString((intHash32(number) % 9) + 4), '_dddddddddddddddddddddddddddddddddddddddd')) AS v + FROM numbers_mt(300000) +); + +SET max_threads = 8, max_block_size = 8192, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes = 1; + +SELECT 'LowCardinality(Nullable(String)), prefetch on'; +SELECT lc, count() FROM t_two_level_few_keys GROUP BY lc ORDER BY lc SETTINGS enable_software_prefetch_in_aggregation = 1; +SELECT 'LowCardinality(Nullable(String)), prefetch off'; +SELECT lc, count() FROM t_two_level_few_keys GROUP BY lc ORDER BY lc SETTINGS enable_software_prefetch_in_aggregation = 0; + +SELECT 'Nullable(String), prefetch on'; +SELECT s, count() FROM t_two_level_few_keys GROUP BY s ORDER BY s SETTINGS enable_software_prefetch_in_aggregation = 1; +SELECT 'Nullable(String), prefetch off'; +SELECT s, count() FROM t_two_level_few_keys GROUP BY s ORDER BY s SETTINGS enable_software_prefetch_in_aggregation = 0; + +SELECT 'keys256, prefetch on'; +SELECT a, b, c, count() FROM t_two_level_few_keys GROUP BY a, b, c ORDER BY a, b, c SETTINGS enable_software_prefetch_in_aggregation = 1; +SELECT 'keys256, prefetch off'; +SELECT a, b, c, count() FROM t_two_level_few_keys GROUP BY a, b, c ORDER BY a, b, c SETTINGS enable_software_prefetch_in_aggregation = 0; + +SELECT 'serialized, prefetch on'; +SELECT str, b, count() FROM t_two_level_few_keys GROUP BY str, b ORDER BY str, b SETTINGS enable_software_prefetch_in_aggregation = 1; +SELECT 'serialized, prefetch off'; +SELECT str, b, count() FROM t_two_level_few_keys GROUP BY str, b ORDER BY str, b SETTINGS enable_software_prefetch_in_aggregation = 0; + +SELECT 'grown two-level, prefetch on and off'; +SELECT count(), sum(cnt), sum(many * cnt) FROM (SELECT many, count() AS cnt FROM t_two_level_few_keys GROUP BY many) SETTINGS enable_software_prefetch_in_aggregation = 1; +SELECT count(), sum(cnt), sum(many * cnt) FROM (SELECT many, count() AS cnt FROM t_two_level_few_keys GROUP BY many) SETTINGS enable_software_prefetch_in_aggregation = 0; + +DROP TABLE t_two_level_few_keys; diff --git a/tests/queries/0_stateless/05498_s3_disk_sse_c_key_validation.reference b/tests/queries/0_stateless/05498_s3_disk_sse_c_key_validation.reference new file mode 100644 index 0000000000000..573541ac9702d --- /dev/null +++ b/tests/queries/0_stateless/05498_s3_disk_sse_c_key_validation.reference @@ -0,0 +1 @@ +0 diff --git a/tests/queries/0_stateless/05498_s3_disk_sse_c_key_validation.sql b/tests/queries/0_stateless/05498_s3_disk_sse_c_key_validation.sql new file mode 100644 index 0000000000000..1247bc62fb1f4 --- /dev/null +++ b/tests/queries/0_stateless/05498_s3_disk_sse_c_key_validation.sql @@ -0,0 +1,31 @@ +-- Tags: no-fasttest +-- no-fasttest: needs S3 support in the build. + +-- An SSE-C key that is not the base64 of a 256-bit key is rejected when the S3 client is created, before any request. + +-- Non-ASCII bytes. +CREATE TABLE t_sse_c (x UInt8) ENGINE = MergeTree ORDER BY x +SETTINGS disk = disk(type = 's3', endpoint = 'http://localhost:11111/test/05498/', no_sign_request = 1, skip_access_check = 1, + server_side_encryption_customer_key_base64 = 'éAAA'); -- { serverError BAD_ARGUMENTS } + +-- Valid base64 of 3 bytes. +CREATE TABLE t_sse_c (x UInt8) ENGINE = MergeTree ORDER BY x +SETTINGS disk = disk(type = 's3', endpoint = 'http://localhost:11111/test/05498/', no_sign_request = 1, skip_access_check = 1, + server_side_encryption_customer_key_base64 = 'AAAA'); -- { serverError BAD_ARGUMENTS } + +-- An invalid character in a key of the right length. +CREATE TABLE t_sse_c (x UInt8) ENGINE = MergeTree ORDER BY x +SETTINGS disk = disk(type = 's3', endpoint = 'http://localhost:11111/test/05498/', no_sign_request = 1, skip_access_check = 1, + server_side_encryption_customer_key_base64 = 'MDEyMzQ1Njc4OTAx!jM0NTY3ODkwMTIzNDU2Nzg5MDE='); -- { serverError BAD_ARGUMENTS } + +-- A valid key followed by more characters. +CREATE TABLE t_sse_c (x UInt8) ENGINE = MergeTree ORDER BY x +SETTINGS disk = disk(type = 's3', endpoint = 'http://localhost:11111/test/05498/', no_sign_request = 1, skip_access_check = 1, + server_side_encryption_customer_key_base64 = 'MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDE=!'); -- { serverError BAD_ARGUMENTS } + +-- A valid key with surrounding whitespace. +CREATE TABLE t_sse_c (x UInt8) ENGINE = MergeTree ORDER BY x +SETTINGS disk = disk(type = 's3', endpoint = 'http://localhost:11111/test/05498/', no_sign_request = 1, skip_access_check = 1, + server_side_encryption_customer_key_base64 = ' MDEyMzQ1Njc4OTAxMjM0NTY3ODkwMTIzNDU2Nzg5MDE= '); -- { serverError BAD_ARGUMENTS } + +SELECT count() FROM system.tables WHERE database = currentDatabase() AND name = 't_sse_c'; diff --git a/tests/queries/0_stateless/05507_mv_source_merge_loop_executable_crash.reference b/tests/queries/0_stateless/05507_mv_source_merge_loop_executable_crash.reference new file mode 100644 index 0000000000000..e1ac9521d29dd --- /dev/null +++ b/tests/queries/0_stateless/05507_mv_source_merge_loop_executable_crash.reference @@ -0,0 +1,9 @@ +insert loop 3 3 +insert merge 3 1051 +append_refresh loop 3 6 +append_refresh merge 3 1066 +refreshable loop 16 109 +refreshable merge 16 1066 +insert executable 3 6 +append_refresh executable 3 15 +refreshable executable 16 1066 diff --git a/tests/queries/0_stateless/05507_mv_source_merge_loop_executable_crash.sh b/tests/queries/0_stateless/05507_mv_source_merge_loop_executable_crash.sh new file mode 100755 index 0000000000000..9ca9db387b390 --- /dev/null +++ b/tests/queries/0_stateless/05507_mv_source_merge_loop_executable_crash.sh @@ -0,0 +1,133 @@ +#!/usr/bin/env bash +# Tags: no-old-analyzer +# no-old-analyzer: the fix is in the analyzer path; the old interpreter reads the view source in its own way. + +# Inside a materialized view query, the source table read by the nested interpreter of `loop` or `executable` +# is the inserted block, like the rest of the view query, while a `merge` child reads the table whatever its +# siblings are. The database name is spelled out: `currentDatabase()` in the view query does not resolve to the +# test database during the push. +# A refreshable view appending to the source table behaves like an INSERT; a refreshable view reading +# the source table reads the table. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +$CLICKHOUSE_CLIENT -m -q " +DROP DATABASE IF EXISTS ${CLICKHOUSE_DATABASE_1}; +DROP TABLE IF EXISTS mv_merge; +DROP TABLE IF EXISTS mv_loop; +DROP TABLE IF EXISTS dst; +DROP TABLE IF EXISTS dist; +DROP TABLE IF EXISTS dist_target; +DROP TABLE IF EXISTS src; + +CREATE TABLE src (x UInt64) ENGINE = MergeTree ORDER BY x; +INSERT INTO src SELECT number + 100 FROM numbers(10); + +CREATE TABLE dist_target (x UInt64) ENGINE = MergeTree ORDER BY x; +CREATE TABLE dist AS dist_target ENGINE = Distributed(test_shard_localhost, ${CLICKHOUSE_DATABASE}, dist_target); + +CREATE TABLE dst (name String, block_rows UInt64, value UInt64) ENGINE = MergeTree ORDER BY name; + +-- 'dist' raises the merge read above FetchColumns, so 'src' is planned by a nested interpreter. +CREATE MATERIALIZED VIEW mv_merge TO dst AS + SELECT 'merge' AS name, count() AS block_rows, + (SELECT sum(x) FROM merge(${CLICKHOUSE_DATABASE}, '^(src|dist)\$') WHERE x > 0) AS value + FROM src; + +CREATE MATERIALIZED VIEW mv_loop TO dst AS + SELECT 'loop' AS name, count() AS block_rows, + (SELECT max(x) FROM (SELECT x FROM loop(${CLICKHOUSE_DATABASE}, src) WHERE x > 0 LIMIT 25)) AS value + FROM src; + +INSERT INTO src VALUES (1), (2), (3); + +SELECT 'insert', * FROM dst ORDER BY name; + +-- A refreshable view appending to the source table pushes its block to these views like an INSERT. It lives in an Atomic +-- database: in a Replicated one an APPEND refresh runs on one replica only. +TRUNCATE TABLE dst; +CREATE DATABASE ${CLICKHOUSE_DATABASE_1} ENGINE = Atomic; +CREATE MATERIALIZED VIEW ${CLICKHOUSE_DATABASE_1}.rmv_append REFRESH EVERY 1 YEAR APPEND TO ${CLICKHOUSE_DATABASE}.src EMPTY AS + SELECT number + 4 AS x FROM numbers(3); +SYSTEM REFRESH VIEW ${CLICKHOUSE_DATABASE_1}.rmv_append; +SYSTEM WAIT VIEW ${CLICKHOUSE_DATABASE_1}.rmv_append; + +SELECT 'append_refresh', * FROM dst ORDER BY name; + +-- A refreshable view reading the source table has no inserted block, so it reads the table. +TRUNCATE TABLE dst; +CREATE MATERIALIZED VIEW ${CLICKHOUSE_DATABASE_1}.rmv_read REFRESH EVERY 1 YEAR APPEND TO ${CLICKHOUSE_DATABASE}.dst EMPTY AS + SELECT 'merge' AS name, count() AS block_rows, + (SELECT sum(x) FROM merge(${CLICKHOUSE_DATABASE}, '^(src|dist)\$') WHERE x > 0) AS value + FROM ${CLICKHOUSE_DATABASE}.src + UNION ALL + SELECT 'loop', count(), (SELECT max(x) FROM (SELECT x FROM loop(${CLICKHOUSE_DATABASE}, src) WHERE x > 0 LIMIT 25)) + FROM ${CLICKHOUSE_DATABASE}.src; +SYSTEM REFRESH VIEW ${CLICKHOUSE_DATABASE_1}.rmv_read; +SYSTEM WAIT VIEW ${CLICKHOUSE_DATABASE_1}.rmv_read; + +SELECT 'refreshable', * FROM dst ORDER BY name; + +DROP DATABASE ${CLICKHOUSE_DATABASE_1}; +DROP TABLE mv_merge; +DROP TABLE mv_loop; +DROP TABLE dst; +DROP TABLE dist; +DROP TABLE dist_target; +DROP TABLE src; +" + +# The input query of `executable` needs `user_scripts_path`, which is a server config option. +WORK_DIR=$(mktemp -d "${CLICKHOUSE_TMP}/05507_XXXXXX") +trap 'rm -rf "${WORK_DIR}"' EXIT +mkdir -p "${WORK_DIR}/user_scripts" "${WORK_DIR}/data" + +cat > "${WORK_DIR}/user_scripts/echo_rows.sh" <<'SCRIPT' +#!/usr/bin/env bash +while IFS= read -r x; do + printf '%s\n' "${x}" +done +SCRIPT +chmod +x "${WORK_DIR}/user_scripts/echo_rows.sh" + +cat > "${WORK_DIR}/config.xml" < + ${WORK_DIR}/user_scripts/ + ${WORK_DIR}/data/ + +EOF + +$CLICKHOUSE_LOCAL --config-file="${WORK_DIR}/config.xml" -m --query " +CREATE DATABASE db_05507; +CREATE TABLE db_05507.src (x UInt64) ENGINE = MergeTree ORDER BY x; +INSERT INTO db_05507.src SELECT number + 100 FROM numbers(10); +CREATE TABLE db_05507.dst (name String, block_rows UInt64, value UInt64) ENGINE = MergeTree ORDER BY name; + +CREATE MATERIALIZED VIEW db_05507.mv_executable TO db_05507.dst AS + SELECT 'executable' AS name, count() AS block_rows, + (SELECT sum(x) FROM executable('echo_rows.sh', 'TSV', 'x UInt64', (SELECT x FROM db_05507.src WHERE x > 0))) AS value + FROM db_05507.src; + +INSERT INTO db_05507.src VALUES (1), (2), (3); + +SELECT 'insert', * FROM db_05507.dst ORDER BY name; + +TRUNCATE TABLE db_05507.dst; +CREATE MATERIALIZED VIEW db_05507.rmv_append REFRESH EVERY 1 YEAR APPEND TO db_05507.src EMPTY AS SELECT number + 4 AS x FROM numbers(3); +SYSTEM REFRESH VIEW db_05507.rmv_append; +SYSTEM WAIT VIEW db_05507.rmv_append; + +SELECT 'append_refresh', * FROM db_05507.dst ORDER BY name; + +TRUNCATE TABLE db_05507.dst; +CREATE MATERIALIZED VIEW db_05507.rmv_read REFRESH EVERY 1 YEAR APPEND TO db_05507.dst EMPTY AS + SELECT 'executable' AS name, count() AS block_rows, + (SELECT sum(x) FROM executable('echo_rows.sh', 'TSV', 'x UInt64', (SELECT x FROM db_05507.src WHERE x > 0))) AS value + FROM db_05507.src; +SYSTEM REFRESH VIEW db_05507.rmv_read; +SYSTEM WAIT VIEW db_05507.rmv_read; + +SELECT 'refreshable', * FROM db_05507.dst ORDER BY name; +" diff --git a/tests/queries/0_stateless/05507_orc_malformed_timestamps.reference b/tests/queries/0_stateless/05507_orc_malformed_timestamps.reference new file mode 100644 index 0000000000000..573a680fbe868 --- /dev/null +++ b/tests/queries/0_stateless/05507_orc_malformed_timestamps.reference @@ -0,0 +1,16 @@ +nanoseconds local push_down=0 lazy=0: (INCORRECT_DATA) +nanoseconds local push_down=0 lazy=1: (INCORRECT_DATA) +nanoseconds local push_down=1 lazy=0: (INCORRECT_DATA) +nanoseconds local push_down=1 lazy=1: (INCORRECT_DATA) +nanoseconds remote push_down=0 lazy=0: (INCORRECT_DATA) +nanoseconds remote push_down=0 lazy=1: (INCORRECT_DATA) +nanoseconds remote push_down=1 lazy=0: (INCORRECT_DATA) +nanoseconds remote push_down=1 lazy=1: (INCORRECT_DATA) +statistics local push_down=0 lazy=0: 3 +statistics local push_down=0 lazy=1: 3 +statistics local push_down=1 lazy=0: 3 +statistics local push_down=1 lazy=1: 3 +statistics remote push_down=0 lazy=0: 3 +statistics remote push_down=0 lazy=1: 3 +statistics remote push_down=1 lazy=0: 3 +statistics remote push_down=1 lazy=1: 3 diff --git a/tests/queries/0_stateless/05507_orc_malformed_timestamps.sh b/tests/queries/0_stateless/05507_orc_malformed_timestamps.sh new file mode 100755 index 0000000000000..53eae4b38eb09 --- /dev/null +++ b/tests/queries/0_stateless/05507_orc_malformed_timestamps.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest, the test uses S3. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh +# shellcheck source=./orc_malformed.lib +. "$CUR_DIR"/orc_malformed.lib + +DATA="$CUR_DIR/data_orc" + +# The nanoseconds of a timestamp with the 7 removed trailing zeros overflow `Int64`. +orc_check nanoseconds "$DATA/malformed_timestamp_nanoseconds.orc" 't Nullable(DateTime64(9))' "SELECT count(), max(t) FROM SOURCE" \ + 'Invalid timestamp in ORC column 1: seconds=[0-9]+, encoded nanoseconds=-?[0-9]+' + +# The minimum nanoseconds in the statistics is 0x80000000: such statistics must not be used by the filter pushdown. +orc_check statistics "$DATA/malformed_timestamp_statistics_nanoseconds.orc" 't DateTime64(9)' \ + "SELECT count() FROM SOURCE WHERE t > toDateTime64('2000-01-01 00:00:00', 9, 'UTC')" diff --git a/tests/queries/0_stateless/05508_orc_writer_out_of_range_values.reference b/tests/queries/0_stateless/05508_orc_writer_out_of_range_values.reference new file mode 100644 index 0000000000000..17ac0b58ed17d --- /dev/null +++ b/tests/queries/0_stateless/05508_orc_writer_out_of_range_values.reference @@ -0,0 +1,7 @@ +1e308 ['0','1','2','0','1','2','0','1','2','0'] +-1e308 ['0','1','2','0','1','2','0','1','2','0'] +(VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE) +(VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE) +(VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE) +9223372036854775807 +-9223372036854775808 diff --git a/tests/queries/0_stateless/05508_orc_writer_out_of_range_values.sh b/tests/queries/0_stateless/05508_orc_writer_out_of_range_values.sh new file mode 100755 index 0000000000000..6da24b0909d78 --- /dev/null +++ b/tests/queries/0_stateless/05508_orc_writer_out_of_range_values.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# A dictionary threshold that does not fit in `size_t` when multiplied by the number of values. +for threshold in 1e308 -1e308; do + $CLICKHOUSE_LOCAL -q "SELECT toString(number % 3) AS s FROM numbers(10) FORMAT ORC SETTINGS output_format_orc_dictionary_key_size_threshold = $threshold" \ + | $CLICKHOUSE_LOCAL --input-format ORC -q "SELECT '$threshold', groupArray(s) FROM table" +done + +# The extreme `DateTime64` values that cannot be stored in ORC with their statistics in milliseconds. +for value in 'reinterpret(toInt64(9223372036854775807), '"'"'DateTime64(0)'"'"')' \ + 'reinterpret(toInt64(-9223372036854775808), '"'"'DateTime64(0)'"'"')' \ + 'reinterpret(toInt64(9223372036854775807), '"'"'DateTime64(1)'"'"')'; do + $CLICKHOUSE_LOCAL -q "SELECT $value AS t FORMAT ORC" 2>&1 >/dev/null | grep -oE '\(VALUE_IS_OUT_OF_RANGE_OF_DATA_TYPE\)' | head -n1 | grep . || echo "$value: no error" +done + +# The extreme values with the nanoseconds can be stored. +for value in 9223372036854775807 -9223372036854775808; do + $CLICKHOUSE_LOCAL -q "SELECT reinterpret(toInt64($value), 'DateTime64(9, \'UTC\')') AS t FORMAT ORC" \ + | $CLICKHOUSE_LOCAL --input-format ORC -q "SELECT reinterpret(t, 'Int64') FROM table" +done diff --git a/tests/queries/0_stateless/05509_jit_llvm_context_does_not_grow_with_distinct_literals.reference b/tests/queries/0_stateless/05509_jit_llvm_context_does_not_grow_with_distinct_literals.reference new file mode 100644 index 0000000000000..9766475a4185a --- /dev/null +++ b/tests/queries/0_stateless/05509_jit_llvm_context_does_not_grow_with_distinct_literals.reference @@ -0,0 +1 @@ +ok diff --git a/tests/queries/0_stateless/05509_jit_llvm_context_does_not_grow_with_distinct_literals.sh b/tests/queries/0_stateless/05509_jit_llvm_context_does_not_grow_with_distinct_literals.sh new file mode 100755 index 0000000000000..91766cd82d837 --- /dev/null +++ b/tests/queries/0_stateless/05509_jit_llvm_context_does_not_grow_with_distinct_literals.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, use_jemalloc +# no-fasttest: requires USE_EMBEDDED_COMPILER, which the fast-test image disables. +# use_jemalloc: measures the dedicated jemalloc arena for JIT allocations. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Every query compiles a filter with different literals. The compiled expression cache holds one entry, +# so the memory used by JIT compilation must stay flat instead of growing with each compiled query. + +JIT_ARENA_BYTES="SELECT sum(curregs * size) FROM system.jemalloc_arena_bins WHERE purpose = 'jit' FORMAT TSV;" + +{ + echo "CREATE TABLE t (key UInt64, value UInt64) ENGINE = Memory;" + echo "INSERT INTO t SELECT number, number FROM numbers(1000);" + for i in $(seq 1 4); do + printf "SELECT count() FROM t WHERE " + for j in $(seq 1 150); do + [ "$j" -gt 1 ] && printf " OR " + printf "(key = %d AND value > %d)" $((i * 100000 + j)) $((i * 100000 + j * 7)) + done + echo " SETTINGS compile_expressions = 1, min_count_to_compile_expression = 0 FORMAT Null;" + [ "$i" -eq 1 ] && echo "$JIT_ARENA_BYTES" + done + echo "$JIT_ARENA_BYTES" +} > "${CLICKHOUSE_TMP}/queries.sql" + +$CLICKHOUSE_LOCAL --queries-file "${CLICKHOUSE_TMP}/queries.sql" -- --compiled_expression_cache_elements_size=1 < /dev/null \ + | tr '\n' ' ' | awk '{ growth = $2 - $1; print (growth < 2000000 ? "ok" : "JIT arena grew by " growth " bytes") }' diff --git a/tests/queries/0_stateless/05509_orc_malformed_dictionaries.reference b/tests/queries/0_stateless/05509_orc_malformed_dictionaries.reference new file mode 100644 index 0000000000000..2d165fb8a6c0b --- /dev/null +++ b/tests/queries/0_stateless/05509_orc_malformed_dictionaries.reference @@ -0,0 +1,16 @@ +empty local push_down=0 lazy=0: (INCORRECT_DATA) +empty local push_down=0 lazy=1: (INCORRECT_DATA) +empty local push_down=1 lazy=0: (INCORRECT_DATA) +empty local push_down=1 lazy=1: (INCORRECT_DATA) +empty remote push_down=0 lazy=0: (INCORRECT_DATA) +empty remote push_down=0 lazy=1: (INCORRECT_DATA) +empty remote push_down=1 lazy=0: (INCORRECT_DATA) +empty remote push_down=1 lazy=1: (INCORRECT_DATA) +nulls local push_down=0 lazy=0: 5 5 +nulls local push_down=0 lazy=1: 5 5 +nulls local push_down=1 lazy=0: 5 5 +nulls local push_down=1 lazy=1: 5 5 +nulls remote push_down=0 lazy=0: 5 5 +nulls remote push_down=0 lazy=1: 5 5 +nulls remote push_down=1 lazy=0: 5 5 +nulls remote push_down=1 lazy=1: 5 5 diff --git a/tests/queries/0_stateless/05509_orc_malformed_dictionaries.sh b/tests/queries/0_stateless/05509_orc_malformed_dictionaries.sh new file mode 100755 index 0000000000000..437d071ce5a5a --- /dev/null +++ b/tests/queries/0_stateless/05509_orc_malformed_dictionaries.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest, the test uses S3. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh +# shellcheck source=./orc_malformed.lib +. "$CUR_DIR"/orc_malformed.lib + +DATA="$CUR_DIR/data_orc" + +# A dictionary without entries referenced by the non-null values. +orc_check empty "$DATA/malformed_dictionary_empty.orc" 's Nullable(String)' "SELECT count(), countIf(s IS NULL) FROM SOURCE" \ + 'ORC dictionary index 0 at row 0 is out of range \[0, 0\)' + +# A valid empty dictionary of a column with only nulls. +FILE="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}_nulls.orc" +trap 'rm -f "$FILE"' EXIT +$CLICKHOUSE_LOCAL -q "SELECT CAST(NULL, 'Nullable(String)') AS s FROM numbers(5) INTO OUTFILE '$FILE' TRUNCATE FORMAT ORC SETTINGS output_format_orc_dictionary_key_size_threshold = 1" +orc_check nulls "$FILE" 's Nullable(String)' "SELECT count(), countIf(s IS NULL) FROM SOURCE" diff --git a/tests/queries/0_stateless/05510_orc_allocation_amplification.reference b/tests/queries/0_stateless/05510_orc_allocation_amplification.reference new file mode 100644 index 0000000000000..1e94a8329c54f --- /dev/null +++ b/tests/queries/0_stateless/05510_orc_allocation_amplification.reference @@ -0,0 +1,32 @@ +compression_block local push_down=0 lazy=0: 1 10000 +compression_block local push_down=0 lazy=1: 1 10000 +compression_block local push_down=1 lazy=0: 1 10000 +compression_block local push_down=1 lazy=1: 1 10000 +compression_block remote push_down=0 lazy=0: 1 10000 +compression_block remote push_down=0 lazy=1: 1 10000 +compression_block remote push_down=1 lazy=0: 1 10000 +compression_block remote push_down=1 lazy=1: 1 10000 +dictionary_size local push_down=0 lazy=0: (INCORRECT_DATA) +dictionary_size local push_down=0 lazy=1: (INCORRECT_DATA) +dictionary_size local push_down=1 lazy=0: (INCORRECT_DATA) +dictionary_size local push_down=1 lazy=1: (INCORRECT_DATA) +dictionary_size remote push_down=0 lazy=0: (INCORRECT_DATA) +dictionary_size remote push_down=0 lazy=1: (INCORRECT_DATA) +dictionary_size remote push_down=1 lazy=0: (INCORRECT_DATA) +dictionary_size remote push_down=1 lazy=1: (INCORRECT_DATA) +snappy_chunk_size local push_down=0 lazy=0: (INCORRECT_DATA) +snappy_chunk_size local push_down=0 lazy=1: (INCORRECT_DATA) +snappy_chunk_size local push_down=1 lazy=0: (INCORRECT_DATA) +snappy_chunk_size local push_down=1 lazy=1: (INCORRECT_DATA) +snappy_chunk_size remote push_down=0 lazy=0: (INCORRECT_DATA) +snappy_chunk_size remote push_down=0 lazy=1: (INCORRECT_DATA) +snappy_chunk_size remote push_down=1 lazy=0: (INCORRECT_DATA) +snappy_chunk_size remote push_down=1 lazy=1: (INCORRECT_DATA) +zstd_chunk_size local push_down=0 lazy=0: (INCORRECT_DATA) +zstd_chunk_size local push_down=0 lazy=1: (INCORRECT_DATA) +zstd_chunk_size local push_down=1 lazy=0: (INCORRECT_DATA) +zstd_chunk_size local push_down=1 lazy=1: (INCORRECT_DATA) +zstd_chunk_size remote push_down=0 lazy=0: (INCORRECT_DATA) +zstd_chunk_size remote push_down=0 lazy=1: (INCORRECT_DATA) +zstd_chunk_size remote push_down=1 lazy=0: (INCORRECT_DATA) +zstd_chunk_size remote push_down=1 lazy=1: (INCORRECT_DATA) diff --git a/tests/queries/0_stateless/05510_orc_allocation_amplification.sh b/tests/queries/0_stateless/05510_orc_allocation_amplification.sh new file mode 100755 index 0000000000000..bc5685ba31872 --- /dev/null +++ b/tests/queries/0_stateless/05510_orc_allocation_amplification.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest, the test uses S3. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh +# shellcheck source=./orc_malformed.lib +. "$CUR_DIR"/orc_malformed.lib + +DATA="$CUR_DIR/data_orc" + +# The memory must be allocated by the size of the data, not by the sizes declared in the file. +LIMIT='max_memory_usage = 100000000' + +# The declared compression block size is 1 GiB, the chunks are small. The schema inference reads the compressed footer. +orc_check compression_block "$DATA/malformed_compression_block_1g.orc" '' "SELECT count(), sum(length(s)) FROM SOURCE" '' "$LIMIT" + +# The declared dictionary size is 2^32 - 1, the stream has a few entries. +orc_check dictionary_size "$DATA/malformed_dictionary_size_max.orc" 's Nullable(String)' "SELECT count(), sum(length(s)) FROM SOURCE" \ + 'Unexpected end of ORC stream column 1 kind 2' "$LIMIT" + +# The sizes stored in the snappy and zstd chunks are 1 GiB, the chunks have a few bytes. +orc_check snappy_chunk_size "$DATA/malformed_snappy_chunk_size.orc" 'x Int64' "SELECT count() FROM SOURCE" \ + 'snappy-compressed chunk of ORC file decompresses to 1073741824 bytes' "$LIMIT" +orc_check zstd_chunk_size "$DATA/malformed_zstd_chunk_size.orc" 'x Int64' "SELECT count() FROM SOURCE" \ + 'Malformed zstd-compressed data in ORC file' "$LIMIT" diff --git a/tests/queries/0_stateless/05511_orc_malformed_stripes_and_streams.reference b/tests/queries/0_stateless/05511_orc_malformed_stripes_and_streams.reference new file mode 100644 index 0000000000000..3c50f86ded0e7 --- /dev/null +++ b/tests/queries/0_stateless/05511_orc_malformed_stripes_and_streams.reference @@ -0,0 +1,16 @@ +stripe local push_down=0 lazy=0: (INCORRECT_DATA) +stripe local push_down=0 lazy=1: (INCORRECT_DATA) +stripe local push_down=1 lazy=0: (INCORRECT_DATA) +stripe local push_down=1 lazy=1: (INCORRECT_DATA) +stripe remote push_down=0 lazy=0: (INCORRECT_DATA) +stripe remote push_down=0 lazy=1: (INCORRECT_DATA) +stripe remote push_down=1 lazy=0: (INCORRECT_DATA) +stripe remote push_down=1 lazy=1: (INCORRECT_DATA) +stream local push_down=0 lazy=0: (INCORRECT_DATA) +stream local push_down=0 lazy=1: (INCORRECT_DATA) +stream local push_down=1 lazy=0: (INCORRECT_DATA) +stream local push_down=1 lazy=1: (INCORRECT_DATA) +stream remote push_down=0 lazy=0: (INCORRECT_DATA) +stream remote push_down=0 lazy=1: (INCORRECT_DATA) +stream remote push_down=1 lazy=0: (INCORRECT_DATA) +stream remote push_down=1 lazy=1: (INCORRECT_DATA) diff --git a/tests/queries/0_stateless/05511_orc_malformed_stripes_and_streams.sh b/tests/queries/0_stateless/05511_orc_malformed_stripes_and_streams.sh new file mode 100755 index 0000000000000..2d3b50f7f31eb --- /dev/null +++ b/tests/queries/0_stateless/05511_orc_malformed_stripes_and_streams.sh @@ -0,0 +1,19 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest, the test uses S3. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh +# shellcheck source=./orc_malformed.lib +. "$CUR_DIR"/orc_malformed.lib + +DATA="$CUR_DIR/data_orc" + +# The sum of the offset and the lengths of the stripe overflows `UInt64`. +orc_check stripe "$DATA/malformed_stripe_offset_overflow.orc" 'a UInt64, b String' "SELECT count(), sum(length(b)) FROM SOURCE" \ + 'Malformed stripe information at stripe index 0' + +# The length of a stream of a column that is not read is 2^64 - 1, the offsets of the next streams wrap around. +orc_check stream "$DATA/malformed_stream_offset_overflow.orc" 'b String' "SELECT count(), sum(length(b)) FROM SOURCE" \ + 'Malformed stream.*11.*18446744073709551615' diff --git a/tests/queries/0_stateless/05512_orc_malformed_row_groups.reference b/tests/queries/0_stateless/05512_orc_malformed_row_groups.reference new file mode 100644 index 0000000000000..1fafd3d526a4f --- /dev/null +++ b/tests/queries/0_stateless/05512_orc_malformed_row_groups.reference @@ -0,0 +1,32 @@ +row_groups local push_down=1 lazy=0: (INCORRECT_DATA) +row_groups local push_down=1 lazy=1: (INCORRECT_DATA) +row_groups remote push_down=1 lazy=0: (INCORRECT_DATA) +row_groups remote push_down=1 lazy=1: (INCORRECT_DATA) +row_groups local push_down=0 lazy=0: (INCORRECT_DATA) +row_groups local push_down=0 lazy=1: (INCORRECT_DATA) +row_groups remote push_down=0 lazy=0: (INCORRECT_DATA) +row_groups remote push_down=0 lazy=1: (INCORRECT_DATA) +patched_base local push_down=0 lazy=0: (INCORRECT_DATA) +patched_base local push_down=0 lazy=1: (INCORRECT_DATA) +patched_base local push_down=1 lazy=0: (INCORRECT_DATA) +patched_base local push_down=1 lazy=1: (INCORRECT_DATA) +patched_base remote push_down=0 lazy=0: (INCORRECT_DATA) +patched_base remote push_down=0 lazy=1: (INCORRECT_DATA) +patched_base remote push_down=1 lazy=0: (INCORRECT_DATA) +patched_base remote push_down=1 lazy=1: (INCORRECT_DATA) +row_index_seek local push_down=0 lazy=0: 1 +row_index_seek local push_down=0 lazy=1: 1 +row_index_seek local push_down=1 lazy=0: (INCORRECT_DATA) +row_index_seek local push_down=1 lazy=1: (INCORRECT_DATA) +row_index_seek remote push_down=0 lazy=0: 1 +row_index_seek remote push_down=0 lazy=1: 1 +row_index_seek remote push_down=1 lazy=0: (INCORRECT_DATA) +row_index_seek remote push_down=1 lazy=1: (INCORRECT_DATA) +row_index_control local push_down=0 lazy=0: 1 +row_index_control local push_down=0 lazy=1: 1 +row_index_control local push_down=1 lazy=0: 1 +row_index_control local push_down=1 lazy=1: 1 +row_index_control remote push_down=0 lazy=0: 1 +row_index_control remote push_down=0 lazy=1: 1 +row_index_control remote push_down=1 lazy=0: 1 +row_index_control remote push_down=1 lazy=1: 1 diff --git a/tests/queries/0_stateless/05512_orc_malformed_row_groups.sh b/tests/queries/0_stateless/05512_orc_malformed_row_groups.sh new file mode 100755 index 0000000000000..163a644dea9f4 --- /dev/null +++ b/tests/queries/0_stateless/05512_orc_malformed_row_groups.sh @@ -0,0 +1,26 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest, the test uses S3. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh +# shellcheck source=./orc_malformed.lib +. "$CUR_DIR"/orc_malformed.lib + +DATA="$CUR_DIR/data_orc" + +# The stripe has 2^64 - 1 rows: rounding up the number of row groups overflows. +ORC_PUSH_DOWN_VALUES=1 orc_check row_groups "$DATA/malformed_row_group_count_overflow.orc" 'x Int64' "SELECT x FROM SOURCE WHERE x = 1" \ + 'The row index of ORC column 1 has 2 entries, less than the [0-9]+ row groups of stripe 0' +ORC_PUSH_DOWN_VALUES=0 orc_check row_groups "$DATA/malformed_row_group_count_overflow.orc" 'x Int64' "SELECT x FROM SOURCE WHERE x = 1" \ + 'Unexpected end of ORC stream column 1 kind 1' + +# A PATCHED_BASE run whose last patch is a continuation (gap 255, patch 0). +orc_check patched_base "$DATA/malformed_patched_base_terminal_continuation.orc" 'x Int64' "SELECT count(), sum(x) FROM SOURCE" \ + 'Corrupt PATCHED_BASE encoded data in ORC stream column 1 kind 1 in stripe 0 \(unterminated gap\)' + +# A row index position beyond the end of the stream, and the same file with the valid position. +orc_check row_index_seek "$DATA/malformed_row_index_seek_past_stream.orc" 'x Int64' "SELECT x FROM SOURCE WHERE x = 1" \ + 'Seek position 127 is beyond the end of ORC stream column 1 kind 1 in stripe 0 of size 3' +orc_check row_index_control "$DATA/row_index_seek_control.orc" 'x Int64' "SELECT x FROM SOURCE WHERE x = 1" diff --git a/tests/queries/0_stateless/05513_orc_metadata_repeated_field_limits.reference b/tests/queries/0_stateless/05513_orc_metadata_repeated_field_limits.reference new file mode 100644 index 0000000000000..bac3e0b9f016c --- /dev/null +++ b/tests/queries/0_stateless/05513_orc_metadata_repeated_field_limits.reference @@ -0,0 +1,16 @@ +footer_statistics local push_down=0 lazy=0: (INCORRECT_DATA) +footer_statistics local push_down=0 lazy=1: (INCORRECT_DATA) +footer_statistics local push_down=1 lazy=0: (INCORRECT_DATA) +footer_statistics local push_down=1 lazy=1: (INCORRECT_DATA) +footer_statistics remote push_down=0 lazy=0: (INCORRECT_DATA) +footer_statistics remote push_down=0 lazy=1: (INCORRECT_DATA) +footer_statistics remote push_down=1 lazy=0: (INCORRECT_DATA) +footer_statistics remote push_down=1 lazy=1: (INCORRECT_DATA) +stripe_statistics local push_down=0 lazy=0: (INCORRECT_DATA) +stripe_statistics local push_down=0 lazy=1: (INCORRECT_DATA) +stripe_statistics local push_down=1 lazy=0: (INCORRECT_DATA) +stripe_statistics local push_down=1 lazy=1: (INCORRECT_DATA) +stripe_statistics remote push_down=0 lazy=0: (INCORRECT_DATA) +stripe_statistics remote push_down=0 lazy=1: (INCORRECT_DATA) +stripe_statistics remote push_down=1 lazy=0: (INCORRECT_DATA) +stripe_statistics remote push_down=1 lazy=1: (INCORRECT_DATA) diff --git a/tests/queries/0_stateless/05513_orc_metadata_repeated_field_limits.sh b/tests/queries/0_stateless/05513_orc_metadata_repeated_field_limits.sh new file mode 100755 index 0000000000000..9c0fb713fb91f --- /dev/null +++ b/tests/queries/0_stateless/05513_orc_metadata_repeated_field_limits.sh @@ -0,0 +1,17 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest, the test uses S3. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh +# shellcheck source=./orc_malformed.lib +. "$CUR_DIR"/orc_malformed.lib + +DATA="$CUR_DIR/data_orc" + +# The number of the repeated objects in the metadata is limited by the numbers of the columns and the stripes. +orc_check footer_statistics "$DATA/malformed_footer_too_many_statistics.orc" 'a UInt64, b String' "SELECT count(), sum(length(b)) FROM SOURCE" \ + 'Too many column statistics in ORC footer: 13, the file has 3 columns' +orc_check stripe_statistics "$DATA/malformed_metadata_too_many_stripes.orc" 'a UInt64, b String' "SELECT count(), sum(length(b)) FROM SOURCE" \ + 'Too many stripe statistics in ORC metadata, at most 1 expected' diff --git a/tests/queries/0_stateless/05514_orc_timestamp_statistics_before_epoch.reference b/tests/queries/0_stateless/05514_orc_timestamp_statistics_before_epoch.reference new file mode 100644 index 0000000000000..7f41d79fbe442 --- /dev/null +++ b/tests/queries/0_stateless/05514_orc_timestamp_statistics_before_epoch.reference @@ -0,0 +1,21 @@ +t < fromUnixTimestamp64Nano(-500000001, 'UTC') +500 +500 0 499 +500 0 499 +t <= fromUnixTimestamp64Nano(-500000001, 'UTC') +600 +501 0 500 +501 0 500 +t = fromUnixTimestamp64Nano(-950000001, 'UTC') +100 +1 50 50 +1 50 50 +t = fromUnixTimestamp64Nano(-999500000, 'UTC') +1 +1 1 1 +1 1 1 +-2000000000 1 +-1 1 +-999500000 1 +-1500000 1 +1000000000 1 diff --git a/tests/queries/0_stateless/05514_orc_timestamp_statistics_before_epoch.sh b/tests/queries/0_stateless/05514_orc_timestamp_statistics_before_epoch.sh new file mode 100755 index 0000000000000..21a4b8bb4428a --- /dev/null +++ b/tests/queries/0_stateless/05514_orc_timestamp_statistics_before_epoch.sh @@ -0,0 +1,45 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +FILE="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}.orc" +trap 'rm -f "$FILE"' EXIT + +# Prints the number of rows that are not pruned, and the result with and without the filter pushdown. +function check() +{ + local condition=$1 + echo "$condition" + $CLICKHOUSE_LOCAL -q " + SELECT count() FROM file('$FILE', 'ORC', 'number UInt64, t DateTime64(9, \'UTC\')') WHERE indexHint($condition); + SELECT count(), min(number), max(number) FROM file('$FILE', 'ORC', 'number UInt64, t DateTime64(9, \'UTC\')') WHERE $condition; + SELECT count(), min(number), max(number) FROM file('$FILE', 'ORC', 'number UInt64, t DateTime64(9, \'UTC\')') WHERE $condition + SETTINGS input_format_orc_filter_push_down = 0" +} + +# The timestamps before the epoch with a negative sub-millisecond part, from -1.000000001 to -0.001000001 seconds, +# in row groups of 100 rows. Their statistics must allow the pruning. +$CLICKHOUSE_LOCAL -q " + SELECT number, fromUnixTimestamp64Nano(toInt64(number) * 1000000 - 1000000001, 'UTC') AS t FROM numbers(1000) + INTO OUTFILE '$FILE' TRUNCATE FORMAT ORC SETTINGS output_format_orc_row_index_stride = 100" +check "t < fromUnixTimestamp64Nano(-500000001, 'UTC')" +check "t <= fromUnixTimestamp64Nano(-500000001, 'UTC')" +check "t = fromUnixTimestamp64Nano(-950000001, 'UTC')" + +# A timestamp in the millisecond after -1 second, in its own row group: the pruning must keep it. +$CLICKHOUSE_LOCAL -q " + SELECT number, fromUnixTimestamp64Nano([-2000000000, -999500000, -500000000][number + 1], 'UTC') AS t FROM numbers(3) + INTO OUTFILE '$FILE' TRUNCATE FORMAT ORC SETTINGS output_format_orc_row_index_stride = 1" +check "t = fromUnixTimestamp64Nano(-999500000, 'UTC')" + +# A file from PyArrow (the C++ writer of ORC) with a bloom filter, which rounds the negative milliseconds toward zero, +# and one row in each row group: the bloom filter must keep the matching rows. +for value in -2000000000 -1 -999500000 -1500000 1000000000; do + $CLICKHOUSE_LOCAL -q " + SELECT $value, count() FROM file('$CUR_DIR/data_orc/timestamps_before_epoch_pyarrow_bloom_filter.orc', 'ORC', 't Nullable(DateTime64(9))') + WHERE t = fromUnixTimestamp64Nano($value)" +done diff --git a/tests/queries/0_stateless/05515_orc_nullable_floats.reference b/tests/queries/0_stateless/05515_orc_nullable_floats.reference new file mode 100644 index 0000000000000..4aae050a32fc2 --- /dev/null +++ b/tests/queries/0_stateless/05515_orc_nullable_floats.reference @@ -0,0 +1,2 @@ +1 +1086 1410 5000 10000 0 diff --git a/tests/queries/0_stateless/05515_orc_nullable_floats.sh b/tests/queries/0_stateless/05515_orc_nullable_floats.sh new file mode 100755 index 0000000000000..c7eba0810a432 --- /dev/null +++ b/tests/queries/0_stateless/05515_orc_nullable_floats.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +FILE="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}.orc" +trap 'rm -f "$FILE"' EXIT + +# Nullable floats with different patterns of nulls, and special values, at the top level and nested. +QUERY=" + SELECT + number, + multiIf(k = 0, NULL, k = 1, nan, k = 2, inf, k = 3, -inf, k = 4, -0., k = 5, toFloat32(1e-42), toFloat32(number) / 7)::Nullable(Float32) AS f32, + multiIf(k2 = 0, NULL, k2 = 1, nan, k2 = 2, -inf, k2 = 3, 5e-324, k2 = 4, -0., number / 3.)::Nullable(Float64) AS f64, + if(intDiv(number, 1000) % 2 = 0, NULL, number / 2.)::Nullable(Float64) AS runs, + NULL::Nullable(Float32) AS all_null, + (number * 1.5)::Nullable(Float64) AS no_null, + arrayMap(x -> if(x % 3 = 0, NULL, toFloat32(x) / 4), range(number % 5))::Array(Nullable(Float32)) AS arr + FROM (SELECT number, cityHash64(number) % 9 AS k, cityHash64(number, 1) % 7 AS k2 FROM numbers(10000))" +STRUCTURE='number UInt64, f32 Nullable(Float32), f64 Nullable(Float64), runs Nullable(Float64), all_null Nullable(Float32), no_null Nullable(Float64), arr Array(Nullable(Float32))' +DIGEST="sum(cityHash64(number, isNull(f32), reinterpretAsUInt32(ifNull(f32, 0)), isNull(f64), reinterpretAsUInt64(ifNull(f64, 0)), + isNull(runs), ifNull(runs, 0), isNull(all_null), no_null, arrayMap(x -> (isNull(x), reinterpretAsUInt32(ifNull(x, 0))), arr)))" + +$CLICKHOUSE_LOCAL -q "$QUERY INTO OUTFILE '$FILE' TRUNCATE FORMAT ORC" +$CLICKHOUSE_LOCAL -q " + SELECT (SELECT $DIGEST FROM ($QUERY)) = (SELECT $DIGEST FROM file('$FILE', 'ORC', '$STRUCTURE')); + SELECT countIf(f32 IS NULL), countIf(f64 IS NULL), countIf(runs IS NULL), countIf(all_null IS NULL), countIf(no_null IS NULL) FROM file('$FILE', 'ORC', '$STRUCTURE')" diff --git a/tests/queries/0_stateless/05546_replicated_database_restore_replica_waits_for_initial_alter.reference b/tests/queries/0_stateless/05546_replicated_database_restore_replica_waits_for_initial_alter.reference new file mode 100644 index 0000000000000..36b34a909261b --- /dev/null +++ b/tests/queries/0_stateless/05546_replicated_database_restore_replica_waits_for_initial_alter.reference @@ -0,0 +1,6 @@ +restore is waiting 1 +alter succeeded +restore finished +x +y +0 diff --git a/tests/queries/0_stateless/05546_replicated_database_restore_replica_waits_for_initial_alter.sh b/tests/queries/0_stateless/05546_replicated_database_restore_replica_waits_for_initial_alter.sh new file mode 100755 index 0000000000000..4083ae7715ff8 --- /dev/null +++ b/tests/queries/0_stateless/05546_replicated_database_restore_replica_waits_for_initial_alter.sh @@ -0,0 +1,58 @@ +#!/usr/bin/env bash +# Tags: zookeeper, no-fasttest, no-parallel +# no-parallel: the `database_replicated_pause_before_initial_query_execution` failpoint fires globally, +# it would pause (or be consumed by) a DDL query of another `Replicated` database. + +# `SYSTEM RESTORE DATABASE REPLICA` waits for the initial DDL queries that the DDL worker has already admitted. +# An `ALTER` of a `ReplicatedMergeTree` table with `alter_sync = 1` waits until the table executes its +# `ALTER_METADATA` entry, which requires an active DDL worker of the database (`canExecuteReplicatedMetadataAlter`). +# The restore must not stop the worker before such a query finishes, otherwise both queries wait for each other. + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +DB="rdb_${CLICKHOUSE_TEST_UNIQUE_NAME}" +ZK_PATH="/test/${CLICKHOUSE_TEST_ZOOKEEPER_PREFIX}/${CLICKHOUSE_TEST_UNIQUE_NAME}" +FAILPOINT="database_replicated_pause_before_initial_query_execution" +RESTORE_QUERY_ID="restore_${CLICKHOUSE_TEST_UNIQUE_NAME}" + +function cleanup() +{ + $CLICKHOUSE_CLIENT --query "SYSTEM DISABLE FAILPOINT $FAILPOINT" >/dev/null 2>&1 ||: + $CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS ${DB} SYNC" >/dev/null 2>&1 ||: + $CLICKHOUSE_CLIENT --query "DROP DATABASE IF EXISTS ${DB}_broken_replicated_tables SYNC" >/dev/null 2>&1 ||: +} +trap cleanup EXIT + +$CLICKHOUSE_CLIENT --query "CREATE DATABASE ${DB} ENGINE = Replicated('${ZK_PATH}', 'shard1', 'replica1')" +$CLICKHOUSE_CLIENT --distributed_ddl_output_mode=none --query "CREATE TABLE ${DB}.t (x UInt64) ENGINE = ReplicatedMergeTree ORDER BY x" + +# Hold the initial `ALTER` after the DDL worker has admitted it, before it executes. +$CLICKHOUSE_CLIENT --query "SYSTEM ENABLE FAILPOINT $FAILPOINT" +$CLICKHOUSE_CLIENT --alter_sync 1 --distributed_ddl_output_mode=none --query "ALTER TABLE ${DB}.t ADD COLUMN y UInt64" & +alter_pid=$! +$CLICKHOUSE_CLIENT --query "SYSTEM WAIT FAILPOINT $FAILPOINT PAUSE" + +$CLICKHOUSE_CLIENT --query_id "$RESTORE_QUERY_ID" --query "SYSTEM RESTORE DATABASE REPLICA ${DB}" >/dev/null 2>&1 & +restore_pid=$! + +# The restore has to wait for the paused `ALTER`. +for _ in $(seq 1 600); do + [ "$($CLICKHOUSE_CLIENT --query "SELECT count() FROM system.processes WHERE query_id = '$RESTORE_QUERY_ID'")" = 1 ] && break + sleep 0.1 +done +sleep 2 +$CLICKHOUSE_CLIENT --query "SELECT 'restore is waiting', count() FROM system.processes WHERE query_id = '$RESTORE_QUERY_ID'" + +# The `ALTER` finishes, including the wait for its `ALTER_METADATA` entry, and then the restore proceeds. +$CLICKHOUSE_CLIENT --query "SYSTEM NOTIFY FAILPOINT $FAILPOINT" +wait $alter_pid && echo "alter succeeded" +# The replica is not lost (its digest is not empty), so the restore may refuse with `REPLICA_ALREADY_EXISTS`; +# either way the DDL worker is re-created. +wait $restore_pid ||: +echo "restore finished" + +$CLICKHOUSE_CLIENT --query "SYSTEM SYNC DATABASE REPLICA ${DB}" +$CLICKHOUSE_CLIENT --query "SELECT name FROM system.columns WHERE database = '${DB}' AND table = 't' ORDER BY name" +$CLICKHOUSE_CLIENT --query "SELECT count() FROM system.tables WHERE database = '${DB}_broken_replicated_tables'" diff --git a/tests/queries/0_stateless/05565_parquet_encoder_shared_thread_pool.reference b/tests/queries/0_stateless/05565_parquet_encoder_shared_thread_pool.reference new file mode 100644 index 0000000000000..69bc311a9dae2 --- /dev/null +++ b/tests/queries/0_stateless/05565_parquet_encoder_shared_thread_pool.reference @@ -0,0 +1,2 @@ +8000 1535808000 4 1 +4800 552844800 24 diff --git a/tests/queries/0_stateless/05565_parquet_encoder_shared_thread_pool.sh b/tests/queries/0_stateless/05565_parquet_encoder_shared_thread_pool.sh new file mode 100755 index 0000000000000..945dea9ed33a5 --- /dev/null +++ b/tests/queries/0_stateless/05565_parquet_encoder_shared_thread_pool.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# - no-fasttest: requires `IcebergLocal` (USE_AVRO build option) + +# Every Parquet writer had its own pool of `max_threads` encoder threads, so many open writers with `max_threads = 32` +# did not fit in a 500-thread global pool: 8 parallel sinks of an Iceberg INSERT with 4 partitions each, and the +# 24 partitions of one stream. +# https://github.com/ClickHouse/ClickHouse/issues/129864 + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +TABLE_DIR="${CLICKHOUSE_USER_FILES}/${CLICKHOUSE_TEST_UNIQUE_NAME}" +rm -rf "${TABLE_DIR}" +mkdir -p "${TABLE_DIR}" +trap 'rm -rf "${TABLE_DIR}"' EXIT + +COLS=$(for i in $(seq 48); do printf ', c%d Int64' "$i"; done) +EXPRS=$(for i in $(seq 48); do printf ', number * %d' "$i"; done) +SETTINGS="SET max_threads = 32, max_threads_min_free_memory_per_thread = 0, max_insert_threads_min_free_memory_per_thread = 0, + output_format_parquet_parallel_encoding = 1, output_format_parquet_row_group_size = 200, + max_block_size = 1000, min_insert_block_size_rows = 1000, min_insert_block_size_bytes = 0" + +# An Iceberg table takes its format settings from its `SETTINGS` clause, not from the session. Small row groups make +# every writer encode while the INSERT runs, not only when its file is finalized. +${CLICKHOUSE_LOCAL} --query " + ${SETTINGS}, allow_insert_into_iceberg = 1, max_insert_threads = 8; + CREATE TABLE t (p Int32 ${COLS}) ENGINE = IcebergLocal('${TABLE_DIR}/iceberg/') PARTITION BY p + SETTINGS output_format_parquet_row_group_size = 200; + INSERT INTO t SELECT number % 4 ${EXPRS} FROM numbers(8000); + SELECT count(), sum(c48), uniqExact(p), uniqExact(_path) > 4 FROM t; +" -- --max_thread_pool_size=500 --thread_pool_queue_size=500 + +${CLICKHOUSE_LOCAL} --query " + ${SETTINGS}, max_insert_threads = 1; + INSERT INTO FUNCTION file('${TABLE_DIR}/file/{_partition_id}.parquet', 'Parquet', 'p UInt8 ${COLS}') PARTITION BY p + SELECT number % 24 ${EXPRS} FROM numbers(4800); + SELECT count(), sum(c48), uniqExact(_path) FROM file('${TABLE_DIR}/file/*.parquet'); +" -- --max_thread_pool_size=500 --thread_pool_queue_size=500 diff --git a/tests/queries/0_stateless/05567_group_array_sample_counter_overflow.reference b/tests/queries/0_stateless/05567_group_array_sample_counter_overflow.reference new file mode 100644 index 0000000000000..ec1c393fef3a5 --- /dev/null +++ b/tests/queries/0_stateless/05567_group_array_sample_counter_overflow.reference @@ -0,0 +1,3 @@ +1 +1 +still alive diff --git a/tests/queries/0_stateless/05567_group_array_sample_counter_overflow.sql b/tests/queries/0_stateless/05567_group_array_sample_counter_overflow.sql new file mode 100644 index 0000000000000..36af866963144 --- /dev/null +++ b/tests/queries/0_stateless/05567_group_array_sample_counter_overflow.sql @@ -0,0 +1,28 @@ +-- A deserialized `groupArraySample` state may carry an arbitrary `total_values` counter. +-- Merging must not wrap it around to zero, which led to a division by zero. + +-- Valid states merge fine. +SELECT length(arrayReduce('groupArraySampleMerge(1, 1)', [ + CAST(unhex('0101410100000000000000173633363431333632323338343637393330303520302033'), 'AggregateFunction(groupArraySample(1, 1), String)'), + CAST(unhex('0101420100000000000000173633363431333632323338343637393330303520302033'), 'AggregateFunction(groupArraySample(1, 1), String)')])); +SELECT length(arrayReduce('groupArraySampleMerge(1, 1)', [ + CAST(unhex('01410100000000000000173633363431333632323338343637393330303520302033'), 'AggregateFunction(groupArraySample(1, 1), UInt8)'), + CAST(unhex('01420100000000000000173633363431333632323338343637393330303520302033'), 'AggregateFunction(groupArraySample(1, 1), UInt8)')])); + +-- total_values = UINT64_MAX, merged with a state with one value. +SELECT arrayReduce('groupArraySampleMerge(1, 1)', [ + CAST(unhex('010141FFFFFFFFFFFFFFFF173633363431333632323338343637393330303520302033'), 'AggregateFunction(groupArraySample(1, 1), String)'), + CAST(unhex('0101420100000000000000173633363431333632323338343637393330303520302033'), 'AggregateFunction(groupArraySample(1, 1), String)')]); -- { serverError INCORRECT_DATA } +SELECT arrayReduce('groupArraySampleMerge(1, 1)', [ + CAST(unhex('0141FFFFFFFFFFFFFFFF173633363431333632323338343637393330303520302033'), 'AggregateFunction(groupArraySample(1, 1), UInt8)'), + CAST(unhex('01420100000000000000173633363431333632323338343637393330303520302033'), 'AggregateFunction(groupArraySample(1, 1), UInt8)')]); -- { serverError INCORRECT_DATA } + +-- Both states have more values than the sample size, and their counters sum to 2^64. +SELECT arrayReduce('groupArraySampleMerge(1, 1)', [ + CAST(unhex('0101410000000000000080173633363431333632323338343637393330303520302033'), 'AggregateFunction(groupArraySample(1, 1), String)'), + CAST(unhex('0101420000000000000080173633363431333632323338343637393330303520302033'), 'AggregateFunction(groupArraySample(1, 1), String)')]); -- { serverError INCORRECT_DATA } +SELECT arrayReduce('groupArraySampleMerge(1, 1)', [ + CAST(unhex('01410000000000000080173633363431333632323338343637393330303520302033'), 'AggregateFunction(groupArraySample(1, 1), UInt8)'), + CAST(unhex('01420000000000000080173633363431333632323338343637393330303520302033'), 'AggregateFunction(groupArraySample(1, 1), UInt8)')]); -- { serverError INCORRECT_DATA } + +SELECT 'still alive'; diff --git a/tests/queries/0_stateless/05567_mongodb_server_credentials.reference b/tests/queries/0_stateless/05567_mongodb_server_credentials.reference new file mode 100644 index 0000000000000..d12c1befdb9c3 --- /dev/null +++ b/tests/queries/0_stateless/05567_mongodb_server_credentials.reference @@ -0,0 +1,27 @@ +Mechanisms that take the credentials from the server, and a client certificate file +refused +refused +refused +refused +refused +refused +refused +refused +Credentials in the connection string +connects +connects +connects +The setting permits the others +connects +connects +A MongoDB table is checked when it is read, with the setting of the session that reads +refused +connects +refused +So is a table created with CREATE TABLE ... AS mongodb(...) +x UInt8 +refused +connects +And the table function with a named collection +refused +connects diff --git a/tests/queries/0_stateless/05567_mongodb_server_credentials.sh b/tests/queries/0_stateless/05567_mongodb_server_credentials.sh new file mode 100755 index 0000000000000..ffd10d4a2a51d --- /dev/null +++ b/tests/queries/0_stateless/05567_mongodb_server_credentials.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: the MongoDB table function and table engine need USE_MONGODB + +# A query that reads with the `mongodb` table function or from a `MongoDB` table is refused with `ACCESS_DENIED` if the +# connection string makes the server authenticate with credentials of the server process, while +# `s3_allow_server_credentials_in_user_queries` is disabled in the session that reads. Nothing listens on 127.0.0.1:1, +# so a query that is not refused fails to connect. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +HOST="127.0.0.1:1" +TIMEOUTS="connectTimeoutMS=100&serverSelectionTimeoutMS=100" +OIDC_URI="mongodb://${HOST}/db?${TIMEOUTS}&authMechanism=MONGODB-OIDC" +NC="nc_05567_${CLICKHOUSE_DATABASE}" + +# Prints `refused` for `ACCESS_DENIED`, `connects` if the query got as far as connecting, or else the output. +classify() +{ + local output + output=$(cat) + if grep -q ACCESS_DENIED <<< "$output"; then + echo refused + elif grep -q 'No suitable servers found' <<< "$output"; then + echo connects + else + echo "$output" + fi +} + +# Runs the query $2 with the setting $1. +check() +{ + $CLICKHOUSE_CLIENT --s3_allow_server_credentials_in_user_queries "$1" -q "$2" 2>&1 | classify +} + +read_uri() +{ + check "$1" "SELECT count() FROM mongodb('mongodb://${HOST}/db?${TIMEOUTS}&$2', 'c', 'x UInt8')" +} + +cleanup() +{ + $CLICKHOUSE_CLIENT -m -q " + SET ast_fuzzer_any_query = 0, ignore_drop_queries_probability = 0; + DROP TABLE IF EXISTS t_engine; + DROP TABLE IF EXISTS t_as_function; + DROP NAMED COLLECTION IF EXISTS ${NC}; + " +} +trap cleanup EXIT +cleanup + +echo 'Mechanisms that take the credentials from the server, and a client certificate file' +read_uri 0 'authMechanism=MONGODB-OIDC' +read_uri 0 'authmechanism=mongodb-oidc&authMechanismProperties=ENVIRONMENT:test' +read_uri 0 'authMechanism=MONGODB-AWS' +read_uri 0 'authMechanism=MONGODB-X509' +check 0 "SELECT count() FROM mongodb('mongodb://user@${HOST}/db?${TIMEOUTS}&authMechanism=GSSAPI', 'c', 'x UInt8')" +read_uri 0 'tls=true&tlsCertificateKeyFile=/nonexistent.pem' +read_uri 0 'ssl=true&sslClientCertificateKeyFile=/nonexistent.pem' +check 0 "SELECT count() FROM mongodb('${HOST}', 'db', 'c', '', '', 'x UInt8', '${TIMEOUTS}&authMechanism=MONGODB-OIDC')" + +echo 'Credentials in the connection string' +check 0 "SELECT count() FROM mongodb('mongodb://user:pass@${HOST}/db?${TIMEOUTS}', 'c', 'x UInt8')" +check 0 "SELECT count() FROM mongodb('mongodb://user:pass@${HOST}/db?${TIMEOUTS}&authMechanism=SCRAM-SHA-256', 'c', 'x UInt8')" +check 0 "SELECT count() FROM mongodb('mongodb://user:pass@${HOST}/db?${TIMEOUTS}&authMechanism=plain', 'c', 'x UInt8')" + +echo 'The setting permits the others' +read_uri 1 'authMechanism=MONGODB-OIDC' +read_uri 1 'tls=true&tlsCertificateKeyFile=/nonexistent.pem' + +echo 'A MongoDB table is checked when it is read, with the setting of the session that reads' +$CLICKHOUSE_CLIENT --s3_allow_server_credentials_in_user_queries 0 -q "CREATE TABLE t_engine (x UInt8) ENGINE = MongoDB('${OIDC_URI}', 'c')" +check 0 "SELECT count() FROM t_engine" +check 1 "SELECT count() FROM t_engine" +$CLICKHOUSE_CLIENT -m -q "DETACH TABLE t_engine; ATTACH TABLE t_engine;" +check 0 "SELECT count() FROM t_engine" + +echo 'So is a table created with CREATE TABLE ... AS mongodb(...)' +$CLICKHOUSE_CLIENT -m -q " + CREATE TABLE t_as_function AS mongodb('${OIDC_URI}', 'c', 'x UInt8'); + DETACH TABLE t_as_function; + ATTACH TABLE t_as_function; + DESCRIBE t_as_function SETTINGS describe_compact_output = 1; +" +check 0 "SELECT count() FROM t_as_function" +check 1 "SELECT count() FROM t_as_function" + +echo 'And the table function with a named collection' +$CLICKHOUSE_CLIENT -q "CREATE NAMED COLLECTION ${NC} AS uri = '${OIDC_URI}', collection = 'c', structure = 'x UInt8'" +check 0 "SELECT count() FROM mongodb(${NC})" +check 1 "SELECT count() FROM mongodb(${NC})" diff --git a/tests/queries/0_stateless/05568_mongodb_dictionary_server_credentials.reference b/tests/queries/0_stateless/05568_mongodb_dictionary_server_credentials.reference new file mode 100644 index 0000000000000..2704615274d8e --- /dev/null +++ b/tests/queries/0_stateless/05568_mongodb_dictionary_server_credentials.reference @@ -0,0 +1,12 @@ +Refused even in a session where the setting is enabled +refused +refused +Credentials in the connection string +connects +The options that a dictionary adds to a named collection +refused +A named collection that changes after the dictionary is created +connects +refused +clickhouse-local is not restricted +connects diff --git a/tests/queries/0_stateless/05568_mongodb_dictionary_server_credentials.sh b/tests/queries/0_stateless/05568_mongodb_dictionary_server_credentials.sh new file mode 100755 index 0000000000000..cd6e077856978 --- /dev/null +++ b/tests/queries/0_stateless/05568_mongodb_dictionary_server_credentials.sh @@ -0,0 +1,85 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: the MongoDB dictionary source needs USE_MONGODB + +# A MongoDB dictionary created with a DDL query may never make the server authenticate with credentials of the server +# process, whatever `s3_allow_server_credentials_in_user_queries` is: the source is built when the server loads the +# dictionary, also after the named collection it refers to has changed. The dictionaries are loaded by `dictGet`. +# Nothing listens on 127.0.0.1:1, so a dictionary that is not refused fails to connect. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +HOST="127.0.0.1:1" +TIMEOUTS="connectTimeoutMS=100&serverSelectionTimeoutMS=100" +OIDC_URI="mongodb://${HOST}/db?${TIMEOUTS}&authMechanism=MONGODB-OIDC" +SCRAM_URI="mongodb://user:pass@${HOST}/db?${TIMEOUTS}&authMechanism=SCRAM-SHA-256" +NC="nc_05568_${CLICKHOUSE_DATABASE}" + +# Prints `refused` for `ACCESS_DENIED`, `connects` if the query got as far as connecting, or else the output. +classify() +{ + local output + output=$(cat) + if grep -q ACCESS_DENIED <<< "$output"; then + echo refused + elif grep -q 'No suitable servers found' <<< "$output"; then + echo connects + else + echo "$output" + fi +} + +# Runs the queries $1 in a session where the setting is enabled. +check() +{ + $CLICKHOUSE_CLIENT --s3_allow_server_credentials_in_user_queries 1 -m -q "$1" 2>&1 | classify +} + +# Creates the dictionary $1 with the source $2 and prints whether reading it is refused. +check_source() +{ + $CLICKHOUSE_CLIENT -q "CREATE DICTIONARY $1 (_id String, v String) PRIMARY KEY _id SOURCE(MONGODB($2 COLLECTION 'c')) LAYOUT(COMPLEX_KEY_DIRECT())" + check "SELECT dictGet('$1', 'v', tuple('k'))" +} + +cleanup() +{ + $CLICKHOUSE_CLIENT -m -q " + SET ast_fuzzer_any_query = 0, ignore_drop_queries_probability = 0; + DROP DICTIONARY IF EXISTS d_uri; + DROP DICTIONARY IF EXISTS d_options; + DROP DICTIONARY IF EXISTS d_scram; + DROP DICTIONARY IF EXISTS d_named_options; + DROP DICTIONARY IF EXISTS d_named_altered; + DROP NAMED COLLECTION IF EXISTS ${NC}; + " +} +trap cleanup EXIT +cleanup + +echo 'Refused even in a session where the setting is enabled' +check_source d_uri "URI '${OIDC_URI}'" +check_source d_options "HOST '127.0.0.1' PORT 1 DB 'db' USER '' PASSWORD '' OPTIONS '${TIMEOUTS}&authMechanism=MONGODB-OIDC'" + +echo 'Credentials in the connection string' +check_source d_scram "URI '${SCRAM_URI}'" + +echo 'The options that a dictionary adds to a named collection' +$CLICKHOUSE_CLIENT -q "CREATE NAMED COLLECTION ${NC} AS host = '127.0.0.1', port = '1', db = 'db', user = '', password = ''" +check_source d_named_options "NAME ${NC} OPTIONS '${TIMEOUTS}&authMechanism=MONGODB-OIDC'" +$CLICKHOUSE_CLIENT -q "DROP DICTIONARY d_named_options" + +echo 'A named collection that changes after the dictionary is created' +$CLICKHOUSE_CLIENT -q "CREATE DICTIONARY d_named_altered (_id String, v String) PRIMARY KEY _id SOURCE(MONGODB(NAME ${NC} COLLECTION 'c')) LAYOUT(COMPLEX_KEY_DIRECT())" +check "SELECT dictGet('d_named_altered', 'v', tuple('k'))" +$CLICKHOUSE_CLIENT -q "ALTER NAMED COLLECTION ${NC} SET options = '${TIMEOUTS}&authMechanism=MONGODB-OIDC'" +check "SYSTEM RELOAD DICTIONARY d_named_altered; SELECT dictGet('d_named_altered', 'v', tuple('k'))" + +echo 'clickhouse-local is not restricted' +$CLICKHOUSE_LOCAL -m -q " + CREATE DICTIONARY d (_id String, v String) PRIMARY KEY _id + SOURCE(MONGODB(URI '${OIDC_URI}' COLLECTION 'c')) LAYOUT(COMPLEX_KEY_DIRECT()); + SELECT dictGet('d', 'v', tuple('k')); +" 2>&1 | classify diff --git a/tests/queries/0_stateless/05574_orc_requested_type_of_another_kind.reference b/tests/queries/0_stateless/05574_orc_requested_type_of_another_kind.reference new file mode 100644 index 0000000000000..338bce460b793 --- /dev/null +++ b/tests/queries/0_stateless/05574_orc_requested_type_of_another_kind.reference @@ -0,0 +1,18 @@ +Code: 53 while converting column `m` from type Map(String, String) to type Tuple(value Nullable(String)) +Code: 53 while converting column `m` from type Map(String, String) to type Tuple(value Nullable(String)) +Code: 53 while converting column `l` from type Array(String) to type Tuple(value Nullable(String)) +Code: 53 while converting column `s` from type Tuple(value String) to type Map(String, String) +Code: 53 while converting column `s` from type Tuple(value String) to type Array(String) +Code: 53 while converting column `lm` from type Array(Map(String, String)) to type Array(Tuple(value String)) +Code: 53 while converting column `sl` from type Tuple(a Array(String)) to type Tuple(a Map(String, String)) +2 {\'key\':\'profile\'} +2 [('key','profile')] +2 {'k':'(\'v\')'} +['(\'v\')'] +2 {'key':'profile'} ['profile'] ('profile') {'k':('v')} [{'k':'v'}] (['x']) +('profile',0) +2 +Code: 53 while converting column `m` from type Map(LowCardinality(String), LowCardinality(Nullable(String))) to type Tuple(value Nullable(String)) +Code: 53 while converting column `lm` from type Array(Map(LowCardinality(String), LowCardinality(Nullable(String)))) to type Array(Tuple(value String)) +{\'key\':\'profile\'} +{'key':'profile'} [{'k':'v'}] diff --git a/tests/queries/0_stateless/05574_orc_requested_type_of_another_kind.sh b/tests/queries/0_stateless/05574_orc_requested_type_of_another_kind.sh new file mode 100755 index 0000000000000..f503b6d888fa4 --- /dev/null +++ b/tests/queries/0_stateless/05574_orc_requested_type_of_another_kind.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: ORC is not built in fasttest. + +# An ORC list, map or struct requested as a type of another kind is cast to it, not read as default values. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Prints the error code and the conversion of a query that has to fail. +function error() +{ + $CLICKHOUSE_LOCAL -q "$1" 2>&1 | tr '\n' ' ' | grep -o -E 'Code: [0-9]+|while converting column [^:]* to type [^:]*' | head -n 2 | paste -sd ' ' +} + +FILE="$(realpath "$CLICKHOUSE_TMP")/${CLICKHOUSE_TEST_UNIQUE_NAME}.orc" +$CLICKHOUSE_LOCAL -q "SELECT 2::Int64 AS id, map('key', 'profile') AS m, ['profile'] AS l, + CAST(tuple('profile'), 'Tuple(value String)') AS s, map('k', CAST(tuple('v'), 'Tuple(value String)')) AS ms, + [map('k', 'v')] AS lm, CAST(tuple(['x']), 'Tuple(a Array(String))') AS sl FORMAT ORC" > "$FILE" +F="file('$FILE', ORC" + +# Conversions the cast cannot do. +error "SELECT * FROM $F, 'id Int64, m Tuple(value Nullable(String))')" +error "SELECT * FROM $F, 'id Int64, m Tuple(value Nullable(String))') SETTINGS input_format_orc_allow_missing_columns = 0" +error "SELECT * FROM $F, 'id Int64, l Tuple(value Nullable(String))')" +error "SELECT * FROM $F, 'id Int64, s Map(String, String)')" +error "SELECT * FROM $F, 'id Int64, s Array(String)')" +error "SELECT * FROM $F, 'id Int64, lm Array(Tuple(value String))')" +error "SELECT * FROM $F, 'id Int64, sl Tuple(a Map(String, String))')" + +# Conversions the cast does, also of a map value. +$CLICKHOUSE_LOCAL -q "SELECT * FROM $F, 'id Int64, m String')" +$CLICKHOUSE_LOCAL -q "SELECT * FROM $F, 'id Int64, m Array(Tuple(String, String))')" +$CLICKHOUSE_LOCAL -q "SELECT * FROM $F, 'id Int64, ms Map(String, String)')" +$CLICKHOUSE_LOCAL -q "SELECT ms.values FROM $F, 'id Int64, ms Map(String, String)')" + +# Matching kinds, a Tuple naming only some of the fields, and the other columns of a mismatched structure. +$CLICKHOUSE_LOCAL -q "SELECT * FROM $F, 'id Int64, m Map(String, String), l Array(String), s Tuple(value String), + ms Map(String, Tuple(value String)), lm Array(Map(String, String)), sl Tuple(a Array(String))')" +$CLICKHOUSE_LOCAL -q "SELECT s FROM $F, 's Tuple(value String, other Int32)')" +$CLICKHOUSE_LOCAL -q "SELECT id FROM $F, 'id Int64, m Tuple(value Nullable(String))')" + +DICT_FILE="$(realpath "$CLICKHOUSE_TMP")/${CLICKHOUSE_TEST_UNIQUE_NAME}_dict.orc" +$CLICKHOUSE_LOCAL --output_format_orc_dictionary_key_size_threshold=1 -q "SELECT map('key', 'profile') AS m, [map('k', 'v')] AS lm FORMAT ORC" > "$DICT_FILE" +D="file('$DICT_FILE', ORC" +S="SETTINGS input_format_orc_dictionary_as_low_cardinality = 1" + +# A dictionary-encoded map key. +error "SELECT * FROM $D, 'm Tuple(value Nullable(String))') $S" +error "SELECT * FROM $D, 'lm Array(Tuple(value String))') $S" +$CLICKHOUSE_LOCAL -q "SELECT * FROM $D, 'm String') $S" +$CLICKHOUSE_LOCAL -q "SELECT * FROM $D, 'm Map(String, String), lm Array(Map(String, String))') $S" + +rm -f "$FILE" +rm -f "$DICT_FILE" diff --git a/tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.reference b/tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.reference new file mode 100644 index 0000000000000..626195e3c0a6a --- /dev/null +++ b/tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.reference @@ -0,0 +1,15 @@ +t: has([NULL, 1], p) [2] [2] 3 +t: has([NULL, 5], p) [1,2] [1,2] 2 +t: NOT has([NULL, 1], p) [1,3,4] [1,3,4] 0 +t: has([NULL, 1]::Array(Nullable(Int64)), p) [2] [2] 3 +t: NOT has([NULL, 1]::Array(Nullable(Int64)), p) [1,3,4] [1,3,4] 1 +t: p IN (NULL, 1), transform_null_in = 1 [2] [2] 3 +t: p IS NULL, delta_lake_enable_engine_predicate = 0 [2] [2] 3 +t: p = 7, delta_lake_enable_engine_predicate = 0 [3] [3] 3 +t: p < 6, delta_lake_enable_engine_predicate = 0 [1] [1] 3 +t: p IS NOT NULL, delta_lake_enable_engine_predicate = 0 [1,3,4] [1,3,4] 1 +t: has([5, 7], p) [1,3] [1,3] 1 +n: has([NULL, 5], p) [1] [1] 2 +n: NOT has([NULL, 5], p) [3,4] [3,4] 1 +t2: has([NULL, 1], a) [1,2,3] [1,2,3] 0 +s: has([NULL, 'a'], s) [1,2] [1,2] 1 diff --git a/tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.sh b/tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.sh new file mode 100755 index 0000000000000..2e388a0166240 --- /dev/null +++ b/tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.sh @@ -0,0 +1,81 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-msan +# Tag no-fasttest: delta-kernel pulls in extra dependencies. +# Tag no-msan: delta-kernel-rs (Rust) is not built under MSan, so DeltaLakeLocal is absent. +# Partition pruning must keep the data file of the NULL partition when the filter's set holds NULL, and prune the other files as before. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh +set -e + +ROOT="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}_delta_null_set" +trap 'rm -rf "${ROOT}" 2>/dev/null' EXIT +rm -rf "${ROOT}" + +# An empty Delta table: a v0 log with only protocol + metaData, as ClickHouse cannot create a Delta log itself. +bootstrap() { + local table=$1 schema=$2 partition_columns=$3 + mkdir -p "${ROOT}/${table}/_delta_log" + cat > "${ROOT}/${table}/_delta_log/00000000000000000000.json" < [settings]: prints the rows read with no filter at the source (LIMIT blocks the push-down), +# the rows read with partition pruning, and the number of data files pruned by the second query. +arm() { + local table=$1 condition=$2 settings=${3:+ SETTINGS $3} + LABELS+=("${table}: ${condition}${3:+, $3}") + QUERIES+=" + SELECT arraySort(groupArray(v)) FROM (SELECT * FROM deltaLakeLocal('${ROOT}/${table}') LIMIT 1000) WHERE ${condition}${settings}; + SELECT sumIf(value, event = 'DeltaLakePartitionPrunedFiles') FROM system.events; + SELECT arraySort(groupArray(v)) FROM deltaLakeLocal('${ROOT}/${table}') WHERE ${condition}${settings}; + SELECT sumIf(value, event = 'DeltaLakePartitionPrunedFiles') FROM system.events;" +} + +arm t "has([NULL, 1], p)" +arm t "has([NULL, 5], p)" +arm t "NOT has([NULL, 1], p)" +arm t "has([NULL, 1]::Array(Nullable(Int64)), p)" +arm t "NOT has([NULL, 1]::Array(Nullable(Int64)), p)" +arm t "p IN (NULL, 1)" "transform_null_in = 1" +# The engine predicate would skip these files before the partition pruner counts them. +arm t "p IS NULL" "delta_lake_enable_engine_predicate = 0" +arm t "p = 7" "delta_lake_enable_engine_predicate = 0" +arm t "p < 6" "delta_lake_enable_engine_predicate = 0" +arm t "p IS NOT NULL" "delta_lake_enable_engine_predicate = 0" +arm t "has([5, 7], p)" +arm n "has([NULL, 5], p)" +arm n "NOT has([NULL, 5], p)" +arm t2 "has([NULL, 1], a)" +arm s "has([NULL, 'a'], s)" + +mapfile -t LINES < <(${CLICKHOUSE_LOCAL} --allow_delta_lake_writes=1 --query "${QUERIES}" < /dev/null) +if [[ "${#LINES[@]}" != "$(( 4 * ${#LABELS[@]} ))" ]]; then + printf 'Expected %s lines, got %s:\n%s\n' "$(( 4 * ${#LABELS[@]} ))" "${#LINES[@]}" "$(printf '%s\n' "${LINES[@]}")" >&2 + exit 1 +fi + +for i in "${!LABELS[@]}"; do + printf '%s\t%s\t%s\t%s\n' "${LABELS[$i]}" "${LINES[4 * i]}" "${LINES[4 * i + 2]}" "$(( LINES[4 * i + 3] - LINES[4 * i + 1] ))" +done diff --git a/tests/queries/0_stateless/05615_formats_accumulating_result_memory_limit_in_local.reference b/tests/queries/0_stateless/05615_formats_accumulating_result_memory_limit_in_local.reference new file mode 100644 index 0000000000000..d0fcfd538e173 --- /dev/null +++ b/tests/queries/0_stateless/05615_formats_accumulating_result_memory_limit_in_local.reference @@ -0,0 +1,6 @@ +MEMORY_LIMIT_EXCEEDED +PrettyCompactMonoBlock 241 +MEMORY_LIMIT_EXCEEDED +PrettyCompact 241 +MEMORY_LIMIT_EXCEEDED +JSONColumns 241 diff --git a/tests/queries/0_stateless/05615_formats_accumulating_result_memory_limit_in_local.sh b/tests/queries/0_stateless/05615_formats_accumulating_result_memory_limit_in_local.sh new file mode 100755 index 0000000000000..551f08b3f82cb --- /dev/null +++ b/tests/queries/0_stateless/05615_formats_accumulating_result_memory_limit_in_local.sh @@ -0,0 +1,18 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Every stored block is larger than the memory limit, so the format that accumulates the result fails to copy it +# when the next block arrives: the query must fail with an error, not crash. +for format in PrettyCompactMonoBlock PrettyCompact JSONColumns; do + $CLICKHOUSE_LOCAL --query " + CREATE TABLE t (x UInt64) ENGINE = Memory; + INSERT INTO t SELECT number FROM numbers(5000000) SETTINGS max_block_size = 5000000, max_insert_block_size = 5000000; + INSERT INTO t SELECT number FROM numbers(5000000) SETTINGS max_block_size = 5000000, max_insert_block_size = 5000000; + SELECT * FROM t FORMAT $format SETTINGS max_threads = 1, max_memory_usage = '20Mi', + output_format_pretty_squash_consecutive_ms = 1000000, output_format_pretty_squash_max_wait_ms = 1000000; + " 2>&1 >/dev/null | grep -o -m1 'MEMORY_LIMIT_EXCEEDED' | head -n1 + echo "$format ${PIPESTATUS[0]}" +done diff --git a/tests/queries/0_stateless/05616_orc_timestamp_before_first_transition.reference b/tests/queries/0_stateless/05616_orc_timestamp_before_first_transition.reference new file mode 100644 index 0000000000000..8c5b5ebbbffc3 --- /dev/null +++ b/tests/queries/0_stateless/05616_orc_timestamp_before_first_transition.reference @@ -0,0 +1,7 @@ +1849-12-31 16:00:00 +1881-07-06 03:13:13 +1949-12-31 16:00:00 +1849-12-31 16:00:00 +1850-01-01 00:36:45 +0 1849-12-31 16:00:00.000000000 +1 1849-12-31 16:00:00.000000000 diff --git a/tests/queries/0_stateless/05616_orc_timestamp_before_first_transition.sh b/tests/queries/0_stateless/05616_orc_timestamp_before_first_transition.sh new file mode 100755 index 0000000000000..8a44944717c9c --- /dev/null +++ b/tests/queries/0_stateless/05616_orc_timestamp_before_first_transition.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# no-fasttest: the fast test build has no ORC format + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# Before the first transition of a time zone, the wall clock is kept with the oldest standard offset +# (Asia/Shanghai +08:00, America/Los_Angeles -08:00), not with the local mean time, as in other ORC readers. + +FILE="${CLICKHOUSE_TMP}/${CLICKHOUSE_DATABASE}.orc" + +$CLICKHOUSE_LOCAL -q " + SELECT arrayJoin([toDateTime64('1850-01-01 00:00:00', 0, 'UTC'), toDateTime64('1881-07-06 11:13:13', 0, 'UTC'), toDateTime64('1950-01-01 00:00:00', 0, 'UTC')]) AS ts + INTO OUTFILE '$FILE' TRUNCATE FORMAT ORC" +$CLICKHOUSE_LOCAL -q "SELECT ts FROM file('$FILE', ORC, 'ts DateTime64(0, \'UTC\')') SETTINGS input_format_orc_reader_time_zone_name = 'Asia/Shanghai'" + +$CLICKHOUSE_LOCAL -q " + SELECT toDateTime64('1850-01-01 00:00:00', 0, 'UTC') AS ts + INTO OUTFILE '$FILE' TRUNCATE FORMAT ORC SETTINGS output_format_orc_writer_time_zone_name = 'America/Los_Angeles'" +$CLICKHOUSE_LOCAL -q "SELECT ts FROM file('$FILE', ORC, 'ts DateTime64(0, \'UTC\')')" + +# Europe/Lisbon starts with a no-op transition record in 1884 that keeps the local mean time (-00:36:45). +$CLICKHOUSE_LOCAL -q " + SELECT toDateTime64('1850-01-01 00:00:00', 0, 'UTC') AS ts + INTO OUTFILE '$FILE' TRUNCATE FORMAT ORC" +$CLICKHOUSE_LOCAL -q "SELECT ts FROM file('$FILE', ORC, 'ts DateTime64(0, \'UTC\')') SETTINGS input_format_orc_reader_time_zone_name = 'Europe/Lisbon'" + +# The timestamp statistics must agree with the values that are read, or the filter pushdown drops the row. +$CLICKHOUSE_LOCAL -q " + SELECT toDateTime64('1850-01-01 00:00:00', 9, 'UTC') AS ts + INTO OUTFILE '$FILE' TRUNCATE FORMAT ORC SETTINGS output_format_orc_writer_time_zone_name = 'America/Los_Angeles'" +for push_down in 0 1; do + $CLICKHOUSE_LOCAL -q " + SELECT $push_down, ts FROM file('$FILE', ORC, 'ts DateTime64(9, \'UTC\')') + WHERE ts = toDateTime64('1849-12-31 16:00:00', 9, 'UTC') SETTINGS input_format_orc_filter_push_down = $push_down" +done + +rm -f "$FILE" diff --git a/tests/queries/0_stateless/05617_gini_deserialize_validation.reference b/tests/queries/0_stateless/05617_gini_deserialize_validation.reference new file mode 100644 index 0000000000000..b1ea233d5eecd --- /dev/null +++ b/tests/queries/0_stateless/05617_gini_deserialize_validation.reference @@ -0,0 +1,3 @@ +0.25 +0.366667 +0.366667 diff --git a/tests/queries/0_stateless/05617_gini_deserialize_validation.sql b/tests/queries/0_stateless/05617_gini_deserialize_validation.sql new file mode 100644 index 0000000000000..bb90839d649e6 --- /dev/null +++ b/tests/queries/0_stateless/05617_gini_deserialize_validation.sql @@ -0,0 +1,15 @@ +-- Serialized `gini` states must be validated like ordinary input values. + +-- Int64 min and max. +SELECT finalizeAggregation(CAST(unhex('020000000000000080FFFFFFFFFFFFFF7F') AS AggregateFunction(gini, Int64))); -- { serverError BAD_ARGUMENTS } +-- Negative value: [-1, 2]. +SELECT finalizeAggregation(CAST(unhex('02FFFFFFFFFFFFFFFF0200000000000000') AS AggregateFunction(gini, Int64))); -- { serverError BAD_ARGUMENTS } +-- Infinity: [0, inf]. +SELECT finalizeAggregation(CAST(unhex('020000000000000000000000000000F07F') AS AggregateFunction(gini, Float64))); -- { serverError BAD_ARGUMENTS } +-- NaN is never stored by `add`: [0, nan]. +SELECT finalizeAggregation(CAST(unhex('020000000000000000000000000000F87F') AS AggregateFunction(gini, Float64))); -- { serverError INCORRECT_DATA } + +-- Valid states still round-trip. +SELECT finalizeAggregation(CAST(unhex('0201000000000000000300000000000000') AS AggregateFunction(gini, Int64))); +SELECT round(finalizeAggregation(CAST(unhex(hex(giniState(number))) AS AggregateFunction(gini, UInt64))), 6) FROM numbers(10); +SELECT round(finalizeAggregation(CAST(unhex(hex(giniState(toFloat64(number)))) AS AggregateFunction(gini, Float64))), 6) FROM numbers(10); diff --git a/tests/queries/0_stateless/05617_orc_nested_struct_without_requested_fields.reference b/tests/queries/0_stateless/05617_orc_nested_struct_without_requested_fields.reference new file mode 100644 index 0000000000000..73a3ed806c7c7 --- /dev/null +++ b/tests/queries/0_stateless/05617_orc_nested_struct_without_requested_fields.reference @@ -0,0 +1,159 @@ +-- { echo } + +SET engine_file_truncate_on_insert = 1; +SET enable_nullable_tuple_type = 1; +INSERT INTO FUNCTION file(currentDatabase() || '_05617.orc', 'ORC') SELECT + number AS id, + if(number = 0, [(1, 'a'), (2, 'b')], [])::Array(Tuple(a Int32, b String)) AS l, + if(number = 0, map('k1', (1, 'a'), 'k2', (2, 'b')), map())::Map(String, Tuple(a Int32, b String)) AS m, + map(CAST(tuple(toInt32(number)), 'Tuple(a Int32)'), 'v') AS mk, + CAST((toInt32(number), [(toInt32(number), 'x')]), 'Tuple(a Int32, l Array(Tuple(x Int32, y String)))') AS t, + [[(toInt32(number), 'x'), (1, 'y')]]::Array(Array(Tuple(a Int32, b String))) AS ll, + map('k', [(toInt32(number), 'x')])::Map(String, Array(Tuple(a Int32, b String))) AS ml, + if(number = 0, [(1, 'a'), NULL, (2, 'b')], [])::Array(Nullable(Tuple(a Int32, b String))) AS ln, + [(toInt32(number), 'c')]::Array(Tuple(A Int32, b String)) AS lc, + CAST((toInt32(number), map('k', (1, 'v'))), 'Tuple(a Int32, m Map(String, Tuple(a Int32, b String)))') AS tm, + [(1, tuple(2)), (3, tuple(4))]::Array(Tuple(x Int32, z Tuple(w Int32))) AS n, + [(1, [(1, 'a'), (2, 'b')])]::Array(Tuple(y Int32, x Array(Tuple(a Int32, b String)))) AS lx, + if(number = 0, [tuple(tuple(1)), tuple(NULL)], [tuple(NULL)])::Array(Tuple(b Nullable(Tuple(a Int32)))) AS nb, + map('k', (if(number = 0, tuple(1), NULL), if(number = 0, NULL, tuple('x'))))::Map(String, Tuple(b Nullable(Tuple(a Int32)), c Nullable(Tuple(x String)))) AS mb, + CAST((toInt32(number), [tuple(if(number = 0, tuple(1), NULL))]), 'Tuple(a Int32, l Array(Tuple(b Nullable(Tuple(a Int32)))))') AS tb, + if(number = 0, tuple(1, 'a'), NULL)::Nullable(Tuple(a Int32, b String)) AS ts, + [CAST(tuple(toDateTime64('3000-01-01 00:00:00', 0, 'UTC')), 'Tuple(a DateTime64(0, \'UTC\'))')] AS ld, + CAST([tuple(11, tuple(5))], 'Array(Tuple(w Int32, x Tuple(v Int32)))') AS an +FROM numbers(2); +-- A list element, a map value and a map key, also nested in a struct, a list or a map, and with NULL elements. +SELECT id, l FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Array(Tuple(q Nullable(Int32)))') ORDER BY id; +0 [(NULL),(NULL)] +1 [] +SELECT id, m FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, m Map(String, Tuple(q Nullable(String)))') ORDER BY id; +0 {'k1':(NULL),'k2':(NULL)} +1 {} +SELECT id, mk FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, mk Map(Tuple(z Nullable(Int32)), String)') ORDER BY id; +0 {(NULL):'v'} +1 {(NULL):'v'} +SELECT id, t FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, t Tuple(a Int32, l Array(Tuple(q Nullable(Int32))))') ORDER BY id; +0 (0,[(NULL)]) +1 (1,[(NULL)]) +SELECT id, ll FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ll Array(Array(Tuple(q Nullable(Int32))))') ORDER BY id; +0 [[(NULL),(NULL)]] +1 [[(NULL),(NULL)]] +SELECT id, ml FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ml Map(String, Array(Tuple(q Nullable(Int32))))') ORDER BY id; +0 {'k':[(NULL)]} +1 {'k':[(NULL)]} +SELECT id, ln FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ln Array(Nullable(Tuple(q Nullable(Int32))))') ORDER BY id; +0 [(NULL),NULL,(NULL)] +1 [] +SELECT id, l FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Array(Tuple(q Nullable(Int32)))') ORDER BY id SETTINGS input_format_orc_row_batch_size = 1; +0 [(NULL),(NULL)] +1 [] +-- The same through a subcolumn of a struct and a subcolumn of a Nested column. +SELECT id, `t.a`, `t.l` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, `t.a` Int32, `t.l` Array(Tuple(q Nullable(Int32)))') ORDER BY id; +0 0 [(NULL)] +1 1 [(NULL)] +SELECT id, `tm.m` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, `tm.m` Map(String, Tuple(q Nullable(String)))') ORDER BY id; +0 {'k':(NULL)} +1 {'k':(NULL)} +SELECT id, `n.z` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, `n.z` Array(Tuple(q Nullable(Int32)))') ORDER BY id; +0 [(NULL),(NULL)] +1 [(NULL),(NULL)] +SELECT id, `n.x`, `n.z` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, `n.x` Array(Int32), `n.z` Array(Tuple(q Nullable(Int32)))') ORDER BY id; +0 [1,3] [(NULL),(NULL)] +1 [1,3] [(NULL),(NULL)] +-- A struct that contains such a list is read with its fields. +SELECT id, lx FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, lx Array(Tuple(x Array(Tuple(q Nullable(Int32)))))') ORDER BY id; +0 [([(NULL),(NULL)])] +1 [([(NULL),(NULL)])] +-- A struct read whole by another column keeps its data. optimize_functions_to_subcolumns = 0 keeps `t.l` itself in the read. +SELECT id, t, length(`t.l`) FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, t String, `t.l` Array(Tuple(q String, r String))') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +0 (0,[(0,\'x\')]) 1 +1 (1,[(1,\'x\')]) 1 +-- A Tuple naming some of the fields, and case-insensitive matching. +SELECT id, l FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Array(Tuple(a Int32, q Nullable(Int32)))') ORDER BY id; +0 [(1,NULL),(2,NULL)] +1 [] +SELECT id, m FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, m Map(String, Tuple(a Int32, q Nullable(String)))') ORDER BY id; +0 {'k1':(1,NULL),'k2':(2,NULL)} +1 {} +SELECT id, lc FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, lc Array(Tuple(a Int32))') ORDER BY id SETTINGS input_format_orc_case_insensitive_column_matching = 1; +0 [(0)] +1 [(1)] +-- A non-nullable Date32 field gets the same default with and without a field that is in the file, an Enum gets its first value. +SELECT id, l, m FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Array(Tuple(q Date32, e Enum8(\'x\' = 1, \'y\' = 2))), m Map(String, Tuple(q Date32))') ORDER BY id; +0 [('1970-01-01','x'),('1970-01-01','x')] {'k1':('1970-01-01'),'k2':('1970-01-01')} +1 [] {} +SELECT id, l, m FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Array(Tuple(a Int32, q Date32)), m Map(String, Tuple(a Int32, q Date32))') ORDER BY id; +0 [(1,'1970-01-01'),(2,'1970-01-01')] {'k1':(1,'1970-01-01'),'k2':(2,'1970-01-01')} +1 [] {} +-- A requested struct field of such a struct keeps its presence, the same as when one of its fields is requested. +SELECT id, nb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, nb Array(Tuple(b Nullable(Tuple(q Int32))))') ORDER BY id; +0 [((0)),(NULL)] +1 [(NULL)] +SELECT id, nb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, nb Array(Tuple(b Nullable(Tuple(a Nullable(Int32), q Int32))))') ORDER BY id; +0 [((1,0)),(NULL)] +1 [(NULL)] +SELECT id, mb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, mb Map(String, Tuple(c Nullable(Tuple(q Int32)), b Nullable(Tuple(r String))))') ORDER BY id; +0 {'k':(NULL,(''))} +1 {'k':((0),NULL)} +SELECT id, mb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, mb Map(String, Tuple(c Nullable(Tuple(x Nullable(String), q Int32)), b Nullable(Tuple(a Nullable(Int32), r String))))') ORDER BY id; +0 {'k':(NULL,(1,''))} +1 {'k':(('x',0),NULL)} +-- A NULL struct requested as non-Nullable is rejected or read as default, the same as when one of its fields is requested. +SELECT id, nb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, nb Array(Tuple(b Tuple(q Int32)))') ORDER BY id SETTINGS input_format_null_as_default = 1; +0 [((0)),((0))] +1 [((0))] +SELECT id, nb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, nb Array(Tuple(b Tuple(a Nullable(Int32), q Int32)))') ORDER BY id SETTINGS input_format_null_as_default = 1; +0 [((1,0)),((NULL,0))] +1 [((NULL,0))] +SELECT id, nb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, nb Array(Tuple(b Tuple(q Int32)))') ORDER BY id SETTINGS input_format_null_as_default = 0; -- { serverError CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN } +SELECT id, nb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, nb Array(Tuple(b Tuple(a Nullable(Int32), q Int32)))') ORDER BY id SETTINGS input_format_null_as_default = 0; -- { serverError CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN } +-- Two columns reading the same struct with different types get the defaults of their own type. +SELECT id, t, `t.l` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, t Tuple(l Array(Tuple(q Nullable(Int32)))), `t.l` Array(Tuple(r Nullable(String), s Nullable(Int32)))') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +0 ([(NULL)]) [(NULL,NULL)] +1 ([(NULL)]) [(NULL,NULL)] +SELECT id, t, `t.l` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, t Tuple(l Array(Tuple(e Enum8(\'x\' = 1)))), `t.l` Array(Tuple(r Nullable(String)))') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +0 ([('x')]) [(NULL)] +1 ([('x')]) [(NULL)] +SELECT id, `t.l`, t FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, `t.l` Array(Tuple(r Nullable(String), s Nullable(Int32))), t Tuple(l Array(Tuple(q Nullable(Int32))))') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +0 [(NULL,NULL)] ([(NULL)]) +1 [(NULL,NULL)] ([(NULL)]) +-- Of two columns reading the same struct, one that requests a nested struct of it keeps that struct's presence. +SELECT id, tb, `tb.l` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, tb Tuple(l Array(Tuple(q Nullable(Int32)))), `tb.l` Array(Tuple(b Nullable(Tuple(q Int32))))') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +0 ([(NULL)]) [((0))] +1 ([(NULL)]) [(NULL)] +SELECT id, `tb.l`, tb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, `tb.l` Array(Tuple(b Nullable(Tuple(q Int32)))), tb Tuple(l Array(Tuple(q Nullable(Int32))))') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +0 [((0))] ([(NULL)]) +1 [(NULL)] ([(NULL)]) +SELECT id, tb, `tb.a` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, tb Tuple(l Array(Tuple(q Nullable(Int32)))), `tb.a` Int32') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +0 ([(NULL)]) 0 +1 ([(NULL)]) 1 +-- A top-level struct keeps its presence too, also when missing columns are not allowed. +SELECT id, ts FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ts Nullable(Tuple(q Int32))') ORDER BY id; +0 (0) +1 \N +SELECT id, ts FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ts Nullable(Tuple(a Int32, q Int32))') ORDER BY id; +0 (1,0) +1 \N +SELECT id, ts FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ts Nullable(Tuple(q Int32))') ORDER BY id SETTINGS input_format_orc_allow_missing_columns = 0; +0 (0) +1 \N +-- A field read only for the size of its struct is not converted. +SELECT id, ld FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ld Array(Tuple(q Int32))') ORDER BY id; +0 [(0)] +1 [(0)] +-- An unnamed Tuple is matched by position and must have as many elements as the struct. +SELECT id, ts FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ts Nullable(Tuple(Int32, String))') ORDER BY id; +0 (1,'a') +1 \N +SELECT id, ts FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ts Nullable(Tuple(Int32, String, Int32))') ORDER BY id; -- { serverError TYPE_MISMATCH } +SELECT id, l FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Array(Tuple(Int32, String, Int32))') ORDER BY id; -- { serverError TYPE_MISMATCH } +-- The same for a Nested column that is not flattened. +SELECT id, l FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Nested(q Int32)') ORDER BY id SETTINGS flatten_nested = 0; +0 [(0),(0)] +1 [] +SELECT id, ln FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ln Nested(q Nullable(Int32))') ORDER BY id SETTINGS flatten_nested = 0; +0 [(NULL),(NULL),(NULL)] +1 [] +SELECT id, ln FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ln Nested(a Nullable(Int32), q Nullable(Int32))') ORDER BY id SETTINGS flatten_nested = 0; +0 [(1,NULL),(NULL,NULL),(2,NULL)] +1 [] diff --git a/tests/queries/0_stateless/05617_orc_nested_struct_without_requested_fields.sql b/tests/queries/0_stateless/05617_orc_nested_struct_without_requested_fields.sql new file mode 100644 index 0000000000000..8dbf07d584af2 --- /dev/null +++ b/tests/queries/0_stateless/05617_orc_nested_struct_without_requested_fields.sql @@ -0,0 +1,101 @@ +-- Tags: no-fasttest +-- no-fasttest: ORC is not available in fasttest builds +-- An ORC list or map whose element, key or value struct has none of the fields of the requested Tuple keeps its +-- elements, and each of them is filled with default values. + +-- { echo } + +SET engine_file_truncate_on_insert = 1; +SET enable_nullable_tuple_type = 1; + +INSERT INTO FUNCTION file(currentDatabase() || '_05617.orc', 'ORC') SELECT + number AS id, + if(number = 0, [(1, 'a'), (2, 'b')], [])::Array(Tuple(a Int32, b String)) AS l, + if(number = 0, map('k1', (1, 'a'), 'k2', (2, 'b')), map())::Map(String, Tuple(a Int32, b String)) AS m, + map(CAST(tuple(toInt32(number)), 'Tuple(a Int32)'), 'v') AS mk, + CAST((toInt32(number), [(toInt32(number), 'x')]), 'Tuple(a Int32, l Array(Tuple(x Int32, y String)))') AS t, + [[(toInt32(number), 'x'), (1, 'y')]]::Array(Array(Tuple(a Int32, b String))) AS ll, + map('k', [(toInt32(number), 'x')])::Map(String, Array(Tuple(a Int32, b String))) AS ml, + if(number = 0, [(1, 'a'), NULL, (2, 'b')], [])::Array(Nullable(Tuple(a Int32, b String))) AS ln, + [(toInt32(number), 'c')]::Array(Tuple(A Int32, b String)) AS lc, + CAST((toInt32(number), map('k', (1, 'v'))), 'Tuple(a Int32, m Map(String, Tuple(a Int32, b String)))') AS tm, + [(1, tuple(2)), (3, tuple(4))]::Array(Tuple(x Int32, z Tuple(w Int32))) AS n, + [(1, [(1, 'a'), (2, 'b')])]::Array(Tuple(y Int32, x Array(Tuple(a Int32, b String)))) AS lx, + if(number = 0, [tuple(tuple(1)), tuple(NULL)], [tuple(NULL)])::Array(Tuple(b Nullable(Tuple(a Int32)))) AS nb, + map('k', (if(number = 0, tuple(1), NULL), if(number = 0, NULL, tuple('x'))))::Map(String, Tuple(b Nullable(Tuple(a Int32)), c Nullable(Tuple(x String)))) AS mb, + CAST((toInt32(number), [tuple(if(number = 0, tuple(1), NULL))]), 'Tuple(a Int32, l Array(Tuple(b Nullable(Tuple(a Int32)))))') AS tb, + if(number = 0, tuple(1, 'a'), NULL)::Nullable(Tuple(a Int32, b String)) AS ts, + [CAST(tuple(toDateTime64('3000-01-01 00:00:00', 0, 'UTC')), 'Tuple(a DateTime64(0, \'UTC\'))')] AS ld, + CAST([tuple(11, tuple(5))], 'Array(Tuple(w Int32, x Tuple(v Int32)))') AS an +FROM numbers(2); + +-- A list element, a map value and a map key, also nested in a struct, a list or a map, and with NULL elements. +SELECT id, l FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Array(Tuple(q Nullable(Int32)))') ORDER BY id; +SELECT id, m FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, m Map(String, Tuple(q Nullable(String)))') ORDER BY id; +SELECT id, mk FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, mk Map(Tuple(z Nullable(Int32)), String)') ORDER BY id; +SELECT id, t FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, t Tuple(a Int32, l Array(Tuple(q Nullable(Int32))))') ORDER BY id; +SELECT id, ll FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ll Array(Array(Tuple(q Nullable(Int32))))') ORDER BY id; +SELECT id, ml FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ml Map(String, Array(Tuple(q Nullable(Int32))))') ORDER BY id; +SELECT id, ln FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ln Array(Nullable(Tuple(q Nullable(Int32))))') ORDER BY id; +SELECT id, l FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Array(Tuple(q Nullable(Int32)))') ORDER BY id SETTINGS input_format_orc_row_batch_size = 1; + +-- The same through a subcolumn of a struct and a subcolumn of a Nested column. +SELECT id, `t.a`, `t.l` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, `t.a` Int32, `t.l` Array(Tuple(q Nullable(Int32)))') ORDER BY id; +SELECT id, `tm.m` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, `tm.m` Map(String, Tuple(q Nullable(String)))') ORDER BY id; +SELECT id, `n.z` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, `n.z` Array(Tuple(q Nullable(Int32)))') ORDER BY id; +SELECT id, `n.x`, `n.z` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, `n.x` Array(Int32), `n.z` Array(Tuple(q Nullable(Int32)))') ORDER BY id; + +-- A struct that contains such a list is read with its fields. +SELECT id, lx FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, lx Array(Tuple(x Array(Tuple(q Nullable(Int32)))))') ORDER BY id; + +-- A struct read whole by another column keeps its data. optimize_functions_to_subcolumns = 0 keeps `t.l` itself in the read. +SELECT id, t, length(`t.l`) FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, t String, `t.l` Array(Tuple(q String, r String))') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; + +-- A Tuple naming some of the fields, and case-insensitive matching. +SELECT id, l FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Array(Tuple(a Int32, q Nullable(Int32)))') ORDER BY id; +SELECT id, m FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, m Map(String, Tuple(a Int32, q Nullable(String)))') ORDER BY id; +SELECT id, lc FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, lc Array(Tuple(a Int32))') ORDER BY id SETTINGS input_format_orc_case_insensitive_column_matching = 1; + +-- A non-nullable Date32 field gets the same default with and without a field that is in the file, an Enum gets its first value. +SELECT id, l, m FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Array(Tuple(q Date32, e Enum8(\'x\' = 1, \'y\' = 2))), m Map(String, Tuple(q Date32))') ORDER BY id; +SELECT id, l, m FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Array(Tuple(a Int32, q Date32)), m Map(String, Tuple(a Int32, q Date32))') ORDER BY id; + +-- A requested struct field of such a struct keeps its presence, the same as when one of its fields is requested. +SELECT id, nb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, nb Array(Tuple(b Nullable(Tuple(q Int32))))') ORDER BY id; +SELECT id, nb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, nb Array(Tuple(b Nullable(Tuple(a Nullable(Int32), q Int32))))') ORDER BY id; +SELECT id, mb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, mb Map(String, Tuple(c Nullable(Tuple(q Int32)), b Nullable(Tuple(r String))))') ORDER BY id; +SELECT id, mb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, mb Map(String, Tuple(c Nullable(Tuple(x Nullable(String), q Int32)), b Nullable(Tuple(a Nullable(Int32), r String))))') ORDER BY id; + +-- A NULL struct requested as non-Nullable is rejected or read as default, the same as when one of its fields is requested. +SELECT id, nb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, nb Array(Tuple(b Tuple(q Int32)))') ORDER BY id SETTINGS input_format_null_as_default = 1; +SELECT id, nb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, nb Array(Tuple(b Tuple(a Nullable(Int32), q Int32)))') ORDER BY id SETTINGS input_format_null_as_default = 1; +SELECT id, nb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, nb Array(Tuple(b Tuple(q Int32)))') ORDER BY id SETTINGS input_format_null_as_default = 0; -- { serverError CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN } +SELECT id, nb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, nb Array(Tuple(b Tuple(a Nullable(Int32), q Int32)))') ORDER BY id SETTINGS input_format_null_as_default = 0; -- { serverError CANNOT_INSERT_NULL_IN_ORDINARY_COLUMN } + +-- Two columns reading the same struct with different types get the defaults of their own type. +SELECT id, t, `t.l` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, t Tuple(l Array(Tuple(q Nullable(Int32)))), `t.l` Array(Tuple(r Nullable(String), s Nullable(Int32)))') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT id, t, `t.l` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, t Tuple(l Array(Tuple(e Enum8(\'x\' = 1)))), `t.l` Array(Tuple(r Nullable(String)))') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT id, `t.l`, t FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, `t.l` Array(Tuple(r Nullable(String), s Nullable(Int32))), t Tuple(l Array(Tuple(q Nullable(Int32))))') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; + +-- Of two columns reading the same struct, one that requests a nested struct of it keeps that struct's presence. +SELECT id, tb, `tb.l` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, tb Tuple(l Array(Tuple(q Nullable(Int32)))), `tb.l` Array(Tuple(b Nullable(Tuple(q Int32))))') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT id, `tb.l`, tb FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, `tb.l` Array(Tuple(b Nullable(Tuple(q Int32)))), tb Tuple(l Array(Tuple(q Nullable(Int32))))') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; +SELECT id, tb, `tb.a` FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, tb Tuple(l Array(Tuple(q Nullable(Int32)))), `tb.a` Int32') ORDER BY id SETTINGS optimize_functions_to_subcolumns = 0; + +-- A top-level struct keeps its presence too, also when missing columns are not allowed. +SELECT id, ts FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ts Nullable(Tuple(q Int32))') ORDER BY id; +SELECT id, ts FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ts Nullable(Tuple(a Int32, q Int32))') ORDER BY id; +SELECT id, ts FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ts Nullable(Tuple(q Int32))') ORDER BY id SETTINGS input_format_orc_allow_missing_columns = 0; + +-- A field read only for the size of its struct is not converted. +SELECT id, ld FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ld Array(Tuple(q Int32))') ORDER BY id; + +-- An unnamed Tuple is matched by position and must have as many elements as the struct. +SELECT id, ts FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ts Nullable(Tuple(Int32, String))') ORDER BY id; +SELECT id, ts FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ts Nullable(Tuple(Int32, String, Int32))') ORDER BY id; -- { serverError TYPE_MISMATCH } +SELECT id, l FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Array(Tuple(Int32, String, Int32))') ORDER BY id; -- { serverError TYPE_MISMATCH } + +-- The same for a Nested column that is not flattened. +SELECT id, l FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, l Nested(q Int32)') ORDER BY id SETTINGS flatten_nested = 0; +SELECT id, ln FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ln Nested(q Nullable(Int32))') ORDER BY id SETTINGS flatten_nested = 0; +SELECT id, ln FROM file(currentDatabase() || '_05617.orc', 'ORC', 'id UInt64, ln Nested(a Nullable(Int32), q Nullable(Int32))') ORDER BY id SETTINGS flatten_nested = 0; diff --git a/tests/queries/0_stateless/05621_delta_lake_column_mapping_swapped_names_order.reference b/tests/queries/0_stateless/05621_delta_lake_column_mapping_swapped_names_order.reference new file mode 100644 index 0000000000000..67e0669d56c6f --- /dev/null +++ b/tests/queries/0_stateless/05621_delta_lake_column_mapping_swapped_names_order.reference @@ -0,0 +1,10 @@ +4 99999 +3 99998 +2 99997 +99999 4 +99998 3 +99997 2 +0 0 0 +1 1 1 +2 2 2 +299995 4999950000 4999950000 diff --git a/tests/queries/0_stateless/05621_delta_lake_column_mapping_swapped_names_order.sh b/tests/queries/0_stateless/05621_delta_lake_column_mapping_swapped_names_order.sh new file mode 100755 index 0000000000000..4b3b3f2de2843 --- /dev/null +++ b/tests/queries/0_stateless/05621_delta_lake_column_mapping_swapped_names_order.sh @@ -0,0 +1,36 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-msan +# Tag no-fasttest: needs Parquet and Delta Lake +# Tag no-msan: delta-kernel-rs (Rust) is not built under MSan, so `deltaLakeLocal` is absent. + +# A Delta Lake table with column mapping where a column is stored under the logical name of another column, read +# with the columns in every order: the renaming of the requested columns to their physical names must not collide +# with a logical name that is not renamed yet (it used to erase the column and read freed memory). + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +DELTA_DIR="${USER_FILES_PATH}/${CLICKHOUSE_DATABASE}_05621" +rm -rf "${DELTA_DIR}" +mkdir -p "${DELTA_DIR}/_delta_log" + +# The data file stores `k` = number % 7, `v` = number and `w` = number: the table's `k_old`, `k` and `w` +# (column mapping enabled on an existing table, then `k` renamed to `k_old` and `v` to `k`). +${CLICKHOUSE_CLIENT} --query " + INSERT INTO FUNCTION file('${DELTA_DIR}/data.parquet', Parquet, 'k Int64, v Int64, w Int64') + SELECT number % 7, number, number FROM numbers(100000)" +DELTA_SCHEMA='{\"type\":\"struct\",\"fields\":[{\"name\":\"k_old\",\"type\":\"long\",\"nullable\":false,\"metadata\":{\"delta.columnMapping.id\":1,\"delta.columnMapping.physicalName\":\"k\"}},{\"name\":\"k\",\"type\":\"long\",\"nullable\":false,\"metadata\":{\"delta.columnMapping.id\":2,\"delta.columnMapping.physicalName\":\"v\"}},{\"name\":\"w\",\"type\":\"long\",\"nullable\":false,\"metadata\":{\"delta.columnMapping.id\":3,\"delta.columnMapping.physicalName\":\"w\"}}]}' +cat > "${DELTA_DIR}/_delta_log/00000000000000000000.json" <&1 | grep -o -m1 -E 'READONLY|QUERY_IS_PROHIBITED' + $CLICKHOUSE_CLIENT $setting -q "ALTER NAMED COLLECTION ${nc}_existing SET key = 'other'" 2>&1 | grep -o -m1 -E 'READONLY|QUERY_IS_PROHIBITED' + $CLICKHOUSE_CLIENT $setting -q "DROP NAMED COLLECTION ${nc}_existing" 2>&1 | grep -o -m1 -E 'READONLY|QUERY_IS_PROHIBITED' +done + +# Nothing was created or dropped. +$CLICKHOUSE_CLIENT -q "SELECT name FROM system.named_collections WHERE name LIKE '${nc}%' ORDER BY name" | sed "s/${nc}/nc/" + +$CLICKHOUSE_CLIENT -q "DROP NAMED COLLECTION ${nc}_existing" diff --git a/tests/queries/0_stateless/data_orc/bad_bloom_filter_1.6.0.orc b/tests/queries/0_stateless/data_orc/bad_bloom_filter_1.6.0.orc new file mode 100644 index 0000000000000000000000000000000000000000..6c52f4f0f98b40dec581f1b02ec3936d34d62796 GIT binary patch literal 709 zcmeYda+YOa;Nsz8VE_Ul77o?`1}g@J$3ah$(i<2Q4f(`X9foPp;g3RC==BExo1XeDT)j`@d*0UouL)T2hk0z`R(Z zLXP?4@vapcj<8#u@|yho1lyA79E~@e53qDH@+2HEcDVcTjX@hD4~ygm1wIL$ga_Wt zzJC_DaqeWa0J?nA^G7KSjEahUVXAU^VaMm3c@m;{OoekM0~>=D(Badqxf%=vTtb6b zHg4E<%CX3zKUbOK)={TKV@@6Y*h6V$&s+aIGd{jSwsL{0WmAA^^S*OhsY*>Z94N8MFvLc#_M{U zPrgjEZ*TIGPvvkpz~A`dxL36L%3Q1DZO2aANS!XVi~Tn%;2*O-D+9y-A7Da(f#LVT zLkEs9u=FG}9eptIV>|;`oST8cjU`yn0T@?~Iho0+PWefRNxqrcsZ$sjOivl|H5l+X zUsQAGG`hGcLZ?7l&5W5; zG3liD8Q;)^3ke1Z0goOeG;Y43sKXqlEyj0^flV7Dfr$jfbTo%{ws&7OR5!K^?J)Fo mQfoMv*fgJAnSmomfKj5Mfklalfk(r{fUW5ZGqZn?vp4|E0T8hO literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_orc/bad_bloom_filter_1.6.11.orc b/tests/queries/0_stateless/data_orc/bad_bloom_filter_1.6.11.orc new file mode 100644 index 0000000000000000000000000000000000000000..fc3ffef8f18c17dc95f532108f94846c6cb9b894 GIT binary patch literal 717 zcmeYda+YOa;Nsz8VE_Ul77o?`1}g@J$3ah$(i<2Q4f(`X9foPp;g3RC==BExo1XeDT)j`@d*0UouL)T2hk0z`R(Z zLXP?4@vapcj<8#u@|yho1lyA79E~@e53qDH@+2HEcDVcTjX@hD4~ygm1wIL$ga_Wt zzJC_DaqeWa0J?nA^G7KSjEahUVXAU^VaMm3c@m;{OoekM0~>=D(Badqxf%=vTtb6b zHg4E<%CX3zKUbOK)={TKV@@6Y*h6V$&s+aIGd{jSwsL{0WmAA^^S*OhsY*>Z94N8MFvLc#_M{U zPrgjEZ*TIGPvvkpz~A`dxL36L%3Q1DZO2aANS!XVi~Tn%;2*O-D+9y-A7Da(f#LVT zLkEs9u=FG}9eptIV>|;`oST8cjU`yn0T@?~Iho0+PWefRNxqrcsZ$sjOivl|H5l+X zUsQAGG`hGcLZ?7l&5W5; zG3liD8Q;)^3ke1Z0goOeG;Y43sKXqlEyj0^flV7Dfr$jfbTo%{ws&7OR5!K^?J)Fo vQfoMv*fgI#S!njcc{ArNRAk^N5nz;PXkbxdV&Ks*F<@)@!p!U+JFZE-iyIBTGB(h%cBfX4BGjAJ&?Gn8U6&@(d& zz$y=vGO&;Wsxk>+P+(wKo$%nn**QlR@+j>TaZurBkYr%+<23NG2w({jkqGi}XJ9z6 zYbW!qRlXhHI#@oOiG6Td^2pN_EKS$M7W4_Nc>Rj~#nl5W{0t0koK8+IN-kjSYyv`D ze0)qod?H{j6B7>yAC~|R3qu>wgGrAQgcBw_c*b{mZuY+kFmEzQ>GXvtHD{W zWs;4rS`L~_X0e}<#`Zdy^-@fN9e2o_(5Co~&Su7rnHMtH82$lGnRVvG(L)DC4;)ZE zeNa{9)WwY{H(#^4HOd*NetdLvKd-FANsdH4L0}-t>A5j0stGG=rYRQ(FVL``E)@{c z*M0ZIflEH;k2rN4K6K*36_4{*Ji0C)IdSR0m4Neim{y!?@a9?3$G>FuO^5RVT}K_x zpW#^}ug521v|$3%saXw27$-3@JWde%aZ=^z*@HR-4S|as4*E8Gdp0$@cTAnwIkPpi z%P_FZ(6h@hvdhr5%P_RFv)Q$?x7ok>cmtzPw}gS?Pk~t{8%{Qy?rLq;ZaC;Qt=ZdS z%Hrgco38WrFxe(}Jv!Q5FKyK@N$^mZl1z-7O-v7)laHEP&>6R9%Nzphv`rVRkP+WK zv2p3ld7Z5S?X43RuADe;;?|1sEk78U&P>8TXu+-id_MC9Fa-<` z?lW*QFhnuTafo1Gc%1Y&K{#Q;gJ(<^dxYv@a%6g#dW4FAv=5MebWNDr~+4;VnvT^!awPNw&jholK~7@| z!qQ)f-`+hkxUucGmEX^Ib+-3k9pAk34oO3dhvpw=li&Ht=fd(`*2q-Zz@MuUHur+;QX7&$q76$-W;u-w_ literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_orc/malformed_dictionary_size_max.orc b/tests/queries/0_stateless/data_orc/malformed_dictionary_size_max.orc new file mode 100644 index 0000000000000000000000000000000000000000..27d3a7a5694c836382a434b145463145bc7e4969 GIT binary patch literal 174 zcmeYdaz2pEz)<19(7?sU!6?8e!2+b2fixqKW&+a8LQEVCLL5M`|3JVm#q91I!X?hd z!jTfdz{SszqQuF?SRuq%A#s8+fB~q=Sc(fsGDvVJXltY>ad7YmF)~Uq7ArAvup6XU xfwftoYja@mU`*n2&dE&9_Q)?SPUUb18qm-npv1($qakU)*7Sv$*+0lx8~}Ex9uxoo literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_orc/malformed_footer_too_many_statistics.orc b/tests/queries/0_stateless/data_orc/malformed_footer_too_many_statistics.orc new file mode 100644 index 0000000000000000000000000000000000000000..00ab37f164451f7d41fdd9918d3a60e6eb3fb9e1 GIT binary patch literal 353 zcmb7!55@c<-_od@Ma)J`I4HxfN+mPlz%p40|nh@xK=W-?pJwH%K)jX%p}<3F&Fh@rNx z?t8Y(IS+(X;JUy~Nt{KsDqAPB`oye0GaG+COmQR4^X~C!`ttU={aZS&{tbFtXANNv_gqUE3}AonBeEZn8f9rlbM|DkzZPz$`J=lJ`D{5 WN=ytq8jF|>*qXjDGy4ZQO8@|~r#gH9 literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_orc/malformed_patched_base_terminal_continuation.orc b/tests/queries/0_stateless/data_orc/malformed_patched_base_terminal_continuation.orc new file mode 100644 index 0000000000000000000000000000000000000000..43552b05c31d975806e7b28bf0348ac911b93320 GIT binary patch literal 83 zcmeYda_(bbf5@QqpMi^wgHeD{fZBafx-+D4Gk42F|jIl^(saf&@Ul}_i@Q96kobdi?g;4J`i%|p7O zHQHy4J^a)g0`Qj6YX%1aWcQxocSeo^^{~g%Xx3tINijrGbQqKXl}3-r5QS%EZ#Tnm$hsO_t_nvG%Tof%Rfa2K6BY3f#1KM=U?J&`Kze_YKPs*0 zx=Eu0j}P9!do!~|CB`MJ=O9i9z@@C6VX*Am!Mha&!ECV3S+H<8^mY#W1gIGH9|rxw zXpDOxF>EY&jzr&%B;N@{i#~~@xF<_qJ7Nf;voiJcyp((SJL0nbs+X1h!K@W3q*{Nxeoi)D+c$m>ztpwO3DRD- MFyFsQ%@$SX1Y@lyc>n+a literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_orc/malformed_timestamp_nanoseconds.orc b/tests/queries/0_stateless/data_orc/malformed_timestamp_nanoseconds.orc new file mode 100644 index 0000000000000000000000000000000000000000..ab57ba54f6cb5f35ce2f2225cedead3e3c24d9e6 GIT binary patch literal 238 zcmeYdaz4P>@V{v>Ba?moe+amFYzD)Ut4a(EH!q|}F!G&RDZuEwo9hH)g+~tq6u9SsRejxb{KXpyLT!00RfJK%^8GkYtdsQIOT(QsUs?5n^PNVk}W&;@~vk yvI3i7h1U!R1`oz0F6W%gPb;1ObElwvGVV&dR5V73CA xVnx6dhY5Zjj7eP1Iho1X9{HuksT|=zmo_vAC^0ecXlNL)HGN@b_78Fv2LL-3L?-|M literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_orc/malformed_zstd_chunk_size.orc b/tests/queries/0_stateless/data_orc/malformed_zstd_chunk_size.orc new file mode 100644 index 0000000000000000000000000000000000000000..4f30ed6c7f1acfc4f1587ecd851ca828b38d2f80 GIT binary patch literal 38 scmeYda+Y9V(AcW~cL4(fg99T20|&nVt3(43uxKzeePL$y4{{a;0H`qt$^ZZW literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_orc/row_index_seek_control.orc b/tests/queries/0_stateless/data_orc/row_index_seek_control.orc new file mode 100644 index 0000000000000000000000000000000000000000..ede347aa81181c970ec5d83161d8595eef07a424 GIT binary patch literal 213 zcmY+7F$=;#426?iIYPMMI7E+5r66?dD4oO#c9E9SA0Uc@C^(4VSp3cY0MA1Nhvb8o z_g>n*7VQ$o4t{h(0M0Nv&0xVpxP3JIYGlb%2P@1(vknFuiM|L#i(WZm2t%pbYzLW0 z!oe?wDK^G5E4xc2?1fj^yr7)fIK&hmIdM`_6*JXeW{C~%rM{dF=jM999eEv|kE6{A OOffC>^QBbV*RwZ`5EX0y literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_orc/timestamps_before_epoch_arrow.orc b/tests/queries/0_stateless/data_orc/timestamps_before_epoch_arrow.orc new file mode 100644 index 0000000000000000000000000000000000000000..d6e0144685e7e91cbc4e2f76aba78db27c503b4f GIT binary patch literal 294 zcmeYda^_+aV&Py3VBnJHVqpLRA#n~CFF}d-S0A;$tXG(DYv0WG<_0&-0vIZo8Q1fF zHU<%<3}+*pj2X=Co-wUs{tp7J@2o(YK>&?!#l^uMlK_Jhv%7By$R%JG;c^KFv%n-qDJ~$TDPf^tsKKJd$H61S$SB2FqQIcR zV8Ecn!NCb(8L(J^&9cH}mcs-;55^=`BRwGGhyc2?p@Bt-iGfE$(SWV#3p2BSkh3@d Dw(m#M literal 0 HcmV?d00001 diff --git a/tests/queries/0_stateless/data_orc/timestamps_before_epoch_pyarrow_bloom_filter.orc b/tests/queries/0_stateless/data_orc/timestamps_before_epoch_pyarrow_bloom_filter.orc new file mode 100644 index 0000000000000000000000000000000000000000..2afd747824ec0515a92db943df384c0a15d9573e GIT binary patch literal 517 zcmah_F-XHu5WUM!;u%X*Gw3%hG-L`zN_4CaLP4z{m7=SIgM;WMjv~5taS^)MxuUCE zXQx6PTq+J-6>$+1LF@hef*^>uyn8P%clTelR(5R9BwT}kal$cy%S#ikkZk6IWVM&3+yxPgX3uK-?@cBI7ut#aQ69Ap@RG z2HVjWORMWDLb5o9G+8=Y{U{Qd6XB0B)D1Zo%tuz3rj&>1G`*BQ+>fo1s*rPY z<~tIo>C52xd@=P*T`HPO=fa~AC5KX$QO)2}nRh1jyG=~AvjcuNo8b}B*Y_dmqTM+s z%Sgh-{5jA?(yFE`wR{4hIO~K=Wjl7=B}Xf}W{@pajjY)=+iq##pbu_Q=h(2_ti3>u z9J)?2=+GRx-X5s=SumAbf+5?n4%qkartR%4)<&I(Tt*DJ7X=wHr|d<1%^YzzE3xM> z-%7E|JP?a`XAz0(J?42JA3%DAus`Z&$y9Xaij(78A=2Oj@9du!#a=!NEc0aI_oNwy zL6aGK>2?l~?dJ7(AxI8I0lE`VGY3E}E@o%dTVTviD!?e2g#g2kAR) [] [] +function orc_check() +{ + local label=$1 file=$2 structure=$3 query=$4 expected_error=${5:-} extra_settings=${6:-} + local name + name=$(basename "$file") + $CLICKHOUSE_CLIENT -q "INSERT INTO FUNCTION s3('${ORC_S3_URL}/${name}', 'test', 'testtest', 'RawBLOB') SETTINGS s3_truncate_on_insert = 1 FORMAT RawBLOB" < "$file" + + local source push_down lazy table client output status + for source in local remote; do + for push_down in $ORC_PUSH_DOWN_VALUES; do + for lazy in 0 1; do + if [ "$source" = local ]; then + table="file('${file}', 'ORC'${structure:+, '${structure}'})" + client=$CLICKHOUSE_LOCAL + else + table="s3('${ORC_S3_URL}/${name}', 'test', 'testtest', 'ORC'${structure:+, '${structure}'})" + client=$CLICKHOUSE_CLIENT + fi + output=$($client -q "${query//SOURCE/$table} SETTINGS input_format_orc_filter_push_down = ${push_down}, input_format_orc_dictionary_as_low_cardinality = ${lazy}${extra_settings:+, $extra_settings}" 2>&1) + status=$? + echo "${label} ${source} push_down=${push_down} lazy=${lazy}: $(orc_result "$output" "$status" "$expected_error")" + done + done + done +} From 16ab31bc37edb9c267c12b7f3d2a9f1f8e9a1cbf Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 14 Nov 2026 10:39:27 +0000 Subject: [PATCH 187/219] Update autogenerated version to 26.8.37.168 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- .../System/StorageSystemContributors.generated.cpp | 1 + 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 2817be684f77a..6218dd0558f10 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54549) +SET(VERSION_REVISION 54550) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 37) -SET(VERSION_GITHASH b47359294fe0f498ee19c5e31f18a9155b6dedae) -SET(VERSION_DESCRIBE v26.8.37.1-lts) -SET(VERSION_STRING 26.8.37.1) +SET(VERSION_PATCH 38) +SET(VERSION_GITHASH 40e366b2afbe912fd1985c300b3ccaae2a2d5b54) +SET(VERSION_DESCRIBE v26.8.38.1-lts) +SET(VERSION_STRING 26.8.38.1) # end of autochange diff --git a/src/Storages/System/StorageSystemContributors.generated.cpp b/src/Storages/System/StorageSystemContributors.generated.cpp index fa3cb25e8b6af..cf5684d39e421 100644 --- a/src/Storages/System/StorageSystemContributors.generated.cpp +++ b/src/Storages/System/StorageSystemContributors.generated.cpp @@ -397,6 +397,7 @@ const char * auto_contributors[] { "Constantine Peresypkin", "CoolT2", "Cory Levy", + "Craig Ringer", "CuiShuoGuo", "Cursor Agent", "CurtizJ", From 873df66706c6fc93c9226676e19f3d755a397aa7 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 14 Nov 2026 15:24:47 +0000 Subject: [PATCH 188/219] Backport #131300 to 26.8: Apply row policies of the current roles of the initial user on remote nodes --- src/Access/ContextAccess.cpp | 79 +++++++++++- src/Access/ContextAccess.h | 8 ++ src/Access/ContextAccessParams.cpp | 10 ++ src/Access/ContextAccessParams.h | 4 + .../ClusterProxy/executeQuery.cpp | 3 + src/Interpreters/Context.cpp | 10 +- src/QueryPipeline/RemoteQueryExecutor.cpp | 49 +++++++- ...policy_set_role_remote_no_secret.reference | 25 ++++ ...34_row_policy_set_role_remote_no_secret.sh | 113 ++++++++++++++++++ 9 files changed, 297 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/05634_row_policy_set_role_remote_no_secret.reference create mode 100755 tests/queries/0_stateless/05634_row_policy_set_role_remote_no_secret.sh diff --git a/src/Access/ContextAccess.cpp b/src/Access/ContextAccess.cpp index 81ed1e44fa2ba..74c4994bb5234 100644 --- a/src/Access/ContextAccess.cpp +++ b/src/Access/ContextAccess.cpp @@ -388,12 +388,15 @@ void ContextAccess::setUser(const UserPtr & user_) const subscription_for_user_change = {}; subscription_for_initial_user_change = {}; subscription_for_roles_changes = {}; + subscription_for_roles_of_initial_user_changes = {}; access = nullptr; access_with_implicit = nullptr; enabled_roles = nullptr; roles_info = nullptr; enabled_row_policies = nullptr; row_policies_of_initial_user = nullptr; + unknown_current_roles_of_initial_user.clear(); + failed_to_find_row_policies_of_initial_user = false; enabled_quota = nullptr; enabled_settings = nullptr; return; @@ -447,6 +450,26 @@ void ContextAccess::setUser(const UserPtr & user_) const ptr->findRowPoliciesOfInitialUser(); } }); + + /// The forwarded current roles of the initial user are found by name, and their row policies depend on the + /// roles granted to them. Both change with the roles, not with the users: a role can be created, dropped, + /// or granted another role. So the forwarded names are found again on any change of roles. + if (params.initial_user_current_roles) + { + subscription_for_roles_of_initial_user_changes = access_control->subscribeForChanges( + AccessEntityType::ROLE, + [weak_ptr = weak_from_this()](const std::vector & changes) + { + if (auto ptr = weak_ptr.lock()) + { + std::lock_guard lock2{ptr->mutex}; + /// Handlers by type are called also without changes, which retries a failed attempt. + if (changes.empty() && !ptr->failed_to_find_row_policies_of_initial_user) + return; + ptr->findRowPoliciesOfInitialUser(); + } + }); + } findRowPoliciesOfInitialUser(); } } @@ -491,7 +514,52 @@ void ContextAccess::calculateAccessRights() const void ContextAccess::findRowPoliciesOfInitialUser() const { - row_policies_of_initial_user = params.initial_user_id ? access_control->tryGetDefaultRowPolicies(*params.initial_user_id) : nullptr; + /// This also runs on changes of users and roles, where an exception is only logged. Until the new row policies are + /// published, reading tables is denied (see `getRowPolicyFilter`), so a failure cannot leave the old or no policies. + failed_to_find_row_policies_of_initial_user = true; + std::shared_ptr new_row_policies; + std::vector new_unknown_current_roles; + findRowPoliciesOfInitialUserImpl(new_row_policies, new_unknown_current_roles); + row_policies_of_initial_user = std::move(new_row_policies); + unknown_current_roles_of_initial_user = std::move(new_unknown_current_roles); + failed_to_find_row_policies_of_initial_user = false; +} + + +void ContextAccess::findRowPoliciesOfInitialUserImpl( + std::shared_ptr & row_policies, std::vector & unknown_current_roles) const +{ + if (!params.initial_user_id) + return; + + if (!params.initial_user_current_roles) + { + row_policies = access_control->tryGetDefaultRowPolicies(*params.initial_user_id); + return; + } + + /// The initiator forwarded the roles its query runs with (after `SET ROLE`), so take the row policies of these roles + /// instead of the default roles. The roles are used as is, the same way as with the interserver secret + /// (see `Session::makeQueryContextImpl`): the initiator has already checked that they are granted to the user. + if (!access_control->tryRead(*params.initial_user_id)) + return; + + std::vector current_roles; + for (const auto & role_name : *params.initial_user_current_roles) + { + if (auto role_id = access_control->find(role_name)) + current_roles.push_back(*role_id); + else + unknown_current_roles.push_back(role_name); + } + + /// Fail closed: a role unknown here cannot be honored, and dropping it could drop a restrictive policy. + /// `getRowPolicyFilter` throws in this case. + if (!unknown_current_roles.empty()) + return; + + auto initial_user_enabled_roles = access_control->getEnabledRolesInfo(current_roles, {})->enabled_roles; + row_policies = access_control->getEnabledRowPolicies(*params.initial_user_id, initial_user_enabled_roles); } @@ -551,6 +619,15 @@ RowPolicyFilterPtr ContextAccess::getRowPolicyFilter(const String & database, co if (initialized && !user && !user_was_dropped) throw Exception(ErrorCodes::LOGICAL_ERROR, "ContextAccess is inconsistent (bug 55041)"); + if (failed_to_find_row_policies_of_initial_user) + throw Exception(ErrorCodes::ACCESS_DENIED, + "The row policies of the initial user could not be found on this node, see the log for the error"); + + if (!unknown_current_roles_of_initial_user.empty()) + throw Exception(ErrorCodes::ACCESS_DENIED, + "Not all of the current roles of the initial user are known on this node: [{}]", + boost::algorithm::join(unknown_current_roles_of_initial_user, ", ")); + if (enabled_row_policies) filter = enabled_row_policies->getFilter(database, table_name, filter_type); diff --git a/src/Access/ContextAccess.h b/src/Access/ContextAccess.h index 5ce4344e50050..631598812e744 100644 --- a/src/Access/ContextAccess.h +++ b/src/Access/ContextAccess.h @@ -156,6 +156,8 @@ class ContextAccess : public std::enable_shared_from_this void setUser(const UserPtr & user_) const TSA_REQUIRES(mutex); void setRolesInfo(const std::shared_ptr & roles_info_) const TSA_REQUIRES(mutex); void findRowPoliciesOfInitialUser() const TSA_REQUIRES(mutex); + void findRowPoliciesOfInitialUserImpl( + std::shared_ptr & row_policies, std::vector & unknown_current_roles) const TSA_REQUIRES(mutex); void calculateAccessRights() const TSA_REQUIRES(mutex); template @@ -217,6 +219,12 @@ class ContextAccess : public std::enable_shared_from_this mutable std::shared_ptr access_with_implicit TSA_GUARDED_BY(mutex); mutable std::shared_ptr enabled_row_policies TSA_GUARDED_BY(mutex); mutable std::shared_ptr row_policies_of_initial_user TSA_GUARDED_BY(mutex); + /// The subscription for changes of roles, which change the forwarded current roles of the initial user. + mutable scope_guard subscription_for_roles_of_initial_user_changes TSA_GUARDED_BY(mutex); + /// Forwarded current roles of the initial user which do not exist on this node. If not empty, reading tables is denied. + mutable std::vector unknown_current_roles_of_initial_user TSA_GUARDED_BY(mutex); + /// Whether the last attempt to find the row policies of the initial user threw. If true, reading tables is denied. + mutable bool failed_to_find_row_policies_of_initial_user TSA_GUARDED_BY(mutex) = false; #if CLICKHOUSE_CLOUD mutable std::shared_ptr enabled_masking_policies TSA_GUARDED_BY(mutex); #endif diff --git a/src/Access/ContextAccessParams.cpp b/src/Access/ContextAccessParams.cpp index bb4dc631a43e2..a6efb7e301313 100644 --- a/src/Access/ContextAccessParams.cpp +++ b/src/Access/ContextAccessParams.cpp @@ -41,6 +41,7 @@ ContextAccessParams::ContextAccessParams( , forwarded_address(client_info_.getLastForwardedForHost()) , quota_key(client_info_.quota_key) , initial_user_id(initial_user_id_) + , initial_user_current_roles(initial_user_id_ ? client_info_.current_roles : std::nullopt) { } @@ -95,6 +96,13 @@ String ContextAccessParams::toString() const out << separator() << "quota_key = " << quota_key; if (initial_user_id) out << separator() << "initial_user_id = " << *initial_user_id; + if (initial_user_current_roles) + { + out << separator() << "initial_user_current_roles = ["; + for (size_t i = 0; i != initial_user_current_roles->size(); ++i) + out << (i ? ", " : "") << (*initial_user_current_roles)[i]; + out << "]"; + } return out.str(); } @@ -135,6 +143,7 @@ bool operator ==(const ContextAccessParams & left, const ContextAccessParams & r CONTEXT_ACCESS_PARAMS_EQUALS(forwarded_address) CONTEXT_ACCESS_PARAMS_EQUALS(quota_key) CONTEXT_ACCESS_PARAMS_EQUALS(initial_user_id) + CONTEXT_ACCESS_PARAMS_EQUALS(initial_user_current_roles) #undef CONTEXT_ACCESS_PARAMS_EQUALS @@ -186,6 +195,7 @@ bool operator <(const ContextAccessParams & left, const ContextAccessParams & ri CONTEXT_ACCESS_PARAMS_LESS(forwarded_address) CONTEXT_ACCESS_PARAMS_LESS(quota_key) CONTEXT_ACCESS_PARAMS_LESS(initial_user_id) + CONTEXT_ACCESS_PARAMS_LESS(initial_user_current_roles) #undef CONTEXT_ACCESS_PARAMS_LESS diff --git a/src/Access/ContextAccessParams.h b/src/Access/ContextAccessParams.h index 6834b2c6171dd..07b76dc6e5a70 100644 --- a/src/Access/ContextAccessParams.h +++ b/src/Access/ContextAccessParams.h @@ -59,6 +59,10 @@ class ContextAccessParams /// Initial user is used to combine row policies with. const std::optional initial_user_id; + /// Names of the roles the initial query was running with (set by `SET ROLE`), as forwarded by the initiator. + /// If not null, the row policies of the initial user are taken for these roles instead of the default roles. + const std::optional> initial_user_current_roles; + /// Outputs `ContextAccessParams` to string for logging. String toString() const; diff --git a/src/Interpreters/ClusterProxy/executeQuery.cpp b/src/Interpreters/ClusterProxy/executeQuery.cpp index 5551f811b8a90..9c69af77ffcdc 100644 --- a/src/Interpreters/ClusterProxy/executeQuery.cpp +++ b/src/Interpreters/ClusterProxy/executeQuery.cpp @@ -338,6 +338,9 @@ static ContextMutablePtr updateSettingsAndClientInfoForCluster(const Cluster & c if (is_remote_function) { const auto & address = cluster.getShardsAddresses().front().front(); + /// The forwarded current roles are the roles of the initial user, so they cannot be passed on for another user. + if (new_client_info.initial_user != address.user) + new_client_info.current_roles.reset(); new_client_info.initial_user = address.user; } diff --git a/src/Interpreters/Context.cpp b/src/Interpreters/Context.cpp index 184dbf95d835e..9c10216da85be 100644 --- a/src/Interpreters/Context.cpp +++ b/src/Interpreters/Context.cpp @@ -2374,8 +2374,13 @@ std::shared_ptr Context::getAccess() const /// If setUserID() was never called then this must be the global context with the full access. bool full_access = !user_id; + /// The row policies of the initial user are added to the row policies of the current user. The initiator of + /// a secondary query also forwards the current roles of the initial user (after `SET ROLE`); if the initial user + /// is the current user, the query still runs with the default roles of the user on a connection without + /// an interserver secret, so the row policies of the forwarded roles are needed then too. std::optional initial_user_id; - if (client_info.initial_user != client_info.current_user) + if (client_info.initial_user != client_info.current_user + || (client_info.query_kind == ClientInfo::QueryKind::SECONDARY_QUERY && client_info.current_roles.has_value())) initial_user_id = getAccessControl().find(client_info.initial_user); return ContextAccessParams{ @@ -8185,6 +8190,9 @@ void Context::setCurrentAddress(const Poco::Net::SocketAddress & current_address void Context::setInitialUserName(const String & initial_user_name) { + /// The forwarded current roles are the roles of the initial user, so they cannot be kept for another user. + if (client_info.initial_user != initial_user_name) + client_info.current_roles.reset(); client_info.initial_user = initial_user_name; need_recalculate_access = true; } diff --git a/src/QueryPipeline/RemoteQueryExecutor.cpp b/src/QueryPipeline/RemoteQueryExecutor.cpp index 53944d7ee9fc2..fa389980983ed 100644 --- a/src/QueryPipeline/RemoteQueryExecutor.cpp +++ b/src/QueryPipeline/RemoteQueryExecutor.cpp @@ -34,6 +34,8 @@ #include #include +#include +#include #include #include @@ -70,6 +72,7 @@ namespace ErrorCodes extern const int UNKNOWN_TABLE; extern const int UNKNOWN_DATABASE; extern const int BAD_ARGUMENTS; + extern const int ACCESS_DENIED; } namespace FailPoints @@ -530,9 +533,51 @@ void RemoteQueryExecutor::sendQueryUnlocked(ClientInfo::QueryKind query_kind, As /// Forward this node's current roles so the remote scopes row policies the same way (gated by the setting). /// Reset first against stale/injected values, and skip when initial_user was rewritten (remote(user=>...)). + /// A node that runs a secondary query passes on the current roles of the initial user it received, so that the next + /// node takes the row policies of the initial user for these roles too. Without an interserver secret, the query + /// runs here as the cluster user, or as the initial user but with its default roles, so the current roles of this + /// node are not the ones of the initial query. The received roles are dropped where the initial user is changed + /// (`remote` with another user), so they always belong to `initial_user`. + std::optional received_roles_of_initial_user; + if (modified_client_info.query_kind == ClientInfo::QueryKind::SECONDARY_QUERY + && context->getClientInfo().query_kind == ClientInfo::QueryKind::SECONDARY_QUERY) + received_roles_of_initial_user = modified_client_info.current_roles; + + const bool push_roles = context->getSettingsRef()[Setting::push_external_roles_in_interserver_queries]; modified_client_info.current_roles.reset(); - if (context->getSettingsRef()[Setting::push_external_roles_in_interserver_queries] - && modified_client_info.initial_user == modified_client_info.current_user) + if (push_roles && received_roles_of_initial_user) + { + /// The next node may be connected with an interserver secret, which is used only if the initial user is + /// the current user (see `Connection::sendQuery`), and then it enables the forwarded roles without checking + /// their grants. Roles received on a connection without a secret come from the client, so they must be granted + /// to the user here, as `SET ROLE` requires. Otherwise they are used only to find row policies. + if (modified_client_info.initial_user == modified_client_info.current_user + && context->getClientInfo().interface != ClientInfo::Interface::TCP_INTERSERVER) + { + /// A context may have no user (`SQL SECURITY NONE`, full access): the query does not run with the roles of + /// the initial user then, so they are not passed on. + const auto user = context->getAccess()->tryGetUser(); + if (!user) + { + received_roles_of_initial_user.reset(); + } + else + { + /// Fail closed: dropping a role could drop a restrictive row policy on the next node. + const auto & access_control = context->getAccessControl(); + for (const auto & role_name : *received_roles_of_initial_user) + { + auto role_id = access_control.find(role_name); + if (!role_id || !user->granted_roles.isGranted(*role_id)) + throw Exception(ErrorCodes::ACCESS_DENIED, + "The current role {} of the initial user is not granted to the user {} on this node", + backQuote(role_name), backQuote(user->getName())); + } + } + } + modified_client_info.current_roles = std::move(received_roles_of_initial_user); + } + else if (push_roles && modified_client_info.initial_user == modified_client_info.current_user) { const auto & access_control = context->getAccessControl(); Strings current_role_names; diff --git a/tests/queries/0_stateless/05634_row_policy_set_role_remote_no_secret.reference b/tests/queries/0_stateless/05634_row_policy_set_role_remote_no_secret.reference new file mode 100644 index 0000000000000..f4e2d3e8901f1 --- /dev/null +++ b/tests/queries/0_stateless/05634_row_policy_set_role_remote_no_secret.reference @@ -0,0 +1,25 @@ +-- 1 +1 6 +-- 2 +2 6 +-- two hops, 1 +1 12 +-- two hops, 2 +2 12 +-- same user, 1 +1 6 +-- same user, 2 +2 6 +-- same user, two hops, 1 +1 12 +-- same user, two hops, 2 +2 12 +-- same user, two hops with a secret, 1 +1 12 +-- same user, two hops with a secret, 2 +2 12 +-- role granted to a role +1 6 +1 6 +3 2 +1 6 diff --git a/tests/queries/0_stateless/05634_row_policy_set_role_remote_no_secret.sh b/tests/queries/0_stateless/05634_row_policy_set_role_remote_no_secret.sh new file mode 100755 index 0000000000000..9dc7766097f6b --- /dev/null +++ b/tests/queries/0_stateless/05634_row_policy_set_role_remote_no_secret.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# A query scoped with SET ROLE must have that scope honored on remote nodes of a cluster without an interserver secret. +# There the remote node runs the query as its own cluster user and adds the row policies of the initial user, +# which must be the policies of the roles the initiator switched to, not of all the default roles of the user. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +USER="user_${CLICKHOUSE_DATABASE}" +ROLE_1="role_1_${CLICKHOUSE_DATABASE}" +ROLE_2="role_2_${CLICKHOUSE_DATABASE}" +ROLE_3="role_3_${CLICKHOUSE_DATABASE}" + +$CLICKHOUSE_CLIENT -m -q " +DROP TABLE IF EXISTS data; +DROP TABLE IF EXISTS data_dist; +DROP TABLE IF EXISTS data_dist_dist; +DROP TABLE IF EXISTS data_dist_secret; +CREATE TABLE data (id UInt32, key UInt32) ENGINE = MergeTree ORDER BY key; +INSERT INTO data VALUES (1, 1), (2, 2), (3, 1), (4, 3), (5, 2), (6, 1), (7, 4), (8, 2); +CREATE TABLE data_dist AS data ENGINE = Distributed(test_cluster_two_shards_localhost, ${CLICKHOUSE_DATABASE}, data); +CREATE TABLE data_dist_dist AS data ENGINE = Distributed(test_cluster_two_shards_localhost, ${CLICKHOUSE_DATABASE}, data_dist); +CREATE TABLE data_dist_secret AS data ENGINE = Distributed(test_cluster_interserver_secret, ${CLICKHOUSE_DATABASE}, data); + +DROP ROLE IF EXISTS ${ROLE_1}, ${ROLE_2}, ${ROLE_3}; +CREATE ROLE ${ROLE_1}; +CREATE ROLE ${ROLE_2}; +CREATE ROLE ${ROLE_3}; +GRANT SELECT ON ${CLICKHOUSE_DATABASE}.* TO ${ROLE_1}, ${ROLE_2}; +CREATE ROW POLICY p_1 ON ${CLICKHOUSE_DATABASE}.data FOR SELECT USING key = 1 TO ${ROLE_1}; +CREATE ROW POLICY p_2 ON ${CLICKHOUSE_DATABASE}.data FOR SELECT USING key = 2 TO ${ROLE_2}; +CREATE ROW POLICY p_3 ON ${CLICKHOUSE_DATABASE}.data FOR SELECT USING key = 3 TO ${ROLE_3}; +DROP USER IF EXISTS ${USER}; +CREATE USER ${USER} IDENTIFIED WITH no_password; +GRANT ${ROLE_1}, ${ROLE_2} TO ${USER}; +GRANT REMOTE, CREATE TEMPORARY TABLE ON *.* TO ${USER}; +ALTER USER ${USER} DEFAULT ROLE ALL; +" + +SETTINGS="SETTINGS prefer_localhost_replica = 0, serialize_query_plan = 0" + +function query_with_role() +{ + local role=$1 + local table=$2 + $CLICKHOUSE_CLIENT --user "${USER}" -m -q " + SET ROLE ${role}; + SELECT key, count() FROM ${table} GROUP BY key ORDER BY key ${SETTINGS}; + " +} + +function role_name() +{ + echo "${1#role_}" | sed "s/_${CLICKHOUSE_DATABASE}//" +} + +# Each shard is read remotely, so every row of the matching key is returned twice. +for role in "${ROLE_1}" "${ROLE_2}" +do + echo "-- $(role_name "${role}")" + query_with_role "${role}" data_dist +done + +# Each shard of the outer table is a node that runs a query as the cluster user and reads both shards again. +for role in "${ROLE_1}" "${ROLE_2}" +do + echo "-- two hops, $(role_name "${role}")" + query_with_role "${role}" data_dist_dist +done + +# The remote nodes authenticate as the user that runs the initial query. +for role in "${ROLE_1}" "${ROLE_2}" +do + echo "-- same user, $(role_name "${role}")" + query_with_role "${role}" "remote('127.0.0.{1,2}', ${CLICKHOUSE_DATABASE}, data, '${USER}', '')" +done + +# The same, and the remote nodes read both shards again as the cluster user. +for role in "${ROLE_1}" "${ROLE_2}" +do + echo "-- same user, two hops, $(role_name "${role}")" + query_with_role "${role}" "remote('127.0.0.{1,2}', ${CLICKHOUSE_DATABASE}, data_dist, '${USER}', '')" +done + +# The same, and the remote nodes read both shards again over a connection with an interserver secret, +# where the next node enables the forwarded roles. +for role in "${ROLE_1}" "${ROLE_2}" +do + echo "-- same user, two hops with a secret, $(role_name "${role}")" + query_with_role "${role}" "remote('127.0.0.{1,2}', ${CLICKHOUSE_DATABASE}, data_dist_secret, '${USER}', '')" +done + +# The row policies of the roles change when another role is granted to a role. +echo "-- role granted to a role" +query_with_role "${ROLE_1}" data_dist +$CLICKHOUSE_CLIENT -q "GRANT ${ROLE_3} TO ${ROLE_1}" +query_with_role "${ROLE_1}" data_dist +$CLICKHOUSE_CLIENT -q "REVOKE ${ROLE_3} FROM ${ROLE_1}" +query_with_role "${ROLE_1}" data_dist + +$CLICKHOUSE_CLIENT -m -q " +DROP ROW POLICY IF EXISTS p_1 ON ${CLICKHOUSE_DATABASE}.data; +DROP ROW POLICY IF EXISTS p_2 ON ${CLICKHOUSE_DATABASE}.data; +DROP ROW POLICY IF EXISTS p_3 ON ${CLICKHOUSE_DATABASE}.data; +DROP TABLE IF EXISTS data_dist_dist; +DROP TABLE IF EXISTS data_dist_secret; +DROP TABLE IF EXISTS data_dist; +DROP TABLE IF EXISTS data; +DROP USER IF EXISTS ${USER}; +DROP ROLE IF EXISTS ${ROLE_1}, ${ROLE_2}, ${ROLE_3}; +" From c8eb5a6ec3f211e93f48fad8b25d7af4408ae9ce Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sat, 14 Nov 2026 15:28:35 +0000 Subject: [PATCH 189/219] Backport #130814 to 26.8: Fix `bloom_filter` index mistaking a column named `mapKeys(m)` for the `Map` index --- .../MergeTree/MergeTreeIndexBloomFilter.cpp | 360 ++++++++++++++++-- .../MergeTree/MergeTreeIndexBloomFilter.h | 13 +- .../MergeTreeIndexJSONSubcolumnHelper.cpp | 22 +- .../MergeTreeIndexJSONSubcolumnHelper.h | 18 +- src/Storages/MergeTree/MergeTreeIndices.cpp | 26 ++ src/Storages/MergeTree/MergeTreeIndices.h | 4 + ...ilter_index_map_keys_column_name.reference | 45 +++ ...loom_filter_index_map_keys_column_name.sql | 171 +++++++++ 8 files changed, 605 insertions(+), 54 deletions(-) create mode 100644 tests/queries/0_stateless/05620_bloom_filter_index_map_keys_column_name.reference create mode 100644 tests/queries/0_stateless/05620_bloom_filter_index_map_keys_column_name.sql diff --git a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp index 7d21b561a7575..5ee251d9e73bc 100644 --- a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.cpp @@ -2,6 +2,7 @@ #include #include +#include #include #include #include @@ -14,10 +15,13 @@ #include #include #include +#include +#include #include #include #include #include +#include #include #include #include @@ -245,16 +249,286 @@ struct MapIndexInfo Field key_field; }; +/// Skips the `ALIAS` nodes above `node`. +const ActionsDAG::Node * skipAlias(const ActionsDAG::Node * node) +{ + while (node->type == ActionsDAG::ActionType::ALIAS) + { + chassert(node->children.size() == 1); + node = node->children.front(); + } + return node; +} + +/// Compares expressions by structure: index expressions come from the old analyzer, so names differ and lambdas may be folded. +class ExpressionMatcher +{ +public: + static bool isSame(const ActionsDAG::Node * query_node, const ActionsDAG::Node * index_node) + { + return match({.node = query_node}, {.node = index_node}); + } + +private: + struct Scope; + + /// Either a node of the DAG of `scope`, or a column captured by a folded lambda. + struct Operand + { + const ActionsDAG::Node * node = nullptr; + const ColumnWithTypeAndName * captured_column = nullptr; + const Scope * scope = nullptr; + }; + + /// The body of a lambda: its inputs are captured operands or lambda arguments. + struct Scope + { + const LambdaCapture * capture = nullptr; + std::vector captured = {}; + size_t depth = 0; + }; + + struct Lambda + { + const LambdaCapture * capture = nullptr; + const ActionsDAG * body = nullptr; + std::vector captured = {}; + }; + + static Operand resolve(Operand operand) + { + while (operand.node) + { + operand.node = skipAlias(operand.node); + if (operand.node->type != ActionsDAG::ActionType::INPUT || !operand.scope) + break; + + const auto & captured_names = operand.scope->capture->captured_names; + auto it = std::ranges::find(captured_names, operand.node->result_name); + if (it == captured_names.end()) + break; + operand = operand.scope->captured[static_cast(it - captured_names.begin())]; + } + return operand; + } + + static std::optional getConstant(const Operand & operand) + { + ColumnWithTypeAndName constant; + if (operand.captured_column) + constant = *operand.captured_column; + else if (operand.node->column && isColumnConst(*operand.node->column)) + constant = {operand.node->column, operand.node->result_type, ""}; + else + return std::nullopt; + + WhichDataType which(constant.type); + if (which.isFunction() || which.isSet()) + return std::nullopt; + + /// Constants of the index expression have no rows. + if (const auto * column_const = typeid_cast(constant.column.get())) + constant.column = column_const->getDataColumnPtr(); + if (constant.column->empty()) + return std::nullopt; + return constant; + } + + static std::optional getLambda(const Operand & operand) + { + if (operand.node && operand.node->type == ActionsDAG::ActionType::FUNCTION) + { + const auto * function = typeid_cast(operand.node->function_base.get()); + if (!function) + return std::nullopt; + + Lambda lambda{.capture = &function->getCapture(), .body = &function->getAcionsDAG()}; + for (const auto * child : operand.node->children) + lambda.captured.push_back({.node = child, .scope = operand.scope}); + return lambda; + } + + ColumnPtr column = operand.captured_column ? operand.captured_column->column : ColumnPtr(operand.node->column); + if (!column) + return std::nullopt; + if (const auto * column_const = typeid_cast(column.get())) + column = column_const->getDataColumnPtr(); + + const auto * column_function = typeid_cast(column.get()); + if (!column_function) + return std::nullopt; + const auto * function = typeid_cast(column_function->getFunction().get()); + if (!function) + return std::nullopt; + + Lambda lambda{.capture = &function->getCapture(), .body = &function->getAcionsDAG()}; + for (const auto & captured_column : column_function->getCapturedColumns()) + lambda.captured.push_back({.captured_column = &captured_column}); + return lambda; + } + + static const ActionsDAG::Node * getLambdaResult(const Lambda & lambda) + { + for (const auto * output : lambda.body->getOutputs()) + if (output->result_name == lambda.capture->return_name) + return output; + return nullptr; + } + + static std::optional getLambdaArgumentIndex(const Operand & operand) + { + size_t index = 0; + for (const auto & argument : operand.scope->capture->lambda_arguments) + { + if (argument.name == operand.node->result_name) + return index; + ++index; + } + return std::nullopt; + } + + static bool matchLambdas(const Lambda & query, const Operand & query_operand, const Lambda & index, const Operand & index_operand) + { + auto same_type = [](const auto & lhs, const auto & rhs) { return lhs.type->equals(*rhs.type); }; + if (!std::ranges::equal(query.capture->lambda_arguments, index.capture->lambda_arguments, same_type) + || !query.capture->return_type->equals(*index.capture->return_type)) + return false; + + const auto * query_result = getLambdaResult(query); + const auto * index_result = getLambdaResult(index); + if (!query_result || !index_result) + return false; + + size_t depth = (query_operand.scope ? query_operand.scope->depth : 0) + 1; + if (depth != (index_operand.scope ? index_operand.scope->depth : 0) + 1) + return false; + + Scope query_scope{.capture = query.capture, .captured = query.captured, .depth = depth}; + Scope index_scope{.capture = index.capture, .captured = index.captured, .depth = depth}; + return match({.node = query_result, .scope = &query_scope}, {.node = index_result, .scope = &index_scope}); + } + + static bool match(Operand query, Operand index) + { + query = resolve(query); + index = resolve(index); + + auto query_constant = getConstant(query); + auto index_constant = getConstant(index); + if (query_constant || index_constant) + return query_constant && index_constant && query_constant->type->equals(*index_constant->type) + && (*query_constant->column)[0] == (*index_constant->column)[0]; + + auto query_lambda = getLambda(query); + auto index_lambda = getLambda(index); + if (query_lambda || index_lambda) + return query_lambda && index_lambda && matchLambdas(*query_lambda, query, *index_lambda, index); + + if (!query.node || !index.node || query.node->type != index.node->type) + return false; + + switch (query.node->type) + { + case ActionsDAG::ActionType::INPUT: + { + if (!query.scope || !index.scope) + return !query.scope && !index.scope && query.node->result_name == index.node->result_name; + + auto query_argument = getLambdaArgumentIndex(query); + return query_argument && query.scope->depth == index.scope->depth && query_argument == getLambdaArgumentIndex(index); + } + case ActionsDAG::ActionType::FUNCTION: + if (query.node->function_base->getName() != index.node->function_base->getName()) + return false; + [[fallthrough]]; + case ActionsDAG::ActionType::ARRAY_JOIN: + { + if (query.node->children.size() != index.node->children.size()) + return false; + for (size_t i = 0; i < query.node->children.size(); ++i) + { + Operand query_child{.node = query.node->children[i], .scope = query.scope}; + Operand index_child{.node = index.node->children[i], .scope = index.scope}; + if (!match(query_child, index_child)) + return false; + } + return true; + } + default: + return false; + } + } +}; + +std::optional findIndexPosition( + const Block & header, const ActionsDAG::NodeRawConstPtrs & index_nodes, const RPNBuilderTreeNode & node) +{ + auto position = header.findPositionByName(node.getColumnName()); + if (position && ExpressionMatcher::isSame(node.getDAGNode(), index_nodes[*position])) + return position; + return std::nullopt; +} + +Names collectJSONIndexColumns(const Block & header, const ActionsDAG::NodeRawConstPtrs & index_nodes) +{ + Names names(header.columns()); + for (size_t position = 0; position < names.size(); ++position) + { + const auto * index_node = skipAlias(index_nodes[position]); + if (index_node->type != ActionsDAG::ActionType::FUNCTION || index_node->function_base->getName() != "JSONAllPaths" + || index_node->children.size() != 1) + continue; + + const auto * argument = skipAlias(index_node->children.front()); + const auto & name = header.getByPosition(position).name; + if (argument->type == ActionsDAG::ActionType::INPUT && name == fmt::format("JSONAllPaths({})", argument->result_name)) + names[position] = name; + } + return names; +} + +/// `map_node` is nullptr for the table column `map_column_name` itself. +std::optional findMapIndexPosition( + const Block & header, + const ActionsDAG::NodeRawConstPtrs & index_nodes, + const String & function_name, + const String & map_column_name, + const ActionsDAG::Node * map_node) +{ + auto position = header.findPositionByName(fmt::format("{}({})", function_name, map_column_name)); + if (!position) + return std::nullopt; + + const auto * index_node = skipAlias(index_nodes[*position]); + if (index_node->type != ActionsDAG::ActionType::FUNCTION || !index_node->function_base + || index_node->function_base->getName() != function_name || index_node->children.size() != 1) + return std::nullopt; + + const auto * argument = skipAlias(index_node->children.front()); + if (!isMap(argument->result_type)) + return std::nullopt; + + bool same_map = map_node + ? ExpressionMatcher::isSame(map_node, argument) + : argument->type == ActionsDAG::ActionType::INPUT && argument->result_name == map_column_name; + if (!same_map) + return std::nullopt; + + return position; +} + /// Try to resolve a Map column against the bloom filter index header by the map column name /// and the key as a Field. Returns std::nullopt if neither `mapKeys()` nor `mapValues()` /// is present in the index. -std::optional tryResolveMapIndexInfo(const String & map_column_name, const Field & key_field, const Block & header) +std::optional tryResolveMapIndexInfo( + const String & map_column_name, + const Field & key_field, + const Block & header, + const ActionsDAG::NodeRawConstPtrs & index_nodes, + const ActionsDAG::Node * map_node) { - auto map_keys_index_column_name = fmt::format("mapKeys({})", map_column_name); - auto map_values_index_column_name = fmt::format("mapValues({})", map_column_name); - - auto keys_position = header.findPositionByName(map_keys_index_column_name); - auto values_position = header.findPositionByName(map_values_index_column_name); + auto keys_position = findMapIndexPosition(header, index_nodes, "mapKeys", map_column_name, map_node); + auto values_position = findMapIndexPosition(header, index_nodes, "mapValues", map_column_name, map_node); if (!keys_position && !values_position) return std::nullopt; @@ -277,7 +551,8 @@ std::optional tryResolveMapIndexInfo(const String & map_column_nam /// Try to parse a Map subcolumn reference like `map.key_` and resolve it /// against the bloom filter index header. The subcolumn name format is produced by /// `FunctionToSubcolumnsPass`. -std::optional tryParseMapSubcolumn(const String & column_name, const Block & header) +std::optional tryParseMapSubcolumn( + const String & column_name, const Block & header, const ActionsDAG::NodeRawConstPtrs & index_nodes) { auto parsed = tryParseMapSubcolumnName(column_name); if (!parsed) @@ -285,13 +560,12 @@ std::optional tryParseMapSubcolumn(const String & column_name, con auto & [map_column_name, serialized_key] = *parsed; - auto map_keys_index_column_name = fmt::format("mapKeys({})", map_column_name); - if (!header.has(map_keys_index_column_name)) - return tryResolveMapIndexInfo(map_column_name, {}, header); + auto keys_position = findMapIndexPosition(header, index_nodes, "mapKeys", map_column_name, nullptr); + if (!keys_position) + return tryResolveMapIndexInfo(map_column_name, {}, header, index_nodes, nullptr); /// Deserialize the key from its text representation using the key type from the index header. - size_t keys_position = header.getPositionByName(map_keys_index_column_name); - const DataTypePtr & index_type = header.getByPosition(keys_position).type; + const DataTypePtr & index_type = header.getByPosition(*keys_position).type; const auto & array_type = assert_cast(*index_type); const auto & key_type = array_type.getNestedType(); @@ -302,12 +576,13 @@ std::optional tryParseMapSubcolumn(const String & column_name, con Field key_field; key_column->get(0, key_field); - return tryResolveMapIndexInfo(map_column_name, key_field, header); + return tryResolveMapIndexInfo(map_column_name, key_field, header, index_nodes, nullptr); } /// Try to resolve a `MapIndexInfo` from a key node that is either an `arrayElement(map, key)` /// function call or a `map.key_` subcolumn reference. -std::optional tryResolveMapInfoFromNode(const RPNBuilderTreeNode & key_node, const Block & header) +std::optional tryResolveMapInfoFromNode( + const RPNBuilderTreeNode & key_node, const Block & header, const ActionsDAG::NodeRawConstPtrs & index_nodes) { if (key_node.isFunction()) { @@ -322,20 +597,28 @@ std::optional tryResolveMapInfoFromNode(const RPNBuilderTreeNode & if (!second_argument.tryGetConstant(constant_value, constant_type)) return std::nullopt; - return tryResolveMapIndexInfo(first_argument.getColumnName(), constant_value, header); + return tryResolveMapIndexInfo(first_argument.getColumnName(), constant_value, header, index_nodes, first_argument.getDAGNode()); } } - return tryParseMapSubcolumn(key_node.getColumnName(), header); + return tryParseMapSubcolumn(key_node.getColumnName(), header, index_nodes); } } MergeTreeIndexConditionBloomFilter::MergeTreeIndexConditionBloomFilter( - const ActionsDAG::Node * predicate, ContextPtr context_, const Block & header_, size_t hash_functions_) + const ActionsDAG::Node * predicate, + ContextPtr context_, + const Block & header_, + size_t hash_functions_, + IsShadowingJSONSubcolumn is_shadowing_json_subcolumn_, + ActionsDAG::NodeRawConstPtrs index_nodes_) : WithContext(context_) , header(header_) , hash_functions(hash_functions_) + , is_shadowing_json_subcolumn(std::move(is_shadowing_json_subcolumn_)) + , index_nodes(std::move(index_nodes_)) + , json_index_columns(collectJSONIndexColumns(header, index_nodes)) , validate_enum_literals_in_operators(context_->getSettingsRef()[Setting::validate_enum_literals_in_operators]) { if (!predicate) @@ -522,7 +805,7 @@ bool MergeTreeIndexConditionBloomFilter::traverseFunction(const RPNBuilderTreeNo if (function_name == "isNotNull" && arguments_size == 1) { auto arg = function.getArgumentAt(0); - if (auto json_info = tryMatchNodeToJSONIndex(arg, header, "JSONAllPaths")) + if (auto json_info = tryMatchNodeToJSONIndex(arg, json_index_columns, "JSONAllPaths", is_shadowing_json_subcolumn)) { auto arg_type = arg.getDAGNode()->result_type; /// It doesn't make sense to use bloom filter for isNotNull on non-Nullable type, as isNotNull will be always true. @@ -619,12 +902,11 @@ bool MergeTreeIndexConditionBloomFilter::traverseTreeIn( RPNElement & out) { const auto key_node = unwrapLosslessConversion(wrapped_key_node); - auto key_node_column_name = key_node.getColumnName(); - if (header.has(key_node_column_name)) + if (auto key_position = findIndexPosition(header, index_nodes, key_node)) { size_t row_size = column->size(); - size_t position = header.getPositionByName(key_node_column_name); + size_t position = *key_position; const DataTypePtr & index_type = header.getByPosition(position).type; /// A NULL of a set built for `toNullable(key)` does not cast to the key type and matches no stored value. @@ -664,7 +946,7 @@ bool MergeTreeIndexConditionBloomFilter::traverseTreeIn( /// tryMatchNodeToJSONIndex handles both plain subcolumns and CAST-wrapped expressions. /// NOT IN is not supported because after BoolMask inversion it never skips any granules. /// nullIn/globalNullIn are deliberately not wired here: JSON paths need per-path NULL checks. - if (auto json_info = tryMatchNodeToJSONIndex(key_node, header, "JSONAllPaths")) + if (auto json_info = tryMatchNodeToJSONIndex(key_node, json_index_columns, "JSONAllPaths", is_shadowing_json_subcolumn)) { if (function_name != "in" && function_name != "globalIn") return false; @@ -721,7 +1003,7 @@ bool MergeTreeIndexConditionBloomFilter::traverseTreeIn( } /// Handle both `arrayElement(map, 'key') IN (set)` and `map.key_ IN (set)`. - if (auto map_info = tryResolveMapInfoFromNode(key_node, header)) + if (auto map_info = tryResolveMapInfoFromNode(key_node, header, index_nodes)) { /** It is important to ignore keys like column_map['Key'] IN ('') because if the key does not exist in the map * we return the default value for arrayElement. @@ -786,11 +1068,11 @@ bool MergeTreeIndexConditionBloomFilter::traverseTreeIn( if (!array_join_argument) return false; - auto array_column_name = array_join_argument->getColumnName(); - if (!header.has(array_column_name)) + auto array_position = findIndexPosition(header, index_nodes, *array_join_argument); + if (!array_position) return false; - size_t position = header.getPositionByName(array_column_name); + size_t position = *array_position; const auto * array_type = typeid_cast(header.getByPosition(position).type.get()); if (!array_type) return false; @@ -1013,10 +1295,9 @@ bool MergeTreeIndexConditionBloomFilter::traverseTreeEquals( { if (auto array_join_argument = key_node.getArrayJoinArgument()) { - auto array_column_name = array_join_argument->getColumnName(); - if (header.has(array_column_name)) + if (auto array_position = findIndexPosition(header, index_nodes, *array_join_argument)) { - size_t position = header.getPositionByName(array_column_name); + size_t position = *array_position; const auto * array_type = typeid_cast(header.getByPosition(position).type.get()); if (array_type && bloomFilterHashDomainMatches(value_type, array_type->getNestedType())) { @@ -1039,9 +1320,9 @@ bool MergeTreeIndexConditionBloomFilter::traverseTreeEquals( } } - if (header.has(key_column_name)) + if (auto key_position = findIndexPosition(header, index_nodes, key_node)) { - size_t position = header.getPositionByName(key_column_name); + size_t position = *key_position; const DataTypePtr & index_type = header.getByPosition(position).type; const auto * array_type = typeid_cast(index_type.get()); @@ -1157,7 +1438,7 @@ bool MergeTreeIndexConditionBloomFilter::traverseTreeEquals( /// Try to match the column name to a JSONAllPaths index for JSON subcolumn filtering. /// tryMatchNodeToJSONIndex handles both plain subcolumns and CAST-wrapped expressions /// like `json.some.path = value`, `json.some.path.:Type = value`, or `json.path::Type = value`. - if (auto json_info = tryMatchNodeToJSONIndex(key_node, header, "JSONAllPaths")) + if (auto json_info = tryMatchNodeToJSONIndex(key_node, json_index_columns, "JSONAllPaths", is_shadowing_json_subcolumn)) { if (function_name != "equals") return false; @@ -1174,14 +1455,12 @@ bool MergeTreeIndexConditionBloomFilter::traverseTreeEquals( if (function_name == "mapContainsValue" || function_name == "mapContainsKey" || function_name == "mapContains" || function_name == "has") { - auto map_keys_index_column_name = fmt::format("mapKeys({})", key_column_name); - if (function_name == "mapContainsValue") - map_keys_index_column_name = fmt::format("mapValues({})", key_column_name); - - if (!header.has(map_keys_index_column_name)) + auto map_index_position = findMapIndexPosition( + header, index_nodes, function_name == "mapContainsValue" ? "mapValues" : "mapKeys", key_column_name, key_node.getDAGNode()); + if (!map_index_position) return false; - size_t position = header.getPositionByName(map_keys_index_column_name); + size_t position = *map_index_position; const DataTypePtr & index_type = header.getByPosition(position).type; const auto * array_type = typeid_cast(index_type.get()); @@ -1254,7 +1533,7 @@ bool MergeTreeIndexConditionBloomFilter::traverseTreeEquals( /// Handle both `arrayElement(map, 'key') = value` and `map.key_ = value`. if (function_name == "equals" || function_name == "notEquals") { - if (auto map_info = tryResolveMapInfoFromNode(key_node, header)) + if (auto map_info = tryResolveMapInfoFromNode(key_node, header, index_nodes)) { /** It is important to ignore keys like column_map['Key'] = '' because if the key does not exist in the map * we return the default value for arrayElement. @@ -1370,7 +1649,8 @@ MergeTreeIndexAggregatorPtr MergeTreeIndexBloomFilter::createIndexAggregator() c MergeTreeIndexConditionPtr MergeTreeIndexBloomFilter::createIndexCondition(const ActionsDAG::Node * predicate, ContextPtr context) const { - return std::make_shared(predicate, context, index.sample_block, hash_functions); + return std::make_shared( + predicate, context, index.sample_block, hash_functions, getJSONSubcolumnShadowingCheck(), index.expression->getActionsDAG().getOutputs()); } static void assertIndexColumnsType(const Block & header) diff --git a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.h b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.h index c978cd7c66869..629a60274e48f 100644 --- a/src/Storages/MergeTree/MergeTreeIndexBloomFilter.h +++ b/src/Storages/MergeTree/MergeTreeIndexBloomFilter.h @@ -74,7 +74,13 @@ class MergeTreeIndexConditionBloomFilter final : public IMergeTreeIndexCondition std::vector> predicate; }; - MergeTreeIndexConditionBloomFilter(const ActionsDAG::Node * predicate, ContextPtr context_, const Block & header_, size_t hash_functions_); + MergeTreeIndexConditionBloomFilter( + const ActionsDAG::Node * predicate, + ContextPtr context_, + const Block & header_, + size_t hash_functions_, + IsShadowingJSONSubcolumn is_shadowing_json_subcolumn_, + ActionsDAG::NodeRawConstPtrs index_nodes_); bool alwaysUnknownOrTrue() const override; @@ -91,6 +97,11 @@ class MergeTreeIndexConditionBloomFilter final : public IMergeTreeIndexCondition private: const Block & header; const size_t hash_functions; + const IsShadowingJSONSubcolumn is_shadowing_json_subcolumn; + /// By position in `header`. + const ActionsDAG::NodeRawConstPtrs index_nodes; + /// Names of `header`, with all but `JSONAllPaths` of a table column left empty. + const Names json_index_columns; const bool validate_enum_literals_in_operators; std::vector rpn; diff --git a/src/Storages/MergeTree/MergeTreeIndexJSONSubcolumnHelper.cpp b/src/Storages/MergeTree/MergeTreeIndexJSONSubcolumnHelper.cpp index dd2647fa71992..a56628fd5d731 100644 --- a/src/Storages/MergeTree/MergeTreeIndexJSONSubcolumnHelper.cpp +++ b/src/Storages/MergeTree/MergeTreeIndexJSONSubcolumnHelper.cpp @@ -34,15 +34,17 @@ static bool isPrefixedSubcolumn(std::string_view subcolumn_name, char prefix) std::optional tryMatchJSONSubcolumnToIndex( const String & column_name, const Block & header, - const String & json_function_name) + const String & json_function_name, + const IsShadowingJSONSubcolumn & is_shadowing_subcolumn) { - return tryMatchJSONSubcolumnToIndex(column_name, header.getNames(), json_function_name); + return tryMatchJSONSubcolumnToIndex(column_name, header.getNames(), json_function_name, is_shadowing_subcolumn); } std::optional tryMatchJSONSubcolumnToIndex( const String & column_name, const Names & index_columns, - const String & json_function_name) + const String & json_function_name, + const IsShadowingJSONSubcolumn & is_shadowing_subcolumn) { /// Scan the index columns, not the dot positions of the name: the name can embed a folded /// constant, so its length is unbounded while `index_columns` is not. @@ -79,7 +81,7 @@ std::optional tryMatchJSONSubcolumnToIndex( matched = true; } - if (!matched) + if (!matched || (is_shadowing_subcolumn && is_shadowing_subcolumn(name, matched_json_column))) return std::nullopt; /// Sub-object (^) and combined literal+sub-object (@) access cannot use the index: such @@ -103,17 +105,19 @@ std::optional tryMatchJSONSubcolumnToIndex( std::optional tryMatchNodeToJSONIndex( const RPNBuilderTreeNode & node, const Block & header, - const String & json_function_name) + const String & json_function_name, + const IsShadowingJSONSubcolumn & is_shadowing_subcolumn) { - return tryMatchNodeToJSONIndex(node, header.getNames(), json_function_name); + return tryMatchNodeToJSONIndex(node, header.getNames(), json_function_name, is_shadowing_subcolumn); } std::optional tryMatchNodeToJSONIndex( const RPNBuilderTreeNode & node, const Names & index_columns, - const String & json_function_name) + const String & json_function_name, + const IsShadowingJSONSubcolumn & is_shadowing_subcolumn) { - auto json_info = tryMatchJSONSubcolumnToIndex(node.getColumnName(), index_columns, json_function_name); + auto json_info = tryMatchJSONSubcolumnToIndex(node.getColumnName(), index_columns, json_function_name, is_shadowing_subcolumn); /// Try CAST unwrapping: CAST(json.path, 'Type') or _CAST(json.path, 'Type') if (!json_info && node.isFunction()) @@ -122,7 +126,7 @@ std::optional tryMatchNodeToJSONIndex( auto fname = func.getFunctionName(); if ((fname == "CAST" || fname == "_CAST") && func.getArgumentsSize() == 2) json_info = tryMatchJSONSubcolumnToIndex( - func.getArgumentAt(0).getColumnName(), index_columns, json_function_name); + func.getArgumentAt(0).getColumnName(), index_columns, json_function_name, is_shadowing_subcolumn); } return json_info; diff --git a/src/Storages/MergeTree/MergeTreeIndexJSONSubcolumnHelper.h b/src/Storages/MergeTree/MergeTreeIndexJSONSubcolumnHelper.h index fdb010f61f8d9..4d26492c8048a 100644 --- a/src/Storages/MergeTree/MergeTreeIndexJSONSubcolumnHelper.h +++ b/src/Storages/MergeTree/MergeTreeIndexJSONSubcolumnHelper.h @@ -1,6 +1,8 @@ #pragma once +#include #include +#include #include #include @@ -9,6 +11,9 @@ namespace DB { +/// Whether another column claims the subcolumn `name` of `json_column`, e.g. a column `j.a` beside `JSON` `j`. +using IsShadowingJSONSubcolumn = std::function; + /// Information extracted from a column name that references a JSON subcolumn /// matched against a JSONAllPaths(...) index column. struct JSONSubcolumnIndexInfo @@ -29,16 +34,19 @@ struct JSONSubcolumnIndexInfo /// Returns nullopt if: /// - No matching index column is found in the header /// - The subcolumn is a sub-object access (^ prefix) or a combined literal+sub-object access (@ prefix) +/// - Another column claims the name (`is_shadowing_subcolumn`) std::optional tryMatchJSONSubcolumnToIndex( const String & column_name, const Block & header, - const String & json_function_name); + const String & json_function_name, + const IsShadowingJSONSubcolumn & is_shadowing_subcolumn = {}); /// Overload that works with a list of index column names instead of a Block. std::optional tryMatchJSONSubcolumnToIndex( const String & column_name, const Names & index_columns, - const String & json_function_name); + const String & json_function_name, + const IsShadowingJSONSubcolumn & is_shadowing_subcolumn = {}); class RPNBuilderTreeNode; /// forward declaration to avoid heavy include @@ -48,13 +56,15 @@ class RPNBuilderTreeNode; /// forward declaration to avoid heavy include std::optional tryMatchNodeToJSONIndex( const RPNBuilderTreeNode & node, const Block & header, - const String & json_function_name); + const String & json_function_name, + const IsShadowingJSONSubcolumn & is_shadowing_subcolumn = {}); /// Overload that works with a list of index column names instead of a Block. std::optional tryMatchNodeToJSONIndex( const RPNBuilderTreeNode & node, const Names & index_columns, - const String & json_function_name); + const String & json_function_name, + const IsShadowingJSONSubcolumn & is_shadowing_subcolumn = {}); /// Check if a JSON path filter is safe to use for index skipping. /// When a JSON path is absent in a granule, the expression evaluates to: diff --git a/src/Storages/MergeTree/MergeTreeIndices.cpp b/src/Storages/MergeTree/MergeTreeIndices.cpp index 7e7f26e1cc416..3be8118564810 100644 --- a/src/Storages/MergeTree/MergeTreeIndices.cpp +++ b/src/Storages/MergeTree/MergeTreeIndices.cpp @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -87,6 +88,31 @@ const NamesAndTypesList & IMergeTreeIndex::getColumnsWithTypesRequiredForIndexCa return index.expression->getRequiredColumnsWithTypes(); } +IsShadowingJSONSubcolumn IMergeTreeIndex::getJSONSubcolumnShadowingCheck() const +{ + /// A column or its regular subcolumn wins over a dynamic path, which belongs to the shortest `JSON` column. + return [metadata = metadata_snapshot](std::string_view name, std::string_view json_column) + { + const auto & columns = metadata->getColumns(); + auto options = GetColumnsOptions(GetColumnsOptions::All).withRegularSubcolumns(); + + if (auto column = columns.tryGetColumn(options, String(name))) + { + auto json = columns.tryGetColumn(options, String(json_column)); + if (!json || column->getNameInStorage() != json->getNameInStorage()) + return true; + } + + for (size_t end = json_column.find('.'); end != std::string_view::npos; end = json_column.find('.', end + 1)) + { + auto prefix = columns.tryGetColumn(options, String(json_column.substr(0, end))); + if (prefix && isObject(removeNullable(removeLowCardinality(prefix->type)))) + return true; + } + return false; + }; +} + namespace { diff --git a/src/Storages/MergeTree/MergeTreeIndices.h b/src/Storages/MergeTree/MergeTreeIndices.h index 6c85669609e34..79ac61107643c 100644 --- a/src/Storages/MergeTree/MergeTreeIndices.h +++ b/src/Storages/MergeTree/MergeTreeIndices.h @@ -6,6 +6,7 @@ #include #include #include +#include #include #include #include @@ -344,6 +345,9 @@ struct IMergeTreeIndex Names getColumnsRequiredForIndexCalc() const; const NamesAndTypesList & getColumnsWithTypesRequiredForIndexCalc() const; + /// Tells whether a name that looks like a subcolumn of a `JSON` column can belong to another column. + IsShadowingJSONSubcolumn getJSONSubcolumnShadowingCheck() const; + StorageMetadataPtr metadata_snapshot; const IndexDescription & index; }; diff --git a/tests/queries/0_stateless/05620_bloom_filter_index_map_keys_column_name.reference b/tests/queries/0_stateless/05620_bloom_filter_index_map_keys_column_name.reference new file mode 100644 index 0000000000000..cd0d59357b36a --- /dev/null +++ b/tests/queries/0_stateless/05620_bloom_filter_index_map_keys_column_name.reference @@ -0,0 +1,45 @@ +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 +1 diff --git a/tests/queries/0_stateless/05620_bloom_filter_index_map_keys_column_name.sql b/tests/queries/0_stateless/05620_bloom_filter_index_map_keys_column_name.sql new file mode 100644 index 0000000000000..815fb2d9495e4 --- /dev/null +++ b/tests/queries/0_stateless/05620_bloom_filter_index_map_keys_column_name.sql @@ -0,0 +1,171 @@ +-- A `bloom_filter` index on a column literally named like an expression is not used for that expression. + +DROP TABLE IF EXISTS t_bf_array; +DROP TABLE IF EXISTS t_bf_lower; +DROP TABLE IF EXISTS t_bf_fixed; +DROP TABLE IF EXISTS t_bf_lc; +DROP TABLE IF EXISTS t_bf_real; +DROP TABLE IF EXISTS t_bf_order; +DROP TABLE IF EXISTS t_bf_json; +DROP TABLE IF EXISTS t_bf_json_shadow; + +CREATE TABLE t_bf_fixed +( + m Map(String, String), + `mapKeys(m)` FixedString(16), + INDEX idx `mapKeys(m)` TYPE bloom_filter +) +ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO t_bf_fixed VALUES (map('a', 'x'), 'deadbeefdeadbeef'); +INSERT INTO t_bf_fixed VALUES (map('k', 'v'), 'deadbeefdeadbeef'); + +SELECT count() FROM t_bf_fixed WHERE m['k'] = 'v'; +SELECT count() FROM t_bf_fixed WHERE m['k'] = 'v' SETTINGS optimize_functions_to_subcolumns = 1; +SELECT count() FROM t_bf_fixed WHERE mapContainsKey(m, 'k'); +SELECT count() FROM t_bf_fixed WHERE has(m, 'k'); + +CREATE TABLE t_bf_lc +( + m Map(String, String), + `mapKeys(m)` LowCardinality(String), + `mapValues(m)` LowCardinality(String), + INDEX idx_k `mapKeys(m)` TYPE bloom_filter, + INDEX idx_v `mapValues(m)` TYPE bloom_filter +) +ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO t_bf_lc VALUES (map('k', 'v'), 'x', 'y'); + +SELECT count() FROM t_bf_lc WHERE m['k'] = 'v'; +SELECT count() FROM t_bf_lc WHERE m['k'] = 'v' SETTINGS optimize_functions_to_subcolumns = 1; +SELECT count() FROM t_bf_lc WHERE mapContainsKey(m, 'k'); +SELECT count() FROM t_bf_lc WHERE mapContainsValue(m, 'v'); +SELECT count() FROM t_bf_lc WHERE has(m, 'k'); + +-- A real `mapKeys(m)` index is still used. +CREATE TABLE t_bf_real +( + m Map(String, String), + INDEX idx mapKeys(m) TYPE bloom_filter +) +ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO t_bf_real VALUES (map('k', 'v')); + +SELECT count() FROM t_bf_real WHERE m['k'] = 'v' SETTINGS force_data_skipping_indices = 'idx'; +SELECT count() FROM t_bf_real WHERE mapContainsKey(m, 'k') SETTINGS force_data_skipping_indices = 'idx'; +SELECT count() FROM t_bf_real WHERE has(mapKeys(m), 'k') SETTINGS force_data_skipping_indices = 'idx', optimize_functions_to_subcolumns = 0; + +CREATE TABLE t_bf_array +( + m Map(String, String), + `mapKeys(m)` Array(String), + `mapValues(m)` Array(String), + INDEX idx_k `mapKeys(m)` TYPE bloom_filter, + INDEX idx_v `mapValues(m)` TYPE bloom_filter +) +ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO t_bf_array VALUES (map('k', 'v'), ['x'], ['y']); + +SET optimize_functions_to_subcolumns = 0; + +SELECT count() FROM t_bf_array WHERE has(mapKeys(m), 'k'); +SELECT count() FROM t_bf_array WHERE mapKeys(m) IN (['k']); +SELECT count() FROM t_bf_array WHERE arrayJoin(mapKeys(m)) = 'k'; +SELECT count() FROM t_bf_array WHERE arrayJoin(mapKeys(m)) IN ('k'); +SELECT count() FROM t_bf_array WHERE has(mapValues(m), 'v'); +SELECT count() FROM t_bf_array WHERE mapValues(m) IN (['v']); + +-- Still used for the column itself. +SELECT count() FROM t_bf_array WHERE has(`mapKeys(m)`, 'x') SETTINGS force_data_skipping_indices = 'idx_k'; +SELECT count() FROM t_bf_array WHERE `mapKeys(m)` IN (['x']) SETTINGS force_data_skipping_indices = 'idx_k'; +SELECT count() FROM t_bf_array WHERE has(`mapValues(m)`, 'y') SETTINGS force_data_skipping_indices = 'idx_v'; + +-- Not specific to `Map`. +CREATE TABLE t_bf_lower +( + s String, + `lower(s)` String, + INDEX idx `lower(s)` TYPE bloom_filter +) +ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO t_bf_lower VALUES ('ABC', 'x'); + +SELECT count() FROM t_bf_lower WHERE lower(s) = 'abc'; +SELECT count() FROM t_bf_lower WHERE lower(s) IN ('abc'); +SELECT count() FROM t_bf_lower WHERE `lower(s)` = 'x' SETTINGS force_data_skipping_indices = 'idx'; + +-- Same columns and names, different argument order. +CREATE TABLE t_bf_order +( + `'k'` String, + arr Array(String), + INDEX idx_map mapKeys(map(`'k'`, 'k')) TYPE bloom_filter, + INDEX idx_concat concat(`'k'`, 'k') TYPE bloom_filter, + INDEX idx_lambda arrayMap(x -> concat(x, `'k'`, 'k'), arr) TYPE bloom_filter +) +ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO t_bf_order VALUES ('x', ['a']); + +SELECT count() FROM t_bf_order WHERE map('k', `'k'`)['k'] = 'x'; +SELECT count() FROM t_bf_order WHERE mapContainsKey(map('k', `'k'`), 'k'); +SELECT count() FROM t_bf_order WHERE has(mapKeys(map('k', `'k'`)), 'k'); +SELECT count() FROM t_bf_order WHERE mapKeys(map('k', `'k'`)) IN (['k']); +SELECT count() FROM t_bf_order WHERE arrayJoin(mapKeys(map('k', `'k'`))) = 'k'; +SELECT count() FROM t_bf_order WHERE concat('k', `'k'`) = 'kx'; +SELECT count() FROM t_bf_order WHERE concat('k', `'k'`) IN ('kx'); +SELECT count() FROM t_bf_order WHERE has(arrayMap(x -> concat(x, 'k', `'k'`), arr), 'akx'); + +SELECT count() FROM t_bf_order WHERE has(mapKeys(map(`'k'`, 'k')), 'x') SETTINGS force_data_skipping_indices = 'idx_map'; +SELECT count() FROM t_bf_order WHERE concat(`'k'`, 'k') = 'xk' SETTINGS force_data_skipping_indices = 'idx_concat'; +SELECT count() FROM t_bf_order WHERE has(arrayMap(x -> concat(x, `'k'`, 'k'), arr), 'axk') SETTINGS force_data_skipping_indices = 'idx_lambda'; + +-- The same for `JSON` subcolumns. +CREATE TABLE t_bf_json +( + j JSON, + `JSONAllPaths(j)` Array(String), + INDEX idx `JSONAllPaths(j)` TYPE bloom_filter +) +ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO t_bf_json VALUES ('{"a":1}', ['x']); + +SELECT count() FROM t_bf_json WHERE j.a = 1; +SELECT count() FROM t_bf_json WHERE j.a::Int64 = 1; +SELECT count() FROM t_bf_json WHERE isNotNull(j.a); +SELECT count() FROM t_bf_json WHERE has(`JSONAllPaths(j)`, 'x') SETTINGS force_data_skipping_indices = 'idx'; + +-- A column named like a `JSON` subcolumn is not that subcolumn. +CREATE TABLE t_bf_json_shadow +( + j JSON, + `j.a` Nullable(String), + `j.t` Tuple(c String), + `j.n` JSON, + INDEX idx JSONAllPaths(j) TYPE bloom_filter, + INDEX idx_n JSONAllPaths(`j.n`) TYPE bloom_filter +) +ENGINE = MergeTree ORDER BY tuple(); + +INSERT INTO t_bf_json_shadow VALUES ('{"b":1,"n":{"c":1}}', 'v', tuple('v'), '{"d":1}'); + +SELECT count() FROM t_bf_json_shadow WHERE `j.a` = 'v'; +SELECT count() FROM t_bf_json_shadow WHERE `j.a` IN ('v'); +SELECT count() FROM t_bf_json_shadow WHERE isNotNull(`j.a`) SETTINGS optimize_functions_to_subcolumns = 0; +SELECT count() FROM t_bf_json_shadow WHERE j.t.c = 'v'; +SELECT count() FROM t_bf_json_shadow WHERE j.n.c = 1; +SELECT count() FROM t_bf_json_shadow WHERE j.b = 1 SETTINGS force_data_skipping_indices = 'idx'; + +DROP TABLE t_bf_fixed; +DROP TABLE t_bf_lc; +DROP TABLE t_bf_real; +DROP TABLE t_bf_array; +DROP TABLE t_bf_lower; +DROP TABLE t_bf_order; +DROP TABLE t_bf_json; +DROP TABLE t_bf_json_shadow; From 1403bef200a02eb1304a2ba3f7cf30c741501f02 Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Sat, 14 Nov 2026 23:37:33 +0100 Subject: [PATCH 190/219] Adjust test for 26.8: parameters of `quantile` are printed without types Parameters of `AggregateFunction` are printed with their types only since https://github.com/ClickHouse/ClickHouse/pull/121725, which is not in 26.8. Co-Authored-By: Claude Opus 5.5 --- ...te_function_parameter_decimal_scale_out_of_bounds.reference | 2 +- ...ggregate_function_parameter_decimal_scale_out_of_bounds.sql | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/05632_aggregate_function_parameter_decimal_scale_out_of_bounds.reference b/tests/queries/0_stateless/05632_aggregate_function_parameter_decimal_scale_out_of_bounds.reference index 9972842f9827a..d00491fd7e5bb 100644 --- a/tests/queries/0_stateless/05632_aggregate_function_parameter_decimal_scale_out_of_bounds.reference +++ b/tests/queries/0_stateless/05632_aggregate_function_parameter_decimal_scale_out_of_bounds.reference @@ -1 +1 @@ -1 1 +1 diff --git a/tests/queries/0_stateless/05632_aggregate_function_parameter_decimal_scale_out_of_bounds.sql b/tests/queries/0_stateless/05632_aggregate_function_parameter_decimal_scale_out_of_bounds.sql index 290390b7a34e8..8e31eb79df3df 100644 --- a/tests/queries/0_stateless/05632_aggregate_function_parameter_decimal_scale_out_of_bounds.sql +++ b/tests/queries/0_stateless/05632_aggregate_function_parameter_decimal_scale_out_of_bounds.sql @@ -11,8 +11,9 @@ SELECT toTypeName(CAST(1 AS AggregateFunction(quantile(CAST('3.46216498963908703 SELECT toTypeName(CAST(1 AS AggregateFunction(quantile(CAST('1', 'Decimal256(4294967295)')), UInt64))); -- { serverError ARGUMENT_OUT_OF_BOUND } -- The maximum scale is still accepted, and the parameter is written back in full. +-- Parameters of `quantile` are printed without their types on this branch, so only the digits are checked. DROP TABLE IF EXISTS t_decimal_scale_parameter; CREATE TABLE t_decimal_scale_parameter (s AggregateFunction(quantile(CAST('1e-76', 'Decimal256(76)')), UInt64)) ENGINE = Memory; -SELECT position(type, '::Decimal256(76)') > 0, position(type, concat('0.', repeat('0', 75), '1')) > 0 +SELECT position(type, concat('0.', repeat('0', 75), '1')) > 0 FROM system.columns WHERE database = currentDatabase() AND table = 't_decimal_scale_parameter'; DROP TABLE t_decimal_scale_parameter; From 645e42ca4a7273fe3d1cdb38e0675f029fdfeef3 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 9 Nov 2026 11:27:10 +0000 Subject: [PATCH 191/219] Backport #128685 to 26.8: Keep the window state layout of `contingency`, `cramersV` and `theilsU` out of `-State` results --- .../AggregateFunctionFactory.cpp | 13 ++- .../Combinators/AggregateFunctionMap.cpp | 36 +++++++- .../Combinators/AggregateFunctionMerge.cpp | 15 ++- .../Combinators/AggregateFunctionMerge.h | 21 +++++ .../Combinators/AggregateFunctionState.cpp | 2 + .../IAggregateFunctionCombinator.h | 11 +++ src/AggregateFunctions/CrossTab.h | 7 ++ src/Columns/ColumnAggregateFunction.cpp | 15 ++- src/DataTypes/DataTypeAggregateFunction.cpp | 12 ++- tests/performance/contingency_functions.xml | 72 ++++++++++++++- .../04204_crosstab_variant_merge.sql | 8 +- ...rosstab_corrupted_state_deserialization.sh | 10 +- ...ed_plan_window_aggregate_state_variant.sql | 19 ++-- ...483_crosstab_window_state_layout.reference | 18 ++++ .../05483_crosstab_window_state_layout.sql | 63 +++++++++++++ ...tab_window_merge_ordinary_states.reference | 9 ++ ..._crosstab_window_merge_ordinary_states.sql | 92 +++++++++++++++++++ ...stab_window_merge_wrapped_states.reference | 13 +++ ...5_crosstab_window_merge_wrapped_states.sql | 79 ++++++++++++++++ 19 files changed, 490 insertions(+), 25 deletions(-) create mode 100644 tests/queries/0_stateless/05483_crosstab_window_state_layout.reference create mode 100644 tests/queries/0_stateless/05483_crosstab_window_state_layout.sql create mode 100644 tests/queries/0_stateless/05484_crosstab_window_merge_ordinary_states.reference create mode 100644 tests/queries/0_stateless/05484_crosstab_window_merge_ordinary_states.sql create mode 100644 tests/queries/0_stateless/05485_crosstab_window_merge_wrapped_states.reference create mode 100644 tests/queries/0_stateless/05485_crosstab_window_merge_wrapped_states.sql diff --git a/src/AggregateFunctions/AggregateFunctionFactory.cpp b/src/AggregateFunctions/AggregateFunctionFactory.cpp index 9ebd8ba3e7f34..e8c1b8260fd1a 100644 --- a/src/AggregateFunctions/AggregateFunctionFactory.cpp +++ b/src/AggregateFunctions/AggregateFunctionFactory.cpp @@ -333,6 +333,10 @@ AggregateFunctionPtr AggregateFunctionFactory::getImpl( Array nested_parameters = combinator->transformParameters(parameters); + /// A state leaves the window function as a value that can be stored, sent, cast or merged with ordinary + /// states, and its type name does not tell the window layout apart. So it always uses the ordinary layout. + const auto nested_state_variant = combinator->returnsNestedState() ? AggregateFunctionStateVariant::Aggregation : state_variant; + AggregateFunctionPtr combined_function; if (combinator->transformsMultipleNestedFunctions()) { @@ -343,7 +347,7 @@ AggregateFunctionPtr AggregateFunctionFactory::getImpl( VectorWithMemoryTracking nested_functions; nested_functions.reserve(nested_arguments_list.size()); for (const auto & nested_arguments : nested_arguments_list) - nested_functions.push_back(get(nested_name, action, nested_arguments, nested_parameters, out_properties, state_variant)); + nested_functions.push_back(get(nested_name, action, nested_arguments, nested_parameters, out_properties, nested_state_variant)); /// A `-State` round-trip reconstructs every element from this one shared name, so it must be /// the action-adjusted base aggregate name, not one element's instantiation (which can collapse @@ -356,7 +360,12 @@ AggregateFunctionPtr AggregateFunctionFactory::getImpl( { DataTypes nested_types = combinator->transformArguments(argument_types); - AggregateFunctionPtr nested_function = get(nested_name, action, nested_types, nested_parameters, out_properties, state_variant); + AggregateFunctionPtr nested_function = get(nested_name, action, nested_types, nested_parameters, out_properties, nested_state_variant); + /// The window layout only makes a window function faster, so use the ordinary layout where the + /// combined function cannot work with the window one. + if (nested_state_variant != AggregateFunctionStateVariant::Aggregation + && !combinator->acceptsNestedFunction(*nested_function, argument_types)) + nested_function = get(nested_name, action, nested_types, nested_parameters, out_properties, AggregateFunctionStateVariant::Aggregation); combined_function = combinator->transformAggregateFunction(nested_function, out_properties, argument_types, parameters); } diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionMap.cpp b/src/AggregateFunctions/Combinators/AggregateFunctionMap.cpp index 1cdc6b1881f23..2dbcfbedf5177 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionMap.cpp +++ b/src/AggregateFunctions/Combinators/AggregateFunctionMap.cpp @@ -220,6 +220,39 @@ class AggregateFunctionMap final } void mergeImpl(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena * arena) const override + { + mergeNestedStates(place, rhs, arena, [&](AggregateDataPtr nested_place, ConstAggregateDataPtr rhs_nested_place) + { + nested_func->merge(nested_place, rhs_nested_place, arena); + }); + } + + bool canMergeStateFromDifferentVariant(const IAggregateFunction & rhs) const override + { + if (!this->haveSameDefinition(rhs)) + return false; + + chassert(rhs.getNestedFunction() != nullptr); + + return nested_func->canMergeStateFromDifferentVariant(*rhs.getNestedFunction()); + } + + void mergeStateFromDifferentVariant( + AggregateDataPtr __restrict place, const IAggregateFunction & rhs, ConstAggregateDataPtr rhs_place, Arena * arena) const override + { + chassert(canMergeStateFromDifferentVariant(rhs)); + auto rhs_nested = rhs.getNestedFunction(); + chassert(rhs_nested != nullptr); + + mergeNestedStates(place, rhs_place, arena, [&](AggregateDataPtr nested_place, ConstAggregateDataPtr rhs_nested_place) + { + nested_func->mergeStateFromDifferentVariant(nested_place, *rhs_nested, rhs_nested_place, arena); + }); + } + +private: + template + void mergeNestedStates(AggregateDataPtr __restrict place, ConstAggregateDataPtr rhs, Arena * arena, MergeNested && merge_nested) const { auto & merged_maps = this->data(place).merged_maps; const auto & rhs_maps = this->data(rhs).merged_maps; @@ -255,10 +288,11 @@ class AggregateFunctionMap final } if (!zero_size_nested) - nested_func->merge(it->second, elem.second, arena); + merge_nested(it->second, elem.second); } } +public: template void destroyImpl(AggregateDataPtr __restrict place) const noexcept { diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionMerge.cpp b/src/AggregateFunctions/Combinators/AggregateFunctionMerge.cpp index eb060617b2dbd..936f6c5402432 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionMerge.cpp +++ b/src/AggregateFunctions/Combinators/AggregateFunctionMerge.cpp @@ -42,7 +42,7 @@ AggregateFunctionMerge::AggregateFunctionMerge(const AggregateFunctionPtr & nest data_type->getFunctionName(), nested_func->getName()); - if (nested_func->canMergeStateFromDifferentVariant(*argument_func)) + if (canMergeStatesOf(*nested_func, *argument_func)) { merge_state_from_different_variant = true; return; @@ -62,6 +62,13 @@ AggregateFunctionMerge::AggregateFunctionMerge(const AggregateFunctionPtr & nest argument_func->getStateType()->getName()); } +bool AggregateFunctionMerge::canMergeStatesOf(const IAggregateFunction & nested, const IAggregateFunction & argument_func) +{ + if (nested.haveSameStateRepresentation(argument_func)) + return true; + return argument_func.getName() == nested.getName() && nested.canMergeStateFromDifferentVariant(argument_func); +} + void AggregateFunctionMerge::add(AggregateDataPtr __restrict place, const IColumn ** columns, size_t row_num, Arena * arena) const { const auto & column = assert_cast(*columns[0]); @@ -86,6 +93,12 @@ class AggregateFunctionCombinatorMerge final : public IAggregateFunctionCombinat bool transformsArgumentTypes() const override { return true; } + bool acceptsNestedFunction(const IAggregateFunction & nested_function, const DataTypes & arguments) const override + { + const auto * argument = typeid_cast(arguments[0].get()); + return argument && AggregateFunctionMerge::canMergeStatesOf(nested_function, *argument->getFunction()); + } + DataTypes transformArguments(const DataTypes & arguments) const override { if (arguments.size() != 1) diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionMerge.h b/src/AggregateFunctions/Combinators/AggregateFunctionMerge.h index 113cb24963d02..82891397cd1a4 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionMerge.h +++ b/src/AggregateFunctions/Combinators/AggregateFunctionMerge.h @@ -94,6 +94,27 @@ class AggregateFunctionMerge final : public IAggregateFunctionHelpermerge(place, rhs, arena); } + bool canMergeStateFromDifferentVariant(const IAggregateFunction & rhs) const override + { + if (!this->haveSameDefinition(rhs)) + return false; + + chassert(rhs.getNestedFunction() != nullptr); + + return nested_func->canMergeStateFromDifferentVariant(*rhs.getNestedFunction()); + } + + void mergeStateFromDifferentVariant( + AggregateDataPtr __restrict place, const IAggregateFunction & rhs, ConstAggregateDataPtr rhs_place, Arena * arena) const override + { + chassert(rhs.getNestedFunction() != nullptr); + + nested_func->mergeStateFromDifferentVariant(place, *rhs.getNestedFunction(), rhs_place, arena); + } + + /// Can `nested` merge the states of `argument_func`, which `-Merge` reads as its argument? + static bool canMergeStatesOf(const IAggregateFunction & nested, const IAggregateFunction & argument_func); + bool isAbleToParallelizeMerge() const override { return nested_func->isAbleToParallelizeMerge(); } bool canOptimizeEqualKeysRanges() const override { return nested_func->canOptimizeEqualKeysRanges(); } diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionState.cpp b/src/AggregateFunctions/Combinators/AggregateFunctionState.cpp index 779184ead0cbc..563a8fbe25600 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionState.cpp +++ b/src/AggregateFunctions/Combinators/AggregateFunctionState.cpp @@ -16,6 +16,8 @@ class AggregateFunctionCombinatorState final : public IAggregateFunctionCombinat public: String getName() const override { return "State"; } + bool returnsNestedState() const override { return true; } + DataTypes transformArguments(const DataTypes & arguments) const override { return arguments; diff --git a/src/AggregateFunctions/Combinators/IAggregateFunctionCombinator.h b/src/AggregateFunctions/Combinators/IAggregateFunctionCombinator.h index 7f65baaa9e5fa..0f4cf8cd21f9a 100644 --- a/src/AggregateFunctions/Combinators/IAggregateFunctionCombinator.h +++ b/src/AggregateFunctions/Combinators/IAggregateFunctionCombinator.h @@ -45,6 +45,17 @@ class IAggregateFunctionCombinator */ virtual bool supportsNesting() const { return false; } + /// Does the combined function return the nested function's state (an `AggregateFunction` value), like State? + virtual bool returnsNestedState() const { return false; } + + /// Can the combined function work with `nested_function` for these arguments? `-Merge` cannot when + /// `nested_function` cannot merge the states that `-Merge` reads, for example when it has the window + /// layout and no support for merging ordinary states into it. + virtual bool acceptsNestedFunction(const IAggregateFunction & /*nested_function*/, const DataTypes & /*arguments*/) const + { + return true; + } + /** Does this combinator transform argument types (like Merge, Array) rather than just adding logic? * * If true, the combinator expects arguments to be of specific types (e.g., Merge expects AggregateFunction type) diff --git a/src/AggregateFunctions/CrossTab.h b/src/AggregateFunctions/CrossTab.h index b983f3dda3fb8..5730bdd0e7b9e 100644 --- a/src/AggregateFunctions/CrossTab.h +++ b/src/AggregateFunctions/CrossTab.h @@ -60,6 +60,13 @@ struct CrossTabCountsState void merge(const CrossTabCountsState & other) { + /// Copying the hash tables whole is much faster than inserting their elements one by one. + if (count == 0) + { + *this = other; + return; + } + count += other.count; for (const auto & [key, value] : other.count_a) count_a[key] += value; diff --git a/src/Columns/ColumnAggregateFunction.cpp b/src/Columns/ColumnAggregateFunction.cpp index 7ac2afa63de38..81bd2e478dedd 100644 --- a/src/Columns/ColumnAggregateFunction.cpp +++ b/src/Columns/ColumnAggregateFunction.cpp @@ -76,19 +76,32 @@ static String getTypeString(const AggregateFunctionPtr & func, std::optional version_) : func(func_), type_string(getTypeString(func, version_)), version(version_) { + checkOrdinaryStateLayout(*func); } ColumnAggregateFunction::ColumnAggregateFunction(const AggregateFunctionPtr & func_, const ConstArenas & arenas_) : foreign_arenas(arenas_), func(func_), type_string(getTypeString(func)) { - + checkOrdinaryStateLayout(*func); } void ColumnAggregateFunction::set(const AggregateFunctionPtr & func_, std::optional version_) { + checkOrdinaryStateLayout(*func_); func = func_; version = version_; type_string = getTypeString(func, version); diff --git a/src/DataTypes/DataTypeAggregateFunction.cpp b/src/DataTypes/DataTypeAggregateFunction.cpp index 74d8401b2208b..e2bfc0240759a 100644 --- a/src/DataTypes/DataTypeAggregateFunction.cpp +++ b/src/DataTypes/DataTypeAggregateFunction.cpp @@ -191,8 +191,18 @@ bool DataTypeAggregateFunction::strictEquals(const DataTypePtr & lhs_state_type, return false; for (size_t i = 0; i < lhs_state->argument_types.size(); ++i) - if (!lhs_state->argument_types[i]->equals(*rhs_state->argument_types[i])) + { + const auto & lhs_argument = lhs_state->argument_types[i]; + const auto & rhs_argument = rhs_state->argument_types[i]; + /// An argument can be a state itself (for example the element states of `-Tuple`), whose layout must be ignored too. + if (ignore_variant && typeid_cast(lhs_argument.get())) + { + if (!strictEquals(lhs_argument, rhs_argument, /*ignore_variant=*/ true)) + return false; + } + else if (!lhs_argument->equals(*rhs_argument)) return false; + } return true; } diff --git a/tests/performance/contingency_functions.xml b/tests/performance/contingency_functions.xml index dae8066a7934a..c5cde1e7695e5 100644 --- a/tests/performance/contingency_functions.xml +++ b/tests/performance/contingency_functions.xml @@ -1,4 +1,21 @@ - + + + CREATE TABLE crosstab_states + ( + k UInt64, + cramers_v AggregateFunction(cramersV, UInt8, UInt8), + theils_u AggregateFunction(theilsU, UInt8, UInt8) + ) + ENGINE = MergeTree ORDER BY k + + + INSERT INTO crosstab_states + SELECT intDiv(number, 100) AS k, cramersVState(number % 100, intDiv(number, 7) % 100), theilsUState(number % 100, intDiv(number, 7) % 100) + FROM numbers(2_000_000) + GROUP BY k + + DROP TABLE IF EXISTS crosstab_states + SELECT cramersV(number, number) FROM numbers(5_000) FORMAT NULL; @@ -58,4 +75,57 @@ FROM numbers(10_000) ) FORMAT NULL; + + + + SELECT finalizeAggregation(cramersVState(a, b) OVER (ORDER BY number ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW)) + FROM (SELECT number, number % 100 AS a, intDiv(number, 7) % 100 AS b FROM numbers(3_000)) FORMAT NULL; + + + + SELECT contingencyState(a, b) OVER (ORDER BY number ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) + FROM (SELECT number, number % 5 AS a, intDiv(number, 3) % 4 AS b FROM numbers(200_000)) FORMAT NULL; + + + + SELECT finalizeAggregation(cramersVState(a, b) OVER (ORDER BY number ROWS BETWEEN 100 PRECEDING AND CURRENT ROW)) + FROM (SELECT number, number % 30 AS a, intDiv(number, 7) % 30 AS b FROM numbers(20_000)) FORMAT NULL; + + + + SELECT theilsUState(a, b) OVER (ORDER BY number ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) + FROM (SELECT number, number % 100 AS a, intDiv(number, 7) % 100 AS b FROM numbers(5_000)) FORMAT NULL; + + + + SELECT finalizeAggregation(theilsUState(a, b) OVER (PARTITION BY p)) + FROM (SELECT number % 100 AS p, intDiv(number, 100) % 10 AS a, (intDiv(number, 100) * 3 + intDiv(number, 1300)) % 8 AS b FROM numbers(20_000)) FORMAT NULL; + + + + + SELECT cramersVMerge(cramers_v) OVER (ORDER BY k ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) FROM crosstab_states FORMAT NULL; + + + + SELECT theilsUMerge(theils_u) OVER (ORDER BY k ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) FROM crosstab_states FORMAT NULL; + + + + SELECT cramersVMerge(s) OVER (PARTITION BY p) + FROM + ( + SELECT p, cramersVState(a, b) OVER (PARTITION BY p ORDER BY number ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS s + FROM (SELECT number, number % 10 AS p, intDiv(number, 10) % 30 AS a, intDiv(number, 70) % 30 AS b FROM numbers(10_000)) + ) FORMAT NULL; + + + + SELECT theilsUMerge(s) OVER (PARTITION BY p) + FROM + ( + SELECT p, theilsUState(a, b) OVER (PARTITION BY p ORDER BY number ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS s + FROM (SELECT number, number % 10 AS p, intDiv(number, 10) % 30 AS a, intDiv(number, 70) % 30 AS b FROM numbers(10_000)) + ) FORMAT NULL; + diff --git a/tests/queries/0_stateless/04204_crosstab_variant_merge.sql b/tests/queries/0_stateless/04204_crosstab_variant_merge.sql index f5a8e28d14f82..a5d58c34a68ca 100644 --- a/tests/queries/0_stateless/04204_crosstab_variant_merge.sql +++ b/tests/queries/0_stateless/04204_crosstab_variant_merge.sql @@ -1,6 +1,6 @@ SET enable_analyzer = 1; -SELECT round(cramersVMerge(s.`AggregateFunction(cramersV, UInt8, UInt8)`), 4) +SELECT round(cramersVMerge(s), 4) FROM ( SELECT cramersVState(toUInt8(number % 10), toUInt8(number % 6)) OVER () AS s FROM numbers(100) LIMIT 1 @@ -8,7 +8,7 @@ FROM SELECT cramersVState(toUInt8(number % 10), toUInt8(number % 6)) AS s FROM numbers(100) ); -SELECT round(contingencyMerge(s.`AggregateFunction(contingency, UInt8, UInt8)`), 4) +SELECT round(contingencyMerge(s), 4) FROM ( SELECT contingencyState(toUInt8(number % 10), toUInt8(number % 6)) OVER () AS s FROM numbers(100) LIMIT 1 @@ -16,7 +16,7 @@ FROM SELECT contingencyState(toUInt8(number % 10), toUInt8(number % 6)) AS s FROM numbers(100) ); -SELECT round(cramersVBiasCorrectedMerge(s.`AggregateFunction(cramersVBiasCorrected, UInt8, UInt8)`), 4) +SELECT round(cramersVBiasCorrectedMerge(s), 4) FROM ( SELECT cramersVBiasCorrectedState(toUInt8(number % 10), toUInt8(number % 6)) OVER () AS s FROM numbers(100) LIMIT 1 @@ -24,7 +24,7 @@ FROM SELECT cramersVBiasCorrectedState(toUInt8(number % 10), toUInt8(number % 6)) AS s FROM numbers(100) ); -SELECT round(theilsUMerge(s.`AggregateFunction(theilsU, UInt8, UInt8)`), 4) +SELECT round(theilsUMerge(s), 4) FROM ( SELECT theilsUState(toUInt8(number % 10), toUInt8(number % 6)) OVER () AS s FROM numbers(100) LIMIT 1 diff --git a/tests/queries/0_stateless/04336_crosstab_corrupted_state_deserialization.sh b/tests/queries/0_stateless/04336_crosstab_corrupted_state_deserialization.sh index 3b4dd6bd55aad..2b0bbf066464e 100755 --- a/tests/queries/0_stateless/04336_crosstab_corrupted_state_deserialization.sh +++ b/tests/queries/0_stateless/04336_crosstab_corrupted_state_deserialization.sh @@ -57,12 +57,14 @@ do done # Genuine states still survive a roundtrip through `String` and finalize to the same results. +# The results are sums over the hash tables, and a state read from `String` can keep its cells +# in another order, so the last digits can differ. $CLICKHOUSE_LOCAL --query " SELECT - finalizeAggregation(st_c) = finalizeAggregation(CAST(CAST(st_c, 'String'), 'AggregateFunction(contingency, UInt16, UInt16)')), - finalizeAggregation(st_v) = finalizeAggregation(CAST(CAST(st_v, 'String'), 'AggregateFunction(cramersV, UInt16, UInt16)')), - finalizeAggregation(st_b) = finalizeAggregation(CAST(CAST(st_b, 'String'), 'AggregateFunction(cramersVBiasCorrected, UInt16, UInt16)')), - finalizeAggregation(st_u) = finalizeAggregation(CAST(CAST(st_u, 'String'), 'AggregateFunction(theilsU, UInt16, UInt16)')) + abs(finalizeAggregation(st_c) - finalizeAggregation(CAST(CAST(st_c, 'String'), 'AggregateFunction(contingency, UInt16, UInt16)'))) < 1e-12, + abs(finalizeAggregation(st_v) - finalizeAggregation(CAST(CAST(st_v, 'String'), 'AggregateFunction(cramersV, UInt16, UInt16)'))) < 1e-12, + abs(finalizeAggregation(st_b) - finalizeAggregation(CAST(CAST(st_b, 'String'), 'AggregateFunction(cramersVBiasCorrected, UInt16, UInt16)'))) < 1e-12, + abs(finalizeAggregation(st_u) - finalizeAggregation(CAST(CAST(st_u, 'String'), 'AggregateFunction(theilsU, UInt16, UInt16)'))) < 1e-12 FROM ( SELECT diff --git a/tests/queries/0_stateless/04734_distributed_plan_window_aggregate_state_variant.sql b/tests/queries/0_stateless/04734_distributed_plan_window_aggregate_state_variant.sql index d3b45af22d960..2db1bbfe3e40a 100644 --- a/tests/queries/0_stateless/04734_distributed_plan_window_aggregate_state_variant.sql +++ b/tests/queries/0_stateless/04734_distributed_plan_window_aggregate_state_variant.sql @@ -1,6 +1,3 @@ --- Tags: no-old-analyzer --- no-old-analyzer: make_distributed_plan requires the analyzer. - -- The plan header encoding does not carry the aggregate function state variant, so a Window-variant -- state (CrossTab family) was decoded as an Aggregation-variant one and the deserialized plan header -- was rejected. Results must match the non-distributed plan. @@ -13,7 +10,9 @@ INSERT INTO t_window_state_variant SELECT number % 4, number % 3 FROM numbers(12 SET make_distributed_plan = 1, enable_parallel_replicas = 0, distributed_plan_execute_locally = 1, distributed_plan_max_rows_to_broadcast = 0, enable_join_runtime_filters = 0; -SELECT round(finalizeAggregation(theilsUState(a, b) OVER (ORDER BY a, b)), 6) AS s +-- `+ 0` turns `-0` into `0`: `ORDER BY` treats them as equal, so otherwise their order in the output would change. + +SELECT round(finalizeAggregation(theilsUState(a, b) OVER (ORDER BY a, b)), 6) + 0 AS s FROM t_window_state_variant ORDER BY s; -- The plan only splits into more than one stage for a multi-stage shape, and a single-stage plan is @@ -26,28 +25,28 @@ SELECT 'plain state distributes' FROM (EXPLAIN PIPELINE SELECT finalizeAggregation(theilsUState(a, b) OVER (ORDER BY a, b)) FROM t_window_state_variant) WHERE explain LIKE '%ReadFromDistributedPlanSource%' LIMIT 1; -SELECT round(finalizeAggregation(cramersVState(a, b) OVER (ORDER BY a, b)), 6) AS s +SELECT round(finalizeAggregation(cramersVState(a, b) OVER (ORDER BY a, b)), 6) + 0 AS s FROM t_window_state_variant ORDER BY s; -SELECT round(finalizeAggregation(contingencyState(a, b) OVER (ORDER BY a, b)), 6) AS s +SELECT round(finalizeAggregation(contingencyState(a, b) OVER (ORDER BY a, b)), 6) + 0 AS s FROM t_window_state_variant ORDER BY s; -- Combinators propagate the variant to the wrapping function, and -ForEach nests the state in an Array. -SELECT round(finalizeAggregation(theilsUStateIf(a, b, a > 1) OVER (ORDER BY a, b)), 6) AS s +SELECT round(finalizeAggregation(theilsUStateIf(a, b, a > 1) OVER (ORDER BY a, b)), 6) + 0 AS s FROM t_window_state_variant ORDER BY s; -SELECT round(finalizeAggregation((theilsUStateForEach([a], [b]) OVER (ORDER BY a, b))[1]), 6) AS s +SELECT round(finalizeAggregation((theilsUStateForEach([a], [b]) OVER (ORDER BY a, b))[1]), 6) + 0 AS s FROM t_window_state_variant ORDER BY s; SELECT 'nested state distributes' FROM (EXPLAIN PIPELINE SELECT finalizeAggregation((theilsUStateForEach([a], [b]) OVER (ORDER BY a, b))[1]) FROM t_window_state_variant) WHERE explain LIKE '%ReadFromDistributedPlanSource%' LIMIT 1; -SELECT round(finalizeAggregation(theilsUArgMaxState(a, b, a) OVER (ORDER BY a, b)), 6) AS s +SELECT round(finalizeAggregation(theilsUArgMaxState(a, b, a) OVER (ORDER BY a, b)), 6) + 0 AS s FROM t_window_state_variant ORDER BY s; -- A step after the window step propagates the state column into its own header check. -SELECT round(finalizeAggregation(s), 6) AS f +SELECT round(finalizeAggregation(s), 6) + 0 AS f FROM (SELECT theilsUState(a, b) OVER (ORDER BY a, b) AS s FROM t_window_state_variant) ORDER BY f; diff --git a/tests/queries/0_stateless/05483_crosstab_window_state_layout.reference b/tests/queries/0_stateless/05483_crosstab_window_state_layout.reference new file mode 100644 index 0000000000000..ddd1e3fe8a09d --- /dev/null +++ b/tests/queries/0_stateless/05483_crosstab_window_state_layout.reference @@ -0,0 +1,18 @@ +-- window states give the same values as ordinary states +0 0.061889 0.061889 +1 0.069208 0.069208 +2 0.061889 0.061889 +3 0.061889 0.061889 +0 0.061889 +1 0.069208 +2 0.061889 +3 0.061889 +0 +-- window state through a spilled sort, in an Array and in a Tuple +3796387405470498412 +3796387405470498412 +-- window state inside another state, cast to its type name +0.007118 +-- window state merged in a distributed plan executed locally +0.007118 0.005033 0.000023 +0.007118 0.005033 0.000023 diff --git a/tests/queries/0_stateless/05483_crosstab_window_state_layout.sql b/tests/queries/0_stateless/05483_crosstab_window_state_layout.sql new file mode 100644 index 0000000000000..d97ef7467ccc6 --- /dev/null +++ b/tests/queries/0_stateless/05483_crosstab_window_state_layout.sql @@ -0,0 +1,63 @@ +-- `contingency` and its relatives keep a window function's state in a separate memory layout. A state +-- that a window function returns with `-State` has the same type name as an ordinary state, so it must +-- use the ordinary layout: otherwise a spill, a `CAST` or a distributed plan reads one layout as the other. + +DROP TABLE IF EXISTS t_crosstab_window_state; +CREATE TABLE t_crosstab_window_state (p UInt32, b UInt32, c UInt32, v UInt32) ENGINE = MergeTree ORDER BY (p, v) +SETTINGS index_granularity = 8; +INSERT INTO t_crosstab_window_state SELECT number % 4, number % 3, intDiv(number, 3) % 5, number FROM numbers(400); + +SELECT '-- window states give the same values as ordinary states'; +SELECT p, round(finalizeAggregation(any(s)), 6), round(contingencyMerge(s), 6) +FROM (SELECT p, contingencyState(b, c) OVER (PARTITION BY p) AS s FROM t_crosstab_window_state) +GROUP BY p ORDER BY p; +SELECT p, round(contingency(b, c), 6) FROM t_crosstab_window_state GROUP BY p ORDER BY p; +SELECT count() +FROM +( + SELECT + round(finalizeAggregation(contingencyState(b, c) OVER (ORDER BY v)), 6) AS from_state, + round(contingency(b, c) OVER (ORDER BY v), 6) AS direct + FROM t_crosstab_window_state +) +WHERE from_state != direct AND NOT (isNaN(from_state) AND isNaN(direct)); + +SELECT '-- window state through a spilled sort, in an Array and in a Tuple'; +-- Without query_plan_enable_optimizations = 0 the Array and the Tuple are built above the sort. +SELECT sum(cityHash64(v, round(finalizeAggregation(a[1]), 6), round(finalizeAggregation(tp.1), 6))) +FROM +( + SELECT v, [s] AS a, tuple(s, v) AS tp + FROM (SELECT v, contingencyState(b, c) OVER (PARTITION BY p ORDER BY v) AS s FROM t_crosstab_window_state) + ORDER BY v DESC +) +SETTINGS query_plan_enable_optimizations = 0, max_bytes_before_external_sort = 1000, max_bytes_ratio_before_external_sort = 0, + max_block_size = 8; +SELECT sum(cityHash64(v, round(finalizeAggregation(a[1]), 6), round(finalizeAggregation(tp.1), 6))) +FROM +( + SELECT v, [s] AS a, tuple(s, v) AS tp + FROM (SELECT v, contingencyState(b, c) OVER (PARTITION BY p ORDER BY v) AS s FROM t_crosstab_window_state) + ORDER BY v DESC +) +SETTINGS query_plan_enable_optimizations = 0, max_bytes_before_external_sort = 0; + +SELECT '-- window state inside another state, cast to its type name'; +SELECT round(finalizeAggregation(finalizeAggregation(CAST(anyState(tuple(s, CAST(1 AS Dynamic))) + AS AggregateFunction(any, Tuple(AggregateFunction(contingency, UInt32, UInt32), Dynamic)))).1), 6) +FROM (SELECT contingencyState(b, c) OVER () AS s FROM t_crosstab_window_state); + +SELECT '-- window state merged in a distributed plan executed locally'; +SET max_rows_to_group_by = 0; +SELECT round(contingencyMerge(CAST(s AS AggregateFunction(contingency, UInt32, UInt32))), 6), + round(cramersVMerge(CAST(s2 AS AggregateFunction(cramersV, UInt32, UInt32))), 6), + round(theilsUMerge(s3), 6) +FROM +( + SELECT contingencyState(b, c) OVER () AS s, cramersVState(b, c) OVER () AS s2, theilsUState(b, c) OVER () AS s3 + FROM t_crosstab_window_state LIMIT 1 +) +SETTINGS make_distributed_plan = 1, distributed_plan_execute_locally = 1; +SELECT round(contingency(b, c), 6), round(cramersV(b, c), 6), round(theilsU(b, c), 6) FROM t_crosstab_window_state; + +DROP TABLE t_crosstab_window_state; diff --git a/tests/queries/0_stateless/05484_crosstab_window_merge_ordinary_states.reference b/tests/queries/0_stateless/05484_crosstab_window_merge_ordinary_states.reference new file mode 100644 index 0000000000000..e60bc8912e0c8 --- /dev/null +++ b/tests/queries/0_stateless/05484_crosstab_window_merge_ordinary_states.reference @@ -0,0 +1,9 @@ +-- each row: window layout equals ordinary layout +60 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +-- whole partition: window merge, then the plain aggregate +0 0.158611 0.158611 +1 0.158611 0.158611 +2 0.163199 0.163199 +0 0.158611 0.158611 +1 0.158611 0.158611 +2 0.163199 0.163199 diff --git a/tests/queries/0_stateless/05484_crosstab_window_merge_ordinary_states.sql b/tests/queries/0_stateless/05484_crosstab_window_merge_ordinary_states.sql new file mode 100644 index 0000000000000..cf8e8b739de50 --- /dev/null +++ b/tests/queries/0_stateless/05484_crosstab_window_merge_ordinary_states.sql @@ -0,0 +1,92 @@ +-- A `-Merge` window function of `contingency` and its relatives keeps its own state in the window layout and +-- merges ordinary states into it, through each combinator between `-Merge` and the function. `-MergeState` +-- builds the same merge with the ordinary layout, so the two must give the same values on every row. +-- The values are compared as text, so `nan` equals `nan`, and `+ 0` turns `-0` into `0`. + +DROP TABLE IF EXISTS t_crosstab_window_merge; +CREATE TABLE t_crosstab_window_merge (p UInt32, k UInt32, b UInt32, c UInt32, n Nullable(UInt32)) +ENGINE = MergeTree ORDER BY (p, k); +INSERT INTO t_crosstab_window_merge +SELECT number % 3, intDiv(number, 3) % 20, number % 4, intDiv(number, 7) % 5, if(number % 11 = 0, NULL, number % 4) +FROM numbers(600); + +SELECT '-- each row: window layout equals ordinary layout'; +SELECT + count(), + countIf(toString(plain) != toString(plain_ordinary)), + countIf(toString(cramers) != toString(cramers_ordinary)), + countIf(toString(cramers_bc) != toString(cramers_bc_ordinary)), + countIf(toString(theils) != toString(theils_ordinary)), + countIf(toString(if_merge) != toString(if_ordinary)), + countIf(toString(array_merge) != toString(array_ordinary)), + countIf(toString(for_each) != toString(for_each_ordinary)), + countIf(toString(nullable) != toString(nullable_ordinary)), + countIf(toString(or_null) != toString(or_null_ordinary)), + countIf(toString(resample) != toString(resample_ordinary)), + countIf(toString(distinct) != toString(distinct_ordinary)), + countIf(toString(tuple_merge) != toString(tuple_ordinary)), + countIf(toString(arg_max) != toString(arg_max_ordinary)), + countIf(toString(map_merge) != toString(map_ordinary)) +FROM +( + SELECT + round(contingencyMerge(s_plain) OVER w, 6) + 0 AS plain, + round(finalizeAggregation(contingencyMergeState(s_plain) OVER w), 6) + 0 AS plain_ordinary, + round(cramersVMerge(s_cramers) OVER w, 6) + 0 AS cramers, + round(finalizeAggregation(cramersVMergeState(s_cramers) OVER w), 6) + 0 AS cramers_ordinary, + round(cramersVBiasCorrectedMerge(s_cramers_bc) OVER w, 6) + 0 AS cramers_bc, + round(finalizeAggregation(cramersVBiasCorrectedMergeState(s_cramers_bc) OVER w), 6) + 0 AS cramers_bc_ordinary, + round(theilsUMerge(s_theils) OVER w, 6) + 0 AS theils, + round(finalizeAggregation(theilsUMergeState(s_theils) OVER w), 6) + 0 AS theils_ordinary, + round(contingencyIfMerge(s_if) OVER w, 6) + 0 AS if_merge, + round(finalizeAggregation(contingencyIfMergeState(s_if) OVER w), 6) + 0 AS if_ordinary, + round(contingencyArrayMerge(s_array) OVER w, 6) + 0 AS array_merge, + round(finalizeAggregation(contingencyArrayMergeState(s_array) OVER w), 6) + 0 AS array_ordinary, + arrayMap(x -> round(x, 6) + 0, contingencyForEachMerge(s_for_each) OVER w) AS for_each, + arrayMap(x -> round(x, 6) + 0, finalizeAggregation(contingencyForEachMergeState(s_for_each) OVER w)) AS for_each_ordinary, + round(contingencyMerge(s_nullable) OVER w, 6) + 0 AS nullable, + round(finalizeAggregation(contingencyMergeState(s_nullable) OVER w), 6) + 0 AS nullable_ordinary, + round(contingencyOrNullMerge(s_or_null) OVER w, 6) + 0 AS or_null, + round(finalizeAggregation(contingencyOrNullMergeState(s_or_null) OVER w), 6) + 0 AS or_null_ordinary, + arrayMap(x -> round(x, 6) + 0, contingencyResampleMerge(0, 20, 10)(s_resample) OVER w) AS resample, + arrayMap(x -> round(x, 6) + 0, finalizeAggregation(contingencyResampleMergeState(0, 20, 10)(s_resample) OVER w)) AS resample_ordinary, + round(contingencyDistinctMerge(s_distinct) OVER w, 6) + 0 AS distinct, + round(finalizeAggregation(contingencyDistinctMergeState(s_distinct) OVER w), 6) + 0 AS distinct_ordinary, + round(tupleElement(contingencyTupleMerge(s_tuple) OVER w, 1), 6) + 0 AS tuple_merge, + round(tupleElement(finalizeAggregation(contingencyTupleMergeState(s_tuple) OVER w), 1), 6) + 0 AS tuple_ordinary, + round(contingencyArgMaxMerge(s_arg_max) OVER w, 6) + 0 AS arg_max, + round(finalizeAggregation(contingencyArgMaxMergeState(s_arg_max) OVER w), 6) + 0 AS arg_max_ordinary, + mapApply((key, value) -> (key, round(value, 6) + 0), contingencyMapMerge(s_map) OVER w) AS map_merge, + mapApply((key, value) -> (key, round(value, 6) + 0), finalizeAggregation(contingencyMapMergeState(s_map) OVER w)) AS map_ordinary + FROM + ( + SELECT + p, + k, + contingencyState(b, c) AS s_plain, + cramersVState(b, c) AS s_cramers, + cramersVBiasCorrectedState(b, c) AS s_cramers_bc, + theilsUState(b, c) AS s_theils, + contingencyIfState(b, c, b != 1) AS s_if, + contingencyArrayState([b], [c]) AS s_array, + contingencyForEachState([b, c], [c, b]) AS s_for_each, + contingencyState(n, c) AS s_nullable, + contingencyOrNullState(b, c) AS s_or_null, + contingencyResampleState(0, 20, 10)(b, c, k) AS s_resample, + contingencyDistinctState(b, c) AS s_distinct, + contingencyTupleState(tuple(b), tuple(c)) AS s_tuple, + contingencyArgMaxState(b, c, k) AS s_arg_max, + contingencyMapState(b, c, p) AS s_map + FROM t_crosstab_window_merge + GROUP BY p, k + ) + WINDOW w AS (PARTITION BY p ORDER BY k ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) +); + +SELECT '-- whole partition: window merge, then the plain aggregate'; +SELECT DISTINCT p, round(contingencyMerge(s) OVER (PARTITION BY p), 6), round((contingencyMapMerge(sm) OVER (PARTITION BY p))[p], 6) +FROM (SELECT p, contingencyState(b, c) AS s, contingencyMapState(b, c, p) AS sm FROM t_crosstab_window_merge GROUP BY p, k) +ORDER BY p; +SELECT p, round(contingency(b, c), 6), round(contingencyMap(b, c, p)[p], 6) FROM t_crosstab_window_merge GROUP BY p ORDER BY p; + +DROP TABLE t_crosstab_window_merge; diff --git a/tests/queries/0_stateless/05485_crosstab_window_merge_wrapped_states.reference b/tests/queries/0_stateless/05485_crosstab_window_merge_wrapped_states.reference new file mode 100644 index 0000000000000..f90e009c927c3 --- /dev/null +++ b/tests/queries/0_stateless/05485_crosstab_window_merge_wrapped_states.reference @@ -0,0 +1,13 @@ +-- each row: window layout equals ordinary layout +60 0 0 0 0 +-- whole partition: window merges of window states, then the plain aggregates +0 0.158611 +1 0.158611 +2 0.163199 +0 0.158611 +1 0.158611 +2 0.163199 +0 0.158611 +1 0.158611 +2 0.163199 +-- a state with other parameters is still rejected diff --git a/tests/queries/0_stateless/05485_crosstab_window_merge_wrapped_states.sql b/tests/queries/0_stateless/05485_crosstab_window_merge_wrapped_states.sql new file mode 100644 index 0000000000000..5d23cdd71bd03 --- /dev/null +++ b/tests/queries/0_stateless/05485_crosstab_window_merge_wrapped_states.sql @@ -0,0 +1,79 @@ +-- States that a window function returns with `-State` have the ordinary layout, and a `-Merge` window +-- function merges them into its window-layout state through every combinator around the function: +-- `-If` and `-Array` around `-Tuple`, and `-Map` and `-ForEach` around another `-Merge`. `-MergeState` +-- builds the same merge with the ordinary layout, so the two must give the same values on every row. +-- The values are compared as text, so `nan` equals `nan`, and `+ 0` turns `-0` into `0`. + +DROP TABLE IF EXISTS t_crosstab_window_wrapped; +CREATE TABLE t_crosstab_window_wrapped (p UInt32, k UInt32, b UInt32, c UInt32, d UInt32) ENGINE = MergeTree ORDER BY (p, k); +INSERT INTO t_crosstab_window_wrapped +SELECT number % 3, intDiv(number, 3) % 20, number % 4, intDiv(number, 7) % 5, intDiv(number, 11) % 3 FROM numbers(600); + +SELECT '-- each row: window layout equals ordinary layout'; +SELECT + count(), + countIf(toString(tuple_if) != toString(tuple_if_ordinary)), + countIf(toString(tuple_array) != toString(tuple_array_ordinary)), + countIf(toString(merge_map) != toString(merge_map_ordinary)), + countIf(toString(merge_for_each) != toString(merge_for_each_ordinary)) +FROM +( + SELECT + (round(r_tuple_if.1, 6) + 0, round(r_tuple_if.2, 6) + 0) AS tuple_if, + (round(r_tuple_if_ordinary.1, 6) + 0, round(r_tuple_if_ordinary.2, 6) + 0) AS tuple_if_ordinary, + (round(r_tuple_array.1, 6) + 0, round(r_tuple_array.2, 6) + 0) AS tuple_array, + (round(r_tuple_array_ordinary.1, 6) + 0, round(r_tuple_array_ordinary.2, 6) + 0) AS tuple_array_ordinary, + mapApply((key, value) -> (key, round(value, 6) + 0), r_merge_map) AS merge_map, + mapApply((key, value) -> (key, round(value, 6) + 0), r_merge_map_ordinary) AS merge_map_ordinary, + arrayMap(x -> round(x, 6) + 0, r_merge_for_each) AS merge_for_each, + arrayMap(x -> round(x, 6) + 0, r_merge_for_each_ordinary) AS merge_for_each_ordinary + FROM + ( + SELECT + contingencyTupleIfMerge(s_tuple_if) OVER w AS r_tuple_if, + finalizeAggregation(contingencyTupleIfMergeState(s_tuple_if) OVER w) AS r_tuple_if_ordinary, + contingencyTupleArrayMerge(s_tuple_array) OVER w AS r_tuple_array, + finalizeAggregation(contingencyTupleArrayMergeState(s_tuple_array) OVER w) AS r_tuple_array_ordinary, + contingencyMergeMapMerge(s_merge_map) OVER w AS r_merge_map, + finalizeAggregation(contingencyMergeMapMergeState(s_merge_map) OVER w) AS r_merge_map_ordinary, + contingencyMergeForEachMerge(s_merge_for_each) OVER w AS r_merge_for_each, + finalizeAggregation(contingencyMergeForEachMergeState(s_merge_for_each) OVER w) AS r_merge_for_each_ordinary + FROM + ( + SELECT + p, + k, + contingencyTupleIfState((b_any, c_any), (c_any, d_any), b_any != 1) OVER w AS s_tuple_if, + contingencyTupleArrayState([(b_any, c_any)], [(c_any, d_any)]) OVER w AS s_tuple_array, + contingencyMergeMapState(s0, k % 3) OVER w AS s_merge_map, + contingencyMergeForEachState(if(k % 2 = 0, [s0], [s0, s1])) OVER w AS s_merge_for_each + FROM + ( + SELECT p, k, any(b) AS b_any, any(c) AS c_any, any(d) AS d_any, contingencyState(b, c) AS s0, contingencyState(c, d) AS s1 + FROM t_crosstab_window_wrapped + GROUP BY p, k + ) + WINDOW w AS (PARTITION BY p ORDER BY k ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) + ) + WINDOW w AS (PARTITION BY p ORDER BY k ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) + ) +); + +SELECT '-- whole partition: window merges of window states, then the plain aggregates'; +SELECT DISTINCT p, round((contingencyMergeMapMerge(s) OVER (PARTITION BY p))[0], 6) +FROM +( + SELECT p, contingencyMergeMapState(s0, 0) OVER (PARTITION BY p) AS s + FROM (SELECT p, k, contingencyState(b, c) AS s0 FROM t_crosstab_window_wrapped GROUP BY p, k) +) +ORDER BY p; +SELECT DISTINCT p, round((contingencyTupleIfMerge(s) OVER (PARTITION BY p)).1, 6) +FROM (SELECT p, contingencyTupleIfState(tuple(b), tuple(c), 1) OVER (PARTITION BY p) AS s FROM t_crosstab_window_wrapped) +ORDER BY p; +SELECT p, round(contingency(b, c), 6) FROM t_crosstab_window_wrapped GROUP BY p ORDER BY p; + +SELECT '-- a state with other parameters is still rejected'; +SELECT contingencyResampleMerge(10, 30, 10)(s) OVER () +FROM (SELECT contingencyResampleState(0, 20, 10)(b, c, k) AS s FROM t_crosstab_window_wrapped GROUP BY p); -- { serverError ILLEGAL_TYPE_OF_ARGUMENT } + +DROP TABLE t_crosstab_window_wrapped; From df6136fcc07bad4f5bbc9faf8ff929535802798d Mon Sep 17 00:00:00 2001 From: Groene AI <270696204+groeneai@users.noreply.github.com> Date: Wed, 11 Nov 2026 08:53:03 +0000 Subject: [PATCH 192/219] Adapt 05484 and 05485 to the `-Map` combinator of 26.8 The `-Map` form with the key as the last argument comes from #123449, which is not in 26.8. 05485 builds its `-Map` state from a `Map` argument instead, and 05484 drops its `contingencyMap` case, which has no equivalent without that form. Co-Authored-By: Claude Opus 5.5 --- ...osstab_window_merge_ordinary_states.reference | 14 +++++++------- ...484_crosstab_window_merge_ordinary_states.sql | 16 ++++++---------- ...5485_crosstab_window_merge_wrapped_states.sql | 4 ++-- 3 files changed, 15 insertions(+), 19 deletions(-) diff --git a/tests/queries/0_stateless/05484_crosstab_window_merge_ordinary_states.reference b/tests/queries/0_stateless/05484_crosstab_window_merge_ordinary_states.reference index e60bc8912e0c8..3248e2f242795 100644 --- a/tests/queries/0_stateless/05484_crosstab_window_merge_ordinary_states.reference +++ b/tests/queries/0_stateless/05484_crosstab_window_merge_ordinary_states.reference @@ -1,9 +1,9 @@ -- each row: window layout equals ordinary layout -60 0 0 0 0 0 0 0 0 0 0 0 0 0 0 +60 0 0 0 0 0 0 0 0 0 0 0 0 0 -- whole partition: window merge, then the plain aggregate -0 0.158611 0.158611 -1 0.158611 0.158611 -2 0.163199 0.163199 -0 0.158611 0.158611 -1 0.158611 0.158611 -2 0.163199 0.163199 +0 0.158611 +1 0.158611 +2 0.163199 +0 0.158611 +1 0.158611 +2 0.163199 diff --git a/tests/queries/0_stateless/05484_crosstab_window_merge_ordinary_states.sql b/tests/queries/0_stateless/05484_crosstab_window_merge_ordinary_states.sql index cf8e8b739de50..d663e4d720660 100644 --- a/tests/queries/0_stateless/05484_crosstab_window_merge_ordinary_states.sql +++ b/tests/queries/0_stateless/05484_crosstab_window_merge_ordinary_states.sql @@ -25,8 +25,7 @@ SELECT countIf(toString(resample) != toString(resample_ordinary)), countIf(toString(distinct) != toString(distinct_ordinary)), countIf(toString(tuple_merge) != toString(tuple_ordinary)), - countIf(toString(arg_max) != toString(arg_max_ordinary)), - countIf(toString(map_merge) != toString(map_ordinary)) + countIf(toString(arg_max) != toString(arg_max_ordinary)) FROM ( SELECT @@ -55,9 +54,7 @@ FROM round(tupleElement(contingencyTupleMerge(s_tuple) OVER w, 1), 6) + 0 AS tuple_merge, round(tupleElement(finalizeAggregation(contingencyTupleMergeState(s_tuple) OVER w), 1), 6) + 0 AS tuple_ordinary, round(contingencyArgMaxMerge(s_arg_max) OVER w, 6) + 0 AS arg_max, - round(finalizeAggregation(contingencyArgMaxMergeState(s_arg_max) OVER w), 6) + 0 AS arg_max_ordinary, - mapApply((key, value) -> (key, round(value, 6) + 0), contingencyMapMerge(s_map) OVER w) AS map_merge, - mapApply((key, value) -> (key, round(value, 6) + 0), finalizeAggregation(contingencyMapMergeState(s_map) OVER w)) AS map_ordinary + round(finalizeAggregation(contingencyArgMaxMergeState(s_arg_max) OVER w), 6) + 0 AS arg_max_ordinary FROM ( SELECT @@ -75,8 +72,7 @@ FROM contingencyResampleState(0, 20, 10)(b, c, k) AS s_resample, contingencyDistinctState(b, c) AS s_distinct, contingencyTupleState(tuple(b), tuple(c)) AS s_tuple, - contingencyArgMaxState(b, c, k) AS s_arg_max, - contingencyMapState(b, c, p) AS s_map + contingencyArgMaxState(b, c, k) AS s_arg_max FROM t_crosstab_window_merge GROUP BY p, k ) @@ -84,9 +80,9 @@ FROM ); SELECT '-- whole partition: window merge, then the plain aggregate'; -SELECT DISTINCT p, round(contingencyMerge(s) OVER (PARTITION BY p), 6), round((contingencyMapMerge(sm) OVER (PARTITION BY p))[p], 6) -FROM (SELECT p, contingencyState(b, c) AS s, contingencyMapState(b, c, p) AS sm FROM t_crosstab_window_merge GROUP BY p, k) +SELECT DISTINCT p, round(contingencyMerge(s) OVER (PARTITION BY p), 6) +FROM (SELECT p, contingencyState(b, c) AS s FROM t_crosstab_window_merge GROUP BY p, k) ORDER BY p; -SELECT p, round(contingency(b, c), 6), round(contingencyMap(b, c, p)[p], 6) FROM t_crosstab_window_merge GROUP BY p ORDER BY p; +SELECT p, round(contingency(b, c), 6) FROM t_crosstab_window_merge GROUP BY p ORDER BY p; DROP TABLE t_crosstab_window_merge; diff --git a/tests/queries/0_stateless/05485_crosstab_window_merge_wrapped_states.sql b/tests/queries/0_stateless/05485_crosstab_window_merge_wrapped_states.sql index 5d23cdd71bd03..3b88c7f2c37c7 100644 --- a/tests/queries/0_stateless/05485_crosstab_window_merge_wrapped_states.sql +++ b/tests/queries/0_stateless/05485_crosstab_window_merge_wrapped_states.sql @@ -45,7 +45,7 @@ FROM k, contingencyTupleIfState((b_any, c_any), (c_any, d_any), b_any != 1) OVER w AS s_tuple_if, contingencyTupleArrayState([(b_any, c_any)], [(c_any, d_any)]) OVER w AS s_tuple_array, - contingencyMergeMapState(s0, k % 3) OVER w AS s_merge_map, + contingencyMergeMapState(map(k % 3, s0)) OVER w AS s_merge_map, contingencyMergeForEachState(if(k % 2 = 0, [s0], [s0, s1])) OVER w AS s_merge_for_each FROM ( @@ -63,7 +63,7 @@ SELECT '-- whole partition: window merges of window states, then the plain aggre SELECT DISTINCT p, round((contingencyMergeMapMerge(s) OVER (PARTITION BY p))[0], 6) FROM ( - SELECT p, contingencyMergeMapState(s0, 0) OVER (PARTITION BY p) AS s + SELECT p, contingencyMergeMapState(map(0, s0)) OVER (PARTITION BY p) AS s FROM (SELECT p, k, contingencyState(b, c) AS s0 FROM t_crosstab_window_wrapped GROUP BY p, k) ) ORDER BY p; From c55da2430d811b5fa2f8c0de5aaa4853d99a7607 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 15 Nov 2026 11:29:27 +0000 Subject: [PATCH 193/219] Backport #131269 to 26.8: Reject negative and overflowing values of `Seconds` settings --- src/Core/SettingsFields.cpp | 10 ++--- src/Core/tests/gtest_settings.cpp | 43 +++++++++++++++++++ .../02294_decimal_second_errors.reference | 2 - .../02294_decimal_second_errors.sql | 4 +- .../02466_distributed_query_profiler.sql | 2 - ...000_too_big_max_execution_time_setting.sql | 1 + ...oo_negative_max_execution_time_setting.sql | 7 +-- ...327_wait_timeout_chrono_overflow.reference | 1 - .../04327_wait_timeout_chrono_overflow.sql | 3 -- 9 files changed, 52 insertions(+), 21 deletions(-) diff --git a/src/Core/SettingsFields.cpp b/src/Core/SettingsFields.cpp index faeaadc691e0a..63f594dbbeb9b 100644 --- a/src/Core/SettingsFields.cpp +++ b/src/Core/SettingsFields.cpp @@ -339,14 +339,14 @@ namespace if (d != 0.0 && !std::isnormal(d)) throw Exception( ErrorCodes::CANNOT_PARSE_NUMBER, "A setting's value in seconds must be a normal floating point number or zero. Got {}", d); -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wimplicit-const-int-float-conversion" - if (d * 1000000 > std::numeric_limits::max() || d * 1000000 < std::numeric_limits::min()) + if (d < 0) + throw Exception(ErrorCodes::BAD_ARGUMENTS, "A setting's value in seconds cannot be negative. Got {}", d); + const Float64 microseconds = d * 1000000; + if (accurate::greaterOp(microseconds, std::numeric_limits::max())) throw Exception( ErrorCodes::BAD_ARGUMENTS, "Cannot convert seconds to microseconds: the setting's value in seconds is too big: {}", d); -#pragma clang diagnostic pop - return static_cast(d * 1000000); + return static_cast(microseconds); } } diff --git a/src/Core/tests/gtest_settings.cpp b/src/Core/tests/gtest_settings.cpp index 963c74dbcbef7..83fd74c4e82ef 100644 --- a/src/Core/tests/gtest_settings.cpp +++ b/src/Core/tests/gtest_settings.cpp @@ -23,6 +23,7 @@ namespace DB::ErrorCodes extern const int ATTEMPT_TO_READ_AFTER_EOF; extern const int TOO_LARGE_STRING_SIZE; extern const int INCORRECT_DATA; + extern const int BAD_ARGUMENTS; extern const int ARGUMENT_OUT_OF_BOUND; } @@ -262,6 +263,48 @@ GTEST_TEST(SettingFieldTimespan, SecondsParseFromStringChecksTheRange) /// A value that does not fit Int64 microseconds is rejected, the same as through Field. ASSERT_THROW(seconds.parseFromString("1e30"), DB::Exception); + ASSERT_THROW(seconds.parseFromString("-1"), DB::Exception); + ASSERT_THROW(seconds.parseFromString("9223372036854.775"), DB::Exception); + ASSERT_EQ(seconds.totalSeconds(), 300); + + seconds.parseFromString("9223372036854"); + ASSERT_GT(seconds.totalSeconds(), 0); +} + +GTEST_TEST(Settings, SecondsOnTheWireRejectNegativeAndOverflowingValues) +{ + auto wire = [](const String & value) + { + WriteBufferFromOwnString out; + BaseSettingsHelpers::writeString("max_execution_time", out); + BaseSettingsHelpers::writeFlags(BaseSettingsHelpers::Flags{}, out); + BaseSettingsHelpers::writeString(value, out); + BaseSettingsHelpers::writeString("", out); /// end of settings + return out.str(); + }; + + for (const String & value : {String("-1"), String("9223372036854.775")}) + { + const String bytes = wire(value); + ReadBufferFromString in(bytes); + Settings settings; + try + { + settings.read(in, SettingsWriteFormat::STRINGS_WITH_FLAGS); + FAIL() << value << " was accepted"; + } + catch (const Exception & e) + { + ASSERT_EQ(e.code(), ErrorCodes::BAD_ARGUMENTS); + } + } + + const String bytes = wire("60"); + ReadBufferFromString in(bytes); + Settings settings; + settings.read(in, SettingsWriteFormat::STRINGS_WITH_FLAGS); + ASSERT_TRUE(settings.isChanged("max_execution_time")); + ASSERT_EQ(settings.get("max_execution_time"), Field(Float64(60))); } GTEST_TEST(SettingsTier, GetTierDecodesEveryEncoding) diff --git a/tests/queries/0_stateless/02294_decimal_second_errors.reference b/tests/queries/0_stateless/02294_decimal_second_errors.reference index a9e2f17562ae7..98fb6a6865639 100644 --- a/tests/queries/0_stateless/02294_decimal_second_errors.reference +++ b/tests/queries/0_stateless/02294_decimal_second_errors.reference @@ -2,5 +2,3 @@ 1 1 1 -1 -1 diff --git a/tests/queries/0_stateless/02294_decimal_second_errors.sql b/tests/queries/0_stateless/02294_decimal_second_errors.sql index b9b6d0a62238f..b7cb062d20fa8 100644 --- a/tests/queries/0_stateless/02294_decimal_second_errors.sql +++ b/tests/queries/0_stateless/02294_decimal_second_errors.sql @@ -1,11 +1,11 @@ SELECT 1 SETTINGS max_execution_time=NaN; -- { clientError CANNOT_PARSE_NUMBER } SELECT 1 SETTINGS max_execution_time=Infinity; -- { clientError CANNOT_PARSE_NUMBER }; SELECT 1 SETTINGS max_execution_time=-Infinity; -- { clientError CANNOT_PARSE_NUMBER }; +SELECT 1 SETTINGS max_execution_time=-0.5; -- { clientError BAD_ARGUMENTS } +SELECT 1 SETTINGS max_execution_time=-1; -- { clientError BAD_ARGUMENTS } -- Ok values -SELECT 1 SETTINGS max_execution_time=-0.5; SELECT 1 SETTINGS max_execution_time=5.5; -SELECT 1 SETTINGS max_execution_time=-1; SELECT 1 SETTINGS max_execution_time=0.0; SELECT 1 SETTINGS max_execution_time=-0.0; SELECT 1 SETTINGS max_execution_time=10; diff --git a/tests/queries/0_stateless/02466_distributed_query_profiler.sql b/tests/queries/0_stateless/02466_distributed_query_profiler.sql index 171cc2a75638c..3f50b79f72772 100644 --- a/tests/queries/0_stateless/02466_distributed_query_profiler.sql +++ b/tests/queries/0_stateless/02466_distributed_query_profiler.sql @@ -7,8 +7,6 @@ select * from remote('127.{2,4}', view( -- LIMIT is to activate query cancellation in case of enough rows already read. limit 10 settings - -- This is to avoid draining in background and got the exception during query execution - drain_timeout=-1, -- This is to activate as much signals as possible to trigger EINTR query_profiler_real_time_period_ns=1, -- This is to use MultiplexedConnections diff --git a/tests/queries/0_stateless/03000_too_big_max_execution_time_setting.sql b/tests/queries/0_stateless/03000_too_big_max_execution_time_setting.sql index 7aa86891b42db..c123ba16ce8dc 100644 --- a/tests/queries/0_stateless/03000_too_big_max_execution_time_setting.sql +++ b/tests/queries/0_stateless/03000_too_big_max_execution_time_setting.sql @@ -1,2 +1,3 @@ select 1 settings max_execution_time = 9223372036854775808; -- {clientError BAD_ARGUMENTS} +select 1 settings max_execution_time = 9223372036854.775; -- {clientError BAD_ARGUMENTS} diff --git a/tests/queries/0_stateless/04201_too_negative_max_execution_time_setting.sql b/tests/queries/0_stateless/04201_too_negative_max_execution_time_setting.sql index 4fdcf3c86d3b8..5c1738da9ea46 100644 --- a/tests/queries/0_stateless/04201_too_negative_max_execution_time_setting.sql +++ b/tests/queries/0_stateless/04201_too_negative_max_execution_time_setting.sql @@ -1,7 +1,2 @@ --- Test: exercises `float64AsSecondsToTimespan` with a very negative value triggering the --- `d * 1000000 < std::numeric_limits::min()` branch. --- Covers: src/Core/SettingsFields.cpp:320 — negative-overflow half of the new range check. --- The PR-shipped test (03000_too_big_max_execution_time_setting.sql) only covers the --- positive-overflow half (`d * 1000000 > max()`); the negative-overflow half is the actual --- UBSan trigger from the linked failure report (-9.22337e+24) but has no test exercising it. +-- A very negative value of a Seconds setting is rejected. SELECT 1 SETTINGS max_execution_time = -1e18; -- { clientError BAD_ARGUMENTS } diff --git a/tests/queries/0_stateless/04327_wait_timeout_chrono_overflow.reference b/tests/queries/0_stateless/04327_wait_timeout_chrono_overflow.reference index 1a787f08934dc..2210e9db18b28 100644 --- a/tests/queries/0_stateless/04327_wait_timeout_chrono_overflow.reference +++ b/tests/queries/0_stateless/04327_wait_timeout_chrono_overflow.reference @@ -1,7 +1,6 @@ 1 1 1 -1 qc_04327_top 1 0 diff --git a/tests/queries/0_stateless/04327_wait_timeout_chrono_overflow.sql b/tests/queries/0_stateless/04327_wait_timeout_chrono_overflow.sql index f4b0d6fe99b84..b0cdb5e5d0616 100644 --- a/tests/queries/0_stateless/04327_wait_timeout_chrono_overflow.sql +++ b/tests/queries/0_stateless/04327_wait_timeout_chrono_overflow.sql @@ -9,9 +9,6 @@ SELECT 1 SETTINGS interactive_delay = 100000000000000000; -- the query just acquires the lock and completes. SELECT count() > 0 FROM numbers(1000) SETTINGS lock_acquire_timeout = 100000000000; --- A negative lock_acquire_timeout must saturate to an immediate deadline, not underflow now() + timeout. -SELECT count() > 0 FROM numbers(1000) SETTINGS lock_acquire_timeout = -100000000000; - -- SYSTEM SYNC MERGES builds its deadline as now() + max_execution_time; a huge value must not overflow. -- SYSTEM commands take no trailing SETTINGS clause, so the timeout is set on the session. With all -- scheduled parts already covered the command returns immediately, so this just exercises the clamp. From 5dd433a97bcca73e3c8b554da9bbac9e56c383e2 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 15 Nov 2026 13:58:19 +0000 Subject: [PATCH 194/219] Backport #131730 to 26.8: Require `${data}` in the result set format of the `Template` input format --- .../Formats/Impl/TemplateRowInputFormat.cpp | 2 ++ ..._template_resultset_without_data.reference | 2 ++ .../05771_template_resultset_without_data.sh | 22 +++++++++++++++++++ 3 files changed, 26 insertions(+) create mode 100644 tests/queries/0_stateless/05771_template_resultset_without_data.reference create mode 100755 tests/queries/0_stateless/05771_template_resultset_without_data.sh diff --git a/src/Processors/Formats/Impl/TemplateRowInputFormat.cpp b/src/Processors/Formats/Impl/TemplateRowInputFormat.cpp index ce110de9a6472..e2e9f2370eee8 100644 --- a/src/Processors/Formats/Impl/TemplateRowInputFormat.cpp +++ b/src/Processors/Formats/Impl/TemplateRowInputFormat.cpp @@ -345,6 +345,8 @@ TemplateFormatReader::TemplateFormatReader( format.throwInvalidFormat("XML deserialization is not supported", i); } } + if (!has_data) + format.throwInvalidFormat("${data} must occur in the format string for result set", format.columnsCount()); /// Validate format string for rows for (size_t i = 0; i < row_format.columnsCount(); ++i) diff --git a/tests/queries/0_stateless/05771_template_resultset_without_data.reference b/tests/queries/0_stateless/05771_template_resultset_without_data.reference new file mode 100644 index 0000000000000..b989368440693 --- /dev/null +++ b/tests/queries/0_stateless/05771_template_resultset_without_data.reference @@ -0,0 +1,2 @@ +1 +INVALID_TEMPLATE_FORMAT diff --git a/tests/queries/0_stateless/05771_template_resultset_without_data.sh b/tests/queries/0_stateless/05771_template_resultset_without_data.sh new file mode 100755 index 0000000000000..43ffdcf70052c --- /dev/null +++ b/tests/queries/0_stateless/05771_template_resultset_without_data.sh @@ -0,0 +1,22 @@ +#!/usr/bin/env bash + +# The format string for the result set of the `Template` input format must contain `${data}`. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +dir="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}" +mkdir -p "$dir" +printf 'prefix ${data}' > "$dir/resultset_ok.format" +printf 'prefix' > "$dir/resultset_without_data.format" +printf '${x:CSV}' > "$dir/row.format" + +printf 'prefix 1' | $CLICKHOUSE_LOCAL --input-format Template --structure 'x UInt64' \ + --format_template_resultset "$dir/resultset_ok.format" --format_template_row "$dir/row.format" -q "SELECT * FROM table" + +printf 'prefix' | $CLICKHOUSE_LOCAL --input-format Template --structure 'x UInt64' \ + --format_template_resultset "$dir/resultset_without_data.format" --format_template_row "$dir/row.format" -q "SELECT * FROM table" 2>&1 \ + | grep -o -m1 'INVALID_TEMPLATE_FORMAT' + +rm -rf "$dir" From 9e17e585183b74186b42c59009886e551f6f5aae Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 15 Nov 2026 17:59:55 +0000 Subject: [PATCH 195/219] Backport #131795 to 26.8: Reserve the appended byte when reading legacy `String` states of `any`/`min`/`max` --- src/AggregateFunctions/SingleValueData.cpp | 8 ++++++++ ...value_string_state_appended_byte.reference | 3 +++ ...ingle_value_string_state_appended_byte.sql | 19 +++++++++++++++++++ 3 files changed, 30 insertions(+) create mode 100644 tests/queries/0_stateless/05800_single_value_string_state_appended_byte.reference create mode 100644 tests/queries/0_stateless/05800_single_value_string_state_appended_byte.sql diff --git a/src/AggregateFunctions/SingleValueData.cpp b/src/AggregateFunctions/SingleValueData.cpp index 032839a38ebac..c60a7f2ec99ad 100644 --- a/src/AggregateFunctions/SingleValueData.cpp +++ b/src/AggregateFunctions/SingleValueData.cpp @@ -1381,6 +1381,14 @@ void SingleValueDataString::read(ReadBuffer & buf, const ISerialization & /*seri } else { + /// The buffer was allocated for `size - 1` bytes above, so it may have no room for the appended byte. + if (capacity < size) + { + /// Arena allocations remain valid until the Arena is destroyed. + const char * old_data = large_data; + allocateLargeDataIfNeeded(size, arena); + memcpy(large_data, old_data, size - 1); + } large_data[size - 1] = last_char; } ++size; diff --git a/tests/queries/0_stateless/05800_single_value_string_state_appended_byte.reference b/tests/queries/0_stateless/05800_single_value_string_state_appended_byte.reference new file mode 100644 index 0000000000000..0893ef348b340 --- /dev/null +++ b/tests/queries/0_stateless/05800_single_value_string_state_appended_byte.reference @@ -0,0 +1,3 @@ +0 1 +1 1 +257 diff --git a/tests/queries/0_stateless/05800_single_value_string_state_appended_byte.sql b/tests/queries/0_stateless/05800_single_value_string_state_appended_byte.sql new file mode 100644 index 0000000000000..fd8b128ec3a10 --- /dev/null +++ b/tests/queries/0_stateless/05800_single_value_string_state_appended_byte.sql @@ -0,0 +1,19 @@ +-- A serialized `String` state of `any`/`min`/`max` from certain old versions ends with a non-zero byte +-- instead of the zero terminator, and that byte is appended to the value. A value whose length without +-- that byte is a power of two needs one more byte than allocated for the string. The adjacent state of +-- the next row would show a corruption of the appended byte. + +SELECT + number, + finalizeAggregation(s) = if(number = 0, concat(repeat('A', 64), 'B'), '') +FROM +( + SELECT + number, + CAST(if(number = 0, concat(unhex('41000000'), repeat('A', 64), 'B'), unhex('FFFFFFFF')), + 'AggregateFunction(any, String)') AS s + FROM numbers(2) +) +ORDER BY number; + +SELECT length(finalizeAggregation(CAST(concat(unhex('01010000'), repeat('A', 256), 'B'), 'AggregateFunction(max, String)'))); From a6e5e2fdb5efba67a141024d227957e4d66c032f Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Sun, 15 Nov 2026 23:10:59 +0000 Subject: [PATCH 196/219] Backport #131366 to 26.8: Check the scale in `writeDecimalFractional` in release builds --- src/IO/WriteHelpers.h | 10 +++++ src/IO/tests/gtest_WriteHelpers.cpp | 64 +++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+) diff --git a/src/IO/WriteHelpers.h b/src/IO/WriteHelpers.h index 9c7106fc03d43..775a63f13195d 100644 --- a/src/IO/WriteHelpers.h +++ b/src/IO/WriteHelpers.h @@ -1354,6 +1354,16 @@ void writeText(Decimal x, UInt32 scale, WriteBuffer & ostr, bool trailing_zer bool fixed_fractional_length = false, UInt32 fractional_length = 0, bool force_decimal_point = false) { + /// The scale of a `DecimalField` is not tied to a data type, so check it in the release build before any + /// arithmetic that depends on it: a scale wider than the type overflows `scaleMultiplier` and the rounding + /// below, and a scale of 2^31 or more becomes a negative exponent, which gives a division by zero. + constexpr size_t max_scale = DecimalUtils::max_precision>; + constexpr size_t max_fractional_length = std::numeric_limits::digits10; + if (scale > max_scale || fractional_length > max_fractional_length) [[unlikely]] + throw Exception(ErrorCodes::ARGUMENT_OUT_OF_BOUND, + "Cannot write a decimal with scale {} and {} fractional digits: the maximum scale is {} and at most {} fractional digits are supported", + scale, fractional_length, max_scale, max_fractional_length); + /// A fixed fractional length narrower than the scale rounds the value away from zero. Round the whole value /// before it is split, so that a carry out of the fractional part reaches the whole part: rounding the /// fractional part alone turns 9.995 into 9.00 instead of 10.00. diff --git a/src/IO/tests/gtest_WriteHelpers.cpp b/src/IO/tests/gtest_WriteHelpers.cpp index b3c7062be5804..7319180f78cda 100644 --- a/src/IO/tests/gtest_WriteHelpers.cpp +++ b/src/IO/tests/gtest_WriteHelpers.cpp @@ -3,6 +3,9 @@ #include #include #include +#include +#include +#include using namespace DB; @@ -30,3 +33,64 @@ TEST(WriteHelpersTest, ToStringWithFinalSeparatorTest) EXPECT_EQ(toStringWithFinalSeparator(v, " or "), "'AAA', 'BBB', 'CCC' or 'DDD'"); } } + + +template +static void expectDecimalTextThrows(UInt32 scale, bool fixed_fractional_length = false, UInt32 fractional_length = 0) +{ + WriteBufferFromOwnString buf; + try + { + writeText(Decimal(1), scale, buf, false, fixed_fractional_length, fractional_length); + FAIL() << "expected ARGUMENT_OUT_OF_BOUND for scale " << scale << ", fractional length " << fractional_length; + } + catch (const Exception & e) + { + EXPECT_EQ(e.code(), ErrorCodes::ARGUMENT_OUT_OF_BOUND); + } +} + +template +static std::string decimalText(T value, UInt32 scale, bool fixed_fractional_length = false, UInt32 fractional_length = 0) +{ + WriteBufferFromOwnString buf; + writeText(Decimal(value), scale, buf, false, fixed_fractional_length, fractional_length); + return buf.str(); +} + +template +static void checkDecimalScaleBoundary() +{ + constexpr UInt32 max_scale = DecimalUtils::max_precision>; + expectDecimalTextThrows(max_scale + 1); + /// Rounding to fewer fractional digits multiplies by a power of ten that depends on the scale. + expectDecimalTextThrows(max_scale + 1, true, 0); + EXPECT_NO_THROW(decimalText(1, max_scale)); + EXPECT_NO_THROW(decimalText(1, max_scale, true, 0)); +} + +/// A `DecimalField` does not validate its scale, and `FieldVisitorToString` formats it with `writeText`. +/// A scale of 2^31 or more used to reach a division by zero in `DecimalUtils::getWholePart`, +/// and a scale wider than the type overflowed the rounding arithmetic. +TEST(WriteHelpersTest, DecimalTextOutOfRangeScaleThrows) +{ + for (UInt32 scale : {77u, 0x80000000u, 0xffffffffu}) + { + expectDecimalTextThrows(scale); + expectDecimalTextThrows(scale); + expectDecimalTextThrows(scale); + expectDecimalTextThrows(scale); + } + expectDecimalTextThrows(0x80000000u, true, 2); + + checkDecimalScaleBoundary(); + checkDecimalScaleBoundary(); + checkDecimalScaleBoundary(); + checkDecimalScaleBoundary(); + + /// 77 fractional digits are supported (`toDecimalString` accepts them), 78 are not. + expectDecimalTextThrows(2, true, 78); + EXPECT_EQ(decimalText(12345, 2, true, 77), "123.45" + std::string(75, '0')); + + EXPECT_EQ(decimalText(12345, 2), "123.45"); +} From aa3aa254144c58bfbf45838e585ac8bf32605ecd Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 16 Nov 2026 12:28:02 +0000 Subject: [PATCH 197/219] Backport #123103 to 26.8: Resolve a JOIN USING key from an alias only for the outermost JOIN --- src/Analyzer/Resolve/QueryAnalyzer.cpp | 77 +++++++++++++++--- src/Core/Settings.cpp | 2 +- ..._using_alias_outermost_join_only.reference | 32 ++++++++ ...6_join_using_alias_outermost_join_only.sql | 78 +++++++++++++++++++ 4 files changed, 178 insertions(+), 11 deletions(-) create mode 100644 tests/queries/0_stateless/05296_join_using_alias_outermost_join_only.reference create mode 100644 tests/queries/0_stateless/05296_join_using_alias_outermost_join_only.sql diff --git a/src/Analyzer/Resolve/QueryAnalyzer.cpp b/src/Analyzer/Resolve/QueryAnalyzer.cpp index 443e177066048..7bd68bd9b7c6e 100644 --- a/src/Analyzer/Resolve/QueryAnalyzer.cpp +++ b/src/Analyzer/Resolve/QueryAnalyzer.cpp @@ -5580,6 +5580,8 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS if (!matched_node) return nullptr; + /// The query node only scopes the resolution: identifier lookups and matchers inside the expression reach the left + /// side through its join tree, and it names the scope in error messages. auto left_subquery = std::make_shared(query_node->getMutableContext()); left_subquery->getProjection().getNodes().push_back(matched_node->clone()); auto subquery_join_tree = left_table_expression; @@ -5589,24 +5591,65 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS IdentifierResolveScope & left_subquery_scope = createIdentifierResolveScope(left_subquery, nullptr /*parent_scope*/); /// We are using alias column mechanism for USING column from projection. - /// It will be calculated right after reading, so column will be not nullable there. + /// It will be calculated right after reading, so column will be not nullable there, and a column of a USING key + /// inside the left side keeps the type it is read with rather than the key's supertype, the same way as in PREWHERE. left_subquery_scope.join_use_nulls = false; + left_subquery_scope.allow_resolve_from_using = false; - resolveQuery(left_subquery, left_subquery_scope); + /// The left side is already resolved (a USING list inside it holds columns, not identifiers), so it must not be + /// resolved again; the scope takes over what resolveQuery builds for a join tree: the table expression data of + /// the leaves, their aliases (a qualified matcher such as `t1.*` finds its table only through them) and the + /// join counters. + for (const auto & table_expression : extractTableExpressions(subquery_join_tree, true /*add_array_join*/)) + { + left_subquery_scope.registered_table_expression_nodes.insert(table_expression); + if (table_expression->hasAlias()) + left_subquery_scope.aliases.alias_name_to_table_expression_node.emplace(table_expression->getAlias(), table_expression); + if (table_expression->getNodeType() != QueryTreeNodeType::ARRAY_JOIN) + left_subquery_scope.table_expression_node_to_data.emplace(table_expression, scope_.getTableExpressionDataOrThrow(table_expression)); + } + + for (std::vector join_tree_nodes{subquery_join_tree.get()}; !join_tree_nodes.empty();) + { + const auto * join_tree_node = join_tree_nodes.back(); + join_tree_nodes.pop_back(); + + if (const auto * join = join_tree_node->as()) + { + ++left_subquery_scope.joins_count; + if (join->isUsingJoinExpression()) + ++left_subquery_scope.using_joins_count; + join_tree_nodes.push_back(join->getLeftTableExpressionNode().get()); + join_tree_nodes.push_back(join->getRightTableExpressionNode().get()); + } + else if (const auto * cross_join = join_tree_node->as()) + { + left_subquery_scope.joins_count += cross_join->getTableExpressions().size() - 1; + for (const auto & cross_join_table_expression : cross_join->getTableExpressions()) + join_tree_nodes.push_back(cross_join_table_expression.get()); + } + else if (const auto * array_join = join_tree_node->as()) + join_tree_nodes.push_back(array_join->getTableExpressionNode().get()); + } - const auto & resolved_nodes = left_subquery->getProjection().getNodes(); - if (resolved_nodes.size() == 1) + auto & projection_node = left_subquery->getProjection().getNodes().front(); + QueryExpressionsAliasVisitor alias_visitor(left_subquery_scope.aliases); + alias_visitor.visit(projection_node); + resolveExpressionNode(projection_node, left_subquery_scope, false /*allow_lambda_expression*/, false /*allow_table_expression*/); + + /// A matcher resolves into a list of columns. + if (projection_node->getNodeType() != QueryTreeNodeType::LIST) { /// Added column should not conflict with existing column names NameSet existing_columns; if (!getColumnsFromTableExpression(left_table_expression, existing_columns)) return nullptr; - NameAndTypePair column_name_type(identifier_full_name_, resolved_nodes.front()->getResultType()); + NameAndTypePair column_name_type(identifier_full_name_, projection_node->getResultType()); while (existing_columns.contains(column_name_type.name)) column_name_type.name = "_" + column_name_type.name; - auto [expression_source, is_single_source] = getExpressionSource(resolved_nodes.front()); + auto [expression_source, is_single_source] = getExpressionSource(projection_node); /// Do not support `SELECT t1.a + t2.a AS id ... USING id` if (!is_single_source) return nullptr; @@ -5621,12 +5664,26 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS return nullptr; /// Create ColumnNode with expression from parent projection - return std::make_shared(std::move(column_name_type), resolved_nodes.front(), + return std::make_shared(std::move(column_name_type), projection_node, expression_source ? expression_source : left_table_expression); } return nullptr; }; + /// The old analyzer rewrote a query with several JOINs into nested subqueries, so an inner JOIN never saw an alias + /// of the query and resolved its key from the left table. Only the outermost JOIN keeps the alias lookup. + auto is_outermost_join = [&join_node, &scope]() -> bool + { + const auto * scope_query_node = scope.scope_node ? scope.scope_node->as() : nullptr; + if (!scope_query_node) + return false; + + IQueryTreeNode * join_tree_root = scope_query_node->getJoinTreeNode().get(); + while (join_tree_root->getNodeType() == QueryTreeNodeType::ARRAY_JOIN) + join_tree_root = join_tree_root->as().getTableExpressionNode().get(); + return join_tree_root == join_node.get(); + }; + for (auto & join_using_node : join_using_list.getNodes()) { nested_alias_matched = false; @@ -5656,9 +5713,9 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS /** With `analyzer_compatibility_join_using_top_level_identifier` alias in projection has higher priority than column from left table. * But if aliased expression cannot be resolved from left table, we get UNKNOW_IDENTIFIER error, * despite the fact that column from USING could be resolved from left table. - * It's compatible with a default behavior for old analyzer. + * It's compatible with a default behavior for old analyzer, which also never used an alias for an inner JOIN. */ - if (settings[Setting::analyzer_compatibility_join_using_top_level_identifier]) + if (settings[Setting::analyzer_compatibility_join_using_top_level_identifier] && is_outermost_join()) result_left_table_expression = try_resolve_identifier_from_query_projection(identifier_full_name, join_node_typed.getLeftTableExpressionNodeTyped(), scope); /// A nested-alias USING key cannot ship to a remote server (rendered SQL keeps only top-level projection aliases), so disable parallel replicas for such a query. @@ -5713,7 +5770,7 @@ void QueryAnalyzer::resolveJoin(QueryTreeNodePtr & join_node, IdentifierResolveS { String extra_message; const QueryNode * query_node = scope.scope_node ? scope.scope_node->as() : nullptr; - if (!settings[Setting::analyzer_compatibility_join_using_top_level_identifier] && query_node) + if (!settings[Setting::analyzer_compatibility_join_using_top_level_identifier] && query_node && is_outermost_join()) { if (auto matched_node = find_aliased_node_in_projection(query_node, identifier_full_name)) extra_message = fmt::format( diff --git a/src/Core/Settings.cpp b/src/Core/Settings.cpp index 21c96f7a0d4cd..43a11d4c3e915 100644 --- a/src/Core/Settings.cpp +++ b/src/Core/Settings.cpp @@ -8354,7 +8354,7 @@ Cloud default value: `1`. Allow the analyzer. )", IMPORTANT, enable_analyzer) \ DECLARE(Bool, analyzer_compatibility_join_using_top_level_identifier, false, R"( -Force to resolve identifier in JOIN USING from projection (for example, in `SELECT a + 1 AS b FROM t1 JOIN t2 USING (b)` join will be performed by `t1.a + 1 = t2.b`, rather then `t1.b = t2.b`). Aliases defined on subexpressions inside the SELECT list are also considered (for example, in `SELECT uniqExact(a + 1 AS b) FROM t1 JOIN t2 USING (b)` the join is performed by `t1.a + 1 = t2.b`). When the matching alias is defined on a subexpression inside the SELECT list rather than as a top-level alias, parallel replicas are disabled for the query. For queries sent to remote servers (`Distributed` tables, the `remote` table function), such a query is rejected with an exception only when the identifier cannot be resolved on the remote server at all; if the alias shadows a real column of the left table, the remote server joins by that column instead, so the results may differ from local execution. +Force to resolve identifier in JOIN USING from projection (for example, in `SELECT a + 1 AS b FROM t1 JOIN t2 USING (b)` join will be performed by `t1.a + 1 = t2.b`, rather then `t1.b = t2.b`). Aliases defined on subexpressions inside the SELECT list are also considered (for example, in `SELECT uniqExact(a + 1 AS b) FROM t1 JOIN t2 USING (b)` the join is performed by `t1.a + 1 = t2.b`). In a query with several `JOIN`s only the outermost `JOIN` resolves its `USING` identifier from an alias; an inner `JOIN` resolves it from its left table, as the old analyzer did. When the matching alias is defined on a subexpression inside the SELECT list rather than as a top-level alias, parallel replicas are disabled for the query. For queries sent to remote servers (`Distributed` tables, the `remote` table function), such a query is rejected with an exception only when the identifier cannot be resolved on the remote server at all; if the alias shadows a real column of the left table, the remote server joins by that column instead, so the results may differ from local execution. )", 0) \ DECLARE(Bool, analyzer_compatibility_allow_compound_identifiers_in_unflatten_nested, true, R"( Allow to add compound identifiers to nested. This is a compatibility setting because it changes the query result. When disabled, `SELECT a.b.c FROM table ARRAY JOIN a` does not work, and `SELECT a FROM table` does not include `a.b.c` column into `Nested a` result. diff --git a/tests/queries/0_stateless/05296_join_using_alias_outermost_join_only.reference b/tests/queries/0_stateless/05296_join_using_alias_outermost_join_only.reference new file mode 100644 index 0000000000000..7bdd6e685505e --- /dev/null +++ b/tests/queries/0_stateless/05296_join_using_alias_outermost_join_only.reference @@ -0,0 +1,32 @@ +single JOIN: the alias is the key even when the left table has a column with the same name +2 2 7 +inner JOIN of two: the left column is the key +2 3 7 +2 0 +outermost JOIN of two: the alias is the key +2 2 7 +inner USING JOIN below the aliased outermost USING key +2 2 7 +three JOINs: inner keys from the left tables, the outermost key from the alias +110 120 2 2 7 +comma JOIN counts as a level +2 3 +ARRAY JOIN above the JOIN is not a level: the JOIN stays outermost +2 2 1 +2 2 2 +2 3 1 +a matcher inside the aliased key expression expands over the left side +(1,(1)) (1) +(1) (1) +(1) (1) +the key is computed before the joins of the left side: a column retyped by an inner USING keeps the type it is read with +(1) Tuple(UInt16) (1) Tuple(UInt16) +(1) Tuple(UInt16) (1) Tuple(UInt16) +2 UInt32 2 UInt16 +join_use_nulls: the key is computed before the inner RIGHT JOIN makes the left columns nullable +2 Nullable(UInt16) 2 UInt16 +a subquery has its own join tree +2 2 7 +2 3 +the reported query: the projection refers to the right side of the inner JOIN +a x 3 2026-01-01 00:00:00 diff --git a/tests/queries/0_stateless/05296_join_using_alias_outermost_join_only.sql b/tests/queries/0_stateless/05296_join_using_alias_outermost_join_only.sql new file mode 100644 index 0000000000000..30e748f4c80b6 --- /dev/null +++ b/tests/queries/0_stateless/05296_join_using_alias_outermost_join_only.sql @@ -0,0 +1,78 @@ +-- With `analyzer_compatibility_join_using_top_level_identifier` a `JOIN USING` identifier is resolved from an alias of the query. +-- The old analyzer rewrote a query with several JOINs into nested subqueries, so an inner JOIN never saw such an alias and +-- took the key from its left table. Only the outermost JOIN keeps the alias lookup; USING keys at different levels of the +-- join tree follow the same rule. + +SET enable_analyzer = 1; +SET analyzer_compatibility_join_using_top_level_identifier = 1; +SET joined_subquery_requires_alias = 0; + +SELECT 'single JOIN: the alias is the key even when the left table has a column with the same name'; +SELECT a + 1 AS b, t2.b AS right_b, t2.c FROM (SELECT 1 AS a, 3 AS b) t1 JOIN (SELECT 2 AS b, 7 AS c) t2 USING (b); + +SELECT 'inner JOIN of two: the left column is the key'; +SELECT a + 1 AS b, t2.b AS right_b, t2.c FROM (SELECT 1 AS a, 3 AS b) t1 JOIN (SELECT 3 AS b, 7 AS c) t2 USING (b) JOIN (SELECT 1 AS z) t3 ON 1 = 1; +-- The alias value would have matched the right side, the left column does not. +SELECT a + 1 AS b, t2.b AS right_b FROM (SELECT 1 AS a, 3 AS b) t1 LEFT JOIN (SELECT 2 AS b) t2 USING (b) JOIN (SELECT 1 AS z) t3 ON 1 = 1; +-- Without a left column the key of an inner JOIN cannot be resolved. +SELECT a + 1 AS b FROM (SELECT 1 AS a) t1 JOIN (SELECT 2 AS b) t2 USING (b) JOIN (SELECT 1 AS z) t3 ON 1 = 1; -- { serverError UNKNOWN_IDENTIFIER } + +SELECT 'outermost JOIN of two: the alias is the key'; +SELECT a + 1 AS b, t3.b AS right_b, t3.c FROM (SELECT 1 AS a) t1 JOIN (SELECT 1 AS z) t2 ON 1 = 1 JOIN (SELECT 2 AS b, 7 AS c) t3 USING (b); + +SELECT 'inner USING JOIN below the aliased outermost USING key'; +SELECT a + 1 AS b, t3.b AS right_b, t3.c FROM (SELECT 1 AS a, 10 AS k) t1 JOIN (SELECT 10 AS k) t2 USING (k) JOIN (SELECT 2 AS b, 7 AS c) t3 USING (b); + +SELECT 'three JOINs: inner keys from the left tables, the outermost key from the alias'; +SELECT k + 100 AS k, m + 100 AS m, a + 1 AS b, t4.b AS right_b, t4.c +FROM (SELECT 1 AS a, 10 AS k) t1 +JOIN (SELECT 10 AS k, 20 AS m) t2 USING (k) +JOIN (SELECT 20 AS m) t3 USING (m) +JOIN (SELECT 2 AS b, 7 AS c) t4 USING (b); + +SELECT 'comma JOIN counts as a level'; +SELECT a + 1 AS b, t2.b AS right_b FROM (SELECT 1 AS a, 3 AS b) t1 JOIN (SELECT 3 AS b) t2 USING (b), (SELECT 1 AS z) t3; + +SELECT 'ARRAY JOIN above the JOIN is not a level: the JOIN stays outermost'; +SELECT a + 1 AS b, t2.b AS right_b, x FROM (SELECT 1 AS a, 3 AS b, [1, 2] AS arr) t1 JOIN (SELECT 2 AS b) t2 USING (b) ARRAY JOIN arr AS x ORDER BY x; +-- A JOIN below an ARRAY JOIN that is itself below another JOIN is an inner JOIN. +SELECT a + 1 AS b, t2.b AS right_b, x FROM (SELECT 1 AS a, 3 AS b, [1] AS arr) t1 JOIN (SELECT 3 AS b) t2 USING (b) ARRAY JOIN arr AS x JOIN (SELECT 1 AS z) t3 ON 1 = 1; + +SELECT 'a matcher inside the aliased key expression expands over the left side'; +SELECT tuple(*) AS k, t2.k AS right_k FROM (SELECT 1 AS a) t1 JOIN (SELECT tuple(1) AS k) t2 USING (k); +SELECT tuple(t1.*) AS k, t2.k AS right_k FROM (SELECT 1 AS a) t1 JOIN (SELECT tuple(1) AS k) t2 USING (k); +SELECT tuple(t1.*) AS k, t3.k AS right_k FROM (SELECT 1 AS a) t1, (SELECT 1 AS z) t2 JOIN (SELECT tuple(1) AS k) t3 USING (k); + +SELECT 'the key is computed before the joins of the left side: a column retyped by an inner USING keeps the type it is read with'; +SELECT tuple(t1.a) AS k, toTypeName(k), t3.k AS right_k, toTypeName(right_k) +FROM (SELECT toUInt8(1) AS a) t1 JOIN (SELECT toUInt16(1) AS a) t2 USING (a) JOIN (SELECT tuple(toUInt16(1)) AS k) t3 USING (k); +SELECT tuple(t1.*) AS k, toTypeName(k), t3.k AS right_k, toTypeName(right_k) +FROM (SELECT toUInt8(1) AS a) t1 JOIN (SELECT toUInt16(1) AS a) t2 USING (a) JOIN (SELECT tuple(toUInt16(1)) AS k) t3 USING (k); +-- An unqualified matcher expands the key column of the inner USING JOIN from both of its tables, as in PREWHERE, +-- so the expression has no single source table and is not a key. +SELECT tuple(*) AS k, toTypeName(k), t3.k AS right_k, toTypeName(right_k) +FROM (SELECT toUInt8(1) AS a) t1 JOIN (SELECT toUInt16(1) AS a) t2 USING (a) JOIN (SELECT tuple(toUInt16(1)) AS k) t3 USING (k); -- { serverError UNKNOWN_IDENTIFIER } +SELECT a + 1 AS b, toTypeName(b), t3.b AS right_b, toTypeName(right_b) +FROM (SELECT toUInt8(1) AS a) t1 JOIN (SELECT toUInt16(1) AS a) t2 USING (a) JOIN (SELECT toUInt16(2) AS b) t3 USING (b); + +SELECT 'join_use_nulls: the key is computed before the inner RIGHT JOIN makes the left columns nullable'; +SET join_use_nulls = 1; +SELECT a + 1 AS b, toTypeName(b), t3.b AS right_b, toTypeName(right_b) +FROM (SELECT 1 AS a, 10 AS x) t1 RIGHT JOIN (SELECT 10 AS x) t2 USING (x) JOIN (SELECT 2 AS b) t3 USING (b); +SET join_use_nulls = 0; + +SELECT 'a subquery has its own join tree'; +SELECT s.b, s.right_b, t.c FROM (SELECT a + 1 AS b, t2.b AS right_b FROM (SELECT 1 AS a, 3 AS b) t1 JOIN (SELECT 2 AS b) t2 USING (b)) s JOIN (SELECT 7 AS c) t ON 1 = 1 JOIN (SELECT 1 AS z) u ON 1 = 1; +SELECT s.b, s.right_b FROM (SELECT a + 1 AS b, t2.b AS right_b FROM (SELECT 1 AS a, 3 AS b) t1 JOIN (SELECT 3 AS b) t2 USING (b) JOIN (SELECT 1 AS z) t3 ON 1 = 1) s; + +SELECT 'the reported query: the projection refers to the right side of the inner JOIN'; +WITH m AS (SELECT 'a' AS symbol, 'x' AS lp, 1 AS pos), + b AS (SELECT 'a' AS symbol, 'x' AS lp, 2 AS net_volume), + dttm AS (SELECT toDateTime('2026-01-01 00:00:00') AS Dttm) +SELECT if(m.symbol = '', b.symbol, m.symbol) AS symbol, + if(m.lp = '', b.lp, m.lp) AS lp, + b.net_volume + m.pos AS pos, + dttm.Dttm AS last_updated +FROM m +FULL OUTER JOIN b USING (symbol, lp) +FULL OUTER JOIN dttm ON 1 = 1; From be65d0e5a011272e60d4f161a3547db273969642 Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Mon, 16 Nov 2026 14:04:38 +0100 Subject: [PATCH 198/219] Accept the `Zlib OR MIT OR Apache-2.0` license of vendored Rust crates The vendored Rust crates of this backport include crates without a license file, such as `lending-iterator-proc_macros`, licensed under `Zlib OR MIT OR Apache-2.0`. `list-licenses.sh` failed on them, breaking the CMake configuration. This is the change made to `list-licenses.sh` on master in d4debff97a30. Co-Authored-By: Claude Opus 5.5 --- utils/list-licenses/list-licenses.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/utils/list-licenses/list-licenses.sh b/utils/list-licenses/list-licenses.sh index d827cebb41c64..009b52b2444f3 100755 --- a/utils/list-licenses/list-licenses.sh +++ b/utils/list-licenses/list-licenses.sh @@ -178,6 +178,7 @@ process_rust_crate() { [ "$LICENSE_TYPE" == "MIT/Apache-2.0" ] || [ "$LICENSE_TYPE" == "MIT OR Apache-2.0 OR LGPL-2.1-or-later" ] || [ "$LICENSE_TYPE" == "Zlib OR Apache-2.0 OR MIT" ] || + [ "$LICENSE_TYPE" == "Zlib OR MIT OR Apache-2.0" ] || [ "$LICENSE_TYPE" == "Apache-2.0 OR BSL-1.0 OR MIT" ] || [ "$LICENSE_TYPE" == "Apache-2.0 WITH LLVM-exception OR Apache-2.0 OR MIT" ] || [ "$LICENSE_TYPE" == "Apache-2.0 WITH LLVM-exception" ] || From 11558bc93535ccf6fe9ccc66b449f02e6f2bd8ca Mon Sep 17 00:00:00 2001 From: Konstantin Bogdanov Date: Mon, 16 Nov 2026 14:40:34 +0100 Subject: [PATCH 199/219] Do not disable `neon` for Rust in the ARMv8.0 compatibility build Rust `nightly-2026-11-02` rejects `-C target_feature=-neon` on `aarch64`, which fails the `arm_v80compat` build. NEON is mandatory in the `aarch64` ABI. This follows the change made to `cmake/cpu_features.cmake` on master in 0101cc633f29. Co-Authored-By: Claude Opus 5.5 --- cmake/cpu_features.cmake | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmake/cpu_features.cmake b/cmake/cpu_features.cmake index 3b1e94bc593f0..9e3c31297ab99 100644 --- a/cmake/cpu_features.cmake +++ b/cmake/cpu_features.cmake @@ -18,7 +18,7 @@ if (ARCH_AARCH64) # crc32 is optional in v8.0 and mandatory in v8.1. Enable it as __crc32()* is used in lot's of places and even very old ARM CPUs # support it. set (COMPILER_FLAGS "${COMPILER_FLAGS} -march=armv8+crc") - list(APPEND RUSTFLAGS_CPU "-C" "target_feature=+crc,-neon") + list(APPEND RUSTFLAGS_CPU "-C" "target_feature=+crc") else () # ARMv8.2 is quite ancient but the lowest common denominator supported by both Graviton 2 and 3 processors [1, 10]. In particular, it # includes LSE (made mandatory with ARMv8.1) which provides nice speedups without having to fall back to compat flag From 80434d416693aa99b874f8ba891a9e043c7d3d10 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 16 Nov 2026 14:47:12 +0000 Subject: [PATCH 200/219] Backport #115665 to 26.8: Fix a wedged mutation after a patch-mode lightweight delete --- src/Storages/MergeTree/MutateTask.cpp | 29 +-- ...ation_row_exists_from_patch_part.reference | 155 ++++++++++++++ ...98_mutation_row_exists_from_patch_part.sql | 195 ++++++++++++++++++ 3 files changed, 351 insertions(+), 28 deletions(-) create mode 100644 tests/queries/0_stateless/04798_mutation_row_exists_from_patch_part.reference create mode 100644 tests/queries/0_stateless/04798_mutation_row_exists_from_patch_part.sql diff --git a/src/Storages/MergeTree/MutateTask.cpp b/src/Storages/MergeTree/MutateTask.cpp index 80ed02c63cdf0..42653cba1036a 100644 --- a/src/Storages/MergeTree/MutateTask.cpp +++ b/src/Storages/MergeTree/MutateTask.cpp @@ -996,28 +996,6 @@ static void addRenamedColumnToColumnsSubstreams( new_columns_substreams.addSubstreamToLastColumn(ISerialization::getFileNameForRenamedColumnStream(old_name, new_name, substream)); } -static bool isDeletedMaskUpdated(const MutationCommand & command, const NameSet & storage_columns_set) -{ - if (storage_columns_set.contains(RowExistsColumn::name)) - return false; - - if (command.type == MutationCommand::READ_COLUMN) - return command.read_for_patch && command.column_name == RowExistsColumn::name; - - if (command.type == MutationCommand::UPDATE) - { - auto alter = command.ast(); - if (!alter || !alter->update_assignments) - return false; - return std::ranges::any_of(alter->update_assignments->children, [](const ASTPtr & child) - { - return child->as().column_name == RowExistsColumn::name; - }); - } - - return false; -} - /// Get the columns list of the resulting part in the same order as storage_columns. static std::tuple getColumnsForNewDataPart( @@ -1040,9 +1018,7 @@ getColumnsForNewDataPart( ColumnsDescription part_columns(source_part->getColumns()); NamesAndTypesList system_columns; - bool deleted_mask_updated = false; bool affects_all_columns = false; - bool supports_lightweight_deletes = source_part->supportLightweightDeleteMutate(); NameSet storage_columns_set; for (const auto & [name, _] : storage_columns) @@ -1052,9 +1028,6 @@ getColumnsForNewDataPart( { affects_all_columns |= command.affectsAllColumns(); - if (supports_lightweight_deletes) - deleted_mask_updated |= isDeletedMaskUpdated(command, storage_columns_set); - /// If we don't have this column in source part, than we don't need to materialize it if (!part_columns.has(command.column_name)) { @@ -1093,7 +1066,7 @@ getColumnsForNewDataPart( bool need_column = false; if (name == RowExistsColumn::name) - need_column = deleted_mask_updated || (part_columns.has(name) && !affects_all_columns); + need_column = updated_header.has(name) || (part_columns.has(name) && !affects_all_columns); else if (name == BlockNumberColumn::name || name == BlockOffsetColumn::name) need_column = part_columns.has(name) || updated_header.has(name); else diff --git a/tests/queries/0_stateless/04798_mutation_row_exists_from_patch_part.reference b/tests/queries/0_stateless/04798_mutation_row_exists_from_patch_part.reference new file mode 100644 index 0000000000000..909dc903c5b7f --- /dev/null +++ b/tests/queries/0_stateless/04798_mutation_row_exists_from_patch_part.reference @@ -0,0 +1,155 @@ +-- lightweight_update: heavyweight DELETE after a patch-mode lightweight DELETE +0 +4 4 +5 5 +7 7 +8 8 +10 10 +11 11 +13 13 +14 14 +16 16 +17 17 +19 19 +20 20 +22 22 +23 23 +0 +-- alter_update mode produces the same rows (oracle) +0 +4 4 +5 5 +7 7 +8 8 +10 10 +11 11 +13 13 +14 14 +16 16 +17 17 +19 19 +20 20 +22 22 +23 23 +-- both modes agree +1 +-- lightweight_update_force +0 +4 4 +5 5 +7 7 +8 8 +10 10 +11 11 +13 13 +14 14 +16 16 +17 17 +19 19 +20 20 +22 22 +23 23 +-- APPLY DELETED MASK and REWRITE PARTS reach the same code path +0 +2 2 +4 4 +5 5 +7 7 +8 8 +10 10 +11 11 +13 13 +14 14 +16 16 +17 17 +19 19 +20 20 +22 22 +23 23 +0 +0 +2 2 +4 4 +5 5 +7 7 +8 8 +10 10 +11 11 +13 13 +14 14 +16 16 +17 17 +19 19 +20 20 +22 22 +23 23 +-- a partial update keeps the mask, so masked rows stay hidden +0 +1 101 +2 2 +4 4 +5 105 +7 107 +8 8 +10 10 +11 111 +13 113 +14 14 +16 16 +17 117 +19 119 +20 20 +22 22 +23 123 +1 +-- a heavyweight DELETE matching no rows keeps the pending patch +0 +16 0 +-- compact parts +0 +4 4 +5 5 +7 7 +8 8 +10 10 +11 11 +13 13 +14 14 +16 16 +17 17 +19 19 +20 20 +22 22 +23 23 +-- with a projection +0 +4 4 +5 5 +7 7 +8 8 +10 10 +11 11 +13 13 +14 14 +16 16 +17 17 +19 19 +20 20 +22 22 +23 23 +-- ReplicatedMergeTree +0 +4 4 +5 5 +7 7 +8 8 +10 10 +11 11 +13 13 +14 14 +16 16 +17 17 +19 19 +20 20 +22 22 +23 23 diff --git a/tests/queries/0_stateless/04798_mutation_row_exists_from_patch_part.sql b/tests/queries/0_stateless/04798_mutation_row_exists_from_patch_part.sql new file mode 100644 index 0000000000000..b6a64baf6abe3 --- /dev/null +++ b/tests/queries/0_stateless/04798_mutation_row_exists_from_patch_part.sql @@ -0,0 +1,195 @@ +-- Tags: no-replicated-database, long +-- long: one run of this file goes past the flaky check's 180s per-run budget under ASan +-- with S3 storage and metadata in Keeper, where every statement pays an object-storage +-- round trip. Untagged, that budget fails the check outright rather than reporting a flake. +-- no-replicated-database: fails due to additional shard. + +SET insert_keeper_fault_injection_probability = 0.0; +SET enable_lightweight_update = 1; +SET mutations_sync = 2, alter_sync = 2, lightweight_deletes_sync = 2; + +-- A lightweight delete in patch mode supplies _row_exists from a patch part rather than from the +-- source part's own columns. A later mutation that rewrites all columns applies that mask on read +-- and physically drops the masked rows, so its result must not be declared to carry the mask. + +SELECT '-- lightweight_update: heavyweight DELETE after a patch-mode lightweight DELETE'; + +DROP TABLE IF EXISTS t_lwu SYNC; +CREATE TABLE t_lwu (k UInt64, v UInt64) ENGINE = MergeTree PARTITION BY (k % 4) ORDER BY k +SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1, + min_bytes_for_wide_part = 0, min_bytes_for_full_part_storage = 0; +INSERT INTO t_lwu SELECT number, number FROM numbers(24); + +DELETE FROM t_lwu WHERE k % 3 = 0 SETTINGS lightweight_delete_mode = 'lightweight_update'; +ALTER TABLE t_lwu UPDATE v = v + 1 WHERE k = 1; +DELETE FROM t_lwu WHERE k = 1 SETTINGS lightweight_delete_mode = 'lightweight_update'; +ALTER TABLE t_lwu DELETE WHERE k = 2; + +SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = 't_lwu' AND NOT is_done; +SELECT k, v FROM t_lwu ORDER BY k; +SELECT count() FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_lwu' AND active + AND column = '_row_exists' AND name NOT LIKE 'patch%' AND name LIKE '2\_%'; + +SELECT '-- alter_update mode produces the same rows (oracle)'; + +DROP TABLE IF EXISTS t_alter SYNC; +CREATE TABLE t_alter (k UInt64, v UInt64) ENGINE = MergeTree PARTITION BY (k % 4) ORDER BY k +SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1, + min_bytes_for_wide_part = 0, min_bytes_for_full_part_storage = 0; +INSERT INTO t_alter SELECT number, number FROM numbers(24); + +DELETE FROM t_alter WHERE k % 3 = 0 SETTINGS lightweight_delete_mode = 'alter_update'; +ALTER TABLE t_alter UPDATE v = v + 1 WHERE k = 1; +DELETE FROM t_alter WHERE k = 1 SETTINGS lightweight_delete_mode = 'alter_update'; +ALTER TABLE t_alter DELETE WHERE k = 2; + +SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = 't_alter' AND NOT is_done; +SELECT k, v FROM t_alter ORDER BY k; + +SELECT '-- both modes agree'; +SELECT (SELECT groupArray((k, v)) FROM (SELECT k, v FROM t_lwu ORDER BY k)) + = (SELECT groupArray((k, v)) FROM (SELECT k, v FROM t_alter ORDER BY k)); + +SELECT '-- lightweight_update_force'; + +DROP TABLE IF EXISTS t_force SYNC; +CREATE TABLE t_force (k UInt64, v UInt64) ENGINE = MergeTree PARTITION BY (k % 4) ORDER BY k +SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1, + min_bytes_for_wide_part = 0, min_bytes_for_full_part_storage = 0; +INSERT INTO t_force SELECT number, number FROM numbers(24); + +DELETE FROM t_force WHERE k % 3 = 0 SETTINGS lightweight_delete_mode = 'lightweight_update_force'; +ALTER TABLE t_force UPDATE v = v + 1 WHERE k = 1; +DELETE FROM t_force WHERE k = 1 SETTINGS lightweight_delete_mode = 'lightweight_update_force'; +ALTER TABLE t_force DELETE WHERE k = 2; + +SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = 't_force' AND NOT is_done; +SELECT k, v FROM t_force ORDER BY k; + +SELECT '-- APPLY DELETED MASK and REWRITE PARTS reach the same code path'; + +DROP TABLE IF EXISTS t_mask SYNC; +CREATE TABLE t_mask (k UInt64, v UInt64) ENGINE = MergeTree PARTITION BY (k % 4) ORDER BY k +SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1, + min_bytes_for_wide_part = 0, min_bytes_for_full_part_storage = 0; +INSERT INTO t_mask SELECT number, number FROM numbers(24); + +DELETE FROM t_mask WHERE k % 3 = 0 SETTINGS lightweight_delete_mode = 'lightweight_update'; +ALTER TABLE t_mask UPDATE v = v + 1 WHERE k = 1; +DELETE FROM t_mask WHERE k = 1 SETTINGS lightweight_delete_mode = 'lightweight_update'; +ALTER TABLE t_mask APPLY DELETED MASK; + +SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = 't_mask' AND NOT is_done; +SELECT k, v FROM t_mask ORDER BY k; +SELECT count() FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_mask' AND active + AND column = '_row_exists' AND name NOT LIKE 'patch%'; + +DROP TABLE IF EXISTS t_rewrite SYNC; +CREATE TABLE t_rewrite (k UInt64, v UInt64) ENGINE = MergeTree PARTITION BY (k % 4) ORDER BY k +SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1, + min_bytes_for_wide_part = 0, min_bytes_for_full_part_storage = 0; +INSERT INTO t_rewrite SELECT number, number FROM numbers(24); + +DELETE FROM t_rewrite WHERE k % 3 = 0 SETTINGS lightweight_delete_mode = 'lightweight_update'; +ALTER TABLE t_rewrite UPDATE v = v + 1 WHERE k = 1; +DELETE FROM t_rewrite WHERE k = 1 SETTINGS lightweight_delete_mode = 'lightweight_update'; +ALTER TABLE t_rewrite REWRITE PARTS; + +SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = 't_rewrite' AND NOT is_done; +SELECT k, v FROM t_rewrite ORDER BY k; + +SELECT '-- a partial update keeps the mask, so masked rows stay hidden'; + +DROP TABLE IF EXISTS t_partial SYNC; +CREATE TABLE t_partial (k UInt64, v UInt64) ENGINE = MergeTree PARTITION BY (k % 4) ORDER BY k +SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1, + min_bytes_for_wide_part = 0, min_bytes_for_full_part_storage = 0; +INSERT INTO t_partial SELECT number, number FROM numbers(24); + +DELETE FROM t_partial WHERE k % 3 = 0 SETTINGS lightweight_delete_mode = 'lightweight_update'; +ALTER TABLE t_partial UPDATE v = v + 100 WHERE k % 2 = 1; + +SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = 't_partial' AND NOT is_done; +SELECT k, v FROM t_partial ORDER BY k; +SELECT count() > 0 FROM system.parts_columns +WHERE database = currentDatabase() AND table = 't_partial' AND active + AND column = '_row_exists' AND name NOT LIKE 'patch%'; + +SELECT '-- a heavyweight DELETE matching no rows keeps the pending patch'; + +DROP TABLE IF EXISTS t_noop SYNC; +CREATE TABLE t_noop (k UInt64, v UInt64) ENGINE = MergeTree PARTITION BY (k % 4) ORDER BY k +SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1, + min_bytes_for_wide_part = 0, min_bytes_for_full_part_storage = 0; +INSERT INTO t_noop SELECT number, number FROM numbers(24); + +DELETE FROM t_noop WHERE k % 3 = 0 SETTINGS lightweight_delete_mode = 'lightweight_update'; +ALTER TABLE t_noop DELETE WHERE 1 = 2; + +SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = 't_noop' AND NOT is_done; +SELECT count(), countIf(k % 3 = 0) FROM t_noop; + +SELECT '-- compact parts'; + +DROP TABLE IF EXISTS t_compact SYNC; +CREATE TABLE t_compact (k UInt64, v UInt64) ENGINE = MergeTree PARTITION BY (k % 4) ORDER BY k +SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1, + min_bytes_for_wide_part = '10G', min_rows_for_wide_part = 1000000, + min_bytes_for_full_part_storage = 0; +INSERT INTO t_compact SELECT number, number FROM numbers(24); + +DELETE FROM t_compact WHERE k % 3 = 0 SETTINGS lightweight_delete_mode = 'lightweight_update'; +ALTER TABLE t_compact UPDATE v = v + 1 WHERE k = 1; +DELETE FROM t_compact WHERE k = 1 SETTINGS lightweight_delete_mode = 'lightweight_update'; +ALTER TABLE t_compact DELETE WHERE k = 2; + +SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = 't_compact' AND NOT is_done; +SELECT k, v FROM t_compact ORDER BY k; + +SELECT '-- with a projection'; + +DROP TABLE IF EXISTS t_proj SYNC; +CREATE TABLE t_proj (k UInt64, v UInt64, PROJECTION p (SELECT v, count() GROUP BY v)) +ENGINE = MergeTree PARTITION BY (k % 4) ORDER BY k +SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1, + min_bytes_for_wide_part = 0, min_bytes_for_full_part_storage = 0, + deduplicate_merge_projection_mode = 'rebuild', lightweight_mutation_projection_mode = 'rebuild'; +INSERT INTO t_proj SELECT number, number FROM numbers(24); + +DELETE FROM t_proj WHERE k % 3 = 0 SETTINGS lightweight_delete_mode = 'lightweight_update'; +ALTER TABLE t_proj UPDATE v = v + 1 WHERE k = 1; +DELETE FROM t_proj WHERE k = 1 SETTINGS lightweight_delete_mode = 'lightweight_update'; +ALTER TABLE t_proj DELETE WHERE k = 2; + +SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = 't_proj' AND NOT is_done; +SELECT k, v FROM t_proj ORDER BY k; + +SELECT '-- ReplicatedMergeTree'; + +DROP TABLE IF EXISTS t_repl SYNC; +CREATE TABLE t_repl (k UInt64, v UInt64) +ENGINE = ReplicatedMergeTree('/zookeeper/{database}/t_repl/', '1') PARTITION BY (k % 4) ORDER BY k +SETTINGS enable_block_number_column = 1, enable_block_offset_column = 1, + min_bytes_for_wide_part = 0, min_bytes_for_full_part_storage = 0; +INSERT INTO t_repl SELECT number, number FROM numbers(24); + +DELETE FROM t_repl WHERE k % 3 = 0 SETTINGS lightweight_delete_mode = 'lightweight_update'; +ALTER TABLE t_repl UPDATE v = v + 1 WHERE k = 1; +DELETE FROM t_repl WHERE k = 1 SETTINGS lightweight_delete_mode = 'lightweight_update'; +ALTER TABLE t_repl DELETE WHERE k = 2; + +SELECT count() FROM system.mutations WHERE database = currentDatabase() AND table = 't_repl' AND NOT is_done; +SELECT k, v FROM t_repl ORDER BY k; + +DROP TABLE t_lwu SYNC; +DROP TABLE t_alter SYNC; +DROP TABLE t_force SYNC; +DROP TABLE t_mask SYNC; +DROP TABLE t_rewrite SYNC; +DROP TABLE t_partial SYNC; +DROP TABLE t_noop SYNC; +DROP TABLE t_compact SYNC; +DROP TABLE t_proj SYNC; +DROP TABLE t_repl SYNC; From 31e7f9017d3676682af98971fe38caa8c89bc594 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 16 Nov 2026 15:30:26 +0000 Subject: [PATCH 201/219] Update autogenerated version to 26.8.38.9 and contributors --- cmake/autogenerated_versions.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/cmake/autogenerated_versions.txt b/cmake/autogenerated_versions.txt index 6218dd0558f10..11e4e6cf18bdd 100644 --- a/cmake/autogenerated_versions.txt +++ b/cmake/autogenerated_versions.txt @@ -2,11 +2,11 @@ # NOTE: VERSION_REVISION has nothing common with DBMS_TCP_PROTOCOL_VERSION, # only DBMS_TCP_PROTOCOL_VERSION should be incremented on protocol changes. -SET(VERSION_REVISION 54550) +SET(VERSION_REVISION 54551) SET(VERSION_MAJOR 26) SET(VERSION_MINOR 8) -SET(VERSION_PATCH 38) -SET(VERSION_GITHASH 40e366b2afbe912fd1985c300b3ccaae2a2d5b54) -SET(VERSION_DESCRIBE v26.8.38.1-lts) -SET(VERSION_STRING 26.8.38.1) +SET(VERSION_PATCH 39) +SET(VERSION_GITHASH 71391c9b2bbc4d2c104996451586a396138a4785) +SET(VERSION_DESCRIBE v26.8.39.1-lts) +SET(VERSION_STRING 26.8.39.1) # end of autochange From 4755d75526ca3f3e83691b5be72e9cebcc3090f5 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 16 Nov 2026 16:53:27 +0000 Subject: [PATCH 202/219] Backport #131732 to 26.8: Check the key size in serialized states of `-Map` aggregate functions with `FixedString` keys --- .../Combinators/AggregateFunctionMap.cpp | 15 ++++++++++++++- ...map_combinator_fixed_string_key_size.reference | 1 + ...05772_map_combinator_fixed_string_key_size.sql | 8 ++++++++ 3 files changed, 23 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05772_map_combinator_fixed_string_key_size.reference create mode 100644 tests/queries/0_stateless/05772_map_combinator_fixed_string_key_size.sql diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionMap.cpp b/src/AggregateFunctions/Combinators/AggregateFunctionMap.cpp index 1cdc6b1881f23..a4b45263ef69c 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionMap.cpp +++ b/src/AggregateFunctions/Combinators/AggregateFunctionMap.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include @@ -313,7 +314,19 @@ class AggregateFunctionMap final KeyType key{}; this->data(place).readKey(key, buf); - + if constexpr (std::is_same_v) + { + /// `keyAt` reads exactly N bytes of a `FixedString(N)` key, so a key of another length + /// could never be found again (e.g. by `rollbackInsertResult`). + if (key_type->getTypeId() == TypeIndex::FixedString) + { + const size_t n = assert_cast(*key_type).getN(); + if (key.size() != n) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Key of size {} in the serialized state of aggregate function {} does not match its type {}", + key.size(), getName(), key_type->getName()); + } + } /// Take the slot before creating the state. `emplace` on a key that is already present /// does not insert, so creating first would abandon the fresh state: nothing would /// reference it and `destroyImpl` walks only `merged_maps`. `serialize` writes each key diff --git a/tests/queries/0_stateless/05772_map_combinator_fixed_string_key_size.reference b/tests/queries/0_stateless/05772_map_combinator_fixed_string_key_size.reference new file mode 100644 index 0000000000000..5c8eba1399661 --- /dev/null +++ b/tests/queries/0_stateless/05772_map_combinator_fixed_string_key_size.reference @@ -0,0 +1 @@ +{'a':5} diff --git a/tests/queries/0_stateless/05772_map_combinator_fixed_string_key_size.sql b/tests/queries/0_stateless/05772_map_combinator_fixed_string_key_size.sql new file mode 100644 index 0000000000000..2e60a3df07283 --- /dev/null +++ b/tests/queries/0_stateless/05772_map_combinator_fixed_string_key_size.sql @@ -0,0 +1,8 @@ +-- A serialized state of a -Map aggregate function with a `FixedString(N)` key must have keys of exactly N bytes. + +SELECT groupArrayStateMapTupleMerge(CAST(unhex('010161012A0101620107') AS AggregateFunction(groupArrayStateMapTuple, Tuple(Map(FixedString(1), UInt8), Map(FixedString(1), UInt8))))) FORMAT Null; +SELECT groupArrayStateMapTupleMerge(CAST(unhex('0100012A010261620107') AS AggregateFunction(groupArrayStateMapTuple, Tuple(Map(FixedString(1), UInt8), Map(FixedString(1), UInt8))))); -- { serverError INCORRECT_DATA } + +SELECT anyMapMerge(CAST(unhex('0101610105') AS AggregateFunction(anyMap, Map(FixedString(1), UInt8)))); +SELECT anyMapMerge(CAST(unhex('010261620105') AS AggregateFunction(anyMap, Map(FixedString(1), UInt8)))); -- { serverError INCORRECT_DATA } +SELECT anyMapMerge(CAST(unhex('01000105') AS AggregateFunction(anyMap, Map(FixedString(1), UInt8)))); -- { serverError INCORRECT_DATA } From 590f187c36c0050627bbd86c1d41ac38275b3e62 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 16 Nov 2026 18:32:17 +0000 Subject: [PATCH 203/219] Backport #131806 to 26.8: Reject encrypted zip entries smaller than the encryption header and footer --- .gitmodules | 2 +- contrib/minizip-ng | 2 +- src/IO/Archives/ZipArchiveReader.cpp | 4 ++ ...re_zip_encrypted_entry_too_short.reference | 1 + ...1_restore_zip_encrypted_entry_too_short.sh | 72 +++++++++++++++++++ 5 files changed, 79 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/05811_restore_zip_encrypted_entry_too_short.reference create mode 100755 tests/queries/0_stateless/05811_restore_zip_encrypted_entry_too_short.sh diff --git a/.gitmodules b/.gitmodules index 8644bac77ee9d..751e6c8100727 100644 --- a/.gitmodules +++ b/.gitmodules @@ -196,7 +196,7 @@ url = https://github.com/ClickHouse/azure-sdk-for-cpp [submodule "contrib/minizip-ng"] path = contrib/minizip-ng - url = https://github.com/zlib-ng/minizip-ng + url = https://github.com/ClickHouse/minizip-ng [submodule "contrib/wyhash"] path = contrib/wyhash url = https://github.com/wangyi-fudan/wyhash diff --git a/contrib/minizip-ng b/contrib/minizip-ng index 95ba7abdd24a9..fd8abd88f4d82 160000 --- a/contrib/minizip-ng +++ b/contrib/minizip-ng @@ -1 +1 @@ -Subproject commit 95ba7abdd24a956bde584db54d1d55e37d511e2f +Subproject commit fd8abd88f4d8215f2becdefe3ef2bfbc06edc3d1 diff --git a/src/IO/Archives/ZipArchiveReader.cpp b/src/IO/Archives/ZipArchiveReader.cpp index 5db78cabf4e53..0776a84382710 100644 --- a/src/IO/Archives/ZipArchiveReader.cpp +++ b/src/IO/Archives/ZipArchiveReader.cpp @@ -179,6 +179,10 @@ namespace { if (strm.at_end) return 0; + /// minizip computes the size as `int32_t` from sizes stored in the archive, and a negative one + /// arrives here converted to a huge number. + if (size > static_cast(std::numeric_limits::max())) // NOLINT(google-runtime-int) + throw Exception(ErrorCodes::CANNOT_UNPACK_ARCHIVE, "Invalid size of a read from a zip archive: {}", size); return strm.read_buffer->read(reinterpret_cast(buf), size); } catch (...) diff --git a/tests/queries/0_stateless/05811_restore_zip_encrypted_entry_too_short.reference b/tests/queries/0_stateless/05811_restore_zip_encrypted_entry_too_short.reference new file mode 100644 index 0000000000000..6ecc93c63ab83 --- /dev/null +++ b/tests/queries/0_stateless/05811_restore_zip_encrypted_entry_too_short.reference @@ -0,0 +1 @@ +CANNOT_UNPACK_ARCHIVE diff --git a/tests/queries/0_stateless/05811_restore_zip_encrypted_entry_too_short.sh b/tests/queries/0_stateless/05811_restore_zip_encrypted_entry_too_short.sh new file mode 100755 index 0000000000000..197b385a1325f --- /dev/null +++ b/tests/queries/0_stateless/05811_restore_zip_encrypted_entry_too_short.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: zip archives need minizip, which is not in the fast test build. + +# An encrypted entry of a zip archive whose compressed size is less than the size of the encryption header +# must be rejected when restoring a backup from the archive. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +WORK_DIR=$(cd "${CLICKHOUSE_TMP}" && pwd)/${CLICKHOUSE_TEST_UNIQUE_NAME} +rm -rf "${WORK_DIR}" +mkdir -p "${WORK_DIR}/backups" + +cat > "${WORK_DIR}/config.xml" < + + ${WORK_DIR}/backups + + +CONFIG + +# A zip archive with one stored entry `.backup` encrypted with the traditional PKWARE encryption and the +# password "password". The compressed size of the entry is 1, less than the 12-byte encryption header. +python3 - "${WORK_DIR}/backups/archive.zip" <<'PYEOF' +import struct +import sys + +table = [] +for n in range(256): + c = n + for _ in range(8): + c = (c >> 1) ^ 0xEDB88320 if c & 1 else c >> 1 + table.append(c) + +def crc32_update(crc, b): + return (crc >> 8) ^ table[(crc ^ b) & 0xFF] + +keys = [0x12345678, 0x23456789, 0x34567890] + +def update_keys(b): + keys[0] = crc32_update(keys[0], b) + keys[1] = ((keys[1] + (keys[0] & 0xFF)) * 134775813 + 1) & 0xFFFFFFFF + keys[2] = crc32_update(keys[2], keys[1] >> 24) + +def encrypt(data): + out = bytearray() + for b in data: + t = (keys[2] | 2) & 0xFFFF + out.append(b ^ (((t * (t ^ 1)) >> 8) & 0xFF)) + update_keys(b) + return bytes(out) + +for b in b'password': + update_keys(b) +# The last byte of the header is the high byte of the CRC (0) to pass the password check. +header = encrypt(bytes(11) + b'\x00') + +name = b'.backup' +local = struct.pack('&1 | grep -o -m1 'CANNOT_UNPACK_ARCHIVE' + +rm -rf "${WORK_DIR}" From d1396f8ea04f2f6d90a430c5c771246f7f631989 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 16 Nov 2026 20:40:57 +0000 Subject: [PATCH 204/219] Backport #131688 to 26.8: Fix `DROP WORKLOAD` and `DROP RESOURCE` removing an entity of the other kind with Keeper workload storage --- .../Workload/WorkloadEntityStorageBase.cpp | 2 +- tests/integration/test_scheduler_io/test.py | 32 +++++++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/src/Common/Scheduler/Workload/WorkloadEntityStorageBase.cpp b/src/Common/Scheduler/Workload/WorkloadEntityStorageBase.cpp index 52eefbc5363af..e5a135477b36c 100644 --- a/src/Common/Scheduler/Workload/WorkloadEntityStorageBase.cpp +++ b/src/Common/Scheduler/Workload/WorkloadEntityStorageBase.cpp @@ -550,7 +550,7 @@ bool WorkloadEntityStorageBase::removeEntity( { std::unique_lock lock(mutex); auto it = entities.find(entity_name); - if (it == entities.end()) + if (it == entities.end() || getEntityType(it->second) != entity_type) { if (throw_if_not_exists) throw Exception(ErrorCodes::BAD_ARGUMENTS, "Workload entity '{}' doesn't exist", entity_name); diff --git a/tests/integration/test_scheduler_io/test.py b/tests/integration/test_scheduler_io/test.py index cb668d3bb412d..579b655179d6a 100644 --- a/tests/integration/test_scheduler_io/test.py +++ b/tests/integration/test_scheduler_io/test.py @@ -1087,6 +1087,38 @@ def check_consistency(): check_consistency() +def test_drop_does_not_remove_entity_of_other_kind(): + node.query( + """ + create resource io_read (read disk s3_no_resource); + create workload all; + create workload development in all; + create user drop_workload_user not identified; + grant drop workload on *.* to drop_workload_user; + create user drop_resource_user not identified; + grant drop resource on *.* to drop_resource_user; + """ + ) + try: + assert "doesn't exist" in node.query_and_get_error("drop workload io_read", user="drop_workload_user") + node.query("drop workload if exists io_read", user="drop_workload_user") + assert "doesn't exist" in node.query_and_get_error("drop resource development", user="drop_resource_user") + node.query("drop resource if exists development", user="drop_resource_user") + + definitions = node.query( + "select value from system.zookeeper where path = '/clickhouse/workload' and name = 'definitions.sql'" + ) + assert "CREATE RESOURCE io_read" in definitions + assert "CREATE WORKLOAD development" in definitions + assert node.query("select count() from system.resources where name = 'io_read'") == "1\n" + assert node.query("select count() from system.workloads where name in ('all', 'development')") == "2\n" + + node.query("drop workload development", user="drop_workload_user") + node.query("drop resource io_read", user="drop_resource_user") + finally: + node.query("drop user if exists drop_workload_user; drop user if exists drop_resource_user") + + def test_throw_on_unknown_workload(): node.query( """ From 79bb0893ede5f876f7111406907d974d5ab3288c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Mon, 16 Nov 2026 22:02:46 +0000 Subject: [PATCH 205/219] Backport #130670 to 26.8: Keep the stress AST fuzzer and ignored DROPs out of 05496_dynamic_disk_wrapper_of_server_credentials_disk --- .../05496_dynamic_disk_wrapper_of_server_credentials_disk.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/queries/0_stateless/05496_dynamic_disk_wrapper_of_server_credentials_disk.sh b/tests/queries/0_stateless/05496_dynamic_disk_wrapper_of_server_credentials_disk.sh index ad47a415bdc3f..a99d5b6e16033 100755 --- a/tests/queries/0_stateless/05496_dynamic_disk_wrapper_of_server_credentials_disk.sh +++ b/tests/queries/0_stateless/05496_dynamic_disk_wrapper_of_server_credentials_disk.sh @@ -7,6 +7,10 @@ CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) # shellcheck source=../shell_config.sh . "$CUR_DIR"/../shell_config.sh +# The stress test fuzzes DDL and ignores some DROPs: a `__fuzz_N` copy or an undropped table would name the disks, +# policies and Keeper nodes removed below, and the server could not start again. +CLICKHOUSE_CLIENT="${CLICKHOUSE_CLIENT} --ast_fuzzer_any_query=0 --ignore_drop_queries_probability=0" + # A session without `s3_allow_server_credentials_in_user_queries` may not wrap, by name, a disk that a session with it # created with the server's own credentials, nor a wrapper of such a disk. A `type` read with `from_zk` counts as a # disk that may use server credentials; here it resolves to an `encrypted` disk, so no object storage is needed. From f76b21343b8f139c1b706e0b62a2e26e9ed668f5 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 17 Nov 2026 03:02:34 +0000 Subject: [PATCH 206/219] Backport #131734 to 26.8: Limit the number of array dimensions reported by PostgreSQL --- .../fetchPostgreSQLTableStructure.cpp | 30 +++++++++++++++---- .../test_storage_postgresql/test.py | 24 +++++++++++++++ 2 files changed, 49 insertions(+), 5 deletions(-) diff --git a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp index c78089ca65b18..bfffeace40829 100644 --- a/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp +++ b/src/Databases/PostgreSQL/fetchPostgreSQLTableStructure.cpp @@ -30,6 +30,29 @@ namespace ErrorCodes extern const int UNKNOWN_TABLE; extern const int BAD_ARGUMENTS; extern const int LOGICAL_ERROR; + extern const int INCORRECT_DATA; +} + +namespace +{ + +/// PostgreSQL itself does not allow more than 6 dimensions (MAXDIM). The number comes from the remote server, +/// and each dimension is one more level of `Array`, so refuse absurd values instead of building a type +/// so deep that working with it exhausts the stack. +constexpr Int64 MAX_POSTGRESQL_ARRAY_DIMENSIONS = 100; + +DataTypePtr wrapInArrays(DataTypePtr type, Int64 dimensions) +{ + if (dimensions < 1 || dimensions > MAX_POSTGRESQL_ARRAY_DIMENSIONS) + throw Exception(ErrorCodes::INCORRECT_DATA, + "PostgreSQL array has unsupported number of dimensions: {} (must be from 1 to {})", + dimensions, MAX_POSTGRESQL_ARRAY_DIMENSIONS); + + for (Int64 i = 0; i < dimensions; ++i) + type = std::make_shared(type); + return type; +} + } @@ -170,8 +193,7 @@ DataTypePtr convertPostgreSQLDataType(String & type, std::function reche } else { - while (dimensions--) - res = std::make_shared(res); + res = wrapInArrays(res, dimensions); } } @@ -290,9 +312,7 @@ PostgreSQLTableStructure::ColumnsInfoPtr readNamesAndTypesList( int dimensions = result[0][1].as(); /// It is always 1d array if it is in recheck. - DataTypePtr type = assert_cast(name_and_type.type.get())->getNestedType(); - while (dimensions--) - type = std::make_shared(type); + DataTypePtr type = wrapInArrays(assert_cast(name_and_type.type.get())->getNestedType(), dimensions); columns[i] = NameAndTypePair(name_and_type.name, type); } diff --git a/tests/integration/test_storage_postgresql/test.py b/tests/integration/test_storage_postgresql/test.py index d3d19bf369b99..9ef71fd66ac11 100644 --- a/tests/integration/test_storage_postgresql/test.py +++ b/tests/integration/test_storage_postgresql/test.py @@ -518,6 +518,30 @@ def test_postgres_ndim(started_cluster): cursor.execute("DROP SCHEMA ndim_schema CASCADE") +def test_postgres_ndim_out_of_range(started_cluster): + cursor = started_cluster.postgres_conn.cursor() + + cursor.execute("DROP TABLE IF EXISTS arr_bad_ndim") + cursor.execute("CREATE TABLE arr_bad_ndim (a integer[])") + cursor.execute("INSERT INTO arr_bad_ndim SELECT '{{1}, {2}}'") + + def query_type(): + return node1.query_and_get_error( + f"""SELECT toTypeName(a) FROM postgresql('postgres1:5432', 'postgres', 'arr_bad_ndim', 'postgres', '{pg_pass}')""" + ) + + # PostgreSQL never reports more than 6 dimensions by itself, so emulate a misbehaving server + # by editing the catalog directly (the connection is a superuser). + cursor.execute( + "UPDATE pg_attribute SET attndims = 30000 WHERE attrelid = 'arr_bad_ndim'::regclass AND attname = 'a'" + ) + error = query_type() + assert "INCORRECT_DATA" in error, error + assert "unsupported number of dimensions" in error, error + + cursor.execute("DROP TABLE arr_bad_ndim") + + def test_postgres_on_conflict(started_cluster): cursor = started_cluster.postgres_conn.cursor() table = "test_conflict" From 3e496ee1bed28902d05f84b36a361dacfaee6658 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 17 Nov 2026 03:05:35 +0000 Subject: [PATCH 207/219] Backport #131733 to 26.8: Make `ColumnAggregateFunction::expand` exception-safe --- src/Columns/ColumnAggregateFunction.cpp | 70 +++++++++++++------ src/Common/FailPoint.cpp | 1 + ...function_column_expand_exception.reference | 1 + ...egate_function_column_expand_exception.sql | 19 +++++ 4 files changed, 71 insertions(+), 20 deletions(-) create mode 100644 tests/queries/0_stateless/05774_aggregate_function_column_expand_exception.reference create mode 100644 tests/queries/0_stateless/05774_aggregate_function_column_expand_exception.sql diff --git a/src/Columns/ColumnAggregateFunction.cpp b/src/Columns/ColumnAggregateFunction.cpp index 81bd2e478dedd..e40c38bb2545c 100644 --- a/src/Columns/ColumnAggregateFunction.cpp +++ b/src/Columns/ColumnAggregateFunction.cpp @@ -39,6 +39,7 @@ namespace ErrorCodes namespace FailPoints { extern const char column_aggregate_function_ensureOwnership_exception[]; + extern const char column_aggregate_function_expand_exception[]; } @@ -405,32 +406,61 @@ void ColumnAggregateFunction::expand(const Filter & mask, bool inverted) if (mask.size() < data.size()) throw Exception(ErrorCodes::LOGICAL_ERROR, "Mask size should be no less than data size."); - ssize_t from = data.size() - 1; - ssize_t index = mask.size() - 1; - data.resize(mask.size()); - while (index >= 0) + const size_t set_bytes = countBytesInFilter(mask); + const size_t selected = inverted ? mask.size() - set_bytes : set_bytes; + if (selected > data.size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Too many bytes in mask"); + if (selected < data.size()) + throw Exception(ErrorCodes::LOGICAL_ERROR, "Not enough bytes in mask"); + + bool inject_memory_limit_exceeded = false; + /// Avoid checking failpoint in a loop, since it uses atomic's + fiu_do_on(FailPoints::column_aggregate_function_expand_exception, { - if (!!mask[index] ^ inverted) - { - if (from < 0) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Too many bytes in mask"); + inject_memory_limit_exceeded = true; + }); - /// Copy only if it makes sense. - if (index != from) - data[index] = data[from]; - --from; - } - else + /// The destructor destroys every state in `data`, so `data` must never hold a pointer to a state + /// that is not created. Build the expanded array aside and publish it only when it is complete. + Container new_data; + new_data.resize_exact(mask.size()); + + size_t from = 0; + size_t index = 0; + try + { + for (; index < mask.size(); ++index) { - data[index] = arena.alignedAlloc(func->sizeOfData(), func->alignOfData()); - func->create(data[index]); - } + if (!!mask[index] ^ inverted) + { + new_data[index] = data[from]; + ++from; + } + else + { + new_data[index] = arena.alignedAlloc(func->sizeOfData(), func->alignOfData()); + func->create(new_data[index]); - --index; + /// Throw after a state is created, so that the cleanup has something to destroy. + if (unlikely(inject_memory_limit_exceeded)) + { + ++index; + throw Exception(ErrorCodes::MEMORY_LIMIT_EXCEEDED, "Failpoint triggered"); + } + } + } + } + catch (...) + { + /// Destroy the states created so far; the ones taken from `data` stay owned by `data`. + if (!func->hasTrivialDestructor()) + for (size_t i = 0; i < index; ++i) + if (!(!!mask[i] ^ inverted)) + func->destroy(new_data[i]); + throw; } - if (from != -1) - throw Exception(ErrorCodes::LOGICAL_ERROR, "Not enough bytes in mask"); + data.swap(new_data); } ColumnPtr ColumnAggregateFunction::permute(const Permutation & perm, size_t limit) const diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index 91b870818faac..5002b68a5b27d 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -197,6 +197,7 @@ static struct InitFiu PAUSEABLE(refresh_mv_pause_after_interrupt_check) \ REGULAR(refresh_mv_skip_execution) \ ONCE(column_aggregate_function_ensureOwnership_exception) \ + ONCE(column_aggregate_function_expand_exception) \ ONCE(space_saving_copy_arena_throw) \ ONCE(adaptive_aggregation_drain_create_states_throw) \ REGULAR(keepermap_fail_drop_data) \ diff --git a/tests/queries/0_stateless/05774_aggregate_function_column_expand_exception.reference b/tests/queries/0_stateless/05774_aggregate_function_column_expand_exception.reference new file mode 100644 index 0000000000000..9766475a4185a --- /dev/null +++ b/tests/queries/0_stateless/05774_aggregate_function_column_expand_exception.reference @@ -0,0 +1 @@ +ok diff --git a/tests/queries/0_stateless/05774_aggregate_function_column_expand_exception.sql b/tests/queries/0_stateless/05774_aggregate_function_column_expand_exception.sql new file mode 100644 index 0000000000000..3b047fa94e0b1 --- /dev/null +++ b/tests/queries/0_stateless/05774_aggregate_function_column_expand_exception.sql @@ -0,0 +1,19 @@ +-- Tags: no-parallel, no-fasttest +-- Tag no-parallel: the failpoint is global and would fire in other queries. +-- Tag no-fasttest: failpoints are not available in the fast test build. + +-- `ColumnAggregateFunction::expand` (used by the short-circuit evaluation of `if`) must stay valid when +-- creating an aggregate function state throws, so that the column can be destroyed. + +SYSTEM ENABLE FAILPOINT column_aggregate_function_expand_exception; + +SELECT finalizeAggregation(if(number % 3 = 0, + initializeAggregation('groupArrayState', number), + initializeAggregation('groupArrayState', number + 1))) +FROM numbers(100) +FORMAT Null +SETTINGS short_circuit_function_evaluation = 'force_enable'; -- { serverError MEMORY_LIMIT_EXCEEDED } + +SYSTEM DISABLE FAILPOINT column_aggregate_function_expand_exception; + +SELECT 'ok'; From 7832913dc19cdc0ee0b018d2d4e57e117e153614 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 17 Nov 2026 03:34:43 +0000 Subject: [PATCH 208/219] Backport #131595 to 26.8: Destroy the queued aggregate states when the merge of GROUP BY tables throws --- .../Combinators/AggregateFunctionTuple.cpp | 17 +- src/AggregateFunctions/IAggregateFunction.cpp | 15 + src/AggregateFunctions/IAggregateFunction.h | 35 ++- src/Common/FailPoint.cpp | 2 + src/Interpreters/Aggregator.cpp | 276 +++++++++++------- src/Interpreters/Aggregator.h | 14 + ...exception_destroys_queued_states.reference | 11 + ..._merge_exception_destroys_queued_states.sh | 66 +++++ 8 files changed, 326 insertions(+), 110 deletions(-) create mode 100644 tests/queries/0_stateless/05638_aggregator_merge_exception_destroys_queued_states.reference create mode 100755 tests/queries/0_stateless/05638_aggregator_merge_exception_destroys_queued_states.sh diff --git a/src/AggregateFunctions/Combinators/AggregateFunctionTuple.cpp b/src/AggregateFunctions/Combinators/AggregateFunctionTuple.cpp index 13b9d034887e5..a2ca866de505b 100644 --- a/src/AggregateFunctions/Combinators/AggregateFunctionTuple.cpp +++ b/src/AggregateFunctions/Combinators/AggregateFunctionTuple.cpp @@ -472,9 +472,20 @@ void AggregateFunctionTuple::mergeAndDestroyBatch( std::atomic & is_cancelled, Arena * arena) const { - for (size_t i = 0; i < nested_functions.size(); ++i) - nested_functions[i]->mergeAndDestroyBatch( - dst_places, rhs_places, size, offset + state_offsets[i], thread_pool, is_cancelled, arena); + size_t i = 0; + try + { + for (; i < nested_functions.size(); ++i) + nested_functions[i]->mergeAndDestroyBatch( + dst_places, rhs_places, size, offset + state_offsets[i], thread_pool, is_cancelled, arena); + } + catch (...) + { + /// The function that threw has destroyed its own states of the batch, the following ones are untouched. + for (size_t k = i + 1; k < nested_functions.size(); ++k) + nested_functions[k]->destroyBatch(0, size, rhs_places, offset + state_offsets[k]); + throw; + } } void AggregateFunctionTuple::serialize(ConstAggregateDataPtr __restrict place, WriteBuffer & buf, std::optional version) const diff --git a/src/AggregateFunctions/IAggregateFunction.cpp b/src/AggregateFunctions/IAggregateFunction.cpp index 50185b83160fc..502a287c7db0e 100644 --- a/src/AggregateFunctions/IAggregateFunction.cpp +++ b/src/AggregateFunctions/IAggregateFunction.cpp @@ -2,6 +2,7 @@ #include #include #include +#include namespace DB @@ -10,6 +11,12 @@ namespace DB namespace ErrorCodes { extern const int NOT_IMPLEMENTED; +extern const int FAULT_INJECTED; +} + +namespace FailPoints +{ +extern const char aggregator_merge_batch_throw[]; } DataTypePtr IAggregateFunction::getStateType() const @@ -145,6 +152,14 @@ void IAggregateFunction::parallelizeMergeMulti( } } +void IAggregateFunction::injectMergeBatchFault() +{ + fiu_do_on(FailPoints::aggregator_merge_batch_throw, + { + throw Exception(ErrorCodes::FAULT_INJECTED, "Injected fault in the batch merge of aggregate function states"); + }); +} + void IAggregateFunction::insertMergeResultInto(AggregateDataPtr __restrict place, IColumn & to, Arena * arena) const { if (isState()) diff --git a/src/AggregateFunctions/IAggregateFunction.h b/src/AggregateFunctions/IAggregateFunction.h index ffa802107efed..6102a5a322036 100644 --- a/src/AggregateFunctions/IAggregateFunction.h +++ b/src/AggregateFunctions/IAggregateFunction.h @@ -263,6 +263,7 @@ class IAggregateFunction : public std::enable_shared_from_this & is_cancelled, Arena * arena) const = 0; /// Serializes state (to transmit it over the network, for example). @@ -507,6 +508,9 @@ class IAggregateFunction : public std::enable_shared_from_this & is_cancelled, Arena * arena) const override { - for (size_t i = 0; i < size; ++i) + size_t i = 0; + try + { + for (; i < size; ++i) + { + if (unlikely(i == 1)) + injectMergeBatchFault(); + + chassert(dst_places[i] + offset != rhs_places[i] + offset, + "IAggregateFunction::mergeAndDestroyBatch called with the same source and destination state"); + if constexpr (Derived::parallelizeMergeWithKey()) + static_cast(this)->mergeImpl(dst_places[i] + offset, rhs_places[i] + offset, thread_pool, is_cancelled, arena); + else + static_cast(this)->mergeImpl(dst_places[i] + offset, rhs_places[i] + offset, arena); + + static_cast(this)->destroy(rhs_places[i] + offset); + } + } + catch (...) { - chassert(dst_places[i] + offset != rhs_places[i] + offset, - "IAggregateFunction::mergeAndDestroyBatch called with the same source and destination state"); - if constexpr (Derived::parallelizeMergeWithKey()) - static_cast(this)->mergeImpl(dst_places[i] + offset, rhs_places[i] + offset, thread_pool, is_cancelled, arena); - else - static_cast(this)->mergeImpl(dst_places[i] + offset, rhs_places[i] + offset, arena); - - static_cast(this)->destroy(rhs_places[i] + offset); + /// `destroy` does not throw, so the sources from `i` on are not destroyed yet. + destroyBatch(i, size, rhs_places, offset); + throw; } } diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index 91b870818faac..2c5c83d583025 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -199,6 +199,8 @@ static struct InitFiu ONCE(column_aggregate_function_ensureOwnership_exception) \ ONCE(space_saving_copy_arena_throw) \ ONCE(adaptive_aggregation_drain_create_states_throw) \ + ONCE(aggregator_merge_collect_states_throw) \ + ONCE(aggregator_merge_batch_throw) \ REGULAR(keepermap_fail_drop_data) \ PAUSEABLE_ONCE(keepermap_create_pause_before_drop_lock_version) \ REGULAR(keeper_fault_on_watch_request) \ diff --git a/src/Interpreters/Aggregator.cpp b/src/Interpreters/Aggregator.cpp index 26f52e0ecdf1f..6074ac9572612 100644 --- a/src/Interpreters/Aggregator.cpp +++ b/src/Interpreters/Aggregator.cpp @@ -34,6 +34,7 @@ #include #include #include +#include #include #include #include @@ -95,6 +96,12 @@ namespace ErrorCodes extern const int CANNOT_MERGE_DIFFERENT_AGGREGATED_DATA_VARIANTS; extern const int LOGICAL_ERROR; extern const int BAD_ARGUMENTS; + extern const int FAULT_INJECTED; +} + +namespace FailPoints +{ + extern const char aggregator_merge_collect_states_throw[]; } } @@ -3186,11 +3193,12 @@ void NO_INLINE Aggregator::mergeSingleLevelPartitionImpl( { if (dst_places.empty()) return; - for (size_t i = 0; i < params.aggregates_size; ++i) - aggregate_functions[i]->mergeAndDestroyBatch( - dst_places.data(), src_places.data(), dst_places.size(), offsets_of_aggregate_states[i], *thread_pool, is_cancelled, arena); - dst_places.clear(); - src_places.clear(); + /// `mergeAndDestroyDetachedBatch` owns the batch from the call on, also when it throws, so the `catch` must not see it. + SCOPE_EXIT({ + dst_places.clear(); + src_places.clear(); + }); + mergeAndDestroyDetachedBatch(dst_places, src_places, arena, is_cancelled, /*skip_compiled=*/ false); }; /// The NULL key of the single-key nullable methods lives in a dedicated slot outside the cells, @@ -3228,98 +3236,113 @@ void NO_INLINE Aggregator::mergeSingleLevelPartitionImpl( const bool owns_null_slot = partition_index == 0; - for (auto * source : sources) + try { - if (is_cancelled.load(std::memory_order_seq_cst)) - return; - - auto & src = getDataVariant(*source).data; + for (auto * source : sources) + { + if (is_cancelled.load(std::memory_order_seq_cst)) + return; - if (owns_null_slot) - merge_null_slot(src); + auto & src = getDataVariant(*source).data; - if constexpr (requires { src.begin() != src.end(); }) - { - static constexpr size_t prefetch_batch_size = 16; - using CellPtr = decltype(&*src.begin()); - std::array, prefetch_batch_size> batch; + if (owns_null_slot) + merge_null_slot(src); - auto it = src.begin(); - const auto end = src.end(); - while (it != end) + if constexpr (requires { src.begin() != src.end(); }) { - size_t batch_size = 0; - for (; it != end && batch_size < prefetch_batch_size; ++it) - { - const size_t hash_value = it.getHash(); - if ((TwoLevelMethod::Data::getBucketFromHash(hash_value) & partition_mask) != partition_index) - continue; - batch[batch_size++] = {&*it, hash_value}; - } + static constexpr size_t prefetch_batch_size = 16; + using CellPtr = decltype(&*src.begin()); + std::array, prefetch_batch_size> batch; - if (params.enable_prefetch) + auto it = src.begin(); + const auto end = src.end(); + while (it != end) { + size_t batch_size = 0; + for (; it != end && batch_size < prefetch_batch_size; ++it) + { + const size_t hash_value = it.getHash(); + if ((TwoLevelMethod::Data::getBucketFromHash(hash_value) & partition_mask) != partition_index) + continue; + batch[batch_size++] = {&*it, hash_value}; + } + + if (params.enable_prefetch) + { + for (size_t i = 0; i < batch_size; ++i) + dst_method.data.prefetchByHash(batch[i].second); + } + for (size_t i = 0; i < batch_size; ++i) - dst_method.data.prefetchByHash(batch[i].second); + merge_cell( + std::decay_t::getKey(batch[i].first->getValue()), + batch[i].first->getMapped(), + batch[i].second); } - - for (size_t i = 0; i < batch_size; ++i) - merge_cell( - std::decay_t::getKey(batch[i].first->getValue()), - batch[i].first->getMapped(), - batch[i].second); } - } - else if constexpr (requires { src.emptyStringSlot(); }) - { - /// A string table dispatches keys over size-class sub-tables and has no unified cell - /// iterator. The empty-string key lives in a dedicated slot whose dispatch hash is 0, - /// so it belongs to partition 0 like the NULL key. - auto & dst_table = dst_method.data; - - if (owns_null_slot) + else if constexpr (requires { src.emptyStringSlot(); }) { - auto & src_slot = src.emptyStringSlot(); - if (src_slot.hasZero() && src_slot.zeroValue()->getMapped()) + /// A string table dispatches keys over size-class sub-tables and has no unified cell + /// iterator. The empty-string key lives in a dedicated slot whose dispatch hash is 0, + /// so it belongs to partition 0 like the NULL key. + auto & dst_table = dst_method.data; + + if (owns_null_slot) { - auto & dst_slot = dst_table.emptyStringSlot(); - typename std::decay_t::LookupResult slot_it; - bool inserted = false; + auto & src_slot = src.emptyStringSlot(); + if (src_slot.hasZero() && src_slot.zeroValue()->getMapped()) + { + auto & dst_slot = dst_table.emptyStringSlot(); + typename std::decay_t::LookupResult slot_it; + bool inserted = false; - /// The empty-string slot ignores the key argument. - dst_slot.emplace(0, slot_it, inserted, 0); - adopt_or_collect(inserted, slot_it->getMapped(), src_slot.zeroValue()->getMapped()); + /// The empty-string slot ignores the key argument. + dst_slot.emplace(0, slot_it, inserted, 0); + adopt_or_collect(inserted, slot_it->getMapped(), src_slot.zeroValue()->getMapped()); + } } - } - std::decay_t::forEachSubMapPair( - src, - dst_table, - [&](auto & src_sub, auto & dst_sub) - { - for (auto it = src_sub.begin(); it != src_sub.end(); ++it) + std::decay_t::forEachSubMapPair( + src, + dst_table, + [&](auto & src_sub, auto & dst_sub) { - const size_t hash_value = it.getHash(); - if ((TwoLevelMethod::Data::getBucketFromHash(hash_value) & partition_mask) != partition_index) - continue; - - AggregateDataPtr & src_data = it->getMapped(); - if (!src_data) - continue; + for (auto it = src_sub.begin(); it != src_sub.end(); ++it) + { + const size_t hash_value = it.getHash(); + if ((TwoLevelMethod::Data::getBucketFromHash(hash_value) & partition_mask) != partition_index) + continue; + + AggregateDataPtr & src_data = it->getMapped(); + if (!src_data) + continue; + + typename std::decay_t::LookupResult dst_it; + bool inserted = false; + dst_sub.emplace(it->value.first, dst_it, inserted, hash_value); + adopt_or_collect(inserted, dst_it->getMapped(), src_data); + } + }); + } + else + { + static_assert(false, "The aggregation method's table supports neither cell iteration nor sub-table pairing"); + } - typename std::decay_t::LookupResult dst_it; - bool inserted = false; - dst_sub.emplace(it->value.first, dst_it, inserted, hash_value); - adopt_or_collect(inserted, dst_it->getMapped(), src_data); - } + if (!src_places.empty()) + fiu_do_on(FailPoints::aggregator_merge_collect_states_throw, + { + throw Exception(ErrorCodes::FAULT_INJECTED, "Injected fault while collecting the states of a merge"); }); - } - else - { - static_assert(false, "The aggregation method's table supports neither cell iteration nor sub-table pairing"); - } - flush_merges(); + flush_merges(); + } + } + catch (...) + { + /// The queued states are detached from their source tables and not merged yet. + destroyDetachedSources(src_places, /*first_function=*/ 0, /*skip_compiled=*/ false); + throw; } } @@ -4296,42 +4319,99 @@ void NO_INLINE Aggregator::mergeDataImpl( src = nullptr; }; - if (parallel_worker) + try { - if constexpr (std::is_same_v || - std::is_same_v) - table_src.mergeToViaIndexFilter(table_dst, std::move(merge), parallel_worker->worker_id, parallel_worker->total_worker); + if (parallel_worker) + { + if constexpr (std::is_same_v || + std::is_same_v) + table_src.mergeToViaIndexFilter(table_dst, std::move(merge), parallel_worker->worker_id, parallel_worker->total_worker); + } + else + { + if (prefetch) + table_src.template mergeToViaEmplace(table_dst, std::move(merge)); + else + table_src.template mergeToViaEmplace(table_dst, std::move(merge)); + table_src.clearAndShrink(); + } + + if (!src_places.empty()) + fiu_do_on(FailPoints::aggregator_merge_collect_states_throw, + { + throw Exception(ErrorCodes::FAULT_INJECTED, "Injected fault while collecting the states of a merge"); + }); } - else + catch (...) { - if (prefetch) - table_src.template mergeToViaEmplace(table_dst, std::move(merge)); - else - table_src.template mergeToViaEmplace(table_dst, std::move(merge)); - table_src.clearAndShrink(); + /// The queued states are detached from `table_src` and not merged yet. + destroyDetachedSources(src_places, /*first_function=*/ 0, /*skip_compiled=*/ false); + throw; } + /// `mergeAndDestroyDetachedBatch` owns the batch from the call on, also when it throws, so the `catch` must not see it. #if USE_EMBEDDED_COMPILER if (use_compiled_functions) { const auto & compiled_functions = compiled_aggregate_functions_holder->compiled_aggregate_functions; callJITFunction(compiled_functions.merge_aggregate_states_function, dst_places.data(), src_places.data(), dst_places.size()); - for (size_t i = 0; i < params.aggregates_size; ++i) - { - if (!is_aggregate_function_compiled[i]) - aggregate_functions[i]->mergeAndDestroyBatch( - dst_places.data(), src_places.data(), dst_places.size(), offsets_of_aggregate_states[i], *thread_pool, is_cancelled, arena); - } - + mergeAndDestroyDetachedBatch(dst_places, src_places, arena, is_cancelled, /*skip_compiled=*/ true); return; } #endif - for (size_t i = 0; i < params.aggregates_size; ++i) + mergeAndDestroyDetachedBatch(dst_places, src_places, arena, is_cancelled, /*skip_compiled=*/ false); +} + +void Aggregator::mergeAndDestroyDetachedBatch( + PaddedPODArray & dst_places, + PaddedPODArray & src_places, + Arena * arena, + std::atomic & is_cancelled, + [[maybe_unused]] bool skip_compiled) const +{ + auto is_skipped = [&]([[maybe_unused]] size_t i) + { +#if USE_EMBEDDED_COMPILER + return skip_compiled && is_aggregate_function_compiled[i]; +#else + return false; +#endif + }; + + size_t i = 0; + try + { + for (; i < params.aggregates_size; ++i) + { + if (is_skipped(i)) + continue; + + aggregate_functions[i]->mergeAndDestroyBatch( + dst_places.data(), src_places.data(), dst_places.size(), offsets_of_aggregate_states[i], *thread_pool, is_cancelled, arena); + } + } + catch (...) { - aggregate_functions[i]->mergeAndDestroyBatch( - dst_places.data(), src_places.data(), dst_places.size(), offsets_of_aggregate_states[i], *thread_pool, is_cancelled, arena); + /// The source states are detached from their hash table, so nobody else will destroy them. The function + /// that threw has destroyed its own states of the batch; the states of the following functions are + /// untouched and are destroyed here. + destroyDetachedSources(src_places, i + 1, skip_compiled); + throw; + } +} + +void Aggregator::destroyDetachedSources( + PaddedPODArray & src_places, size_t first_function, [[maybe_unused]] bool skip_compiled) const noexcept +{ + for (size_t k = first_function; k < params.aggregates_size; ++k) + { +#if USE_EMBEDDED_COMPILER + if (skip_compiled && is_aggregate_function_compiled[k]) + continue; +#endif + aggregate_functions[k]->destroyBatch(0, src_places.size(), src_places.data(), offsets_of_aggregate_states[k]); } } diff --git a/src/Interpreters/Aggregator.h b/src/Interpreters/Aggregator.h index 42f0787448bcb..bd112cbd1391e 100644 --- a/src/Interpreters/Aggregator.h +++ b/src/Interpreters/Aggregator.h @@ -954,6 +954,10 @@ class Aggregator final UInt32 total_worker; }; + /// Destroy the states of the aggregate functions from `first_function` on of the detached source rows `src_places`, + /// except, with `skip_compiled`, those of the compiled functions (see `mergeAndDestroyDetachedBatch`). + void destroyDetachedSources(PaddedPODArray & src_places, size_t first_function, bool skip_compiled) const noexcept; + /// Merge NULL key data from hash table `src` into `dst`. template void mergeDataNullKey( @@ -969,6 +973,16 @@ class Aggregator final std::atomic & is_cancelled, const ParallelMergeWorker * parallel_worker = nullptr) const; + /// Merge the batch of source rows `src_places` into `dst_places` and destroy them. The source rows are detached + /// from their hash table: if it throws, every state of the batch has been destroyed, except, with `skip_compiled`, + /// the states of the compiled functions, which the caller has merged by the JIT-compiled function. + void mergeAndDestroyDetachedBatch( + PaddedPODArray & dst_places, + PaddedPODArray & src_places, + Arena * arena, + std::atomic & is_cancelled, + bool skip_compiled) const; + /// Merge data from hash table `src` into `dst`, but only for keys that already exist in dst. In other cases, merge the data into `overflows`. template requires MapAggregationMethod diff --git a/tests/queries/0_stateless/05638_aggregator_merge_exception_destroys_queued_states.reference b/tests/queries/0_stateless/05638_aggregator_merge_exception_destroys_queued_states.reference new file mode 100644 index 0000000000000..f410742dc3838 --- /dev/null +++ b/tests/queries/0_stateless/05638_aggregator_merge_exception_destroys_queued_states.reference @@ -0,0 +1,11 @@ +partition 1 +partition_string 2 +partition_nullable 3 +serial 4 +bucket 5 +fixed_map 6 +collect leaks: 0 +partition_batch 1 +serial_batch 2 +tuple_batch 3 +batch leaks: 0 diff --git a/tests/queries/0_stateless/05638_aggregator_merge_exception_destroys_queued_states.sh b/tests/queries/0_stateless/05638_aggregator_merge_exception_destroys_queued_states.sh new file mode 100755 index 0000000000000..88e631094d45d --- /dev/null +++ b/tests/queries/0_stateless/05638_aggregator_merge_exception_destroys_queued_states.sh @@ -0,0 +1,66 @@ +#!/usr/bin/env bash + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +# The merge of the per-thread tables of GROUP BY detaches the states of a key found in several tables +# from the source table and queues them for a batch merge. A failpoint throws either while states are +# queued (as a memory limit does when the destination table grows) or inside the batch merge after its +# first row, and every queued state must be destroyed exactly once. The array of each `quantileExact` +# state is on the heap, so a leaked state is reported by LeakSanitizer at exit, and a state destroyed +# twice makes AddressSanitizer abort before the remaining cases print. Each case prints the count of +# injected faults so far, so a case that does not throw shows too. +# +# A leak reports only at process exit, so each `clickhouse local` gets its own sanitizer log and the +# leak reports in it are counted: 0 on every build. The log also keeps the report out of the runner's +# fatal log, and the pipeline keeps the shell from printing a job status on an abort. +run() +{ + local log="${CLICKHOUSE_TMP}/05638_$1_${CLICKHOUSE_TEST_UNIQUE_NAME}" + rm -f "$log"* + local o="log_path=$log" + ASAN_OPTIONS="${ASAN_OPTIONS:+$ASAN_OPTIONS:}$o" MSAN_OPTIONS="${MSAN_OPTIONS:+$MSAN_OPTIONS:}$o" \ + TSAN_OPTIONS="${TSAN_OPTIONS:+$TSAN_OPTIONS:}$o" UBSAN_OPTIONS="${UBSAN_OPTIONS:+$UBSAN_OPTIONS:}$o" \ + ${CLICKHOUSE_LOCAL} --ignore-error --query " + SET max_threads = 4, max_threads_min_free_memory_per_thread = 0, max_block_size = 1000, + collect_hash_table_stats_during_aggregation = 0, compile_aggregate_expressions = 0, + max_bytes_before_external_group_by = 0, max_bytes_ratio_before_external_group_by = 0; + $2" 2>/dev/null | cat + echo "$1 leaks: $(cat "$log"* 2>/dev/null | grep -c 'detected memory leaks')" + rm -f "$log"* +} + +# One case: name, failpoint, settings, key, aggregate functions. +query() +{ + echo "SYSTEM ENABLE FAILPOINT $2; + SELECT k, ${5:-quantileExact(number), quantileExact(number + 1)} + FROM (SELECT number, $4 AS k FROM numbers_mt(400000)) GROUP BY k SETTINGS $3 FORMAT Null; + SELECT '$1', value FROM system.errors WHERE name = 'FAULT_INJECTED';" +} + +single="enable_adaptive_aggregator = 0, group_by_two_level_threshold = 0, group_by_two_level_threshold_bytes = 0" +partition="$single, enable_parallel_single_level_merge = 1" +serial="$single, enable_parallel_single_level_merge = 0" +bucket="enable_adaptive_aggregator = 0, group_by_two_level_threshold = 1, group_by_two_level_threshold_bytes = 0" +collect=aggregator_merge_collect_states_throw +batch=aggregator_merge_batch_throw + +# A throw after states are queued: the parallel hash-partition merge of a table with cell iteration, of a +# string table and of a table with a NULL key, the serial merge of single-level tables, the merge of a +# two-level bucket and the parallel merge of fixed hash maps. +run collect " + $(query partition $collect "$partition" "toUInt64(number % 1000)") + $(query partition_string $collect "$partition" "toString(number % 1000)") + $(query partition_nullable $collect "$partition" "if(number % 10 = 0, NULL, toUInt64(number % 1000))") + $(query serial $collect "$serial" "toUInt64(number % 1000)") + $(query bucket $collect "$bucket" "toUInt64(number % 1000)") + $(query fixed_map $collect "$single" "toUInt16(number % 1000)")" + +# A throw inside the batch merge of the first function after its first row, in both merges, and inside +# the first nested function of a `-Tuple` combinator. +run batch " + $(query partition_batch $batch "$partition" "toUInt64(number % 1000)") + $(query serial_batch $batch "$serial" "toUInt64(number % 1000)") + $(query tuple_batch $batch "$partition" "toUInt64(number % 1000)" "quantileExactTuple((number, number + 1))")" From f222a4affed90ade6aa81c36fdf6d624f9b22720 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 17 Nov 2026 04:46:03 +0000 Subject: [PATCH 209/219] Backport #131787 to 26.8: Check that DWARF attributes do not go past the end of the unit --- .../Formats/Impl/DWARFBlockInputFormat.cpp | 80 +++++++++++++- .../Formats/Impl/DWARFBlockInputFormat.h | 7 +- ...rf_attribute_past_end_of_section.reference | 13 +++ ...810_dwarf_attribute_past_end_of_section.sh | 100 ++++++++++++++++++ 4 files changed, 194 insertions(+), 6 deletions(-) create mode 100644 tests/queries/0_stateless/05810_dwarf_attribute_past_end_of_section.reference create mode 100755 tests/queries/0_stateless/05810_dwarf_attribute_past_end_of_section.sh diff --git a/src/Processors/Formats/Impl/DWARFBlockInputFormat.cpp b/src/Processors/Formats/Impl/DWARFBlockInputFormat.cpp index 11099194927c1..2d8e395dd779e 100644 --- a/src/Processors/Formats/Impl/DWARFBlockInputFormat.cpp +++ b/src/Processors/Formats/Impl/DWARFBlockInputFormat.cpp @@ -136,8 +136,53 @@ static size_t getColumnIdx(const std::string & name) return it->second; } -DWARFBlockInputFormat::UnitState::UnitState(llvm::DWARFUnit * u) +/// `tryExtractDIEsIfNeeded` makes LLVM parse the unit DIE. LLVM advances the offset by the length of a block or +/// a fixed-size form without checking it against the section (see `checkAttributeEnd`), so a malformed length +/// made it read past the end of the section on the next attribute. Walk the unit DIE the same way with checks first. +/// `data` is truncated at the end of the unit, so this walk itself does not read the next unit, and LLVM parses +/// the unit DIE only if all its attributes are within the unit. +static void checkUnitDie( + const llvm::DWARFUnit & unit, const llvm::DWARFAbbreviationDeclarationSet & abbrevs, const llvm::DWARFDataExtractor & data, + uint64_t end_offset) +{ + const uint64_t end = data.size(); + uint64_t offset = unit.getOffset() + unit.getHeaderSize(); + /// Leave other errors to LLVM, which reports them. + if (offset >= end) + return; + auto err = llvm::Error::success(); + uint64_t abbrev_code = data.getULEB128(&offset, &err); + if (err) + throw Exception(ErrorCodes::CANNOT_PARSE_DWARF, "Failed to parse the unit entry at offset {}: {}", unit.getOffset(), llvm::toString(std::move(err))); + if (abbrev_code == 0 || abbrev_code > UINT32_MAX) + return; + const auto * abbrev = abbrevs.getAbbreviationDeclaration(static_cast(abbrev_code)); + if (!abbrev) + return; + for (const auto & attr : abbrev->attributes()) + { + const uint64_t attr_offset = offset; + if (auto size = attr.getByteSize(unit)) + offset += *size; + else if (!llvm::DWARFFormValue::skipValue(attr.Form, data, &offset, unit.getFormParams())) + return; + /// `skipValue` does not advance the offset if the string is not terminated within the unit. + if (attr.Form == llvm::dwarf::DW_FORM_string && offset == attr_offset) + throw Exception(ErrorCodes::CANNOT_PARSE_DWARF, + "String attribute {} of the unit entry is not terminated before the end of the unit {}", + llvm::dwarf::AttributeString(attr.Attr).operator std::string_view(), end_offset); + /// A huge length makes the offset wrap around. + if (offset < attr_offset || offset > end) + throw Exception(ErrorCodes::CANNOT_PARSE_DWARF, + "Attribute {} of form {} of the unit entry goes past the end of the unit: offset {}, end of unit {}", + llvm::dwarf::AttributeString(attr.Attr).operator std::string_view(), attr.Form, offset, end_offset); + } +} + +DWARFBlockInputFormat::UnitState::UnitState(llvm::DWARFUnit * u, const llvm::DWARFDataExtractor & section_data) : dwarf_unit(u), end_offset(dwarf_unit->getNextUnitOffset()) + /// Offsets stay relative to the start of the section. + , data(section_data, std::min(end_offset, section_data.size())) , offset(dwarf_unit->getOffset() + dwarf_unit->getHeaderSize()) { /// This call is not thread safe, so we do it during initialization. @@ -145,6 +190,8 @@ DWARFBlockInputFormat::UnitState::UnitState(llvm::DWARFUnit * u) if (abbrevs == nullptr) throw Exception(ErrorCodes::CANNOT_PARSE_DWARF, "Couldn't find abbreviation set for unit at offset {}", dwarf_unit->getOffset()); + checkUnitDie(*u, *abbrevs, data, end_offset); + /// This call initializes some data structures inside DWARFUnit that are needed for parsing attributes. auto err = u->tryExtractDIEsIfNeeded(/*CUDieOnly*/ true); if (err) @@ -285,7 +332,7 @@ void DWARFBlockInputFormat::initializeIfNeeded() dwarf_context = llvm::DWARFContext::create(sections, /*AddrSize*/ 8); for (std::unique_ptr & unit : dwarf_context->info_section_units()) - units_queue.emplace_back(unit.get()); + units_queue.emplace_back(unit.get(), *extractor); LOG_DEBUG(getLogger("DWARF"), "{} units, reading in {} threads", units_queue.size(), num_threads); @@ -353,16 +400,32 @@ static inline void throwIfError(llvm::Error & e, const char * what) throw Exception(ErrorCodes::CANNOT_PARSE_DWARF, "Failed to parse {}: {}", what, llvm::toString(std::move(e))); } +/// LLVM advances the offset by the length of a block or a fixed-size form without checking it against the +/// section (LEB128 decoding checks the offset only with an assertion), so every following read would start +/// past the end of the section. Attributes are read from `UnitState::data`, which ends at the end of the unit, +/// so the bytes of the next unit are never read, and the offset is checked after each attribute. +/// The offset is advanced with plain unsigned addition, so a huge length makes it wrap around to before the start of the attribute. +void DWARFBlockInputFormat::checkAttributeEnd( + const llvm::DWARFAbbreviationDeclaration::AttributeSpec & attr, uint64_t attr_offset, uint64_t offset, const UnitState & unit) const +{ + if (offset < attr_offset || offset > unit.data.size()) + throw Exception(ErrorCodes::CANNOT_PARSE_DWARF, + "Attribute {} of form {} goes past the end of the unit: offset {}, end of unit {}", + llvm::dwarf::AttributeString(attr.Attr).operator std::string_view(), attr.Form, offset, unit.end_offset); +} + llvm::DWARFFormValue DWARFBlockInputFormat::parseAttribute( const llvm::DWARFAbbreviationDeclaration::AttributeSpec & attr, uint64_t * offset, const UnitState & unit) const { + const uint64_t attr_offset = *offset; auto val = llvm::DWARFFormValue::createFromSValue( attr.Form, attr.isImplicitConst() ? attr.getImplicitConstValue() : 0); - if (!val.extractValue(*extractor, offset, unit.dwarf_unit->getFormParams(), unit.dwarf_unit)) + if (!val.extractValue(unit.data, offset, unit.dwarf_unit->getFormParams(), unit.dwarf_unit)) throw Exception(ErrorCodes::CANNOT_PARSE_DWARF, "Failed to parse attribute {} of form {} at offset {}", llvm::dwarf::AttributeString(attr.Attr).operator std::string_view(), attr.Form, *offset); + checkAttributeEnd(attr, attr_offset, *offset, unit); return val; } @@ -379,11 +442,18 @@ void DWARFBlockInputFormat::skipAttribute( } else { + const uint64_t attr_offset = *offset; if (!llvm::DWARFFormValue::skipValue( - attr.Form, *extractor, offset, unit.dwarf_unit->getFormParams())) + attr.Form, unit.data, offset, unit.dwarf_unit->getFormParams())) throw Exception(ErrorCodes::CANNOT_PARSE_DWARF, "Failed to skip attribute {} of form {} at offset {}", llvm::dwarf::AttributeString(attr.Attr).operator std::string_view(), attr.Form, *offset); + /// `skipValue` does not advance the offset if the string is not terminated within the unit. + if (attr.Form == llvm::dwarf::DW_FORM_string && *offset == attr_offset) + throw Exception(ErrorCodes::CANNOT_PARSE_DWARF, + "String attribute {} at offset {} is not terminated before the end of the unit {}", + llvm::dwarf::AttributeString(attr.Attr).operator std::string_view(), attr_offset, unit.end_offset); + checkAttributeEnd(attr, attr_offset, *offset, unit); } } @@ -452,7 +522,7 @@ Chunk DWARFBlockInputFormat::parseEntries(UnitState & unit) col_ancestor_array_offsets->insertValue(col_ancestor_tags->size()); } - uint64_t abbrev_code = extractor->getULEB128(&unit.offset, &err); + uint64_t abbrev_code = unit.data.getULEB128(&unit.offset, &err); throwIfError(err, "DIE header"); if (abbrev_code == 0) diff --git a/src/Processors/Formats/Impl/DWARFBlockInputFormat.h b/src/Processors/Formats/Impl/DWARFBlockInputFormat.h index 0aad4631279fb..6becbc0b41515 100644 --- a/src/Processors/Formats/Impl/DWARFBlockInputFormat.h +++ b/src/Processors/Formats/Impl/DWARFBlockInputFormat.h @@ -49,6 +49,8 @@ class DWARFBlockInputFormat final : public IInputFormat const llvm::DWARFUnit * dwarf_unit; const llvm::DWARFAbbreviationDeclarationSet * abbrevs; uint64_t end_offset; + /// `.debug_info` truncated at the end of this unit, so that parsing its entries never reads the next unit. + llvm::DWARFDataExtractor data; std::string unit_name; ColumnPtr filename_table; // from .debug_line @@ -64,7 +66,7 @@ class DWARFBlockInputFormat final : public IInputFormat bool eof() const { return offset == end_offset; } - explicit UnitState(llvm::DWARFUnit * u); + UnitState(llvm::DWARFUnit * u, const llvm::DWARFDataExtractor & section_data); }; const FormatSettings format_settings; @@ -105,6 +107,9 @@ class DWARFBlockInputFormat final : public IInputFormat void parseFilenameTable(UnitState & unit, uint64_t offset); Chunk parseEntries(UnitState & unit); + void checkAttributeEnd( + const llvm::DWARFAbbreviationDeclaration::AttributeSpec & attr, uint64_t attr_offset, uint64_t offset, + const UnitState & unit) const; llvm::DWARFFormValue parseAttribute( const llvm::DWARFAbbreviationDeclaration::AttributeSpec & attr, uint64_t * offset, const UnitState & unit) const; diff --git a/tests/queries/0_stateless/05810_dwarf_attribute_past_end_of_section.reference b/tests/queries/0_stateless/05810_dwarf_attribute_past_end_of_section.reference new file mode 100644 index 0000000000000..be0a0fd5bdfcd --- /dev/null +++ b/tests/queries/0_stateless/05810_dwarf_attribute_past_end_of_section.reference @@ -0,0 +1,13 @@ +compile_unit producer exprloc +compile_unit name exprloc +CANNOT_PARSE_DWARF +CANNOT_PARSE_DWARF +variable location exprloc +variable name exprloc +CANNOT_PARSE_DWARF +CANNOT_PARSE_DWARF +compile_unit a +compile_unit b +CANNOT_PARSE_DWARF +CANNOT_PARSE_DWARF +CANNOT_PARSE_DWARF diff --git a/tests/queries/0_stateless/05810_dwarf_attribute_past_end_of_section.sh b/tests/queries/0_stateless/05810_dwarf_attribute_past_end_of_section.sh new file mode 100755 index 0000000000000..6b44a7ba836dd --- /dev/null +++ b/tests/queries/0_stateless/05810_dwarf_attribute_past_end_of_section.sh @@ -0,0 +1,100 @@ +#!/usr/bin/env bash +# Tags: no-fasttest, no-msan +# Tag no-fasttest: the `DWARF` format is not built in the fast test. +# Tag no-msan: the `DWARF` format is not built with MSan. + +# An attribute whose length goes past the end of `.debug_info` must be rejected by the `DWARF` format. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +prefix="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}" + +# A minimal ELF with one compilation unit with two attributes of form DW_FORM_exprloc either in the unit entry +# or in its child. The length of the first expression is either 1 (valid), 0xFFFFFFFF (past the end of the section) +# or 0xFFFFFFFFFFFFFFFF (makes the offset wrap around to before the attribute). +python3 - "$prefix" <<'PYEOF' +import struct +import sys + +ABBREV = bytes([ + # Abbreviation 1: DW_TAG_compile_unit without children, with DW_AT_producer and DW_AT_name, both of form DW_FORM_exprloc. + 0x01, 0x11, 0x00, 0x25, 0x18, 0x03, 0x18, 0x00, 0x00, + # Abbreviation 2: DW_TAG_compile_unit with children and without attributes. + 0x02, 0x11, 0x01, 0x00, 0x00, + # Abbreviation 3: DW_TAG_variable without children, with DW_AT_location and DW_AT_name, both of form DW_FORM_exprloc. + 0x03, 0x34, 0x00, 0x02, 0x18, 0x03, 0x18, 0x00, 0x00, + # Abbreviation 4: DW_TAG_compile_unit without children, with DW_AT_name of form DW_FORM_string. + 0x04, 0x11, 0x00, 0x03, 0x08, 0x00, 0x00, + # Abbreviation 5: DW_TAG_compile_unit without children, with DW_AT_language of form DW_FORM_data4. + 0x05, 0x11, 0x00, 0x13, 0x06, 0x00, 0x00, + # Abbreviation 6: DW_TAG_variable without children, with DW_AT_name of form DW_FORM_string. + 0x06, 0x34, 0x00, 0x03, 0x08, 0x00, 0x00, + 0x00]) + +def make_unit(entries): + header = bytes([0x05, 0x00, 0x01, 0x08, 0, 0, 0, 0]) # version 5, DW_UT_compile, address size 8, abbrev offset 0 + return struct.pack('&1 | grep -o -m1 'CANNOT_PARSE_DWARF' + $CLICKHOUSE_LOCAL -q "SELECT count() FROM file('${prefix}${kind}_wraparound.elf', DWARF)" 2>&1 | grep -o -m1 'CANNOT_PARSE_DWARF' + rm -f "${prefix}${kind}_valid.elf" "${prefix}${kind}_invalid.elf" "${prefix}${kind}_wraparound.elf" +done + +$CLICKHOUSE_LOCAL -q "SELECT tag, name FROM file('${prefix}_two_units_valid.elf', DWARF) ORDER BY offset" +for kind in string data4 nested_string; do + $CLICKHOUSE_LOCAL -q "SELECT count() FROM file('${prefix}_two_units_${kind}.elf', DWARF)" 2>&1 | grep -o -m1 'CANNOT_PARSE_DWARF' +done +rm -f "${prefix}"_two_units_*.elf From 09f1fe56b84e26382cecb8b5588f72b0653e2216 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 17 Nov 2026 04:56:56 +0000 Subject: [PATCH 210/219] Backport #131792 to 26.8: Reject repeated paths in serialized states of `mergedJSONPatch` --- .../AggregateFunctionMergedJSONPatch.cpp | 11 +++++++- ...json_patch_state_duplicate_paths.reference | 2 ++ ...erged_json_patch_state_duplicate_paths.sql | 28 +++++++++++++++++++ 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05781_merged_json_patch_state_duplicate_paths.reference create mode 100644 tests/queries/0_stateless/05781_merged_json_patch_state_duplicate_paths.sql diff --git a/src/AggregateFunctions/AggregateFunctionMergedJSONPatch.cpp b/src/AggregateFunctions/AggregateFunctionMergedJSONPatch.cpp index 2410e08ca2f58..21396ecc1efce 100644 --- a/src/AggregateFunctions/AggregateFunctionMergedJSONPatch.cpp +++ b/src/AggregateFunctions/AggregateFunctionMergedJSONPatch.cpp @@ -37,6 +37,7 @@ namespace ErrorCodes { extern const int NUMBER_OF_ARGUMENTS_DOESNT_MATCH; extern const int ILLEGAL_TYPE_OF_ARGUMENT; + extern const int INCORRECT_DATA; } @@ -281,8 +282,9 @@ struct AggregateFunctionMergedJSONPatchData size_t size = 0; readVarUInt(size, buf); + /// Do not reserve `size` entries upfront: it comes from untrusted data and can be arbitrarily large. + /// A state with a wrong size fails on reading after the end of the buffer instead. VectorWithMemoryTracking batch; - batch.reserve(size); for (size_t i = 0; i < size; ++i) { @@ -290,6 +292,13 @@ struct AggregateFunctionMergedJSONPatchData readStringBinary(lv.path, buf); readStringBinary(lv.value_blob, buf); lv.sort_key.deserialize(buf); + + /// `serialize` writes `entries`, which are sorted by path and have no duplicate paths. + /// A duplicate path would be inserted into the result twice for a single row of JSON. + if (i > 0 && batch[i - 1].path >= lv.path) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Paths in a serialized state of aggregate function mergedJSONPatch are not sorted or have duplicates (at path '{}')", + lv.path); } insertBatchAtomic(batch); diff --git a/tests/queries/0_stateless/05781_merged_json_patch_state_duplicate_paths.reference b/tests/queries/0_stateless/05781_merged_json_patch_state_duplicate_paths.reference new file mode 100644 index 0000000000000..936a03c1b20f3 --- /dev/null +++ b/tests/queries/0_stateless/05781_merged_json_patch_state_duplicate_paths.reference @@ -0,0 +1,2 @@ +{"a":1,"b":2} +{"a":1,"b":3,"c":{"d":2}} diff --git a/tests/queries/0_stateless/05781_merged_json_patch_state_duplicate_paths.sql b/tests/queries/0_stateless/05781_merged_json_patch_state_duplicate_paths.sql new file mode 100644 index 0000000000000..f67de3d5a7a7a --- /dev/null +++ b/tests/queries/0_stateless/05781_merged_json_patch_state_duplicate_paths.sql @@ -0,0 +1,28 @@ +-- A serialized state of `mergedJSONPatch` has its paths sorted and without duplicates. +-- A repeated path must be rejected instead of inserting several values of the path into one row of `JSON`. + +-- The state: the number of entries, then the path, the value and the sort key (`UInt64`) of every entry. +-- `a` is a typed path (`Nullable(Int64)`), `b` is a dynamic path (Dynamic binary: type `Int64` and the number). +WITH + '01' || hex('a') || '09' || '000100000000000000' || '0100000000000000' AS entry_a, + '01' || hex('b') || '09' || '0A0200000000000000' || '0100000000000000' AS entry_b +SELECT finalizeAggregation(CAST(unhex('02' || entry_a || entry_b), 'AggregateFunction(mergedJSONPatch, JSON(a Nullable(Int64)), UInt64)')); + +WITH + '01' || hex('b') || '09' || '0A0200000000000000' || '0100000000000000' AS entry_b +SELECT finalizeAggregation(CAST(unhex('02' || entry_b || entry_b), 'AggregateFunction(mergedJSONPatch, JSON(a Nullable(Int64)), UInt64)')); -- { serverError INCORRECT_DATA } + +WITH + '01' || hex('a') || '09' || '000100000000000000' || '0100000000000000' AS entry_a +SELECT finalizeAggregation(CAST(unhex('02' || entry_a || entry_a), 'AggregateFunction(mergedJSONPatch, JSON(a Nullable(Int64)), UInt64)')); -- { serverError INCORRECT_DATA } + +WITH + '01' || hex('a') || '09' || '000100000000000000' || '0100000000000000' AS entry_a, + '01' || hex('b') || '09' || '0A0200000000000000' || '0100000000000000' AS entry_b +SELECT finalizeAggregation(CAST(unhex('02' || entry_b || entry_a), 'AggregateFunction(mergedJSONPatch, JSON(a Nullable(Int64)), UInt64)')); -- { serverError INCORRECT_DATA } + +-- A huge number of entries must not be trusted for a preallocation. +SELECT finalizeAggregation(CAST(unhex('FFFFFFFFFFFFFFFF7F'), 'AggregateFunction(mergedJSONPatch, JSON(a Nullable(Int64)), UInt64)')); -- { serverError ATTEMPT_TO_READ_AFTER_EOF } + +-- A state produced by the function itself is accepted. +SELECT finalizeAggregation(mergedJSONPatchState('{"a" : 1, "c" : {"d" : 2}, "b" : 3}'::JSON(a Nullable(Int64)), 1::UInt64)); From 00cdf97d4256043f2257cc28d61a58c7bd519c91 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 17 Nov 2026 05:07:23 +0000 Subject: [PATCH 211/219] Backport #131797 to 26.8: Bounds-check skipping fully filtered values in the Parquet `PLAIN` string decoder --- .../Formats/Impl/Parquet/Decoding.cpp | 17 +++--- ...lain_string_fully_filtered_pages.reference | 7 +++ ...rquet_plain_string_fully_filtered_pages.sh | 57 +++++++++++++++++++ 3 files changed, 72 insertions(+), 9 deletions(-) create mode 100644 tests/queries/0_stateless/05802_parquet_plain_string_fully_filtered_pages.reference create mode 100755 tests/queries/0_stateless/05802_parquet_plain_string_fully_filtered_pages.sh diff --git a/src/Processors/Formats/Impl/Parquet/Decoding.cpp b/src/Processors/Formats/Impl/Parquet/Decoding.cpp index 87924b120ed70..a20f27cb38ccb 100644 --- a/src/Processors/Formats/Impl/Parquet/Decoding.cpp +++ b/src/Processors/Formats/Impl/Parquet/Decoding.cpp @@ -331,8 +331,9 @@ struct PlainStringDecoder : public PageDecoder { for (size_t i = 0; i < num_values; ++i) { + requireRemainingBytes(4); UInt32 x = 0; - memcpy(&x, data, 4); /// omitting range check because input is padded + memcpy(&x, data, 4); size_t len = 4 + size_t(x); requireRemainingBytes(len); data += len; @@ -353,8 +354,9 @@ struct PlainStringDecoder : public PageDecoder col_str.reserve(col_str.size() + to_reserve); for (size_t i = 0; i < num_values; ++i) { + requireRemainingBytes(4); UInt32 x = 0; - memcpy(&x, data, 4); /// omitting range check because input is padded + memcpy(&x, data, 4); size_t len = 4 + size_t(x); requireRemainingBytes(len); if (!filter || filter[filter_offset + i]) @@ -373,12 +375,7 @@ struct PlainStringDecoder : public PageDecoder num_pass += filter[filter_offset + i]; if (num_pass == 0) { - for (size_t i = 0; i < num_values; ++i) - { - UInt32 x = 0; - memcpy(&x, data, 4); - data += 4 + size_t(x); - } + skip(num_values); return; } chars_buf.reserve(num_pass * 32); // rough estimate @@ -386,6 +383,7 @@ struct PlainStringDecoder : public PageDecoder size_t offset = 0; for (size_t i = 0; i < num_values; ++i) { + requireRemainingBytes(4); UInt32 x = 0; memcpy(&x, data, 4); size_t len = 4 + size_t(x); @@ -408,8 +406,9 @@ struct PlainStringDecoder : public PageDecoder size_t offset = 0; for (size_t i = 0; i < num_values; ++i) { + requireRemainingBytes(4); UInt32 x = 0; - memcpy(&x, data, 4); /// omitting range check because input is padded + memcpy(&x, data, 4); size_t len = 4 + size_t(x); requireRemainingBytes(len); offset += len; diff --git a/tests/queries/0_stateless/05802_parquet_plain_string_fully_filtered_pages.reference b/tests/queries/0_stateless/05802_parquet_plain_string_fully_filtered_pages.reference new file mode 100644 index 0000000000000..d678195608b20 --- /dev/null +++ b/tests/queries/0_stateless/05802_parquet_plain_string_fully_filtered_pages.reference @@ -0,0 +1,7 @@ +10000 94995000 -94995000 0 +0 (0,0) +999 (999,-999) +2000 (2000,-2000) +2999 (2999,-2999) +3000 30748500 +INCORRECT_DATA diff --git a/tests/queries/0_stateless/05802_parquet_plain_string_fully_filtered_pages.sh b/tests/queries/0_stateless/05802_parquet_plain_string_fully_filtered_pages.sh new file mode 100755 index 0000000000000..85f88c52cb0b0 --- /dev/null +++ b/tests/queries/0_stateless/05802_parquet_plain_string_fully_filtered_pages.sh @@ -0,0 +1,57 @@ +#!/usr/bin/env bash + +# `PREWHERE` over a Parquet file with many small pages, where some pages lie entirely inside runs of +# filtered-out rows. Such pages are passed to the page decoders with an all-zero filter, and the decoders +# skip their values. Covers `PLAIN` `BYTE_ARRAY` read through a non-trivial string converter (WKB into +# `Geometry`). The values that pass must be decoded from the right positions. There is no page index, +# so the filter is applied inside the decoders. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +file="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}.parquet" + +$CLICKHOUSE_LOCAL -q " + SELECT number AS k, concat(unhex('0101000000'), reinterpretAsFixedString(toFloat64(number)), reinterpretAsFixedString(toFloat64(-number))) AS g + FROM numbers(20000) + FORMAT Parquet + SETTINGS output_format_parquet_string_as_string = 0, output_format_parquet_compression_method = 'none', + output_format_parquet_data_page_size = 1024, output_format_parquet_batch_size = 64, + output_format_parquet_write_page_index = 0, output_format_parquet_max_dictionary_size = 0, + output_format_parquet_write_bloom_filter = 0, output_format_parquet_row_group_size = 20000" > "$file" + +S="input_format_parquet_max_block_size = 4096, max_threads = 1" +F="file('$file', 'Parquet', 'k UInt64, g Geometry')" + +$CLICKHOUSE_LOCAL -q "SELECT count(), sum(variantElement(g, 'Point').1), sum(variantElement(g, 'Point').2), countIf(variantElement(g, 'Point') != (toFloat64(k), toFloat64(-k))) FROM $F PREWHERE intDiv(k, 1000) % 2 = 0 SETTINGS $S" +$CLICKHOUSE_LOCAL -q "SELECT k, g FROM $F PREWHERE intDiv(k, 1000) % 2 = 0 AND k % 1000 IN (0, 999) ORDER BY k LIMIT 4 SETTINGS $S" +$CLICKHOUSE_LOCAL -q "SELECT count(), sum(variantElement(g, 'Point').1) FROM $F PREWHERE k % 3000 >= 2500 SETTINGS $S" + +# Corrupt the length of the value for k = 5500, which lies in the middle of a range of filtered-out rows, +# so its page is fully filtered while other rows of the block pass (the filter is applied inside the +# decoders starting from the second block). Skipping the values of that page must detect the invalid +# length instead of reading out of bounds. Checksums are not written, so that the corruption is not +# caught by the page checksum. +corrupted="${CLICKHOUSE_TMP}/${CLICKHOUSE_TEST_UNIQUE_NAME}_corrupted.parquet" +$CLICKHOUSE_LOCAL -q " + SELECT number AS k, concat(unhex('0101000000'), reinterpretAsFixedString(toFloat64(number)), reinterpretAsFixedString(toFloat64(-number))) AS g + FROM numbers(20000) + FORMAT Parquet + SETTINGS output_format_parquet_string_as_string = 0, output_format_parquet_compression_method = 'none', + output_format_parquet_data_page_size = 1024, output_format_parquet_batch_size = 64, + output_format_parquet_write_page_index = 0, output_format_parquet_max_dictionary_size = 0, + output_format_parquet_write_bloom_filter = 0, output_format_parquet_row_group_size = 20000, + output_format_parquet_write_checksums = 0" > "$file" +$CLICKHOUSE_LOCAL -q " + WITH concat(unhex('0101000000'), reinterpretAsFixedString(toFloat64(5500)), reinterpretAsFixedString(toFloat64(-5500))) AS v + SELECT replaceOne(raw, concat(unhex('15000000'), v), concat(unhex('FFFFFF7F'), v)) + FROM file('$file', 'RawBLOB', 'raw String') + FORMAT RawBLOB" > "$corrupted" +cmp -s "$file" "$corrupted" && echo "the file was not corrupted" +F="file('$corrupted', 'Parquet', 'k UInt64, g Geometry')" +$CLICKHOUSE_LOCAL -q "SELECT count(), sum(variantElement(g, 'Point').1) FROM $F PREWHERE intDiv(k, 1000) % 2 = 0 SETTINGS $S" 2>&1 | grep -o -m1 'INCORRECT_DATA' + +rm -f "$corrupted" + +rm -f "$file" From fe93ba0fe3fed0a74e1d5713f12814b75a7a68c0 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 17 Nov 2026 08:16:11 +0000 Subject: [PATCH 212/219] Backport #130893 to 26.8: Keep the read-in-order virtual row for a key column fixed by `WHERE` and listed in `ORDER BY` --- .../Optimizations/optimizeReadInOrder.cpp | 22 +- ...d_in_order_virtual_row_fixed_key.reference | 94 ++++++++ ...20_read_in_order_virtual_row_fixed_key.sql | 200 ++++++++++++++++++ 3 files changed, 312 insertions(+), 4 deletions(-) create mode 100644 tests/queries/0_stateless/05620_read_in_order_virtual_row_fixed_key.reference create mode 100644 tests/queries/0_stateless/05620_read_in_order_virtual_row_fixed_key.sql diff --git a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp index c5e2f3bf05fb7..18516e6e709c2 100644 --- a/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp +++ b/src/Processors/QueryPlan/Optimizations/optimizeReadInOrder.cpp @@ -498,6 +498,9 @@ SortingInputOrder buildInputOrderFromSortDescription( std::vector match_infos; match_infos.reserve(description.size()); + /// For every fixed `ORDER BY` column: the read direction in which it is ordered like its key column, 0 if there is none. + std::vector fixed_column_directions; + while (next_description_column < description.size() && next_sort_key < sorting_key.column_names.size()) { const auto & sorting_key_column = sorting_key.column_names[next_sort_key]; @@ -564,14 +567,20 @@ SortingInputOrder buildInputOrderFromSortDescription( /// don't let it determine read direction - skip to next columns. /// Example: ORDER BY tenant, event_time DESC with WHERE tenant='42' /// tenant is constant, so read direction should come from event_time DESC. - if (fixed_columns.contains(sort_node)) + /// The key column has to be fixed as well: `toDate(ts) = '2024-01-01'` fixes `toDate(ts)` but not the key column `ts`. + if (fixed_columns.contains(sort_node) && fixed_key_columns.contains(sort_column_node)) { - /// Virtual row for fixed column from order by is not supported now. - can_optimize_virtual_row = false; + int direction = sort_column_description.direction * reverse_indicator; + if (match.monotonicity) + direction = match.monotonicity->strict ? direction * match.monotonicity->direction : 0; + const bool requests_nulls_last = sort_column_description.direction == sort_column_description.nulls_direction; + if (column_is_nullable && (direction * reverse_indicator == 1) != requests_nulls_last) + direction = 0; + fixed_column_directions.push_back(direction); /// Still add the fixed column to the sort description (required for sort matching) /// but don't set current_direction so it won't influence read_direction. - match_infos.push_back({.source = sort_node, .fixed_column = sort_node}); + match_infos.push_back({.source = sort_node, .monotonic = match.monotonicity ? &match : nullptr}); order_key_prefix_descr.push_back(sort_column_description); ++next_description_column; ++next_sort_key; @@ -665,6 +674,11 @@ SortingInputOrder buildInputOrderFromSortDescription( if (pk_column_names.size() < next_sort_key) can_optimize_virtual_row = false; + /// The virtual row takes a fixed `ORDER BY` column from the index like any other key column. + /// It bounds the rows only if the read goes in the direction of the column, as if the column was not fixed. + if (std::ranges::any_of(fixed_column_directions, [&](int direction) { return direction != read_direction; })) + can_optimize_virtual_row = false; + auto order_info = std::make_shared(order_key_prefix_descr, next_sort_key, read_direction, limit); std::optional virtual_row_conversion; diff --git a/tests/queries/0_stateless/05620_read_in_order_virtual_row_fixed_key.reference b/tests/queries/0_stateless/05620_read_in_order_virtual_row_fixed_key.reference new file mode 100644 index 0000000000000..4037806115c61 --- /dev/null +++ b/tests/queries/0_stateless/05620_read_in_order_virtual_row_fixed_key.reference @@ -0,0 +1,94 @@ +--- key (a, b): plan +-- WHERE a = 1 ORDER BY a, b +Read type: InOrder +Virtual row conversions +-- WHERE a = 1 ORDER BY a DESC, b DESC +Read type: InReverseOrder +Virtual row conversions +-- WHERE a IN (1) ORDER BY a, b +Read type: InOrder +Virtual row conversions +-- WHERE a = 1 ORDER BY a DESC, b: the read does not go in the direction of the fixed column, so its index values bound nothing +Read type: InOrder +--- key (a, b): result +1 +2 +5 +11 +12 +8063 +8062 +8061 +8060 +8059 +1 +1 +1 +1 +1 +-- the fixed column in the other direction, or not listed +1 +1 +1 +-- a virtual row after each block, several threads +1 +1 +1 +1 +--- key (a, b): the parts which begin with `a = 1` far from the result are not read +1 +--- key (a, b, c) +-- fixed middle key: WHERE b = 1 ORDER BY a, b, c +Read type: InOrder +Virtual row conversions +1 +1 +1 +-- two fixed keys: WHERE a = 1 AND b = 2 ORDER BY a, b, c +Read type: InOrder +Virtual row conversions +1 +1 +--- key (toDate(ts), b) +Read type: InOrder +Virtual row conversions +1 +1 +--- key (ts, b): `toDate(ts) = ...` fixes `toDate(ts)` but not the key column `ts`, which still determines the order +977 +978 +979 +980 +981 +977 +978 +979 +980 +981 +--- INNER JOIN: reading in order needs the virtual row +Read type: InOrder +Virtual row conversions +Read type: Default +Read type: InReverseOrder +Virtual row conversions +Read type: Default +21 63 +35 105 +42 126 +91 273 +105 315 +8057 24171 +8050 24150 +8043 24129 +8036 24108 +8029 24087 +-- with FINAL +Read type: InOrder +Virtual row conversions +Read type: Default +0 2 0 +7 2 21 +14 2 42 +21 2 63 +28 2 84 +1 diff --git a/tests/queries/0_stateless/05620_read_in_order_virtual_row_fixed_key.sql b/tests/queries/0_stateless/05620_read_in_order_virtual_row_fixed_key.sql new file mode 100644 index 0000000000000..7ceb63aa0cbe2 --- /dev/null +++ b/tests/queries/0_stateless/05620_read_in_order_virtual_row_fixed_key.sql @@ -0,0 +1,200 @@ +-- A key column fixed by the filter (`WHERE key = constant`) and listed in `ORDER BY` does not determine the direction of reading. +-- If the read goes in the direction of that column anyway, `ORDER BY` follows the sorting key as it is, and the virtual row +-- of reading in order stays enabled. Reading in order through an `INNER JOIN` needs it. + +DROP TABLE IF EXISTS t_fixed_key; +DROP TABLE IF EXISTS t_fixed_key_3; +DROP TABLE IF EXISTS t_fixed_key_expr; +DROP TABLE IF EXISTS t_fixed_key_ts; +DROP TABLE IF EXISTS t_fixed_key_right; +DROP TABLE IF EXISTS t_fixed_key_final; + +SET optimize_read_in_order = 1, read_in_order_use_virtual_row = 1, read_in_order_use_virtual_row_per_block = 0; +SET read_in_order_two_level_merge_threshold = 100, max_threads = 1, max_block_size = 8, enable_parallel_replicas = 0; +SET max_bytes_ratio_before_external_join = 0, max_bytes_before_external_join = 0, query_plan_read_in_order_through_join = 1, join_runtime_filter_min_probe_rows = 0; +SET query_plan_optimize_join_order_limit = 1, query_plan_optimize_join_order_randomize = 0, query_plan_join_swap_table = 0; + +CREATE TABLE t_fixed_key (a UInt32, b UInt32) ENGINE = MergeTree ORDER BY (a, b) SETTINGS index_granularity = 8; +SYSTEM STOP MERGES t_fixed_key; + +-- The granule where `a = 1` begins starts with `a = 0`, the one where it ends is followed by `a = 2`. +INSERT INTO t_fixed_key SELECT intDiv(number, 100) AS a, if(a = 1, 10 * (number % 100) + 1, 100000 + number) FROM numbers(300); +-- Begins with `a = 1`. +INSERT INTO t_fixed_key SELECT if(number < 80, 1, 2) AS a, if(a = 1, 10 * number + 2, 200000 + number) FROM numbers(120); +-- Ends with `a = 1`. +INSERT INTO t_fixed_key SELECT if(number < 44, 0, 1) AS a, if(a = 1, 10 * (number - 44) + 5, 400000 + number) FROM numbers(128); +-- No rows with `a = 1`, but the granule where `a` changes from 0 to 2 is selected. +INSERT INTO t_fixed_key SELECT if(number < 50, 0, 2), 300000 + number FROM numbers(100); +-- Only `a = 1`, far from both ends of the result. +INSERT INTO t_fixed_key SELECT 1, 5000 + number FROM numbers(64); +INSERT INTO t_fixed_key SELECT 1, 6000 + number FROM numbers(64); +INSERT INTO t_fixed_key SELECT 1, 7000 + number FROM numbers(64); +INSERT INTO t_fixed_key SELECT 1, 8000 + number FROM numbers(64); + +SELECT '--- key (a, b): plan'; + +SELECT '-- WHERE a = 1 ORDER BY a, b'; +SELECT extract(explain, '(?:Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 SELECT b FROM t_fixed_key WHERE a = 1 ORDER BY a, b LIMIT 5) WHERE e != ''; + +SELECT '-- WHERE a = 1 ORDER BY a DESC, b DESC'; +SELECT extract(explain, '(?:Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 SELECT b FROM t_fixed_key WHERE a = 1 ORDER BY a DESC, b DESC LIMIT 5) WHERE e != ''; + +SELECT '-- WHERE a IN (1) ORDER BY a, b'; +SELECT extract(explain, '(?:Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 SELECT b FROM t_fixed_key WHERE a IN (1) ORDER BY a, b LIMIT 5) WHERE e != ''; + +SELECT '-- WHERE a = 1 ORDER BY a DESC, b: the read does not go in the direction of the fixed column, so its index values bound nothing'; +SELECT extract(explain, '(?:Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 SELECT b FROM t_fixed_key WHERE a = 1 ORDER BY a DESC, b LIMIT 5) WHERE e != ''; + +SELECT '--- key (a, b): result'; + +SELECT b FROM t_fixed_key WHERE a = 1 ORDER BY a, b LIMIT 5; +SELECT b FROM t_fixed_key WHERE a = 1 ORDER BY a DESC, b DESC LIMIT 5; + +SELECT (SELECT groupArray(b) FROM (SELECT b FROM t_fixed_key WHERE a = 1 ORDER BY a, b LIMIT 40)) + = (SELECT arraySlice(arraySort(groupArray(b)), 1, 40) FROM t_fixed_key WHERE a = 1); +SELECT (SELECT groupArray(b) FROM (SELECT b FROM t_fixed_key WHERE a = 1 ORDER BY a, b)) + = (SELECT arraySort(groupArray(b)) FROM t_fixed_key WHERE a = 1); +SELECT (SELECT groupArray(b) FROM (SELECT b FROM t_fixed_key WHERE a = 1 ORDER BY a DESC, b DESC LIMIT 40)) + = (SELECT arraySlice(arrayReverseSort(groupArray(b)), 1, 40) FROM t_fixed_key WHERE a = 1); +SELECT (SELECT groupArray(b) FROM (SELECT b FROM t_fixed_key WHERE a = 1 ORDER BY a DESC, b DESC)) + = (SELECT arrayReverseSort(groupArray(b)) FROM t_fixed_key WHERE a = 1); +SELECT (SELECT groupArray(b) FROM (SELECT b FROM t_fixed_key WHERE a IN (1) ORDER BY a, b LIMIT 40)) + = (SELECT arraySlice(arraySort(groupArray(b)), 1, 40) FROM t_fixed_key WHERE a = 1); + +SELECT '-- the fixed column in the other direction, or not listed'; +SELECT (SELECT groupArray(b) FROM (SELECT b FROM t_fixed_key WHERE a = 1 ORDER BY a DESC, b LIMIT 40)) + = (SELECT arraySlice(arraySort(groupArray(b)), 1, 40) FROM t_fixed_key WHERE a = 1); +SELECT (SELECT groupArray(b) FROM (SELECT b FROM t_fixed_key WHERE a = 1 ORDER BY a, b DESC LIMIT 40)) + = (SELECT arraySlice(arrayReverseSort(groupArray(b)), 1, 40) FROM t_fixed_key WHERE a = 1); +SELECT (SELECT groupArray(b) FROM (SELECT b FROM t_fixed_key WHERE a = 1 ORDER BY b LIMIT 40)) + = (SELECT arraySlice(arraySort(groupArray(b)), 1, 40) FROM t_fixed_key WHERE a = 1); + +SELECT '-- a virtual row after each block, several threads'; +SELECT (SELECT groupArray(b) FROM (SELECT b FROM t_fixed_key WHERE a = 1 ORDER BY a, b LIMIT 40)) + = (SELECT arraySlice(arraySort(groupArray(b)), 1, 40) FROM t_fixed_key WHERE a = 1) +SETTINGS read_in_order_use_virtual_row_per_block = 1; +SELECT (SELECT groupArray(b) FROM (SELECT b FROM t_fixed_key WHERE a = 1 ORDER BY a DESC, b DESC LIMIT 40)) + = (SELECT arraySlice(arrayReverseSort(groupArray(b)), 1, 40) FROM t_fixed_key WHERE a = 1) +SETTINGS read_in_order_use_virtual_row_per_block = 1; +SELECT (SELECT groupArray(b) FROM (SELECT b FROM t_fixed_key WHERE a = 1 ORDER BY a, b)) + = (SELECT arraySort(groupArray(b)) FROM t_fixed_key WHERE a = 1) +SETTINGS max_threads = 4; +SELECT (SELECT groupArray(b) FROM (SELECT b FROM t_fixed_key WHERE a = 1 ORDER BY a DESC, b DESC)) + = (SELECT arrayReverseSort(groupArray(b)) FROM t_fixed_key WHERE a = 1) +SETTINGS max_threads = 4, read_in_order_use_virtual_row_per_block = 1; + +SELECT '--- key (a, b): the parts which begin with `a = 1` far from the result are not read'; + +SELECT b FROM t_fixed_key WHERE a = 1 ORDER BY a, b LIMIT 5 FORMAT Null SETTINGS log_comment = '05620_virtual_row'; +SELECT b FROM t_fixed_key WHERE a = 1 ORDER BY a, b LIMIT 5 FORMAT Null SETTINGS read_in_order_use_virtual_row = 0, log_comment = '05620_no_virtual_row'; +SYSTEM FLUSH LOGS query_log; + +SELECT + (SELECT read_rows FROM system.query_log + WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND log_comment = '05620_virtual_row' AND event_date >= yesterday() + ORDER BY event_time_microseconds DESC LIMIT 1) + < (SELECT read_rows FROM system.query_log + WHERE current_database = currentDatabase() AND type = 'QueryFinish' AND log_comment = '05620_no_virtual_row' AND event_date >= yesterday() + ORDER BY event_time_microseconds DESC LIMIT 1); + +SELECT '--- key (a, b, c)'; + +CREATE TABLE t_fixed_key_3 (a UInt32, b UInt32, c UInt32) ENGINE = MergeTree ORDER BY (a, b, c) SETTINGS index_granularity = 8; +SYSTEM STOP MERGES t_fixed_key_3; + +INSERT INTO t_fixed_key_3 SELECT intDiv(number, 40), intDiv(number, 10) % 4, ((number * 37) % 120) * 10 + 1 FROM numbers(120); +INSERT INTO t_fixed_key_3 SELECT 1 + intDiv(number, 40), intDiv(number, 10) % 4, ((number * 37) % 80) * 10 + 2 FROM numbers(80); +INSERT INTO t_fixed_key_3 SELECT intDiv(number, 30), 1 + intDiv(number, 10) % 3, ((number * 37) % 90) * 10 + 3 FROM numbers(90); +INSERT INTO t_fixed_key_3 SELECT if(number < 32, 1, 2), if(number < 16, 2, 3), ((number * 37) % 64) * 10 + 4 FROM numbers(64); +INSERT INTO t_fixed_key_3 SELECT if(number < 40, 0, 2), intDiv(number, 10) % 4, ((number * 37) % 80) * 10 + 5 FROM numbers(80); + +SELECT '-- fixed middle key: WHERE b = 1 ORDER BY a, b, c'; +SELECT extract(explain, '(?:Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 SELECT a, c FROM t_fixed_key_3 WHERE b = 1 ORDER BY a, b, c LIMIT 5) WHERE e != ''; + +SELECT (SELECT groupArray((a, c)) FROM (SELECT a, c FROM t_fixed_key_3 WHERE b = 1 ORDER BY a, b, c LIMIT 20)) + = (SELECT arraySlice(arraySort(groupArray((a, c))), 1, 20) FROM t_fixed_key_3 WHERE b = 1); +SELECT (SELECT groupArray((a, c)) FROM (SELECT a, c FROM t_fixed_key_3 WHERE b = 1 ORDER BY a DESC, b DESC, c DESC)) + = (SELECT arrayReverseSort(groupArray((a, c))) FROM t_fixed_key_3 WHERE b = 1); +SELECT (SELECT groupArray((a, c)) FROM (SELECT a, c FROM t_fixed_key_3 WHERE b = 1 ORDER BY a, b DESC, c)) + = (SELECT arraySort(groupArray((a, c))) FROM t_fixed_key_3 WHERE b = 1); + +SELECT '-- two fixed keys: WHERE a = 1 AND b = 2 ORDER BY a, b, c'; +SELECT extract(explain, '(?:Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 SELECT c FROM t_fixed_key_3 WHERE a = 1 AND b = 2 ORDER BY a, b, c LIMIT 5) WHERE e != ''; + +SELECT (SELECT groupArray(c) FROM (SELECT c FROM t_fixed_key_3 WHERE a = 1 AND b = 2 ORDER BY a, b, c LIMIT 20)) + = (SELECT arraySlice(arraySort(groupArray(c)), 1, 20) FROM t_fixed_key_3 WHERE a = 1 AND b = 2); +SELECT (SELECT groupArray(c) FROM (SELECT c FROM t_fixed_key_3 WHERE a = 1 AND b = 2 ORDER BY a DESC, b, c DESC)) + = (SELECT arrayReverseSort(groupArray(c)) FROM t_fixed_key_3 WHERE a = 1 AND b = 2); + +SELECT '--- key (toDate(ts), b)'; + +CREATE TABLE t_fixed_key_expr (ts DateTime('UTC'), b UInt32) ENGINE = MergeTree ORDER BY (toDate(ts), b) SETTINGS index_granularity = 8; +SYSTEM STOP MERGES t_fixed_key_expr; + +INSERT INTO t_fixed_key_expr SELECT toDateTime('2024-01-01 00:00:00', 'UTC') + intDiv(number, 60) * 86400 + number % 60 * 600, if(intDiv(number, 60) = 1, 10 * (number % 60) + 1, 100000 + number) FROM numbers(180); +INSERT INTO t_fixed_key_expr SELECT toDateTime('2024-01-02 00:00:00', 'UTC') + intDiv(number, 48) * 86400 + number % 48 * 600, if(intDiv(number, 48) = 0, 10 * (number % 48) + 2, 200000 + number) FROM numbers(96); +INSERT INTO t_fixed_key_expr SELECT toDateTime('2024-01-02 00:00:00', 'UTC') + number * 600, 5000 + number FROM numbers(64); + +SELECT extract(explain, '(?:Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 SELECT b FROM t_fixed_key_expr WHERE toDate(ts) = '2024-01-02' ORDER BY toDate(ts), b LIMIT 5) WHERE e != ''; + +SELECT (SELECT groupArray(b) FROM (SELECT b FROM t_fixed_key_expr WHERE toDate(ts) = '2024-01-02' ORDER BY toDate(ts), b LIMIT 20)) + = (SELECT arraySlice(arraySort(groupArray(b)), 1, 20) FROM t_fixed_key_expr WHERE toDate(ts) = '2024-01-02'); +SELECT (SELECT groupArray(b) FROM (SELECT b FROM t_fixed_key_expr WHERE toDate(ts) = '2024-01-02' ORDER BY toDate(ts) DESC, b)) + = (SELECT arraySort(groupArray(b)) FROM t_fixed_key_expr WHERE toDate(ts) = '2024-01-02'); + +SELECT '--- key (ts, b): `toDate(ts) = ...` fixes `toDate(ts)` but not the key column `ts`, which still determines the order'; + +CREATE TABLE t_fixed_key_ts (ts DateTime('UTC'), b UInt32) ENGINE = MergeTree ORDER BY (ts, b) SETTINGS index_granularity = 8; +SYSTEM STOP MERGES t_fixed_key_ts; + +INSERT INTO t_fixed_key_ts SELECT toDateTime('2024-01-01 00:00:00', 'UTC') + number * 3600, 1000 - number FROM numbers(60); +INSERT INTO t_fixed_key_ts SELECT toDateTime('2024-01-01 00:30:00', 'UTC') + number * 3600, 2000 - number FROM numbers(60); + +SELECT b FROM t_fixed_key_ts WHERE toDate(ts) = '2024-01-01' ORDER BY toDate(ts), b LIMIT 5; +SELECT b FROM t_fixed_key_ts WHERE toDate(ts) = '2024-01-01' ORDER BY toDate(ts) DESC, b LIMIT 5; + +SELECT '--- INNER JOIN: reading in order needs the virtual row'; + +CREATE TABLE t_fixed_key_right (b UInt32, w UInt32) ENGINE = MergeTree ORDER BY b; +INSERT INTO t_fixed_key_right SELECT number, number * 3 FROM numbers(9000) WHERE number % 7 = 0; + +SELECT extract(explain, '(?:Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 + SELECT l.b, r.w FROM t_fixed_key AS l INNER JOIN t_fixed_key_right AS r ON l.b = r.b WHERE l.a = 1 ORDER BY l.a, l.b LIMIT 5) WHERE e != ''; +SELECT extract(explain, '(?:Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 + SELECT l.b, r.w FROM t_fixed_key AS l INNER JOIN t_fixed_key_right AS r ON l.b = r.b WHERE l.a = 1 ORDER BY l.a DESC, l.b DESC LIMIT 5) WHERE e != ''; + +SELECT l.b, r.w FROM t_fixed_key AS l INNER JOIN t_fixed_key_right AS r ON l.b = r.b WHERE l.a = 1 ORDER BY l.a, l.b LIMIT 5; +SELECT l.b, r.w FROM t_fixed_key AS l INNER JOIN t_fixed_key_right AS r ON l.b = r.b WHERE l.a = 1 ORDER BY l.a DESC, l.b DESC LIMIT 5; + +SELECT '-- with FINAL'; + +CREATE TABLE t_fixed_key_final (a UInt32, b UInt32, ver UInt32) ENGINE = ReplacingMergeTree(ver) ORDER BY (a, b) SETTINGS index_granularity = 8; +SYSTEM STOP MERGES t_fixed_key_final; + +INSERT INTO t_fixed_key_final SELECT intDiv(number, 100) AS a, if(a = 1, 7 * (number % 100), 100000 + number), 1 FROM numbers(300); +INSERT INTO t_fixed_key_final SELECT if(number < 44, 0, 1) AS a, if(a = 1, 7 * (number - 44), 400000 + number), 2 FROM numbers(128); +INSERT INTO t_fixed_key_final SELECT 1, 7 * number + 3500, 3 FROM numbers(64); + +SELECT extract(explain, '(?:Read type|Virtual row conversions).*') AS e +FROM (EXPLAIN PLAN actions = 1, indexes = 0 + SELECT l.b, l.ver, r.w FROM t_fixed_key_final AS l FINAL INNER JOIN t_fixed_key_right AS r ON l.b = r.b WHERE l.a = 1 ORDER BY l.a, l.b LIMIT 5) WHERE e != ''; + +SELECT l.b, l.ver, r.w FROM t_fixed_key_final AS l FINAL INNER JOIN t_fixed_key_right AS r ON l.b = r.b WHERE l.a = 1 ORDER BY l.a, l.b LIMIT 5; +SELECT (SELECT groupArray((b, ver)) FROM (SELECT l.b AS b, l.ver AS ver FROM t_fixed_key_final AS l FINAL INNER JOIN t_fixed_key_right AS r ON l.b = r.b WHERE l.a = 1 ORDER BY l.a, l.b)) + = (SELECT arraySort(groupArray((b, ver))) FROM (SELECT l.b AS b, l.ver AS ver FROM t_fixed_key_final AS l FINAL INNER JOIN t_fixed_key_right AS r ON l.b = r.b WHERE l.a = 1)); + +DROP TABLE t_fixed_key; +DROP TABLE t_fixed_key_3; +DROP TABLE t_fixed_key_expr; +DROP TABLE t_fixed_key_ts; +DROP TABLE t_fixed_key_right; +DROP TABLE t_fixed_key_final; From 415b7e18bb8e12f03d30c107dac697e174c14a4c Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 17 Nov 2026 08:22:43 +0000 Subject: [PATCH 213/219] Backport #131736 to 26.8: Reject serialized query plan steps without their inputs --- .../QueryPlan/BlocksMarshallingStep.cpp | 8 +- src/Processors/QueryPlan/Serialization.cpp | 22 ++ src/Processors/QueryPlan/Serialization.h | 27 +- src/Processors/QueryPlan/UnionStep.cpp | 4 + ...5_query_plan_step_missing_inputs.reference | 6 + .../05775_query_plan_step_missing_inputs.sh | 274 ++++++++++++++++++ 6 files changed, 339 insertions(+), 2 deletions(-) create mode 100644 tests/queries/0_stateless/05775_query_plan_step_missing_inputs.reference create mode 100755 tests/queries/0_stateless/05775_query_plan_step_missing_inputs.sh diff --git a/src/Processors/QueryPlan/BlocksMarshallingStep.cpp b/src/Processors/QueryPlan/BlocksMarshallingStep.cpp index d2858b19fce39..5f843d63ed9e8 100644 --- a/src/Processors/QueryPlan/BlocksMarshallingStep.cpp +++ b/src/Processors/QueryPlan/BlocksMarshallingStep.cpp @@ -8,6 +8,11 @@ namespace DB { +namespace ErrorCodes +{ + extern const int INCORRECT_DATA; +} + static ITransformingStep::Traits getTraits() { return ITransformingStep::Traits @@ -73,7 +78,8 @@ void BlocksMarshallingStep::transformPipeline(QueryPipelineBuilder & pipeline, c QueryPlanStepPtr BlocksMarshallingStep::deserialize(Deserialization & ctx) { - chassert(ctx.input_headers.size() == 1); + if (ctx.input_headers.size() != 1) + throw Exception(ErrorCodes::INCORRECT_DATA, "BlocksMarshallingStep must have exactly one input, got {}", ctx.input_headers.size()); return std::make_unique(ctx.input_headers.front()); } diff --git a/src/Processors/QueryPlan/Serialization.cpp b/src/Processors/QueryPlan/Serialization.cpp index bf08e08aa2836..dfd8163c68d9d 100644 --- a/src/Processors/QueryPlan/Serialization.cpp +++ b/src/Processors/QueryPlan/Serialization.cpp @@ -27,6 +27,14 @@ namespace ErrorCodes extern const int LOGICAL_ERROR; } +const SharedHeader & DeserializedInputHeaders::at(size_t i) const +{ + if (i >= headers.size()) + throw Exception(ErrorCodes::INCORRECT_DATA, + "A step of the serialized query plan has {} input(s), but the input {} is required", headers.size(), i); + return headers[i]; +} + static void serializeHeader(const Block & header, WriteBuffer & out) { /// Write only names and types. @@ -270,12 +278,26 @@ QueryPlanAndSets QueryPlan::deserialize(ReadBuffer & in, const ContextPtr & cont SharedHeaders input_headers; input_headers.reserve(frame.children.size()); for (const auto & child : frame.children) + { + /// A sink (e.g. `GatherSend`) has no output, so it cannot be an input of another step. + if (!child->step->hasOutputHeader()) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Deserialized step {} has a child step {} without an output stream", + step_name, child->step->getName()); input_headers.push_back(child->step->getOutputHeader()); + } IQueryPlanStep::Deserialization ctx{ in, sets_registry, {}, context, input_headers, output_header, settings, max_type_complexity, flags.version, flags.skip_data}; auto step = step_registry.createStep(step_name, ctx); + /// The plan tree assumes that a step has exactly one input per child, e.g. optimizations iterate + /// over the children and access the input header of each of them. + if (step->getInputHeaders().size() != frame.children.size()) + throw Exception(ErrorCodes::INCORRECT_DATA, + "Deserialized step {} has {} input(s), but the serialized query plan has {} child(ren) of it", + step_name, step->getInputHeaders().size(), frame.children.size()); + if (step->hasOutputHeader()) { /// Headers encoding to the same bytes are indistinguishable to this serializer, so their diff --git a/src/Processors/QueryPlan/Serialization.h b/src/Processors/QueryPlan/Serialization.h index 42fca57ce0ca4..b085101b2c83e 100644 --- a/src/Processors/QueryPlan/Serialization.h +++ b/src/Processors/QueryPlan/Serialization.h @@ -31,6 +31,31 @@ struct IQueryPlanStep::Serialization struct SerializedSetsRegistry; +/// The output headers of the children of a step being deserialized. The number of children comes from +/// the serialized plan, so it may not be what the step expects: accessing a missing child throws +/// instead of reading out of bounds. Converts to `const SharedHeaders &` to pass all of them on. +class DeserializedInputHeaders +{ +public: + DeserializedInputHeaders(const SharedHeaders & headers_) : headers(headers_) {} /// NOLINT(google-explicit-constructor) + /// Only a reference is stored, so a temporary vector would dangle right after the initialization. + DeserializedInputHeaders(SharedHeaders && headers_) = delete; + operator const SharedHeaders &() const { return headers; } /// NOLINT(google-explicit-constructor) + + [[nodiscard]] size_t size() const { return headers.size(); } + [[nodiscard]] bool empty() const { return headers.empty(); } + [[nodiscard]] auto begin() const { return headers.begin(); } + [[nodiscard]] auto end() const { return headers.end(); } + + [[nodiscard]] const SharedHeader & at(size_t i) const; + [[nodiscard]] const SharedHeader & operator[](size_t i) const { return at(i); } + [[nodiscard]] const SharedHeader & front() const { return at(0); } + [[nodiscard]] const SharedHeader & back() const { return at(headers.empty() ? 0 : headers.size() - 1); } + +private: + const SharedHeaders & headers; +}; + /// Deserialization context passed to `IQueryPlanStep::deserialize`. struct IQueryPlanStep::Deserialization { @@ -40,7 +65,7 @@ struct IQueryPlanStep::Deserialization const ContextPtr & context; - const SharedHeaders & input_headers; + DeserializedInputHeaders input_headers; const SharedHeader & output_header; const QueryPlanSerializationSettings & settings; diff --git a/src/Processors/QueryPlan/UnionStep.cpp b/src/Processors/QueryPlan/UnionStep.cpp index bf6189acc0423..21f16a4ba4b14 100644 --- a/src/Processors/QueryPlan/UnionStep.cpp +++ b/src/Processors/QueryPlan/UnionStep.cpp @@ -23,6 +23,7 @@ namespace ErrorCodes { extern const int LOGICAL_ERROR; extern const int PARAMETER_OUT_OF_BOUND; + extern const int INCORRECT_DATA; } static bool containsAggregateStateColumn(const IColumn & column) @@ -232,6 +233,9 @@ void UnionStep::serialize(Serialization & ctx) const QueryPlanStepPtr UnionStep::deserialize(Deserialization & ctx) { + if (ctx.input_headers.empty()) + throw Exception(ErrorCodes::INCORRECT_DATA, "UnionStep must have at least one input"); + UInt8 flags = 0; readIntBinary(flags, ctx.in); return std::make_unique(ctx.input_headers, /*max_threads_=*/0, /*allow_narrowing_=*/flags & 1); diff --git a/tests/queries/0_stateless/05775_query_plan_step_missing_inputs.reference b/tests/queries/0_stateless/05775_query_plan_step_missing_inputs.reference new file mode 100644 index 0000000000000..fe7ad40f7f5c2 --- /dev/null +++ b/tests/queries/0_stateless/05775_query_plan_step_missing_inputs.reference @@ -0,0 +1,6 @@ +Extremes 117 +Limit 117 +Union 117 +Extremes with two children 117 +Extremes over GatherSend 117 +server alive 1 diff --git a/tests/queries/0_stateless/05775_query_plan_step_missing_inputs.sh b/tests/queries/0_stateless/05775_query_plan_step_missing_inputs.sh new file mode 100755 index 0000000000000..74dfbdc3bfb53 --- /dev/null +++ b/tests/queries/0_stateless/05775_query_plan_step_missing_inputs.sh @@ -0,0 +1,274 @@ +#!/usr/bin/env bash +# A step of a `QueryPlan` packet (native-protocol packet type 13) needs as many children as it reads input +# headers, but the number of children comes from the packet. Steps used to read the input headers without +# checking it, so a plan with a step lacking its children made the server read out of bounds. Now such a +# plan is rejected with `INCORRECT_DATA` and the server keeps serving. +# +# The packet is only reachable for a query sent with the `QueryPlan` processing stage while the +# server has `process_query_plan_packet` enabled (see tests/config/config.d/process_query_plan_packet.xml). +# The client here speaks the raw native protocol pinned to revision 54482 so the revision-gated +# fields of the handshake are fixed. + +CUR_DIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CUR_DIR"/../shell_config.sh + +python3 - <<'PYEOF' +import os +import socket +import uuid + +CLICKHOUSE_HOST = os.environ.get("CLICKHOUSE_HOST", "127.0.0.1") +CLICKHOUSE_PORT = int(os.environ.get("CLICKHOUSE_PORT_TCP", "9000")) +CLICKHOUSE_USER = os.environ.get("CLICKHOUSE_USER", "default") +CLICKHOUSE_PASSWORD = os.environ.get("CLICKHOUSE_PASSWORD", "") +CLICKHOUSE_DATABASE = os.environ.get("CLICKHOUSE_DATABASE", "default") + +CLIENT_NAME = "query plan missing inputs test" +REVISION = 54482 + +# Plan serialization version gates for header fields (see Core/ProtocolDefines.h). +WITH_EXECUTION_LIMITS = 10 # max_threads + concurrency_control +WITH_STEP_VERSIONS = 19 # per-step version + + +def writeVarUInt(x, ba): + for _ in range(0, 9): + byte = x & 0x7F + if x > 0x7F: + byte |= 0x80 + ba.append(byte) + x >>= 7 + if x == 0: + return + + +def varuint(x): + ba = bytearray() + writeVarUInt(x, ba) + return bytes(ba) + + +def writeStringBinary(s, ba): + b = bytes(s, "utf-8") if isinstance(s, str) else s + writeVarUInt(len(b), ba) + ba.extend(b) + + +def strbin(s): + ba = bytearray() + writeStringBinary(s, ba) + return bytes(ba) + + +def readStrict(s, size=1): + res = bytearray() + while size: + cur = s.recv(size) + if not cur: + raise EOFError("Connection closed by server") + size -= len(cur) + res.extend(cur) + return res + + +def readUInt(s, size=1): + res = readStrict(s, size) + val = 0 + for i in range(len(res)): + val += res[i] << (i * 8) + return val + + +def readUInt32(s): + return readUInt(s, 4) + + +def readVarUInt(s): + x = 0 + for i in range(9): + byte = readStrict(s)[0] + x |= (byte & 0x7F) << (7 * i) + if not byte & 0x80: + return x + return x + + +def readStringBinary(s): + size = readVarUInt(s) + return readStrict(s, size).decode("utf-8") + + +def sendHello(s): + ba = bytearray() + writeVarUInt(0, ba) # Hello + writeStringBinary(CLIENT_NAME, ba) + writeVarUInt(24, ba) # major + writeVarUInt(9, ba) # minor + writeVarUInt(REVISION, ba) + writeStringBinary(CLICKHOUSE_DATABASE, ba) + writeStringBinary(CLICKHOUSE_USER, ba) + writeStringBinary(CLICKHOUSE_PASSWORD, ba) + s.sendall(ba) + + +def receiveHello(s): + assert readVarUInt(s) == 0 # Hello + readStringBinary(s) # server name + readVarUInt(s) # major + readVarUInt(s) # minor + readVarUInt(s) # revision + readVarUInt(s) # parallel replicas protocol version + readStringBinary(s) # timezone + readStringBinary(s) # display name + readVarUInt(s) # version patch + readStringBinary(s) # proto_send chunked + readStringBinary(s) # proto_recv chunked + for _ in range(readVarUInt(s)): # password complexity rules + readStringBinary(s) + readStringBinary(s) + readStrict(s, 8) # nonce + while True: # server settings + if readStringBinary(s) == "": + break + readVarUInt(s) + readStringBinary(s) + query_plan_version = readVarUInt(s) # query plan serialization version + readVarUInt(s) # cluster function protocol version + return query_plan_version + + +def sendAddendum(s): + ba = bytearray() + writeStringBinary("", ba) # quota key + writeStringBinary("notchunked", ba) # proto_send chunked + writeStringBinary("notchunked", ba) # proto_recv chunked + writeVarUInt(0, ba) # parallel replicas protocol version + s.sendall(ba) + + +def serializeClientInfo(ba, query_id): + ba.append(1) # INITIAL_QUERY + writeStringBinary(CLICKHOUSE_USER, ba) # initial_user + writeStringBinary(query_id, ba) # initial_query_id + writeStringBinary("127.0.0.1:9000", ba) # initial_address + ba.extend([0] * 8) # initial_query_start_time_microseconds + ba.append(1) # interface = TCP + writeStringBinary("os_user", ba) + writeStringBinary("client_hostname", ba) + writeStringBinary(CLIENT_NAME, ba) + writeVarUInt(24, ba) # client major + writeVarUInt(9, ba) # client minor + writeVarUInt(REVISION, ba) # client tcp protocol version + writeStringBinary("", ba) # quota key + writeVarUInt(0, ba) # distributed_depth + writeVarUInt(1, ba) # client version patch + ba.append(0) # opentelemetry: no trace id + writeVarUInt(0, ba) # parallel replicas: collaborate_with_initiator + writeVarUInt(0, ba) # parallel replicas: obsolete count + writeVarUInt(0, ba) # parallel replicas: number_of_current_replica + writeVarUInt(0, ba) # script query number + writeVarUInt(0, ba) # script line number + ba.append(0) # jwt: none + + +def sendQuery(s, query, stage): + ba = bytearray() + query_id = uuid.uuid4().hex + writeVarUInt(1, ba) # Query + writeStringBinary(query_id, ba) + serializeClientInfo(ba, query_id) + writeStringBinary("", ba) # empty per-query settings + writeStringBinary("", ba) # interserver externally granted roles + writeStringBinary("", ba) # interserver secret + writeVarUInt(stage, ba) # query processing stage + ba.append(0) # no compression + writeStringBinary(query, ba) + writeStringBinary("", ba) # query parameters terminator + s.sendall(ba) + + +def buildNode(plan_version, step_name, payload, children): + """Serializes a plan node: its children first, then the step itself with an empty output header.""" + node = bytearray() + writeVarUInt(len(children), node) # num_children + for child in children: + node += child + writeStringBinary(step_name, node) + if plan_version >= WITH_STEP_VERSIONS: + writeVarUInt(0, node) # step_version + writeStringBinary("", node) # step_description + writeVarUInt(0, node) # output header: 0 columns + writeVarUInt(0, node) # settings: 0 changed + node += payload + return bytes(node) + + +def buildTree(plan_version, tree): + """Serializes a plan subtree given as `(step_name, payload, [subtrees of the children])`.""" + step_name, payload, children = tree + return buildNode(plan_version, step_name, payload, [buildTree(plan_version, child) for child in children]) + + +def buildQueryPlanPacket(plan_version, step_name, payload, num_children, children): + body = bytearray() + writeVarUInt(plan_version, body) + if plan_version >= WITH_EXECUTION_LIMITS: + writeVarUInt(0, body) # max_threads + body.append(0) # concurrency_control + # Node tree: a root step with the given children, or with the given number of `ReadNothing` children. + if children is None: + children = [("ReadNothing", b"", [])] * num_children + body += buildTree(plan_version, (step_name, payload, children)) + + packet = bytearray() + writeVarUInt(13, packet) # Client::QueryPlan + packet += body + return bytes(packet) + + +def readException(s): + code = readUInt32(s) + readStringBinary(s) # name + text = readStringBinary(s) + readStringBinary(s) # trace + readStrict(s) # has_nested + return code, text + + +def sendPlan(step_name, payload, num_children=0, children=None): + """Sends a `QueryPlan` packet with a step having `children` (or `num_children` `ReadNothing` children) and returns the exception code, or None if the query is not rejected.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.settimeout(60) + s.connect((CLICKHOUSE_HOST, CLICKHOUSE_PORT)) + sendHello(s) + plan_version = receiveHello(s) + sendAddendum(s) + # Stage 7 == QueryProcessingStage::QueryPlan: the server then expects a QueryPlan packet. + sendQuery(s, "SELECT 1", 7) + s.sendall(buildQueryPlanPacket(plan_version, step_name, payload, num_children, children)) + + # Only the first packet matters: either the exception or the start of the query result. + packet_type = readVarUInt(s) + if packet_type == 2: # Exception + return readException(s)[0] + return None + + +def main(): + # 117 == INCORRECT_DATA + for step_name, payload in [("Extremes", b""), ("Limit", b"\x00\x01\x00"), ("Union", b"\x00")]: + print(step_name, sendPlan(step_name, payload)) + # A step with more children than it has inputs. + print("Extremes with two children", sendPlan("Extremes", b"", num_children=2)) + # A sink has no output stream, so it cannot be a child of another step. + # `GatherSend` payload: exchange ID and no sort description. + gather_send = ("GatherSend", strbin("x") + b"\x00", [("ReadNothing", b"", [])]) + print("Extremes over GatherSend", sendPlan("Extremes", b"", children=[gather_send])) + + +main() +PYEOF + +# The server must still be alive and serving after rejecting the malicious packet. +${CLICKHOUSE_CLIENT} --query "SELECT 'server alive', 1" From 11949380d2c6772c1a219c514eebd6983051e664 Mon Sep 17 00:00:00 2001 From: robot-clickhouse Date: Tue, 17 Nov 2026 09:46:08 +0000 Subject: [PATCH 214/219] Backport #129818 to 26.8: Merge tables report rows and sizes only of the tables the user can see --- src/Storages/IStorage.h | 5 +- src/Storages/StorageAlias.cpp | 12 ++ src/Storages/StorageAlias.h | 9 +- src/Storages/StorageMerge.cpp | 69 +++++++++-- src/Storages/StorageMerge.h | 7 +- src/Storages/System/StorageSystemColumns.cpp | 2 +- ...e_storage_hidden_tables_metadata.reference | 32 ++++++ ...16_merge_storage_hidden_tables_metadata.sh | 108 ++++++++++++++++++ 8 files changed, 224 insertions(+), 20 deletions(-) create mode 100644 tests/queries/0_stateless/05616_merge_storage_hidden_tables_metadata.reference create mode 100755 tests/queries/0_stateless/05616_merge_storage_hidden_tables_metadata.sh diff --git a/src/Storages/IStorage.h b/src/Storages/IStorage.h index ee27ad3647b4e..323a6c67108f6 100644 --- a/src/Storages/IStorage.h +++ b/src/Storages/IStorage.h @@ -241,8 +241,9 @@ class IStorage : public std::enable_shared_from_this, public TypePromo /// The default implementation falls back to the parameterless version. virtual ColumnSizeByName getColumnSizes(const Names & /*columns*/, bool /*calculate_subcolumn_sizes*/) const { return getColumnSizes(); } - /// Same as getColumnSizes() but may return nullopt in some specific engines like Merge/Alias - virtual std::optional tryGetColumnSizes() const { return getColumnSizes(); } + /// Same as getColumnSizes() but may return nullopt in some specific engines like Merge/Alias, + /// and covers only the data visible to the user of `query_context`. + virtual std::optional tryGetColumnSizes(const ContextPtr & /*query_context*/) const { return getColumnSizes(); } /// Optional size information of each secondary index. /// Valid only for MergeTree family. diff --git a/src/Storages/StorageAlias.cpp b/src/Storages/StorageAlias.cpp index dff2af5530826..dc6695c6cf026 100644 --- a/src/Storages/StorageAlias.cpp +++ b/src/Storages/StorageAlias.cpp @@ -75,6 +75,18 @@ bool StorageAlias::isTargetTableGranted(ContextPtr query_context, AccessType acc return access->isGranted(access_type, target_database, target_table, column_name); } +std::optional StorageAlias::tryGetColumnSizes(const ContextPtr & query_context) const +{ + auto target = tryGetTargetTable(); + if (!target) + return std::nullopt; + + auto sizes = target->tryGetColumnSizes(query_context); + if (sizes) + std::erase_if(*sizes, [&](const auto & column_size) { return !isTargetTableGranted(query_context, AccessType::SHOW_COLUMNS, column_size.first); }); + return sizes; +} + /// AliasSink: Writes data to the target table using full INSERT pipeline /// which triggers materialized views on the target table. class AliasSink final : public SinkToStorage, WithContext diff --git a/src/Storages/StorageAlias.h b/src/Storages/StorageAlias.h index 7c60ab2006db1..8bf6d7dae7bf7 100644 --- a/src/Storages/StorageAlias.h +++ b/src/Storages/StorageAlias.h @@ -288,14 +288,7 @@ class StorageAlias final : public IStorage, WithContext ColumnSizeByName getColumnSizes() const override { auto target = tryGetTargetTable(); return target ? target->getColumnSizes() : ColumnSizeByName{}; } ColumnSizeByName getColumnSizes(const Names & columns, bool calculate_subcolumn_sizes) const override { auto target = tryGetTargetTable(); return target ? target->getColumnSizes(columns, calculate_subcolumn_sizes) : ColumnSizeByName{}; } - std::optional tryGetColumnSizes() const override - { - auto target = tryGetTargetTable(); - if (!target) - return std::nullopt; - - return target->getColumnSizes(); - } + std::optional tryGetColumnSizes(const ContextPtr & query_context) const override; IndexSizeByName getSecondaryIndexSizes() const override { auto target = tryGetTargetTable(); return target ? target->getSecondaryIndexSizes() : IndexSizeByName{}; } diff --git a/src/Storages/StorageMerge.cpp b/src/Storages/StorageMerge.cpp index 69c5fce2f15ba..891fcbe513e88 100644 --- a/src/Storages/StorageMerge.cpp +++ b/src/Storages/StorageMerge.cpp @@ -342,6 +342,25 @@ StoragePtr StorageMerge::traverseTablesUntilImpl(const ContextPtr & query_contex return {}; } +template +StoragePtr StorageMerge::traverseVisibleTablesUntil(const ContextPtr & query_context, F && predicate) const +{ + auto access = query_context->getAccess(); + std::optional last_database; + bool granted_show_on_all_tables = false; + return traverseTablesUntilImpl(query_context, this, database_name_or_regexp, [&](const StoragePtr & table) + { + auto id = table->getStorageID(); + if (id.database_name != last_database) + { + last_database = id.database_name; + granted_show_on_all_tables = access->isGranted(AccessType::SHOW_TABLES, id.database_name); + } + return (granted_show_on_all_tables || access->isGranted(AccessType::SHOW_TABLES, id.database_name, id.table_name)) + && predicate(table); + }); +} + template void StorageMerge::forEachTable(F && func) const { @@ -2180,11 +2199,27 @@ IStorage::ColumnSizeByName StorageMerge::getColumnSizes(const Names & columns, b return column_sizes; } -std::optional StorageMerge::tryGetColumnSizes() const +std::optional StorageMerge::tryGetColumnSizes(const ContextPtr & query_context) const { try { - return getColumnSizes(); + auto access = query_context->getAccess(); + ColumnSizeByName column_sizes; + auto first_table_without_sizes = traverseVisibleTablesUntil(query_context, [&](const auto & table) + { + auto sizes = table->tryGetColumnSizes(query_context); + if (!sizes) + return true; + auto id = table->getStorageID(); + bool show_all_columns = access->isGranted(AccessType::SHOW_COLUMNS, id.database_name, id.table_name); + for (const auto & [name, size] : *sizes) + if (show_all_columns || access->isGranted(AccessType::SHOW_COLUMNS, id.database_name, id.table_name, name)) + column_sizes[name].add(size); + return false; + }); + if (first_table_without_sizes) + return std::nullopt; + return column_sizes; } catch (const Exception & e) { @@ -2222,24 +2257,44 @@ bool StorageMerge::supportsTrivialCountOptimization(const StorageSnapshotPtr &, { /// Here we actually need storage snapshot of all nested tables. /// But to avoid complexity pass nullptr to make more lightweight check in MergeTreeData. - return traverseTablesUntil([&](const auto & table) { return !table->supportsTrivialCountOptimization(nullptr, ctx); }) == nullptr; + auto access = ctx->getAccess(); + return traverseVisibleTablesUntil(ctx, [&](const auto & table) + { + /// Like a direct `SELECT count()`, counting a child needs `SELECT` on at least one of its columns. + auto id = table->getStorageID(); + bool has_select_access = access->isGranted(AccessType::SELECT, id.database_name); + if (!has_select_access) + { + const auto metadata = table->getInMemoryMetadataPtr(ctx, false); + for (const auto & column : metadata->getColumns()) + { + if (access->isGranted(AccessType::SELECT, id.database_name, id.table_name, column.name)) + { + has_select_access = true; + break; + } + } + } + + return !has_select_access || !table->supportsTrivialCountOptimization(nullptr, ctx); + }) == nullptr; } std::optional StorageMerge::totalRows(ContextPtr query_context) const { - return totalRowsOrBytes([&](const auto & table) { return table->totalRows(query_context); }); + return totalRowsOrBytes(query_context, [&](const auto & table) { return table->totalRows(query_context); }); } std::optional StorageMerge::totalBytes(ContextPtr query_context) const { - return totalRowsOrBytes([&](const auto & table) { return table->totalBytes(query_context); }); + return totalRowsOrBytes(query_context, [&](const auto & table) { return table->totalBytes(query_context); }); } template -std::optional StorageMerge::totalRowsOrBytes(F && func) const +std::optional StorageMerge::totalRowsOrBytes(const ContextPtr & query_context, F && func) const { UInt64 total_rows_or_bytes = 0; - auto first_table = traverseTablesUntil([&](const auto & table) + auto first_table = traverseVisibleTablesUntil(query_context, [&](const auto & table) { if (auto rows_or_bytes = func(table)) { diff --git a/src/Storages/StorageMerge.h b/src/Storages/StorageMerge.h index e5fdaecaf70e9..072046829b5e9 100644 --- a/src/Storages/StorageMerge.h +++ b/src/Storages/StorageMerge.h @@ -140,6 +140,9 @@ class StorageMerge final : public IStorage, WithContext template StoragePtr traverseTablesUntil(F && predicate) const; + template + StoragePtr traverseVisibleTablesUntil(const ContextPtr & query_context, F && predicate) const; + template void forEachTable(F && func) const; @@ -156,7 +159,7 @@ class StorageMerge final : public IStorage, WithContext ColumnSizeByName getColumnSizes() const override; ColumnSizeByName getColumnSizes(const Names & columns, bool calculate_subcolumn_sizes) const override; - std::optional tryGetColumnSizes() const override; + std::optional tryGetColumnSizes(const ContextPtr & query_context) const override; ColumnsDescription getColumnsDescriptionFromSourceTables(const ContextPtr & context) const; @@ -165,7 +168,7 @@ class StorageMerge final : public IStorage, WithContext bool tableSupportsPrewhere() const; template - std::optional totalRowsOrBytes(F && func) const; + std::optional totalRowsOrBytes(const ContextPtr & query_context, F && func) const; friend class ReadFromMerge; }; diff --git a/src/Storages/System/StorageSystemColumns.cpp b/src/Storages/System/StorageSystemColumns.cpp index 63e0da72493c3..31bd431464be6 100644 --- a/src/Storages/System/StorageSystemColumns.cpp +++ b/src/Storages/System/StorageSystemColumns.cpp @@ -179,7 +179,7 @@ class ColumnsSource final : public ISource /// Certain information about a table - should be calculated only when the corresponding columns are queried. if (can_expose_any_column_metadata && (columns_mask[7] || columns_mask[8] || columns_mask[9])) { - if (auto sizes = storage->tryGetColumnSizes()) + if (auto sizes = storage->tryGetColumnSizes(context)) column_sizes = std::move(*sizes); } diff --git a/tests/queries/0_stateless/05616_merge_storage_hidden_tables_metadata.reference b/tests/queries/0_stateless/05616_merge_storage_hidden_tables_metadata.reference new file mode 100644 index 0000000000000..aa470e20a5752 --- /dev/null +++ b/tests/queries/0_stateless/05616_merge_storage_hidden_tables_metadata.reference @@ -0,0 +1,32 @@ +1 +m 3 +merge() 3 +m_func 3 +a_m 3 +mm 3 +m 3 1 +m id 1 1 +m s 1 1 +a_m 3 1 +a_m id 1 1 +a_m s 1 1 +mm 3 1 +mm id 1 1 +mm s 1 1 +1 +m_show 8 +m_col 10 +u2 t_hidden 0 +u2 m 3 +u2 m 3 +u2 m 3 +m_show id 1 1 +m_show s 1 1 +m_col id 1 1 +m_col s 1 1 +m_alias id 1 1 +m_alias s 1 1 +m_ashow id 1 1 +m_ashow s 1 1 +admin m 1003 +admin m 1003 diff --git a/tests/queries/0_stateless/05616_merge_storage_hidden_tables_metadata.sh b/tests/queries/0_stateless/05616_merge_storage_hidden_tables_metadata.sh new file mode 100755 index 0000000000000..5d03d178b2f7a --- /dev/null +++ b/tests/queries/0_stateless/05616_merge_storage_hidden_tables_metadata.sh @@ -0,0 +1,108 @@ +#!/usr/bin/env bash + +CURDIR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +# shellcheck source=../shell_config.sh +. "$CURDIR"/../shell_config.sh + +user="u_${CLICKHOUSE_DATABASE}" +user2="u2_${CLICKHOUSE_DATABASE}" +# The settings under which `count()` is answered from metadata (trivial count). +trivial="apply_patch_parts = 0, apply_mutations_on_fly = 0, optimize_trivial_count_query = 1, enable_parallel_replicas = 0, make_distributed_plan = 0, empty_result_for_aggregation_by_empty_set = 0" + +${CLICKHOUSE_CLIENT} --multiline -q " +DROP USER IF EXISTS ${user}; +DROP USER IF EXISTS ${user2}; +CREATE TABLE t_visible (id UInt32, s String) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; +CREATE TABLE t_hidden (id UInt32, s String) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; +CREATE TABLE t_show (id UInt32, s String) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; +CREATE TABLE t_col (id UInt32, s String) ENGINE = MergeTree ORDER BY id SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; +INSERT INTO t_visible SELECT number, 'v' FROM numbers(3); +INSERT INTO t_hidden SELECT number, repeat('h', 100) FROM numbers(1000); +INSERT INTO t_show SELECT number, 's' FROM numbers(5); +INSERT INTO t_col SELECT number, 'c' FROM numbers(7); +CREATE TABLE m (id UInt32, s String) ENGINE = Merge(currentDatabase(), '^t_(visible|hidden)\$'); +CREATE TABLE m_show (id UInt32, s String) ENGINE = Merge(currentDatabase(), '^t_(visible|show)\$'); +CREATE TABLE m_col (id UInt32, s String) ENGINE = Merge(currentDatabase(), '^t_(visible|col)\$'); +CREATE TABLE m_func AS merge(currentDatabase(), '^t_(visible|hidden)\$'); +CREATE TABLE a_m ENGINE = Alias(currentDatabase(), m); +CREATE TABLE mm (id UInt32, s String) ENGINE = Merge(currentDatabase(), '^m\$'); +CREATE TABLE a_hidden ENGINE = Alias(currentDatabase(), t_hidden); +CREATE TABLE m_alias (id UInt32, s String) ENGINE = Merge(currentDatabase(), '^(t_visible|a_hidden)\$'); +CREATE TABLE a_show ENGINE = Alias(currentDatabase(), t_show); +CREATE TABLE m_ashow (id UInt32, s String) ENGINE = Merge(currentDatabase(), '^(t_visible|a_show)\$'); +CREATE USER ${user} IDENTIFIED WITH plaintext_password BY '${user}'; +GRANT SELECT ON t_visible TO ${user}; +GRANT SELECT ON m TO ${user}; +GRANT SELECT ON m_show TO ${user}; +GRANT SELECT ON m_col TO ${user}; +GRANT SELECT ON m_func TO ${user}; +GRANT SELECT ON a_m TO ${user}; +GRANT SELECT ON mm TO ${user}; +GRANT SHOW TABLES ON t_show TO ${user}; +GRANT SELECT(id) ON t_col TO ${user}; +GRANT SELECT ON a_hidden TO ${user}; +GRANT SELECT ON m_alias TO ${user}; +GRANT SELECT ON a_show TO ${user}; +GRANT SELECT ON m_ashow TO ${user}; +CREATE USER ${user2} IDENTIFIED WITH plaintext_password BY '${user2}'; +GRANT SELECT ON ${CLICKHOUSE_DATABASE}.* TO ${user2}; +REVOKE SELECT ON ${CLICKHOUSE_DATABASE}.t_hidden FROM ${user2}; +SELECT min(data_compressed_bytes) > 0 FROM system.columns WHERE database = currentDatabase() AND table = 't_visible'; +" + +# Rows and sizes of a matched table the user cannot see are not reported. +${CLICKHOUSE_CLIENT} --user "${user}" --password "${user}" --multiline -q " +SELECT 'm', count() FROM m SETTINGS ${trivial}; +SELECT 'merge()', count() FROM merge(currentDatabase(), '^t_(visible|hidden)\$') SETTINGS ${trivial}; +SELECT 'm_func', count() FROM m_func; +SELECT 'a_m', count() FROM a_m SETTINGS ${trivial}; +SELECT 'mm', count() FROM mm SETTINGS ${trivial}; +" + +for table in m a_m mm +do + ${CLICKHOUSE_CLIENT} --user "${user}" --password "${user}" --multiline -q " + SELECT '${table}', total_rows, total_bytes = (SELECT total_bytes FROM system.tables WHERE database = currentDatabase() AND name = 't_visible') + FROM system.tables WHERE database = currentDatabase() AND name = '${table}'; + SELECT '${table}', c.name, c.data_compressed_bytes = v.data_compressed_bytes, c.data_uncompressed_bytes = v.data_uncompressed_bytes + FROM system.columns AS c INNER JOIN (SELECT name, data_compressed_bytes, data_uncompressed_bytes FROM system.columns WHERE database = currentDatabase() AND table = 't_visible') AS v USING (name) + WHERE c.database = currentDatabase() AND c.table = '${table}' ORDER BY c.name; + " +done + +# The trivial count is still used for a partially granted user. +${CLICKHOUSE_CLIENT} --user "${user}" --password "${user}" -q "EXPLAIN SELECT count() FROM m SETTINGS ${trivial}" | grep -c 'Optimized trivial count' + +# A visible table the user cannot read is not counted from metadata either. +${CLICKHOUSE_CLIENT} --user "${user}" --password "${user}" --multiline -q " +SELECT count() FROM m_show SETTINGS ${trivial}; -- { serverError ACCESS_DENIED } +SELECT count() FROM m_show; -- { serverError ACCESS_DENIED } +SELECT 'm_show', total_rows FROM system.tables WHERE database = currentDatabase() AND name = 'm_show'; +SELECT 'm_col', count() FROM m_col SETTINGS ${trivial}; +" + +# A table-level revoke inside a database-wide grant hides that table. +${CLICKHOUSE_CLIENT} --user "${user2}" --password "${user2}" --multiline -q " +SELECT 'u2 t_hidden', count() FROM system.tables WHERE database = currentDatabase() AND name = 't_hidden'; +SELECT 'u2 m', count() FROM m; +SELECT 'u2 m', count() FROM m SETTINGS ${trivial}; +SELECT 'u2 m', total_rows FROM system.tables WHERE database = currentDatabase() AND name = 'm'; +" + +# Column sizes of a Merge table are the sums of the column sizes the user sees on its children. +for table_children in "m_show:'t_visible','t_show'" "m_col:'t_visible','t_col'" "m_alias:'t_visible','a_hidden'" "m_ashow:'t_visible','a_show'" +do + table=${table_children%%:*} + ${CLICKHOUSE_CLIENT} --user "${user}" --password "${user}" -q " + SELECT '${table}', c.name, c.data_compressed_bytes = v.compressed, c.data_uncompressed_bytes = v.uncompressed + FROM system.columns AS c INNER JOIN (SELECT name, sum(data_compressed_bytes) AS compressed, sum(data_uncompressed_bytes) AS uncompressed + FROM system.columns WHERE database = currentDatabase() AND table IN (${table_children#*:}) GROUP BY name) AS v USING (name) + WHERE c.database = currentDatabase() AND c.table = '${table}' ORDER BY c.name" +done + +${CLICKHOUSE_CLIENT} --multiline -q " +SELECT 'admin m', count() FROM m SETTINGS ${trivial}; +SELECT 'admin m', total_rows FROM system.tables WHERE database = currentDatabase() AND name = 'm'; +DROP USER IF EXISTS ${user}; +DROP USER IF EXISTS ${user2}; +" From 8a8d27fe6bb29a6eda1c9530c8d96f66d542dac9 Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Tue, 17 Nov 2026 16:40:59 +0100 Subject: [PATCH 215/219] prewhere: skip DAG-only grouping for AST conditions The old analyzer's AST-based `PREWHERE` path has no `ActionsDAG` node. The throwing-condition grouping dereferences that null pointer when either plan-level `PREWHERE` optimization or all plan optimizations are disabled. Skip the DAG-only grouping for AST conditions, keeping the old grouping behavior. https://github.com/Altinity/ClickHouse/pull/2870#issuecomment-6948872234 --- .../MergeTree/MergeTreeWhereOptimizer.cpp | 4 +++- .../05801_prewhere_ast_null_dag.reference | 6 +++++ .../05801_prewhere_ast_null_dag.sql | 23 +++++++++++++++++++ 3 files changed, 32 insertions(+), 1 deletion(-) create mode 100644 tests/queries/0_stateless/05801_prewhere_ast_null_dag.reference create mode 100644 tests/queries/0_stateless/05801_prewhere_ast_null_dag.sql diff --git a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp index c7f4576b487a9..d0e6cbd2e9a39 100644 --- a/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp +++ b/src/Storages/MergeTree/MergeTreeWhereOptimizer.cpp @@ -466,7 +466,9 @@ void MergeTreeWhereOptimizer::analyzeImpl(Conditions & res, const RPNBuilderTree std::vector storage_columns_guarding_throwing_condition; for (size_t i = 0; i < infos.size(); ++i) { - if (!infos[i].viable || infos[i].expensive || !conditionMayThrow(infos[i].node.getDAGNode())) + /// The AST-based optimizer has no DAG node for this condition. + const auto * dag_node = infos[i].node.getDAGNode(); + if (!infos[i].viable || infos[i].expensive || !dag_node || !conditionMayThrow(dag_node)) continue; for (size_t j = 0; j < i; ++j) { diff --git a/tests/queries/0_stateless/05801_prewhere_ast_null_dag.reference b/tests/queries/0_stateless/05801_prewhere_ast_null_dag.reference new file mode 100644 index 0000000000000..b50a06b75db95 --- /dev/null +++ b/tests/queries/0_stateless/05801_prewhere_ast_null_dag.reference @@ -0,0 +1,6 @@ +0 +3 +0 +3 +0 +3 diff --git a/tests/queries/0_stateless/05801_prewhere_ast_null_dag.sql b/tests/queries/0_stateless/05801_prewhere_ast_null_dag.sql new file mode 100644 index 0000000000000..46730f1bf8518 --- /dev/null +++ b/tests/queries/0_stateless/05801_prewhere_ast_null_dag.sql @@ -0,0 +1,23 @@ +-- AST-backed predicates do not have an `ActionsDAG` node. +-- Ordinary comparisons make the result independent of predicate ordering. +SET enable_analyzer = 0, optimize_move_to_prewhere = 1; +SET allow_reorder_prewhere_conditions = 1, move_all_conditions_to_prewhere = 1, use_statistics = 0; + +DROP TABLE IF EXISTS t_prewhere_ast_null_dag; +CREATE TABLE t_prewhere_ast_null_dag (id UInt64, a UInt64, b UInt64) +ENGINE = MergeTree ORDER BY id +SETTINGS min_bytes_for_wide_part = 0, min_rows_for_wide_part = 0; +INSERT INTO t_prewhere_ast_null_dag VALUES (0, 1, 2), (1, 1, 3), (2, 0, 2), (3, 1, 2); + +-- With both plan optimizations enabled, the old analyzer still uses a DAG. +SELECT id FROM t_prewhere_ast_null_dag WHERE a = 1 AND b = 2 ORDER BY id +SETTINGS query_plan_optimize_prewhere = 1, query_plan_enable_optimizations = 1; + +-- Disabling either setting selects AST-based PREWHERE optimization. +SELECT id FROM t_prewhere_ast_null_dag WHERE a = 1 AND b = 2 ORDER BY id +SETTINGS query_plan_optimize_prewhere = 0, query_plan_enable_optimizations = 1; + +SELECT id FROM t_prewhere_ast_null_dag WHERE a = 1 AND b = 2 ORDER BY id +SETTINGS query_plan_optimize_prewhere = 1, query_plan_enable_optimizations = 0; + +DROP TABLE t_prewhere_ast_null_dag; From 99409d7d3a506b1287f62346208245fdc1511e3d Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Tue, 17 Nov 2026 16:47:16 +0100 Subject: [PATCH 216/219] tests: pin filter merging in the guard-ordering test With `query_plan_merge_filters = 0`, the 26.8 reader can evaluate a throwing condition before applying its guard's filter. Keep this ordering test on the default until the reader-safety fix and its performance follow-up are ported and verified in this branch. https://github.com/Altinity/ClickHouse/pull/2870#issuecomment-6948872234 --- .../0_stateless/05627_prewhere_guard_sibling_subcolumn.sql | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/queries/0_stateless/05627_prewhere_guard_sibling_subcolumn.sql b/tests/queries/0_stateless/05627_prewhere_guard_sibling_subcolumn.sql index 6319990ed7914..d39b56dedb1c6 100644 --- a/tests/queries/0_stateless/05627_prewhere_guard_sibling_subcolumn.sql +++ b/tests/queries/0_stateless/05627_prewhere_guard_sibling_subcolumn.sql @@ -2,6 +2,12 @@ -- when WHERE is moved to PREWHERE, although the throwing condition reads less and would be ordered first. -- https://github.com/ClickHouse/ClickHouse/issues/130086 +-- With `query_plan_merge_filters = 0`, this ordering test also needs a reader-safety fix. +-- Keep the default until that fix and its performance follow-up are ported and verified here: +-- https://github.com/ClickHouse/ClickHouse/pull/111954 +-- https://github.com/ClickHouse/ClickHouse/pull/132112 +SET query_plan_merge_filters = 1; + SET optimize_move_to_prewhere = 1, query_plan_optimize_prewhere = 1, allow_reorder_prewhere_conditions = 1; SET optimize_functions_to_subcolumns = 1, use_statistics = 0, move_all_conditions_to_prewhere = 1; From 5cb783694e3c75b98359c3a7819d13cd287b0881 Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Tue, 17 Nov 2026 18:23:33 +0100 Subject: [PATCH 217/219] tests: enable retries in the CAS throttle fixture The scripted client bypasses SDK retries, so disabling coordinated retries makes `Default` issue one request, just like `SingleAttempt`. Enable coordination only in the throttle test, with zero backoff, to restore its three-versus-one check without changing shared defaults. https://github.com/Altinity/ClickHouse/pull/2870#issuecomment-6948872234 --- src/Disks/tests/gtest_cas_upstream_slice.cpp | 37 ++++++++++++++------ 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/src/Disks/tests/gtest_cas_upstream_slice.cpp b/src/Disks/tests/gtest_cas_upstream_slice.cpp index 2012ac3f9d665..e39154d611a07 100644 --- a/src/Disks/tests/gtest_cas_upstream_slice.cpp +++ b/src/Disks/tests/gtest_cas_upstream_slice.cpp @@ -264,14 +264,18 @@ class ScriptedGetObjectClient : public DB::S3::Client const std::shared_ptr state; public: - ScriptedGetObjectClient() : ScriptedGetObjectClient(std::make_shared(), GetClientConfiguration()) { } + using ConfigureClient = std::function; - static DB::S3::PocoHTTPClientConfiguration GetClientConfiguration() + explicit ScriptedGetObjectClient(const ConfigureClient & configure = {}) + : ScriptedGetObjectClient(std::make_shared(), GetClientConfiguration(configure)) + { + } + + static DB::S3::PocoHTTPClientConfiguration GetClientConfiguration(const ConfigureClient & configure) { DB::RemoteHostFilter remote_host_filter; - /// max_retries is deliberately nonzero: it is the disk client's own attempt loop, and the - /// only thing that distinguishes it from the single-attempt clone. The two slow-down flags - /// are off so that loop spins without waiting out a real backoff. + /// Keep a nonzero retry budget so retry-enabled tests can distinguish this client + /// from its single-attempt clone. auto configuration = DB::S3::ClientFactory::instance().createClientConfiguration( "some-region", remote_host_filter, @@ -283,6 +287,8 @@ class ScriptedGetObjectClient : public DB::S3::Client /* for_disk_s3 = */ false, /* opt_disk_name = */ {}, /* request_throttler = */ {}); + if (configure) + configure(configuration); /// `ClientFactory::create` normally fills `retryStrategy`; this fake bypasses the factory. configuration.retryStrategy = std::make_shared(configuration.retry_strategy); return configuration; @@ -444,9 +450,10 @@ class ScriptedGetObjectClient : public DB::S3::Client std::shared_ptr makeScriptedS3ObjectStorage( ScriptedGetObjectClient *& out_client, - DB::S3ObjectStorage::S3CredentialsRefreshCallback credentials_refresh_callback = {}) + DB::S3ObjectStorage::S3CredentialsRefreshCallback credentials_refresh_callback = {}, + const ScriptedGetObjectClient::ConfigureClient & configure_client = {}) { - auto owned_client = std::make_unique(); + auto owned_client = std::make_unique(configure_client); out_client = owned_client.get(); DB::S3::URI uri; @@ -703,7 +710,7 @@ TEST(CASUpstreamSlice, ConditionalRemoveRecoversFromAnExpiredTokenAndInstallsThe EXPECT_EQ(refreshed->deleteRequestTimeouts().size(), 1u); } -/// The client the conditional DELETE selects is what decides whether the SDK reissues a throttled +/// The client the conditional DELETE selects is what decides whether ClickHouse reissues a throttled /// request. The `Default` half is what makes "exactly one" mean something: the disk client here does /// retry a throttle, so a single attempt is a property of the clone, not of the fake. /// @@ -714,8 +721,18 @@ TEST(CASUpstreamSlice, SingleAttemptConditionalRemoveIssuesExactlyOneRequestOnTh { (void)getContext(); + /// The fake bypasses SDK retries. Since https://github.com/ClickHouse/ClickHouse/pull/124557, + /// ClickHouse's loop retries throttle errors only with `s3_slow_all_threads_after_retryable_error` enabled. + /// Zero delays avoid a real backoff. + const auto configure_retries = [](DB::S3::PocoHTTPClientConfiguration & configuration) + { + configuration.s3_slow_all_threads_after_retryable_error = true; + configuration.retry_strategy.initial_delay_ms = 0; + configuration.retry_strategy.max_delay_ms = 0; + }; + ScriptedGetObjectClient * retrying = nullptr; - auto retrying_storage = makeScriptedS3ObjectStorage(retrying); + auto retrying_storage = makeScriptedS3ObjectStorage(retrying, {}, configure_retries); retrying->scriptDelete({controlThrottle()}); EXPECT_ANY_THROW(retrying_storage->removeObjectIfTokenMatches( @@ -723,7 +740,7 @@ TEST(CASUpstreamSlice, SingleAttemptConditionalRemoveIssuesExactlyOneRequestOnTh EXPECT_EQ(retrying->deleteRequestTimeouts().size(), 3u); /// max_retries = 2, so three attempts ScriptedGetObjectClient * client = nullptr; - auto storage = makeScriptedS3ObjectStorage(client); + auto storage = makeScriptedS3ObjectStorage(client, {}, configure_retries); client->scriptDelete({controlThrottle()}); EXPECT_ANY_THROW(storage->removeObjectIfTokenMatches( From 395d806cfd36b0242104a98fb0ed4b7afd16b522 Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Tue, 17 Nov 2026 18:36:49 +0100 Subject: [PATCH 218/219] tests: refresh Delta Lake pruning counts The bumped reader reports higher pruning counters for two predicates, while returning the same rows in all 15 cases. Align the expected counts so the regression test keeps checking `NULL` handling. https://github.com/Altinity/ClickHouse/pull/2870#issuecomment-6948872234 --- .../05576_delta_lake_partition_pruning_null_in_set.reference | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.reference b/tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.reference index 626195e3c0a6a..9bb359e9df5ae 100644 --- a/tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.reference +++ b/tests/queries/0_stateless/05576_delta_lake_partition_pruning_null_in_set.reference @@ -1,6 +1,6 @@ t: has([NULL, 1], p) [2] [2] 3 t: has([NULL, 5], p) [1,2] [1,2] 2 -t: NOT has([NULL, 1], p) [1,3,4] [1,3,4] 0 +t: NOT has([NULL, 1], p) [1,3,4] [1,3,4] 1 t: has([NULL, 1]::Array(Nullable(Int64)), p) [2] [2] 3 t: NOT has([NULL, 1]::Array(Nullable(Int64)), p) [1,3,4] [1,3,4] 1 t: p IN (NULL, 1), transform_null_in = 1 [2] [2] 3 @@ -8,7 +8,7 @@ t: p IS NULL, delta_lake_enable_engine_predicate = 0 [2] [2] 3 t: p = 7, delta_lake_enable_engine_predicate = 0 [3] [3] 3 t: p < 6, delta_lake_enable_engine_predicate = 0 [1] [1] 3 t: p IS NOT NULL, delta_lake_enable_engine_predicate = 0 [1,3,4] [1,3,4] 1 -t: has([5, 7], p) [1,3] [1,3] 1 +t: has([5, 7], p) [1,3] [1,3] 2 n: has([NULL, 5], p) [1] [1] 2 n: NOT has([NULL, 5], p) [3,4] [3,4] 1 t2: has([NULL, 1], a) [1,2,3] [1,2,3] 0 From 997e7a3ae390f383d7a4334cdd62173ad8cf563b Mon Sep 17 00:00:00 2001 From: Mikhail Koviazin Date: Tue, 17 Nov 2026 23:28:11 +0100 Subject: [PATCH 219/219] tests: exclude Parquet regression from fast tests The fast build has no `Parquet` support. Keep this regression in Parquet-enabled jobs. https://github.com/Altinity/ClickHouse/actions/runs/44141297255/job/134310276886?pr=2870 https://github.com/Altinity/ClickHouse/pull/2870 --- .../05802_parquet_plain_string_fully_filtered_pages.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/queries/0_stateless/05802_parquet_plain_string_fully_filtered_pages.sh b/tests/queries/0_stateless/05802_parquet_plain_string_fully_filtered_pages.sh index 85f88c52cb0b0..fcaaad2030700 100755 --- a/tests/queries/0_stateless/05802_parquet_plain_string_fully_filtered_pages.sh +++ b/tests/queries/0_stateless/05802_parquet_plain_string_fully_filtered_pages.sh @@ -1,4 +1,6 @@ #!/usr/bin/env bash +# Tags: no-fasttest +# Tag no-fasttest: `Parquet` is not built in the fast test. # `PREWHERE` over a Parquet file with many small pages, where some pages lie entirely inside runs of # filtered-out rows. Such pages are passed to the page decoders with an all-zero filter, and the decoders