From b3f14c5c018e0c323f9c57a1d7f293833e845c1c Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Wed, 18 Nov 2026 12:06:26 +0100 Subject: [PATCH 1/7] Ping remote shards of cluster reads before reusing a pooled connection Since https://github.com/ClickHouse/ClickHouse/pull/110068 a pooled connection is validated with a zero-timeout poll and not with a `Ping`. The poll sees a closed socket, but not a server that stopped without closing it (for example, `SIGSTOP`). For a cluster table function such as `fileCluster`, `s3Cluster` or a swarm read, such a replica stays in the pipeline and the query waits for `receive_timeout` (300 seconds by default) before it fails with `SOCKET_TIMEOUT`. Before that PR the `Ping` skipped the replica after `sync_request_timeout`. `ReadFromCluster::initializePipeline` now passes `ping_pooled_connection` to `getMany` for every shard that is not local. A local shard is excluded because `ReplicasAsShards` gives this server's own address a real TCP pool, and pinging that address does not reveal a dead peer. The ping is limited by `sync_request_timeout`. A failed ping raises `SOCKET_TIMEOUT` inside the synchronous `ConnectionEstablisher::run`, and that path already drops the socket and reconnects once. A connection that is not established yet is still checked by the handshake, so no extra round trip is added. The asynchronous path (hedged connections and `tryGetEntry` with an async callback) keeps the poll, because a blocking ping would stall the fiber. Other callers are unchanged and keep the poll-based validation. Adds the `unresponsive_pooled_connection` failpoint, which makes `Connection::ping` fail, and the stateless test `05812_ping_pooled_connection_on_cluster_read`. Issue: https://github.com/Altinity/ClickHouse/issues/2896 Co-authored-by: Cursor --- src/Client/Connection.cpp | 8 ++++ src/Client/ConnectionEstablisher.cpp | 46 +++++++++++++++---- src/Client/ConnectionEstablisher.h | 23 +++++++--- src/Client/ConnectionPoolWithFailover.cpp | 13 ++++-- src/Client/ConnectionPoolWithFailover.h | 9 +++- src/Common/FailPoint.cpp | 1 + src/Storages/IStorageCluster.cpp | 11 ++++- ...ooled_connection_on_cluster_read.reference | 4 ++ ...ping_pooled_connection_on_cluster_read.sql | 39 ++++++++++++++++ 9 files changed, 130 insertions(+), 24 deletions(-) create mode 100644 tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.reference create mode 100644 tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.sql diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 71df187ab8cfe..fccefeafec0f4 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -84,6 +84,7 @@ namespace FailPoints { extern const char receive_timeout_on_table_status_response[]; extern const char unexpected_packet_in_table_status_response[]; + extern const char unresponsive_pooled_connection[]; } namespace ErrorCodes @@ -915,6 +916,13 @@ void Connection::sendClusterNameAndSalt() bool Connection::ping(const ConnectionTimeouts & timeouts) { + /// For tests: a pooled peer that does not answer. `disconnect` matches the failure path below, + /// so the caller reconnects instead of reusing this socket. + fiu_do_on(FailPoints::unresponsive_pooled_connection, { + disconnect(); + return false; + }); + try { TimeoutSetter timeout_setter(*socket, timeouts.sync_request_timeout, true); diff --git a/src/Client/ConnectionEstablisher.cpp b/src/Client/ConnectionEstablisher.cpp index 121a4d16ac01f..24c4bd74765a4 100644 --- a/src/Client/ConnectionEstablisher.cpp +++ b/src/Client/ConnectionEstablisher.cpp @@ -45,8 +45,14 @@ ConnectionEstablisher::ConnectionEstablisher( const ConnectionTimeouts * timeouts_, const Settings & settings_, LoggerPtr log_, - const QualifiedTableName * table_to_check_) - : pool(std::move(pool_)), timeouts(timeouts_), settings(settings_), log(log_), table_to_check(table_to_check_) + const QualifiedTableName * table_to_check_, + bool ping_pooled_connection_) + : pool(std::move(pool_)) + , timeouts(timeouts_) + , settings(settings_) + , log(log_) + , table_to_check(table_to_check_) + , ping_pooled_connection(ping_pooled_connection_) { } @@ -55,12 +61,17 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std:: /// A pooled connection is handed out without pinging it first - a ping would add an unnecessary /// Ping-Pong round trip when the connection is still alive. A connection that the server has /// closed while it was idle in the pool is detected by a zero-timeout poll and recovered by - /// reconnecting (see Connection::forceConnected). If a pooled connection nevertheless fails its + /// reconnecting (see `Connection::forceConnected`). If a pooled connection nevertheless fails its /// very first request (e.g. the server went away without closing it), in the synchronous case we /// reconnect once and retry, so that a single available replica is not dropped just because its /// pooled connection went stale. In the asynchronous case we do not retry here - stale - /// connections are handled by retrying the next replica (see ConnectionPoolWithFailover). + /// connections are handled by retrying the next replica (see `ConnectionPoolWithFailover`). + /// + /// Cluster reads of a remote shard opt into a ping (`ping_pooled_connection`). The poll cannot + /// see a server that stopped without closing the socket, and the query would then block until + /// `receive_timeout`. The asynchronous path keeps the poll: a ping there would block the fiber. const bool can_reconnect = !async_callback; + const bool ping_pooled = ping_pooled_connection && can_reconnect; /// Whether the connection was taken from the pool already established. Only such a connection /// can be stale; a failure of a freshly established one is a genuine replica problem, and @@ -85,12 +96,28 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std:: }); /// Establish the connection, or revalidate a pooled one, under the async callback: with the - /// callback installed, Connection::connect uses a non-blocking connect and yields on it, so + /// callback installed, `Connection::connect` uses a non-blocking connect and yields on it, so /// a caller that multiplexes several connection attempts (e.g. hedged connections) can /// preempt a slow connect or handshake and switch to another replica. `result.entry` is /// assigned before connecting, so the connection being established is visible to the - /// multi-address timeout handling of ConnectionEstablisherAsync. - result.entry->forceConnected(*timeouts); + /// multi-address timeout handling of `ConnectionEstablisherAsync`. + /// + /// A pooled connection that the caller asked to ping is checked with `Ping` instead of the + /// poll. `Ping` fails when the server stopped without closing the socket; the poll reports + /// that socket as idle. A connection that is not established yet is connected, and the + /// handshake is the check. A failed ping is a soft error, so the retry below reconnects once. + if (ping_pooled && result.entry->isConnected()) + { + if (!result.entry->ping(*timeouts)) + { + LOG_TRACE(log, "Pooled connection did not answer a ping, will reconnect."); + throw Exception(ErrorCodes::SOCKET_TIMEOUT, "Ping of a pooled connection failed"); + } + } + else + { + result.entry->forceConnected(*timeouts); + } UInt64 server_revision = 0; if (table_to_check) @@ -219,9 +246,10 @@ ConnectionEstablisherAsync::ConnectionEstablisherAsync( const ConnectionTimeouts * timeouts_, const Settings & settings_, LoggerPtr log_, - const QualifiedTableName * table_to_check_) + const QualifiedTableName * table_to_check_, + bool ping_pooled_connection_) : AsyncTaskExecutor(std::make_unique(*this), "ConnectionEstablisherAsync") - , connection_establisher(std::move(pool_), timeouts_, settings_, log_, table_to_check_) + , connection_establisher(std::move(pool_), timeouts_, settings_, log_, table_to_check_, ping_pooled_connection_) { epoll.add(timeout_descriptor.getDescriptor()); } diff --git a/src/Client/ConnectionEstablisher.h b/src/Client/ConnectionEstablisher.h index c25d255b3c742..13cb73d0c329c 100644 --- a/src/Client/ConnectionEstablisher.h +++ b/src/Client/ConnectionEstablisher.h @@ -17,19 +17,26 @@ class ConnectionEstablisher public: using TryResult = PoolWithFailoverBase::TryResult; + /// `ping_pooled_connection` sends a `Ping` on a connection taken from the pool already + /// established. A zero-timeout poll cannot see a server that stopped without closing the + /// socket, and a cluster read would then wait for `receive_timeout`. The ping is bounded by + /// `sync_request_timeout`. Callers that can still run without this replica (cluster reads of + /// a remote shard) pass true. The default keeps the poll, with no round trip. ConnectionEstablisher(ConnectionPoolPtr pool_, const ConnectionTimeouts * timeouts_, const Settings & settings_, LoggerPtr log, - const QualifiedTableName * table_to_check = nullptr); + const QualifiedTableName * table_to_check = nullptr, + bool ping_pooled_connection_ = false); /// Establish connection and save it in result, write possible exception message in fail_message. /// The connection is returned from the connection pool and it can be stale (the server may have - /// closed it while it was idle). The pooled connection is not pinged before use: pinging it adds - /// an unnecessary Ping-Pong round trip when the connection is still alive. Instead, a stale - /// connection is detected by a zero-timeout poll (see Connection::forceConnected) or, failing - /// that, by retrying - either here (a pooled connection that fails its first request is - /// reconnected once) or at a higher level (ConnectionPoolWithFailover retries the next replica). + /// closed it while it was idle). Unless `ping_pooled_connection` was set, the pooled connection + /// is not pinged before use: pinging it adds an unnecessary Ping-Pong round trip when the + /// connection is still alive. Instead, a stale connection is detected by a zero-timeout poll + /// (see `Connection::forceConnected`) or, failing that, by retrying - either here (a pooled + /// connection that fails its first request is reconnected once) or at a higher level + /// (`ConnectionPoolWithFailover` retries the next replica). void run(TryResult & result, std::string & fail_message); /// Set async callback that will be called when reading from socket blocks. @@ -41,6 +48,7 @@ class ConnectionEstablisher const Settings & settings; LoggerPtr log; const QualifiedTableName * table_to_check; + bool ping_pooled_connection; AsyncCallback async_callback = {}; }; @@ -62,7 +70,8 @@ class ConnectionEstablisherAsync : public AsyncTaskExecutor const ConnectionTimeouts * timeouts_, const Settings & settings_, LoggerPtr log_, - const QualifiedTableName * table_to_check_ = nullptr); + const QualifiedTableName * table_to_check_ = nullptr, + bool ping_pooled_connection_ = false); /// Get file descriptor that can be added in epoll and be polled, /// when this fd becomes ready, you call resume establishing connection. diff --git a/src/Client/ConnectionPoolWithFailover.cpp b/src/Client/ConnectionPoolWithFailover.cpp index 0f18eaf45647f..90656f938b0ff 100644 --- a/src/Client/ConnectionPoolWithFailover.cpp +++ b/src/Client/ConnectionPoolWithFailover.cpp @@ -134,10 +134,11 @@ std::vector ConnectionPoolWithFailover::getMany( PoolMode pool_mode, AsyncCallback async_callback, std::optional skip_unavailable_endpoints, - GetPriorityForLoadBalancing::Func priority_func) + GetPriorityForLoadBalancing::Func priority_func, + bool ping_pooled_connection) { TryGetEntryFunc try_get_entry = [&](const NestedPoolPtr & pool, std::string & fail_message) - { return tryGetEntry(pool, timeouts, fail_message, settings, nullptr, async_callback); }; + { return tryGetEntry(pool, timeouts, fail_message, settings, nullptr, async_callback, ping_pooled_connection); }; std::vector results = getManyImpl(settings, pool_mode, try_get_entry, skip_unavailable_endpoints, priority_func); @@ -251,12 +252,14 @@ ConnectionPoolWithFailover::tryGetEntry( std::string & fail_message, const Settings & settings, const QualifiedTableName * table_to_check, - [[maybe_unused]] AsyncCallback async_callback) + [[maybe_unused]] AsyncCallback async_callback, + bool ping_pooled_connection) { #if defined(OS_LINUX) if (async_callback) { - ConnectionEstablisherAsync connection_establisher_async(pool, &timeouts, settings, log, table_to_check); + ConnectionEstablisherAsync connection_establisher_async( + pool, &timeouts, settings, log, table_to_check, ping_pooled_connection); while (true) { connection_establisher_async.resume(); @@ -277,7 +280,7 @@ ConnectionPoolWithFailover::tryGetEntry( } #endif - ConnectionEstablisher connection_establisher(pool, &timeouts, settings, log, table_to_check); + ConnectionEstablisher connection_establisher(pool, &timeouts, settings, log, table_to_check, ping_pooled_connection); TryResult result; connection_establisher.run(result, fail_message); return result; diff --git a/src/Client/ConnectionPoolWithFailover.h b/src/Client/ConnectionPoolWithFailover.h index eb4b28895b718..f528ad0a6c7c2 100644 --- a/src/Client/ConnectionPoolWithFailover.h +++ b/src/Client/ConnectionPoolWithFailover.h @@ -54,13 +54,17 @@ class ConnectionPoolWithFailover : public IConnectionPool, private PoolWithFailo /** Allocates up to the specified number of connections to work. * Connections provide access to different replicas of one shard. */ + /// `ping_pooled_connection` pings a connection that the pool returned already established. + /// Used by cluster reads of a remote shard: a server that stopped without closing the socket + /// is skipped after `sync_request_timeout` instead of blocking the query for `receive_timeout`. std::vector getMany( const ConnectionTimeouts & timeouts, const Settings & settings, PoolMode pool_mode, AsyncCallback async_callback = {}, std::optional skip_unavailable_endpoints = std::nullopt, - GetPriorityForLoadBalancing::Func priority_func = {}); + GetPriorityForLoadBalancing::Func priority_func = {}, + bool ping_pooled_connection = false); /// The same as getMany(), but return std::vector. std::vector getManyForTableFunction(const ConnectionTimeouts & timeouts, @@ -132,7 +136,8 @@ class ConnectionPoolWithFailover : public IConnectionPool, private PoolWithFailo std::string & fail_message, const Settings & settings, const QualifiedTableName * table_to_check = nullptr, - AsyncCallback async_callback = {}); + AsyncCallback async_callback = {}, + bool ping_pooled_connection = false); GetPriorityForLoadBalancing::Func makeGetPriorityFunc(const Settings & settings); diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index 41ba90c529d8c..fc4b33ac6c546 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -48,6 +48,7 @@ static struct InitFiu PAUSEABLE_ONCE(remote_query_executor_receive_packet_pause) \ PAUSEABLE_ONCE(remote_query_executor_finish_drain_pause) \ ONCE(connection_stale_on_establish) \ + REGULAR(unresponsive_pooled_connection) \ REGULAR(cluster_discovery_faults) \ REGULAR(cluster_discovery_unregister_fail) \ ONCE(cluster_discovery_retry_signal_fail) \ diff --git a/src/Storages/IStorageCluster.cpp b/src/Storages/IStorageCluster.cpp index f696540c68212..8ae4c38a93c9c 100644 --- a/src/Storages/IStorageCluster.cpp +++ b/src/Storages/IStorageCluster.cpp @@ -937,12 +937,21 @@ void ReadFromCluster::initializePipeline(QueryPipelineBuilder & pipeline, const /// We're taking all replicas as shards, /// so each shard will have only one address to connect to. + /// + /// A remote shard is pinged before the pooled connection is used. `ReplicasAsShards` gives + /// this server's own address a real TCP pool too, and `isLocal` is how that shard is told + /// apart: pinging it cannot discover a dead peer. A remote peer that stopped without + /// closing the socket otherwise stays in the pipeline and the query waits for + /// `receive_timeout`. + const bool ping_pooled_connection = !shard_info.isLocal(); auto try_results = shard_info.pool->getMany( timeouts, current_settings, PoolMode::GET_ONE, {}, - /*skip_unavailable_endpoints=*/true); + /*skip_unavailable_endpoints=*/true, + /*priority_func=*/{}, + ping_pooled_connection); if (try_results.empty()) continue; diff --git a/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.reference b/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.reference new file mode 100644 index 0000000000000..6813c34cb05cd --- /dev/null +++ b/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.reference @@ -0,0 +1,4 @@ +127.0.0.1 1 +127.0.0.2 0 +3 +0 diff --git a/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.sql b/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.sql new file mode 100644 index 0000000000000..5511fd0656b6e --- /dev/null +++ b/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.sql @@ -0,0 +1,39 @@ +-- Tags: no-fasttest, no-parallel +-- no-fasttest: the Fast test build has no libfiu, so SYSTEM ENABLE FAILPOINT is unsupported. +-- no-parallel: the fail point below fails every Ping in the server process. + +-- `test_cluster_two_shards` has one local address (127.0.0.1) and one that is not local +-- (127.0.0.2). A cluster read pings a pooled connection only for the remote shard. The fail +-- point makes that Ping fail. The read must still return every row: the remote shard is +-- reconnected, and the local shard is not pinged. `DistributedConnectionFailTry` is what shows +-- the Ping ran. The zero-timeout poll used for every other pooled connection does not call it. + +SELECT host_name, is_local +FROM system.clusters +WHERE cluster = 'test_cluster_two_shards' +ORDER BY host_name; + +INSERT INTO FUNCTION file(currentDatabase() || '_05812_data.tsv', 'TSV', 'x UInt8') +SELECT number FROM numbers(3) +SETTINGS engine_file_truncate_on_insert = 1; + +-- Fill the connection pool. The next read is the one that takes a pooled connection. +SELECT count() +FROM fileCluster('test_cluster_two_shards', currentDatabase() || '_05812_data.tsv', 'TSV', 'x UInt8') +FORMAT Null; + +CREATE TEMPORARY TABLE ping_fail_try_before (value UInt64); +INSERT INTO ping_fail_try_before +SELECT ifNull((SELECT value FROM system.events WHERE event = 'DistributedConnectionFailTry'), 0); + +SYSTEM ENABLE FAILPOINT unresponsive_pooled_connection; + +SELECT count() +FROM fileCluster('test_cluster_two_shards', currentDatabase() || '_05812_data.tsv', 'TSV', 'x UInt8'); + +SYSTEM DISABLE FAILPOINT unresponsive_pooled_connection; + +SELECT throwIf( + ifNull((SELECT value FROM system.events WHERE event = 'DistributedConnectionFailTry'), 0) + <= (SELECT value FROM ping_fail_try_before), + 'Remote shard was not pinged'); From 5ab6731f878e79dadd5666b3013f8996775af10d Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Wed, 18 Nov 2026 13:06:48 +0100 Subject: [PATCH 2/7] Fix build: call the public checkConnected for the pooled ping `Connection::ping` is private, so `ConnectionEstablisher::run` failed to compile in b3f14c5c018e. Call `Connection::checkConnected`, the public wrapper around `ping`. The branch that calls it is already known to be connected, so the call only runs the ping. Co-authored-by: Cursor --- src/Client/ConnectionEstablisher.cpp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Client/ConnectionEstablisher.cpp b/src/Client/ConnectionEstablisher.cpp index 24c4bd74765a4..93a5ae7c87277 100644 --- a/src/Client/ConnectionEstablisher.cpp +++ b/src/Client/ConnectionEstablisher.cpp @@ -108,7 +108,9 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std:: /// handshake is the check. A failed ping is a soft error, so the retry below reconnects once. if (ping_pooled && result.entry->isConnected()) { - if (!result.entry->ping(*timeouts)) + /// `ping` is private. `checkConnected` is the public wrapper, and this branch + /// already knows the connection is established, so it only runs the ping. + if (!result.entry->checkConnected(*timeouts)) { LOG_TRACE(log, "Pooled connection did not answer a ping, will reconnect."); throw Exception(ErrorCodes::SOCKET_TIMEOUT, "Ping of a pooled connection failed"); From c3462f343fb27df951110393cad1870f62e735ac Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Wed, 18 Nov 2026 13:50:11 +0100 Subject: [PATCH 3/7] Skip a cluster shard whose pooled ping times out A cluster read pings a pooled connection to a remote shard before using it. If the peer kept the socket open and sent no `Pong` within `sync_request_timeout`, the establisher reconnected once and waited up to `handshake_timeout_ms` for a `Hello` that a stopped process never sends. `Connection::checkConnectedThrowOnTimeout` lets a `SOCKET_TIMEOUT` out of `Connection::ping`. `ConnectionEstablisher::run` then skips the replica and does not reconnect. A socket the server has already closed still returns false from `ping`, so it still reconnects once. `checkConnected` keeps its old contract for the client resync path. Adds the `pooled_connection_ping_timeout` failpoint and two checks to `05812_ping_pooled_connection_on_cluster_read`: a timed-out ping does not increase `DistributedConnectionReconnectCount`, and it does increase `DistributedConnectionFailTry`. Related: https://github.com/Altinity/ClickHouse/issues/2896 Co-authored-by: Cursor --- src/Client/Connection.cpp | 27 +++++++++++++-- src/Client/Connection.h | 13 ++++++- src/Client/ConnectionEstablisher.cpp | 34 +++++++++++++++---- src/Client/ConnectionEstablisher.h | 7 ++-- src/Client/ConnectionPoolWithFailover.h | 6 ++-- src/Common/FailPoint.cpp | 1 + src/Storages/IStorageCluster.cpp | 3 +- ...ooled_connection_on_cluster_read.reference | 3 ++ ...ping_pooled_connection_on_cluster_read.sql | 31 ++++++++++++++++- 9 files changed, 108 insertions(+), 17 deletions(-) diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index fccefeafec0f4..cf0de08529bf6 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -85,6 +85,7 @@ namespace FailPoints extern const char receive_timeout_on_table_status_response[]; extern const char unexpected_packet_in_table_status_response[]; extern const char unresponsive_pooled_connection[]; + extern const char pooled_connection_ping_timeout[]; } namespace ErrorCodes @@ -914,15 +915,22 @@ void Connection::sendClusterNameAndSalt() } #endif -bool Connection::ping(const ConnectionTimeouts & timeouts) +bool Connection::ping(const ConnectionTimeouts & timeouts, bool rethrow_on_timeout) { - /// For tests: a pooled peer that does not answer. `disconnect` matches the failure path below, - /// so the caller reconnects instead of reusing this socket. + /// For tests: a pooled socket the server has already closed. `disconnect` matches the failure + /// path below, so the caller reconnects instead of reusing this socket. fiu_do_on(FailPoints::unresponsive_pooled_connection, { disconnect(); return false; }); + /// For tests: a peer that keeps the socket open and sends no `Pong`. Callers that asked to + /// see the timeout skip this replica instead of opening another connection. + fiu_do_on(FailPoints::pooled_connection_ping_timeout, { + disconnect(); + throw Exception(ErrorCodes::SOCKET_TIMEOUT, "Injected ping timeout"); + }); + try { TimeoutSetter timeout_setter(*socket, timeouts.sync_request_timeout, true); @@ -951,6 +959,19 @@ bool Connection::ping(const ConnectionTimeouts & timeouts) if (pong != Protocol::Server::Pong) throwUnexpectedPacket(pong, "Pong", &timeout_setter); } + catch (const Exception & e) + { + /// `Exception` extends `Poco::Exception`, so this arm has to come first. A read or write + /// that reaches `sync_request_timeout` is `SOCKET_TIMEOUT`. With `rethrow_on_timeout` the + /// caller skips the replica. Every other failure still returns false, and `checkConnected` + /// keeps its old contract. + const bool timed_out = rethrow_on_timeout && e.code() == ErrorCodes::SOCKET_TIMEOUT; + disconnect(); + LOG_TRACE(log_wrapper.get(), fmt::runtime(e.displayText())); + if (timed_out) + throw; + return false; + } catch (const Poco::Exception & e) { /// Explicitly disconnect since ping() can receive EndOfStream, diff --git a/src/Client/Connection.h b/src/Client/Connection.h index 3ec99be2cf9c7..0c1bd025598fa 100644 --- a/src/Client/Connection.h +++ b/src/Client/Connection.h @@ -173,6 +173,15 @@ class Connection : public IServerConnection bool checkConnected(const ConnectionTimeouts & timeouts) override { return isConnected() && ping(timeouts); } + /// Same round trip as `checkConnected`, but a peer that keeps the socket open and sends no + /// `Pong` within `sync_request_timeout` throws `SOCKET_TIMEOUT`. A socket the server has + /// already closed still returns false. Cluster reads use the difference so a silent peer is + /// skipped instead of opening another connection and waiting out `handshake_timeout_ms`. + bool checkConnectedThrowOnTimeout(const ConnectionTimeouts & timeouts) + { + return isConnected() && ping(timeouts, /*rethrow_on_timeout=*/ true); + } + /// Note that a server that went away without closing the connection is not detected here, and /// neither is a close that has not arrived yet; that only shows up when the connection is used. /// Pinging the server to find out would add a round trip to every query, and a pong that does @@ -372,7 +381,9 @@ class Connection : public IServerConnection #if USE_SSL void sendClusterNameAndSalt(); #endif - bool ping(const ConnectionTimeouts & timeouts); + /// `rethrow_on_timeout` lets `SOCKET_TIMEOUT` out. Other failures still return false after + /// `disconnect`, which is what `checkConnected` and the clickhouse-client resync rely on. + bool ping(const ConnectionTimeouts & timeouts, bool rethrow_on_timeout = false); /// Whether the connection can no longer serve a request, checked without a round trip. bool isStale(); diff --git a/src/Client/ConnectionEstablisher.cpp b/src/Client/ConnectionEstablisher.cpp index 93a5ae7c87277..8e2df0c865265 100644 --- a/src/Client/ConnectionEstablisher.cpp +++ b/src/Client/ConnectionEstablisher.cpp @@ -77,12 +77,16 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std:: /// can be stale; a failure of a freshly established one is a genuine replica problem, and /// retrying it is left to the caller (with its error accounting). bool had_pooled_connection = false; + /// Set when `Ping` reached `sync_request_timeout`. The peer kept the socket open, so a new + /// handshake would wait for `handshake_timeout_ms` and then fail the same way. + bool pooled_ping_timed_out = false; auto try_establish = [&]() { ProfileEvents::increment(ProfileEvents::DistributedConnectionTries); had_pooled_connection = false; + pooled_ping_timed_out = false; result.entry = pool->getUnchecked(*timeouts, settings); had_pooled_connection = result.entry->isConnected(); @@ -105,12 +109,28 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std:: /// A pooled connection that the caller asked to ping is checked with `Ping` instead of the /// poll. `Ping` fails when the server stopped without closing the socket; the poll reports /// that socket as idle. A connection that is not established yet is connected, and the - /// handshake is the check. A failed ping is a soft error, so the retry below reconnects once. + /// handshake is the check. A dead socket is a soft error, so the retry below reconnects + /// once. A timeout means the peer kept the socket and sent nothing, and the retry is + /// skipped: the next handshake would block for `handshake_timeout_ms`. if (ping_pooled && result.entry->isConnected()) { - /// `ping` is private. `checkConnected` is the public wrapper, and this branch - /// already knows the connection is established, so it only runs the ping. - if (!result.entry->checkConnected(*timeouts)) + /// `ping` is private. `checkConnectedThrowOnTimeout` is the wrapper that lets a + /// `SOCKET_TIMEOUT` out. This branch already knows the connection is established. + bool answered = false; + try + { + answered = result.entry->checkConnectedThrowOnTimeout(*timeouts); + } + catch (const Exception & e) + { + if (e.code() == ErrorCodes::SOCKET_TIMEOUT) + { + pooled_ping_timed_out = true; + LOG_TRACE(log, "Pooled connection timed out on ping, will skip it."); + } + throw; + } + if (!answered) { LOG_TRACE(log, "Pooled connection did not answer a ping, will reconnect."); throw Exception(ErrorCodes::SOCKET_TIMEOUT, "Ping of a pooled connection failed"); @@ -221,8 +241,10 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std:: /// Reconnect and retry once if a previously established (pooled) connection went stale. /// The failed request is safe to repeat: it is either a read-only status request, or the - /// connection was closed before the request reached a live server. - if (can_reconnect && tries == 0 && had_pooled_connection) + /// connection was closed before the request reached a live server. A ping that timed out + /// is the other case: the peer still holds the socket, and another handshake would wait + /// for `handshake_timeout_ms` with the same result, so the replica is skipped here. + if (can_reconnect && tries == 0 && had_pooled_connection && !pooled_ping_timed_out) { ProfileEvents::increment(ProfileEvents::DistributedConnectionReconnectCount); continue; diff --git a/src/Client/ConnectionEstablisher.h b/src/Client/ConnectionEstablisher.h index 13cb73d0c329c..d89b671061230 100644 --- a/src/Client/ConnectionEstablisher.h +++ b/src/Client/ConnectionEstablisher.h @@ -19,9 +19,10 @@ class ConnectionEstablisher /// `ping_pooled_connection` sends a `Ping` on a connection taken from the pool already /// established. A zero-timeout poll cannot see a server that stopped without closing the - /// socket, and a cluster read would then wait for `receive_timeout`. The ping is bounded by - /// `sync_request_timeout`. Callers that can still run without this replica (cluster reads of - /// a remote shard) pass true. The default keeps the poll, with no round trip. + /// socket, and a cluster read would then wait for `receive_timeout`. No `Pong` within + /// `sync_request_timeout` skips the replica, with no further handshake. A socket the server + /// has already closed still reconnects once. Callers that can run without this replica + /// (cluster reads of a remote shard) pass true. The default keeps the poll, with no round trip. ConnectionEstablisher(ConnectionPoolPtr pool_, const ConnectionTimeouts * timeouts_, const Settings & settings_, diff --git a/src/Client/ConnectionPoolWithFailover.h b/src/Client/ConnectionPoolWithFailover.h index f528ad0a6c7c2..ab91734f71ca1 100644 --- a/src/Client/ConnectionPoolWithFailover.h +++ b/src/Client/ConnectionPoolWithFailover.h @@ -55,8 +55,10 @@ class ConnectionPoolWithFailover : public IConnectionPool, private PoolWithFailo * Connections provide access to different replicas of one shard. */ /// `ping_pooled_connection` pings a connection that the pool returned already established. - /// Used by cluster reads of a remote shard: a server that stopped without closing the socket - /// is skipped after `sync_request_timeout` instead of blocking the query for `receive_timeout`. + /// Used by cluster reads of a remote shard. A server that stopped without closing the socket + /// sends no `Pong` within `sync_request_timeout`, and that shard is skipped. A new handshake + /// is not attempted: it would wait for `handshake_timeout_ms` and get no `Hello` either. + /// A socket the server has already closed still reconnects once. std::vector getMany( const ConnectionTimeouts & timeouts, const Settings & settings, diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index fc4b33ac6c546..86ada6f74c813 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -49,6 +49,7 @@ static struct InitFiu PAUSEABLE_ONCE(remote_query_executor_finish_drain_pause) \ ONCE(connection_stale_on_establish) \ REGULAR(unresponsive_pooled_connection) \ + REGULAR(pooled_connection_ping_timeout) \ REGULAR(cluster_discovery_faults) \ REGULAR(cluster_discovery_unregister_fail) \ ONCE(cluster_discovery_retry_signal_fail) \ diff --git a/src/Storages/IStorageCluster.cpp b/src/Storages/IStorageCluster.cpp index 8ae4c38a93c9c..4d1a622617d9c 100644 --- a/src/Storages/IStorageCluster.cpp +++ b/src/Storages/IStorageCluster.cpp @@ -942,7 +942,8 @@ void ReadFromCluster::initializePipeline(QueryPipelineBuilder & pipeline, const /// this server's own address a real TCP pool too, and `isLocal` is how that shard is told /// apart: pinging it cannot discover a dead peer. A remote peer that stopped without /// closing the socket otherwise stays in the pipeline and the query waits for - /// `receive_timeout`. + /// `receive_timeout`. No `Pong` within `sync_request_timeout` skips the shard. The + /// establisher does not open a second connection in that case. const bool ping_pooled_connection = !shard_info.isLocal(); auto try_results = shard_info.pool->getMany( timeouts, diff --git a/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.reference b/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.reference index 6813c34cb05cd..207dac7880712 100644 --- a/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.reference +++ b/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.reference @@ -2,3 +2,6 @@ 127.0.0.2 0 3 0 +3 +0 +0 diff --git a/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.sql b/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.sql index 5511fd0656b6e..36c5ce516b16c 100644 --- a/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.sql +++ b/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.sql @@ -1,6 +1,6 @@ -- Tags: no-fasttest, no-parallel -- no-fasttest: the Fast test build has no libfiu, so SYSTEM ENABLE FAILPOINT is unsupported. --- no-parallel: the fail point below fails every Ping in the server process. +-- no-parallel: the fail points below fail every Ping in the server process. -- `test_cluster_two_shards` has one local address (127.0.0.1) and one that is not local -- (127.0.0.2). A cluster read pings a pooled connection only for the remote shard. The fail @@ -37,3 +37,32 @@ SELECT throwIf( ifNull((SELECT value FROM system.events WHERE event = 'DistributedConnectionFailTry'), 0) <= (SELECT value FROM ping_fail_try_before), 'Remote shard was not pinged'); + +-- A peer that keeps the socket open and sends no Pong is skipped after the ping. Opening another +-- connection would wait for handshake_timeout_ms and get no Hello either. `ReconnectCount` stays +-- put; `FailTry` still moves. The local shard is not pinged, so it still reads the file. + +CREATE TEMPORARY TABLE ping_reconnect_before (value UInt64); +INSERT INTO ping_reconnect_before +SELECT ifNull((SELECT value FROM system.events WHERE event = 'DistributedConnectionReconnectCount'), 0); + +CREATE TEMPORARY TABLE ping_fail_try_before_timeout (value UInt64); +INSERT INTO ping_fail_try_before_timeout +SELECT ifNull((SELECT value FROM system.events WHERE event = 'DistributedConnectionFailTry'), 0); + +SYSTEM ENABLE FAILPOINT pooled_connection_ping_timeout; + +SELECT count() +FROM fileCluster('test_cluster_two_shards', currentDatabase() || '_05812_data.tsv', 'TSV', 'x UInt8'); + +SYSTEM DISABLE FAILPOINT pooled_connection_ping_timeout; + +SELECT throwIf( + ifNull((SELECT value FROM system.events WHERE event = 'DistributedConnectionReconnectCount'), 0) + != (SELECT value FROM ping_reconnect_before), + 'Ping timeout reconnected'); + +SELECT throwIf( + ifNull((SELECT value FROM system.events WHERE event = 'DistributedConnectionFailTry'), 0) + <= (SELECT value FROM ping_fail_try_before_timeout), + 'Ping timeout was not applied'); From 21f11d361b3c54a1458ddfab152629fe60636425 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Wed, 18 Nov 2026 17:03:22 +0100 Subject: [PATCH 4/7] Reconnect once after a cluster read ping times out A pooled remote connection that does not answer `Ping` within `sync_request_timeout` was skipped. A live server behind a dropped flow was then dropped from the cluster read, and the query could fail with `ALL_CONNECTION_TRIES_FAILED` even though a new connection would succeed. The establisher now reconnects once after a `SOCKET_TIMEOUT` from the ping, the same way it already does for a dead socket. The handshake of that retry is capped by `sync_request_timeout`, so a stopped server does not wait out `handshake_timeout_ms`. The cap only shortens `handshake_timeout` on that attempt and never raises it. The `pooled_connection_ping_timeout` failpoint test now expects `DistributedConnectionReconnectCount` to increase, and both remote and local shards to be usable after the reconnect. Related: https://github.com/Altinity/ClickHouse/issues/2896 Co-authored-by: Cursor --- src/Client/Connection.cpp | 4 +- src/Client/Connection.h | 4 +- src/Client/ConnectionEstablisher.cpp | 40 +++++++++++++------ src/Client/ConnectionEstablisher.h | 8 ++-- src/Client/ConnectionPoolWithFailover.h | 7 ++-- src/Storages/IStorageCluster.cpp | 4 +- ...ooled_connection_on_cluster_read.reference | 1 + ...ping_pooled_connection_on_cluster_read.sql | 20 +++++++--- 8 files changed, 58 insertions(+), 30 deletions(-) diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index cf0de08529bf6..26a1d17e56a4c 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -924,8 +924,8 @@ bool Connection::ping(const ConnectionTimeouts & timeouts, bool rethrow_on_timeo return false; }); - /// For tests: a peer that keeps the socket open and sends no `Pong`. Callers that asked to - /// see the timeout skip this replica instead of opening another connection. + /// For tests: a peer that keeps the socket open and sends no `Pong`. The establisher + /// reconnects once, and caps that handshake by `sync_request_timeout`. fiu_do_on(FailPoints::pooled_connection_ping_timeout, { disconnect(); throw Exception(ErrorCodes::SOCKET_TIMEOUT, "Injected ping timeout"); diff --git a/src/Client/Connection.h b/src/Client/Connection.h index 0c1bd025598fa..25b610c960326 100644 --- a/src/Client/Connection.h +++ b/src/Client/Connection.h @@ -175,8 +175,8 @@ class Connection : public IServerConnection /// Same round trip as `checkConnected`, but a peer that keeps the socket open and sends no /// `Pong` within `sync_request_timeout` throws `SOCKET_TIMEOUT`. A socket the server has - /// already closed still returns false. Cluster reads use the difference so a silent peer is - /// skipped instead of opening another connection and waiting out `handshake_timeout_ms`. + /// already closed still returns false. Cluster reads reconnect either way. After a timeout + /// the new handshake is capped by `sync_request_timeout`. bool checkConnectedThrowOnTimeout(const ConnectionTimeouts & timeouts) { return isConnected() && ping(timeouts, /*rethrow_on_timeout=*/ true); diff --git a/src/Client/ConnectionEstablisher.cpp b/src/Client/ConnectionEstablisher.cpp index 8e2df0c865265..1e8b086919028 100644 --- a/src/Client/ConnectionEstablisher.cpp +++ b/src/Client/ConnectionEstablisher.cpp @@ -77,17 +77,28 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std:: /// can be stale; a failure of a freshly established one is a genuine replica problem, and /// retrying it is left to the caller (with its error accounting). bool had_pooled_connection = false; - /// Set when `Ping` reached `sync_request_timeout`. The peer kept the socket open, so a new - /// handshake would wait for `handshake_timeout_ms` and then fail the same way. + /// Set when `Ping` reached `sync_request_timeout` on this attempt. bool pooled_ping_timed_out = false; + /// The next attempt is the one reconnect after a ping timeout. Its `Hello` wait is capped + /// by `sync_request_timeout` so a stopped process does not sit through `handshake_timeout_ms`. + bool cap_handshake_after_ping_timeout = false; auto try_establish = [&]() { ProfileEvents::increment(ProfileEvents::DistributedConnectionTries); + ConnectionTimeouts attempt_timeouts = *timeouts; + if (cap_handshake_after_ping_timeout) + { + attempt_timeouts.handshake_timeout = ConnectionTimeouts::saturate( + attempt_timeouts.handshake_timeout, attempt_timeouts.sync_request_timeout); + cap_handshake_after_ping_timeout = false; + LOG_TRACE(log, "Reconnecting after a ping timeout with the handshake capped by `sync_request_timeout`."); + } + had_pooled_connection = false; pooled_ping_timed_out = false; - result.entry = pool->getUnchecked(*timeouts, settings); + result.entry = pool->getUnchecked(attempt_timeouts, settings); had_pooled_connection = result.entry->isConnected(); AsyncCallbackSetter async_setter(&*result.entry, std::move(async_callback)); @@ -110,8 +121,8 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std:: /// poll. `Ping` fails when the server stopped without closing the socket; the poll reports /// that socket as idle. A connection that is not established yet is connected, and the /// handshake is the check. A dead socket is a soft error, so the retry below reconnects - /// once. A timeout means the peer kept the socket and sent nothing, and the retry is - /// skipped: the next handshake would block for `handshake_timeout_ms`. + /// once with the usual handshake timeout. A timeout also reconnects once: the old flow may + /// be dead while the server accepts a new connection. That handshake is capped above. if (ping_pooled && result.entry->isConnected()) { /// `ping` is private. `checkConnectedThrowOnTimeout` is the wrapper that lets a @@ -119,14 +130,14 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std:: bool answered = false; try { - answered = result.entry->checkConnectedThrowOnTimeout(*timeouts); + answered = result.entry->checkConnectedThrowOnTimeout(attempt_timeouts); } catch (const Exception & e) { if (e.code() == ErrorCodes::SOCKET_TIMEOUT) { pooled_ping_timed_out = true; - LOG_TRACE(log, "Pooled connection timed out on ping, will skip it."); + LOG_TRACE(log, "Pooled connection timed out on ping, will reconnect."); } throw; } @@ -138,12 +149,12 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std:: } else { - result.entry->forceConnected(*timeouts); + result.entry->forceConnected(attempt_timeouts); } UInt64 server_revision = 0; if (table_to_check) - server_revision = result.entry->getServerRevision(*timeouts); + server_revision = result.entry->getServerRevision(attempt_timeouts); if (!table_to_check || server_revision < DBMS_MIN_REVISION_WITH_TABLES_STATUS) { @@ -158,7 +169,7 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std:: TablesStatusRequest status_request; status_request.tables.emplace(*table_to_check); - TablesStatusResponse status_response = result.entry->getTablesStatus(*timeouts, status_request); + TablesStatusResponse status_response = result.entry->getTablesStatus(attempt_timeouts, status_request); auto table_status_it = status_response.table_states_by_id.find(*table_to_check); if (table_status_it == status_response.table_states_by_id.end()) { @@ -242,10 +253,13 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std:: /// Reconnect and retry once if a previously established (pooled) connection went stale. /// The failed request is safe to repeat: it is either a read-only status request, or the /// connection was closed before the request reached a live server. A ping that timed out - /// is the other case: the peer still holds the socket, and another handshake would wait - /// for `handshake_timeout_ms` with the same result, so the replica is skipped here. - if (can_reconnect && tries == 0 && had_pooled_connection && !pooled_ping_timed_out) + /// is included: a new connection can succeed when the old flow was dropped and the server + /// is up. `cap_handshake_after_ping_timeout` limits that `Hello` wait. + if (can_reconnect && tries == 0 && had_pooled_connection) { + if (pooled_ping_timed_out) + cap_handshake_after_ping_timeout = true; + ProfileEvents::increment(ProfileEvents::DistributedConnectionReconnectCount); continue; } diff --git a/src/Client/ConnectionEstablisher.h b/src/Client/ConnectionEstablisher.h index d89b671061230..d029cb48d241c 100644 --- a/src/Client/ConnectionEstablisher.h +++ b/src/Client/ConnectionEstablisher.h @@ -20,9 +20,11 @@ class ConnectionEstablisher /// `ping_pooled_connection` sends a `Ping` on a connection taken from the pool already /// established. A zero-timeout poll cannot see a server that stopped without closing the /// socket, and a cluster read would then wait for `receive_timeout`. No `Pong` within - /// `sync_request_timeout` skips the replica, with no further handshake. A socket the server - /// has already closed still reconnects once. Callers that can run without this replica - /// (cluster reads of a remote shard) pass true. The default keeps the poll, with no round trip. + /// `sync_request_timeout` reconnects once. That handshake is capped by `sync_request_timeout`: + /// a live server finishes `Hello` in one round trip, and a stopped server does not wait out + /// `handshake_timeout_ms`. A socket the server has already closed also reconnects once, with + /// the usual handshake timeout. Callers that can run without this replica (cluster reads of + /// a remote shard) pass true. The default keeps the poll, with no round trip. ConnectionEstablisher(ConnectionPoolPtr pool_, const ConnectionTimeouts * timeouts_, const Settings & settings_, diff --git a/src/Client/ConnectionPoolWithFailover.h b/src/Client/ConnectionPoolWithFailover.h index ab91734f71ca1..79f57b4d28279 100644 --- a/src/Client/ConnectionPoolWithFailover.h +++ b/src/Client/ConnectionPoolWithFailover.h @@ -56,9 +56,10 @@ class ConnectionPoolWithFailover : public IConnectionPool, private PoolWithFailo */ /// `ping_pooled_connection` pings a connection that the pool returned already established. /// Used by cluster reads of a remote shard. A server that stopped without closing the socket - /// sends no `Pong` within `sync_request_timeout`, and that shard is skipped. A new handshake - /// is not attempted: it would wait for `handshake_timeout_ms` and get no `Hello` either. - /// A socket the server has already closed still reconnects once. + /// sends no `Pong` within `sync_request_timeout`. The pooled socket is dropped and one new + /// connection is opened, with the handshake capped by `sync_request_timeout`. A live server + /// accepts that connection. A stopped server stops at the cap instead of `handshake_timeout_ms`. + /// A socket the server has already closed reconnects once with the usual handshake timeout. std::vector getMany( const ConnectionTimeouts & timeouts, const Settings & settings, diff --git a/src/Storages/IStorageCluster.cpp b/src/Storages/IStorageCluster.cpp index 4d1a622617d9c..229447e2257fd 100644 --- a/src/Storages/IStorageCluster.cpp +++ b/src/Storages/IStorageCluster.cpp @@ -942,8 +942,8 @@ void ReadFromCluster::initializePipeline(QueryPipelineBuilder & pipeline, const /// this server's own address a real TCP pool too, and `isLocal` is how that shard is told /// apart: pinging it cannot discover a dead peer. A remote peer that stopped without /// closing the socket otherwise stays in the pipeline and the query waits for - /// `receive_timeout`. No `Pong` within `sync_request_timeout` skips the shard. The - /// establisher does not open a second connection in that case. + /// `receive_timeout`. No `Pong` within `sync_request_timeout` drops that socket and opens + /// one new connection. The handshake on that connection is capped by `sync_request_timeout`. const bool ping_pooled_connection = !shard_info.isLocal(); auto try_results = shard_info.pool->getMany( timeouts, diff --git a/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.reference b/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.reference index 207dac7880712..c4389efae0b92 100644 --- a/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.reference +++ b/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.reference @@ -5,3 +5,4 @@ 3 0 0 +0 diff --git a/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.sql b/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.sql index 36c5ce516b16c..baad2c4e31ec8 100644 --- a/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.sql +++ b/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.sql @@ -38,9 +38,10 @@ SELECT throwIf( <= (SELECT value FROM ping_fail_try_before), 'Remote shard was not pinged'); --- A peer that keeps the socket open and sends no Pong is skipped after the ping. Opening another --- connection would wait for handshake_timeout_ms and get no Hello either. `ReconnectCount` stays --- put; `FailTry` still moves. The local shard is not pinged, so it still reads the file. +-- A peer that keeps the socket open and sends no Pong still reconnects once. The new connection +-- succeeds here, because this server is up and only the ping was failed. `ReconnectCount` and +-- `DistributedConnectionUsable` both move: the local shard and the reconnected remote shard are +-- usable. `FailTry` moves too. A stopped server would stop at the capped handshake instead. CREATE TEMPORARY TABLE ping_reconnect_before (value UInt64); INSERT INTO ping_reconnect_before @@ -50,6 +51,10 @@ CREATE TEMPORARY TABLE ping_fail_try_before_timeout (value UInt64); INSERT INTO ping_fail_try_before_timeout SELECT ifNull((SELECT value FROM system.events WHERE event = 'DistributedConnectionFailTry'), 0); +CREATE TEMPORARY TABLE ping_usable_before (value UInt64); +INSERT INTO ping_usable_before +SELECT ifNull((SELECT value FROM system.events WHERE event = 'DistributedConnectionUsable'), 0); + SYSTEM ENABLE FAILPOINT pooled_connection_ping_timeout; SELECT count() @@ -59,10 +64,15 @@ SYSTEM DISABLE FAILPOINT pooled_connection_ping_timeout; SELECT throwIf( ifNull((SELECT value FROM system.events WHERE event = 'DistributedConnectionReconnectCount'), 0) - != (SELECT value FROM ping_reconnect_before), - 'Ping timeout reconnected'); + <= (SELECT value FROM ping_reconnect_before), + 'Ping timeout did not reconnect'); SELECT throwIf( ifNull((SELECT value FROM system.events WHERE event = 'DistributedConnectionFailTry'), 0) <= (SELECT value FROM ping_fail_try_before_timeout), 'Ping timeout was not applied'); + +SELECT throwIf( + ifNull((SELECT value FROM system.events WHERE event = 'DistributedConnectionUsable'), 0) + < (SELECT value FROM ping_usable_before) + 2, + 'Ping timeout did not bring the remote shard back'); From 95eb850deb82851602fc9c8924a02663fe5c9f9c Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Thu, 19 Nov 2026 12:15:46 +0100 Subject: [PATCH 5/7] Cap a zero handshake timeout on the reconnect after a ping timeout The reconnect after a `Ping` timeout caps the handshake timeout by `sync_request_timeout` with `ConnectionTimeouts::saturate`. `saturate` keeps any value smaller than the limit, so a zero `handshake_timeout_ms` stayed zero. A zero socket timeout never fires, so a peer that accepts TCP and never sends `Hello` could block the retry indefinitely instead of stopping at `sync_request_timeout`. Treat a zero handshake timeout as no limit and replace it with `sync_request_timeout`. Any other value is still capped to the minimum of the two. Related: https://github.com/Altinity/ClickHouse/pull/2912 Co-authored-by: Cursor --- src/Client/ConnectionEstablisher.cpp | 18 +++++++++++++++--- src/Client/ConnectionEstablisher.h | 11 ++++++----- 2 files changed, 21 insertions(+), 8 deletions(-) diff --git a/src/Client/ConnectionEstablisher.cpp b/src/Client/ConnectionEstablisher.cpp index 1e8b086919028..d5845087137c6 100644 --- a/src/Client/ConnectionEstablisher.cpp +++ b/src/Client/ConnectionEstablisher.cpp @@ -80,7 +80,7 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std:: /// Set when `Ping` reached `sync_request_timeout` on this attempt. bool pooled_ping_timed_out = false; /// The next attempt is the one reconnect after a ping timeout. Its `Hello` wait is capped - /// by `sync_request_timeout` so a stopped process does not sit through `handshake_timeout_ms`. + /// by `sync_request_timeout`, including when `handshake_timeout_ms` is `0` (no socket timeout). bool cap_handshake_after_ping_timeout = false; auto try_establish = [&]() @@ -90,8 +90,20 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std:: ConnectionTimeouts attempt_timeouts = *timeouts; if (cap_handshake_after_ping_timeout) { - attempt_timeouts.handshake_timeout = ConnectionTimeouts::saturate( - attempt_timeouts.handshake_timeout, attempt_timeouts.sync_request_timeout); + /// `saturate` keeps a zero timeout: it is smaller than any positive limit. A zero + /// socket timeout never fires, and `handshake_timeout_ms = 0` means no limit, so + /// that value is replaced by `sync_request_timeout`. Any other handshake timeout + /// is the minimum of the two. + const Poco::Timespan & limit = attempt_timeouts.sync_request_timeout; + if (attempt_timeouts.handshake_timeout.totalMicroseconds() == 0) + { + attempt_timeouts.handshake_timeout = limit; + } + else + { + attempt_timeouts.handshake_timeout = ConnectionTimeouts::saturate(attempt_timeouts.handshake_timeout, limit); + } + cap_handshake_after_ping_timeout = false; LOG_TRACE(log, "Reconnecting after a ping timeout with the handshake capped by `sync_request_timeout`."); } diff --git a/src/Client/ConnectionEstablisher.h b/src/Client/ConnectionEstablisher.h index d029cb48d241c..9ea5abd4b3eb5 100644 --- a/src/Client/ConnectionEstablisher.h +++ b/src/Client/ConnectionEstablisher.h @@ -20,11 +20,12 @@ class ConnectionEstablisher /// `ping_pooled_connection` sends a `Ping` on a connection taken from the pool already /// established. A zero-timeout poll cannot see a server that stopped without closing the /// socket, and a cluster read would then wait for `receive_timeout`. No `Pong` within - /// `sync_request_timeout` reconnects once. That handshake is capped by `sync_request_timeout`: - /// a live server finishes `Hello` in one round trip, and a stopped server does not wait out - /// `handshake_timeout_ms`. A socket the server has already closed also reconnects once, with - /// the usual handshake timeout. Callers that can run without this replica (cluster reads of - /// a remote shard) pass true. The default keeps the poll, with no round trip. + /// `sync_request_timeout` reconnects once. That handshake is capped by `sync_request_timeout`, + /// also when `handshake_timeout_ms` is `0`: a live server finishes `Hello` in one round trip, + /// and a stopped server stops at the cap. A socket the server has already closed also + /// reconnects once, with the usual handshake timeout. Callers that can run without this + /// replica (cluster reads of a remote shard) pass true. The default keeps the poll, with no + /// round trip. ConnectionEstablisher(ConnectionPoolPtr pool_, const ConnectionTimeouts * timeouts_, const Settings & settings_, From 547ccfb64620bb8dc275238239e51aae37973af2 Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Thu, 19 Nov 2026 12:27:58 +0100 Subject: [PATCH 6/7] Install the requested timeout when the socket timeout is zero `TimeoutSetter` with `limit_max_timeout` keeps the current socket timeout when it is not longer than the requested one. A zero socket timeout means no limit, but it is not longer than any finite value, so it was kept. `Connection::ping` uses this setter with `sync_request_timeout`, so a pooled connection with `receive_timeout = 0` could wait for a `Pong` forever when the peer accepted the connection and never answered. Treat a zero send or receive timeout as no limit and install the requested timeout in its place. A finite timeout that is already shorter is still kept. The destructor restores the original zero timeout, so the query itself stays unlimited. Related: https://github.com/Altinity/ClickHouse/pull/2912 Co-authored-by: Cursor --- src/IO/TimeoutSetter.cpp | 15 +++++++++++++-- src/IO/TimeoutSetter.h | 1 + 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/src/IO/TimeoutSetter.cpp b/src/IO/TimeoutSetter.cpp index 2e7327827008d..c5324776bd6b9 100644 --- a/src/IO/TimeoutSetter.cpp +++ b/src/IO/TimeoutSetter.cpp @@ -6,6 +6,15 @@ namespace DB { +namespace +{ +/// A zero socket timeout never fires. It is not a limit shorter than `timeout`. +bool isUnlimited(const Poco::Timespan & timeout) +{ + return timeout.totalMicroseconds() == 0; +} +} + TimeoutSetter::TimeoutSetter(Poco::Net::StreamSocket & socket_, Poco::Timespan send_timeout_, Poco::Timespan receive_timeout_, @@ -15,10 +24,12 @@ TimeoutSetter::TimeoutSetter(Poco::Net::StreamSocket & socket_, old_send_timeout = socket.getSendTimeout(); old_receive_timeout = socket.getReceiveTimeout(); - if (!limit_max_timeout || old_send_timeout > send_timeout) + /// `limit_max_timeout` keeps a finite timeout that is already shorter. Zero is no timeout, + /// so the requested timeout is installed instead of leaving the socket to block forever. + if (!limit_max_timeout || isUnlimited(old_send_timeout) || old_send_timeout > send_timeout) socket.setSendTimeout(send_timeout); - if (!limit_max_timeout || old_receive_timeout > receive_timeout) + if (!limit_max_timeout || isUnlimited(old_receive_timeout) || old_receive_timeout > receive_timeout) socket.setReceiveTimeout(receive_timeout); } diff --git a/src/IO/TimeoutSetter.h b/src/IO/TimeoutSetter.h index 3479986d7fe45..d4e26ee4a5353 100644 --- a/src/IO/TimeoutSetter.h +++ b/src/IO/TimeoutSetter.h @@ -8,6 +8,7 @@ namespace DB { /// Temporarily overrides socket send/receive timeouts and reset them back into destructor (or manually by calling reset method) /// If "limit_max_timeout" is true, timeouts could be only decreased (maxed by previous value). +/// A zero timeout means no limit, so a finite value still replaces it. struct TimeoutSetter { TimeoutSetter(Poco::Net::StreamSocket & socket_, From 8be0a0f986bc672f0c188143b3df3c00d13ce1cc Mon Sep 17 00:00:00 2001 From: Anton Ivashkin Date: Thu, 19 Nov 2026 12:32:48 +0100 Subject: [PATCH 7/7] Fix the comment on the ping timeout The comment in `Connection::ping` said a timeout skips the replica. The cluster-read caller does not skip it on the first timeout: it reconnects once with the handshake capped by `sync_request_timeout`, and the replica is skipped only if that reconnect fails. Describe the actual flow. No code change. Related: https://github.com/Altinity/ClickHouse/pull/2912 Co-authored-by: Cursor --- src/Client/Connection.cpp | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 26a1d17e56a4c..7535a1d95856c 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -962,9 +962,10 @@ bool Connection::ping(const ConnectionTimeouts & timeouts, bool rethrow_on_timeo catch (const Exception & e) { /// `Exception` extends `Poco::Exception`, so this arm has to come first. A read or write - /// that reaches `sync_request_timeout` is `SOCKET_TIMEOUT`. With `rethrow_on_timeout` the - /// caller skips the replica. Every other failure still returns false, and `checkConnected` - /// keeps its old contract. + /// that reaches `sync_request_timeout` is `SOCKET_TIMEOUT`. With `rethrow_on_timeout` that + /// timeout is rethrown: the cluster-read caller reconnects once and caps that handshake by + /// `sync_request_timeout`. The replica is skipped only if the reconnect fails. Every other + /// failure still returns false, and `checkConnected` keeps its old contract. const bool timed_out = rethrow_on_timeout && e.code() == ErrorCodes::SOCKET_TIMEOUT; disconnect(); LOG_TRACE(log_wrapper.get(), fmt::runtime(e.displayText()));