diff --git a/src/Client/Connection.cpp b/src/Client/Connection.cpp index 71df187ab8cfe..7535a1d95856c 100644 --- a/src/Client/Connection.cpp +++ b/src/Client/Connection.cpp @@ -84,6 +84,8 @@ namespace FailPoints { extern const char receive_timeout_on_table_status_response[]; extern const char unexpected_packet_in_table_status_response[]; + extern const char unresponsive_pooled_connection[]; + extern const char pooled_connection_ping_timeout[]; } namespace ErrorCodes @@ -913,8 +915,22 @@ void Connection::sendClusterNameAndSalt() } #endif -bool Connection::ping(const ConnectionTimeouts & timeouts) +bool Connection::ping(const ConnectionTimeouts & timeouts, bool rethrow_on_timeout) { + /// For tests: a pooled socket the server has already closed. `disconnect` matches the failure + /// path below, so the caller reconnects instead of reusing this socket. + fiu_do_on(FailPoints::unresponsive_pooled_connection, { + disconnect(); + return false; + }); + + /// For tests: a peer that keeps the socket open and sends no `Pong`. The establisher + /// reconnects once, and caps that handshake by `sync_request_timeout`. + fiu_do_on(FailPoints::pooled_connection_ping_timeout, { + disconnect(); + throw Exception(ErrorCodes::SOCKET_TIMEOUT, "Injected ping timeout"); + }); + try { TimeoutSetter timeout_setter(*socket, timeouts.sync_request_timeout, true); @@ -943,6 +959,20 @@ bool Connection::ping(const ConnectionTimeouts & timeouts) if (pong != Protocol::Server::Pong) throwUnexpectedPacket(pong, "Pong", &timeout_setter); } + catch (const Exception & e) + { + /// `Exception` extends `Poco::Exception`, so this arm has to come first. A read or write + /// that reaches `sync_request_timeout` is `SOCKET_TIMEOUT`. With `rethrow_on_timeout` that + /// timeout is rethrown: the cluster-read caller reconnects once and caps that handshake by + /// `sync_request_timeout`. The replica is skipped only if the reconnect fails. Every other + /// failure still returns false, and `checkConnected` keeps its old contract. + const bool timed_out = rethrow_on_timeout && e.code() == ErrorCodes::SOCKET_TIMEOUT; + disconnect(); + LOG_TRACE(log_wrapper.get(), fmt::runtime(e.displayText())); + if (timed_out) + throw; + return false; + } catch (const Poco::Exception & e) { /// Explicitly disconnect since ping() can receive EndOfStream, diff --git a/src/Client/Connection.h b/src/Client/Connection.h index 3ec99be2cf9c7..25b610c960326 100644 --- a/src/Client/Connection.h +++ b/src/Client/Connection.h @@ -173,6 +173,15 @@ class Connection : public IServerConnection bool checkConnected(const ConnectionTimeouts & timeouts) override { return isConnected() && ping(timeouts); } + /// Same round trip as `checkConnected`, but a peer that keeps the socket open and sends no + /// `Pong` within `sync_request_timeout` throws `SOCKET_TIMEOUT`. A socket the server has + /// already closed still returns false. Cluster reads reconnect either way. After a timeout + /// the new handshake is capped by `sync_request_timeout`. + bool checkConnectedThrowOnTimeout(const ConnectionTimeouts & timeouts) + { + return isConnected() && ping(timeouts, /*rethrow_on_timeout=*/ true); + } + /// Note that a server that went away without closing the connection is not detected here, and /// neither is a close that has not arrived yet; that only shows up when the connection is used. /// Pinging the server to find out would add a round trip to every query, and a pong that does @@ -372,7 +381,9 @@ class Connection : public IServerConnection #if USE_SSL void sendClusterNameAndSalt(); #endif - bool ping(const ConnectionTimeouts & timeouts); + /// `rethrow_on_timeout` lets `SOCKET_TIMEOUT` out. Other failures still return false after + /// `disconnect`, which is what `checkConnected` and the clickhouse-client resync rely on. + bool ping(const ConnectionTimeouts & timeouts, bool rethrow_on_timeout = false); /// Whether the connection can no longer serve a request, checked without a round trip. bool isStale(); diff --git a/src/Client/ConnectionEstablisher.cpp b/src/Client/ConnectionEstablisher.cpp index 121a4d16ac01f..d5845087137c6 100644 --- a/src/Client/ConnectionEstablisher.cpp +++ b/src/Client/ConnectionEstablisher.cpp @@ -45,8 +45,14 @@ ConnectionEstablisher::ConnectionEstablisher( const ConnectionTimeouts * timeouts_, const Settings & settings_, LoggerPtr log_, - const QualifiedTableName * table_to_check_) - : pool(std::move(pool_)), timeouts(timeouts_), settings(settings_), log(log_), table_to_check(table_to_check_) + const QualifiedTableName * table_to_check_, + bool ping_pooled_connection_) + : pool(std::move(pool_)) + , timeouts(timeouts_) + , settings(settings_) + , log(log_) + , table_to_check(table_to_check_) + , ping_pooled_connection(ping_pooled_connection_) { } @@ -55,24 +61,56 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std:: /// A pooled connection is handed out without pinging it first - a ping would add an unnecessary /// Ping-Pong round trip when the connection is still alive. A connection that the server has /// closed while it was idle in the pool is detected by a zero-timeout poll and recovered by - /// reconnecting (see Connection::forceConnected). If a pooled connection nevertheless fails its + /// reconnecting (see `Connection::forceConnected`). If a pooled connection nevertheless fails its /// very first request (e.g. the server went away without closing it), in the synchronous case we /// reconnect once and retry, so that a single available replica is not dropped just because its /// pooled connection went stale. In the asynchronous case we do not retry here - stale - /// connections are handled by retrying the next replica (see ConnectionPoolWithFailover). + /// connections are handled by retrying the next replica (see `ConnectionPoolWithFailover`). + /// + /// Cluster reads of a remote shard opt into a ping (`ping_pooled_connection`). The poll cannot + /// see a server that stopped without closing the socket, and the query would then block until + /// `receive_timeout`. The asynchronous path keeps the poll: a ping there would block the fiber. const bool can_reconnect = !async_callback; + const bool ping_pooled = ping_pooled_connection && can_reconnect; /// Whether the connection was taken from the pool already established. Only such a connection /// can be stale; a failure of a freshly established one is a genuine replica problem, and /// retrying it is left to the caller (with its error accounting). bool had_pooled_connection = false; + /// Set when `Ping` reached `sync_request_timeout` on this attempt. + bool pooled_ping_timed_out = false; + /// The next attempt is the one reconnect after a ping timeout. Its `Hello` wait is capped + /// by `sync_request_timeout`, including when `handshake_timeout_ms` is `0` (no socket timeout). + bool cap_handshake_after_ping_timeout = false; auto try_establish = [&]() { ProfileEvents::increment(ProfileEvents::DistributedConnectionTries); + ConnectionTimeouts attempt_timeouts = *timeouts; + if (cap_handshake_after_ping_timeout) + { + /// `saturate` keeps a zero timeout: it is smaller than any positive limit. A zero + /// socket timeout never fires, and `handshake_timeout_ms = 0` means no limit, so + /// that value is replaced by `sync_request_timeout`. Any other handshake timeout + /// is the minimum of the two. + const Poco::Timespan & limit = attempt_timeouts.sync_request_timeout; + if (attempt_timeouts.handshake_timeout.totalMicroseconds() == 0) + { + attempt_timeouts.handshake_timeout = limit; + } + else + { + attempt_timeouts.handshake_timeout = ConnectionTimeouts::saturate(attempt_timeouts.handshake_timeout, limit); + } + + cap_handshake_after_ping_timeout = false; + LOG_TRACE(log, "Reconnecting after a ping timeout with the handshake capped by `sync_request_timeout`."); + } + had_pooled_connection = false; - result.entry = pool->getUnchecked(*timeouts, settings); + pooled_ping_timed_out = false; + result.entry = pool->getUnchecked(attempt_timeouts, settings); had_pooled_connection = result.entry->isConnected(); AsyncCallbackSetter async_setter(&*result.entry, std::move(async_callback)); @@ -85,16 +123,50 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std:: }); /// Establish the connection, or revalidate a pooled one, under the async callback: with the - /// callback installed, Connection::connect uses a non-blocking connect and yields on it, so + /// callback installed, `Connection::connect` uses a non-blocking connect and yields on it, so /// a caller that multiplexes several connection attempts (e.g. hedged connections) can /// preempt a slow connect or handshake and switch to another replica. `result.entry` is /// assigned before connecting, so the connection being established is visible to the - /// multi-address timeout handling of ConnectionEstablisherAsync. - result.entry->forceConnected(*timeouts); + /// multi-address timeout handling of `ConnectionEstablisherAsync`. + /// + /// A pooled connection that the caller asked to ping is checked with `Ping` instead of the + /// poll. `Ping` fails when the server stopped without closing the socket; the poll reports + /// that socket as idle. A connection that is not established yet is connected, and the + /// handshake is the check. A dead socket is a soft error, so the retry below reconnects + /// once with the usual handshake timeout. A timeout also reconnects once: the old flow may + /// be dead while the server accepts a new connection. That handshake is capped above. + if (ping_pooled && result.entry->isConnected()) + { + /// `ping` is private. `checkConnectedThrowOnTimeout` is the wrapper that lets a + /// `SOCKET_TIMEOUT` out. This branch already knows the connection is established. + bool answered = false; + try + { + answered = result.entry->checkConnectedThrowOnTimeout(attempt_timeouts); + } + catch (const Exception & e) + { + if (e.code() == ErrorCodes::SOCKET_TIMEOUT) + { + pooled_ping_timed_out = true; + LOG_TRACE(log, "Pooled connection timed out on ping, will reconnect."); + } + throw; + } + if (!answered) + { + LOG_TRACE(log, "Pooled connection did not answer a ping, will reconnect."); + throw Exception(ErrorCodes::SOCKET_TIMEOUT, "Ping of a pooled connection failed"); + } + } + else + { + result.entry->forceConnected(attempt_timeouts); + } UInt64 server_revision = 0; if (table_to_check) - server_revision = result.entry->getServerRevision(*timeouts); + server_revision = result.entry->getServerRevision(attempt_timeouts); if (!table_to_check || server_revision < DBMS_MIN_REVISION_WITH_TABLES_STATUS) { @@ -109,7 +181,7 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std:: TablesStatusRequest status_request; status_request.tables.emplace(*table_to_check); - TablesStatusResponse status_response = result.entry->getTablesStatus(*timeouts, status_request); + TablesStatusResponse status_response = result.entry->getTablesStatus(attempt_timeouts, status_request); auto table_status_it = status_response.table_states_by_id.find(*table_to_check); if (table_status_it == status_response.table_states_by_id.end()) { @@ -192,9 +264,14 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std:: /// Reconnect and retry once if a previously established (pooled) connection went stale. /// The failed request is safe to repeat: it is either a read-only status request, or the - /// connection was closed before the request reached a live server. + /// connection was closed before the request reached a live server. A ping that timed out + /// is included: a new connection can succeed when the old flow was dropped and the server + /// is up. `cap_handshake_after_ping_timeout` limits that `Hello` wait. if (can_reconnect && tries == 0 && had_pooled_connection) { + if (pooled_ping_timed_out) + cap_handshake_after_ping_timeout = true; + ProfileEvents::increment(ProfileEvents::DistributedConnectionReconnectCount); continue; } @@ -219,9 +296,10 @@ ConnectionEstablisherAsync::ConnectionEstablisherAsync( const ConnectionTimeouts * timeouts_, const Settings & settings_, LoggerPtr log_, - const QualifiedTableName * table_to_check_) + const QualifiedTableName * table_to_check_, + bool ping_pooled_connection_) : AsyncTaskExecutor(std::make_unique(*this), "ConnectionEstablisherAsync") - , connection_establisher(std::move(pool_), timeouts_, settings_, log_, table_to_check_) + , connection_establisher(std::move(pool_), timeouts_, settings_, log_, table_to_check_, ping_pooled_connection_) { epoll.add(timeout_descriptor.getDescriptor()); } diff --git a/src/Client/ConnectionEstablisher.h b/src/Client/ConnectionEstablisher.h index c25d255b3c742..9ea5abd4b3eb5 100644 --- a/src/Client/ConnectionEstablisher.h +++ b/src/Client/ConnectionEstablisher.h @@ -17,19 +17,30 @@ class ConnectionEstablisher public: using TryResult = PoolWithFailoverBase::TryResult; + /// `ping_pooled_connection` sends a `Ping` on a connection taken from the pool already + /// established. A zero-timeout poll cannot see a server that stopped without closing the + /// socket, and a cluster read would then wait for `receive_timeout`. No `Pong` within + /// `sync_request_timeout` reconnects once. That handshake is capped by `sync_request_timeout`, + /// also when `handshake_timeout_ms` is `0`: a live server finishes `Hello` in one round trip, + /// and a stopped server stops at the cap. A socket the server has already closed also + /// reconnects once, with the usual handshake timeout. Callers that can run without this + /// replica (cluster reads of a remote shard) pass true. The default keeps the poll, with no + /// round trip. ConnectionEstablisher(ConnectionPoolPtr pool_, const ConnectionTimeouts * timeouts_, const Settings & settings_, LoggerPtr log, - const QualifiedTableName * table_to_check = nullptr); + const QualifiedTableName * table_to_check = nullptr, + bool ping_pooled_connection_ = false); /// Establish connection and save it in result, write possible exception message in fail_message. /// The connection is returned from the connection pool and it can be stale (the server may have - /// closed it while it was idle). The pooled connection is not pinged before use: pinging it adds - /// an unnecessary Ping-Pong round trip when the connection is still alive. Instead, a stale - /// connection is detected by a zero-timeout poll (see Connection::forceConnected) or, failing - /// that, by retrying - either here (a pooled connection that fails its first request is - /// reconnected once) or at a higher level (ConnectionPoolWithFailover retries the next replica). + /// closed it while it was idle). Unless `ping_pooled_connection` was set, the pooled connection + /// is not pinged before use: pinging it adds an unnecessary Ping-Pong round trip when the + /// connection is still alive. Instead, a stale connection is detected by a zero-timeout poll + /// (see `Connection::forceConnected`) or, failing that, by retrying - either here (a pooled + /// connection that fails its first request is reconnected once) or at a higher level + /// (`ConnectionPoolWithFailover` retries the next replica). void run(TryResult & result, std::string & fail_message); /// Set async callback that will be called when reading from socket blocks. @@ -41,6 +52,7 @@ class ConnectionEstablisher const Settings & settings; LoggerPtr log; const QualifiedTableName * table_to_check; + bool ping_pooled_connection; AsyncCallback async_callback = {}; }; @@ -62,7 +74,8 @@ class ConnectionEstablisherAsync : public AsyncTaskExecutor const ConnectionTimeouts * timeouts_, const Settings & settings_, LoggerPtr log_, - const QualifiedTableName * table_to_check_ = nullptr); + const QualifiedTableName * table_to_check_ = nullptr, + bool ping_pooled_connection_ = false); /// Get file descriptor that can be added in epoll and be polled, /// when this fd becomes ready, you call resume establishing connection. diff --git a/src/Client/ConnectionPoolWithFailover.cpp b/src/Client/ConnectionPoolWithFailover.cpp index 0f18eaf45647f..90656f938b0ff 100644 --- a/src/Client/ConnectionPoolWithFailover.cpp +++ b/src/Client/ConnectionPoolWithFailover.cpp @@ -134,10 +134,11 @@ std::vector ConnectionPoolWithFailover::getMany( PoolMode pool_mode, AsyncCallback async_callback, std::optional skip_unavailable_endpoints, - GetPriorityForLoadBalancing::Func priority_func) + GetPriorityForLoadBalancing::Func priority_func, + bool ping_pooled_connection) { TryGetEntryFunc try_get_entry = [&](const NestedPoolPtr & pool, std::string & fail_message) - { return tryGetEntry(pool, timeouts, fail_message, settings, nullptr, async_callback); }; + { return tryGetEntry(pool, timeouts, fail_message, settings, nullptr, async_callback, ping_pooled_connection); }; std::vector results = getManyImpl(settings, pool_mode, try_get_entry, skip_unavailable_endpoints, priority_func); @@ -251,12 +252,14 @@ ConnectionPoolWithFailover::tryGetEntry( std::string & fail_message, const Settings & settings, const QualifiedTableName * table_to_check, - [[maybe_unused]] AsyncCallback async_callback) + [[maybe_unused]] AsyncCallback async_callback, + bool ping_pooled_connection) { #if defined(OS_LINUX) if (async_callback) { - ConnectionEstablisherAsync connection_establisher_async(pool, &timeouts, settings, log, table_to_check); + ConnectionEstablisherAsync connection_establisher_async( + pool, &timeouts, settings, log, table_to_check, ping_pooled_connection); while (true) { connection_establisher_async.resume(); @@ -277,7 +280,7 @@ ConnectionPoolWithFailover::tryGetEntry( } #endif - ConnectionEstablisher connection_establisher(pool, &timeouts, settings, log, table_to_check); + ConnectionEstablisher connection_establisher(pool, &timeouts, settings, log, table_to_check, ping_pooled_connection); TryResult result; connection_establisher.run(result, fail_message); return result; diff --git a/src/Client/ConnectionPoolWithFailover.h b/src/Client/ConnectionPoolWithFailover.h index eb4b28895b718..79f57b4d28279 100644 --- a/src/Client/ConnectionPoolWithFailover.h +++ b/src/Client/ConnectionPoolWithFailover.h @@ -54,13 +54,20 @@ class ConnectionPoolWithFailover : public IConnectionPool, private PoolWithFailo /** Allocates up to the specified number of connections to work. * Connections provide access to different replicas of one shard. */ + /// `ping_pooled_connection` pings a connection that the pool returned already established. + /// Used by cluster reads of a remote shard. A server that stopped without closing the socket + /// sends no `Pong` within `sync_request_timeout`. The pooled socket is dropped and one new + /// connection is opened, with the handshake capped by `sync_request_timeout`. A live server + /// accepts that connection. A stopped server stops at the cap instead of `handshake_timeout_ms`. + /// A socket the server has already closed reconnects once with the usual handshake timeout. std::vector getMany( const ConnectionTimeouts & timeouts, const Settings & settings, PoolMode pool_mode, AsyncCallback async_callback = {}, std::optional skip_unavailable_endpoints = std::nullopt, - GetPriorityForLoadBalancing::Func priority_func = {}); + GetPriorityForLoadBalancing::Func priority_func = {}, + bool ping_pooled_connection = false); /// The same as getMany(), but return std::vector. std::vector getManyForTableFunction(const ConnectionTimeouts & timeouts, @@ -132,7 +139,8 @@ class ConnectionPoolWithFailover : public IConnectionPool, private PoolWithFailo std::string & fail_message, const Settings & settings, const QualifiedTableName * table_to_check = nullptr, - AsyncCallback async_callback = {}); + AsyncCallback async_callback = {}, + bool ping_pooled_connection = false); GetPriorityForLoadBalancing::Func makeGetPriorityFunc(const Settings & settings); diff --git a/src/Common/FailPoint.cpp b/src/Common/FailPoint.cpp index 22c4f510b274b..d708bd59e67d5 100644 --- a/src/Common/FailPoint.cpp +++ b/src/Common/FailPoint.cpp @@ -48,6 +48,8 @@ static struct InitFiu PAUSEABLE_ONCE(remote_query_executor_receive_packet_pause) \ PAUSEABLE_ONCE(remote_query_executor_finish_drain_pause) \ ONCE(connection_stale_on_establish) \ + REGULAR(unresponsive_pooled_connection) \ + REGULAR(pooled_connection_ping_timeout) \ PAUSEABLE(swarm_mode_before_task_request) \ REGULAR(cluster_discovery_faults) \ REGULAR(cluster_discovery_unregister_fail) \ diff --git a/src/IO/TimeoutSetter.cpp b/src/IO/TimeoutSetter.cpp index 2e7327827008d..c5324776bd6b9 100644 --- a/src/IO/TimeoutSetter.cpp +++ b/src/IO/TimeoutSetter.cpp @@ -6,6 +6,15 @@ namespace DB { +namespace +{ +/// A zero socket timeout never fires. It is not a limit shorter than `timeout`. +bool isUnlimited(const Poco::Timespan & timeout) +{ + return timeout.totalMicroseconds() == 0; +} +} + TimeoutSetter::TimeoutSetter(Poco::Net::StreamSocket & socket_, Poco::Timespan send_timeout_, Poco::Timespan receive_timeout_, @@ -15,10 +24,12 @@ TimeoutSetter::TimeoutSetter(Poco::Net::StreamSocket & socket_, old_send_timeout = socket.getSendTimeout(); old_receive_timeout = socket.getReceiveTimeout(); - if (!limit_max_timeout || old_send_timeout > send_timeout) + /// `limit_max_timeout` keeps a finite timeout that is already shorter. Zero is no timeout, + /// so the requested timeout is installed instead of leaving the socket to block forever. + if (!limit_max_timeout || isUnlimited(old_send_timeout) || old_send_timeout > send_timeout) socket.setSendTimeout(send_timeout); - if (!limit_max_timeout || old_receive_timeout > receive_timeout) + if (!limit_max_timeout || isUnlimited(old_receive_timeout) || old_receive_timeout > receive_timeout) socket.setReceiveTimeout(receive_timeout); } diff --git a/src/IO/TimeoutSetter.h b/src/IO/TimeoutSetter.h index 3479986d7fe45..d4e26ee4a5353 100644 --- a/src/IO/TimeoutSetter.h +++ b/src/IO/TimeoutSetter.h @@ -8,6 +8,7 @@ namespace DB { /// Temporarily overrides socket send/receive timeouts and reset them back into destructor (or manually by calling reset method) /// If "limit_max_timeout" is true, timeouts could be only decreased (maxed by previous value). +/// A zero timeout means no limit, so a finite value still replaces it. struct TimeoutSetter { TimeoutSetter(Poco::Net::StreamSocket & socket_, diff --git a/src/Storages/IStorageCluster.cpp b/src/Storages/IStorageCluster.cpp index f696540c68212..229447e2257fd 100644 --- a/src/Storages/IStorageCluster.cpp +++ b/src/Storages/IStorageCluster.cpp @@ -937,12 +937,22 @@ void ReadFromCluster::initializePipeline(QueryPipelineBuilder & pipeline, const /// We're taking all replicas as shards, /// so each shard will have only one address to connect to. + /// + /// A remote shard is pinged before the pooled connection is used. `ReplicasAsShards` gives + /// this server's own address a real TCP pool too, and `isLocal` is how that shard is told + /// apart: pinging it cannot discover a dead peer. A remote peer that stopped without + /// closing the socket otherwise stays in the pipeline and the query waits for + /// `receive_timeout`. No `Pong` within `sync_request_timeout` drops that socket and opens + /// one new connection. The handshake on that connection is capped by `sync_request_timeout`. + const bool ping_pooled_connection = !shard_info.isLocal(); auto try_results = shard_info.pool->getMany( timeouts, current_settings, PoolMode::GET_ONE, {}, - /*skip_unavailable_endpoints=*/true); + /*skip_unavailable_endpoints=*/true, + /*priority_func=*/{}, + ping_pooled_connection); if (try_results.empty()) continue; diff --git a/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.reference b/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.reference new file mode 100644 index 0000000000000..c4389efae0b92 --- /dev/null +++ b/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.reference @@ -0,0 +1,8 @@ +127.0.0.1 1 +127.0.0.2 0 +3 +0 +3 +0 +0 +0 diff --git a/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.sql b/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.sql new file mode 100644 index 0000000000000..baad2c4e31ec8 --- /dev/null +++ b/tests/queries/0_stateless/05812_ping_pooled_connection_on_cluster_read.sql @@ -0,0 +1,78 @@ +-- Tags: no-fasttest, no-parallel +-- no-fasttest: the Fast test build has no libfiu, so SYSTEM ENABLE FAILPOINT is unsupported. +-- no-parallel: the fail points below fail every Ping in the server process. + +-- `test_cluster_two_shards` has one local address (127.0.0.1) and one that is not local +-- (127.0.0.2). A cluster read pings a pooled connection only for the remote shard. The fail +-- point makes that Ping fail. The read must still return every row: the remote shard is +-- reconnected, and the local shard is not pinged. `DistributedConnectionFailTry` is what shows +-- the Ping ran. The zero-timeout poll used for every other pooled connection does not call it. + +SELECT host_name, is_local +FROM system.clusters +WHERE cluster = 'test_cluster_two_shards' +ORDER BY host_name; + +INSERT INTO FUNCTION file(currentDatabase() || '_05812_data.tsv', 'TSV', 'x UInt8') +SELECT number FROM numbers(3) +SETTINGS engine_file_truncate_on_insert = 1; + +-- Fill the connection pool. The next read is the one that takes a pooled connection. +SELECT count() +FROM fileCluster('test_cluster_two_shards', currentDatabase() || '_05812_data.tsv', 'TSV', 'x UInt8') +FORMAT Null; + +CREATE TEMPORARY TABLE ping_fail_try_before (value UInt64); +INSERT INTO ping_fail_try_before +SELECT ifNull((SELECT value FROM system.events WHERE event = 'DistributedConnectionFailTry'), 0); + +SYSTEM ENABLE FAILPOINT unresponsive_pooled_connection; + +SELECT count() +FROM fileCluster('test_cluster_two_shards', currentDatabase() || '_05812_data.tsv', 'TSV', 'x UInt8'); + +SYSTEM DISABLE FAILPOINT unresponsive_pooled_connection; + +SELECT throwIf( + ifNull((SELECT value FROM system.events WHERE event = 'DistributedConnectionFailTry'), 0) + <= (SELECT value FROM ping_fail_try_before), + 'Remote shard was not pinged'); + +-- A peer that keeps the socket open and sends no Pong still reconnects once. The new connection +-- succeeds here, because this server is up and only the ping was failed. `ReconnectCount` and +-- `DistributedConnectionUsable` both move: the local shard and the reconnected remote shard are +-- usable. `FailTry` moves too. A stopped server would stop at the capped handshake instead. + +CREATE TEMPORARY TABLE ping_reconnect_before (value UInt64); +INSERT INTO ping_reconnect_before +SELECT ifNull((SELECT value FROM system.events WHERE event = 'DistributedConnectionReconnectCount'), 0); + +CREATE TEMPORARY TABLE ping_fail_try_before_timeout (value UInt64); +INSERT INTO ping_fail_try_before_timeout +SELECT ifNull((SELECT value FROM system.events WHERE event = 'DistributedConnectionFailTry'), 0); + +CREATE TEMPORARY TABLE ping_usable_before (value UInt64); +INSERT INTO ping_usable_before +SELECT ifNull((SELECT value FROM system.events WHERE event = 'DistributedConnectionUsable'), 0); + +SYSTEM ENABLE FAILPOINT pooled_connection_ping_timeout; + +SELECT count() +FROM fileCluster('test_cluster_two_shards', currentDatabase() || '_05812_data.tsv', 'TSV', 'x UInt8'); + +SYSTEM DISABLE FAILPOINT pooled_connection_ping_timeout; + +SELECT throwIf( + ifNull((SELECT value FROM system.events WHERE event = 'DistributedConnectionReconnectCount'), 0) + <= (SELECT value FROM ping_reconnect_before), + 'Ping timeout did not reconnect'); + +SELECT throwIf( + ifNull((SELECT value FROM system.events WHERE event = 'DistributedConnectionFailTry'), 0) + <= (SELECT value FROM ping_fail_try_before_timeout), + 'Ping timeout was not applied'); + +SELECT throwIf( + ifNull((SELECT value FROM system.events WHERE event = 'DistributedConnectionUsable'), 0) + < (SELECT value FROM ping_usable_before) + 2, + 'Ping timeout did not bring the remote shard back');