Skip to content
Open
32 changes: 31 additions & 1 deletion src/Client/Connection.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,8 @@ namespace FailPoints
{
extern const char receive_timeout_on_table_status_response[];
extern const char unexpected_packet_in_table_status_response[];
extern const char unresponsive_pooled_connection[];
extern const char pooled_connection_ping_timeout[];
}

namespace ErrorCodes
Expand Down Expand Up @@ -913,8 +915,22 @@ void Connection::sendClusterNameAndSalt()
}
#endif

bool Connection::ping(const ConnectionTimeouts & timeouts)
bool Connection::ping(const ConnectionTimeouts & timeouts, bool rethrow_on_timeout)
{
/// For tests: a pooled socket the server has already closed. `disconnect` matches the failure
/// path below, so the caller reconnects instead of reusing this socket.
fiu_do_on(FailPoints::unresponsive_pooled_connection, {
disconnect();
return false;
});

/// For tests: a peer that keeps the socket open and sends no `Pong`. The establisher
/// reconnects once, and caps that handshake by `sync_request_timeout`.
fiu_do_on(FailPoints::pooled_connection_ping_timeout, {
disconnect();
throw Exception(ErrorCodes::SOCKET_TIMEOUT, "Injected ping timeout");
});

try
{
TimeoutSetter timeout_setter(*socket, timeouts.sync_request_timeout, true);
Expand Down Expand Up @@ -943,6 +959,20 @@ bool Connection::ping(const ConnectionTimeouts & timeouts)
if (pong != Protocol::Server::Pong)
throwUnexpectedPacket(pong, "Pong", &timeout_setter);
}
catch (const Exception & e)
{
/// `Exception` extends `Poco::Exception`, so this arm has to come first. A read or write
/// that reaches `sync_request_timeout` is `SOCKET_TIMEOUT`. With `rethrow_on_timeout` that
/// timeout is rethrown: the cluster-read caller reconnects once and caps that handshake by
/// `sync_request_timeout`. The replica is skipped only if the reconnect fails. Every other
/// failure still returns false, and `checkConnected` keeps its old contract.
const bool timed_out = rethrow_on_timeout && e.code() == ErrorCodes::SOCKET_TIMEOUT;
disconnect();
LOG_TRACE(log_wrapper.get(), fmt::runtime(e.displayText()));
if (timed_out)
throw;
return false;
}
catch (const Poco::Exception & e)
{
/// Explicitly disconnect since ping() can receive EndOfStream,
Expand Down
13 changes: 12 additions & 1 deletion src/Client/Connection.h
Original file line number Diff line number Diff line change
Expand Up @@ -173,6 +173,15 @@ class Connection : public IServerConnection

bool checkConnected(const ConnectionTimeouts & timeouts) override { return isConnected() && ping(timeouts); }

/// Same round trip as `checkConnected`, but a peer that keeps the socket open and sends no
/// `Pong` within `sync_request_timeout` throws `SOCKET_TIMEOUT`. A socket the server has
/// already closed still returns false. Cluster reads reconnect either way. After a timeout
/// the new handshake is capped by `sync_request_timeout`.
bool checkConnectedThrowOnTimeout(const ConnectionTimeouts & timeouts)
{
return isConnected() && ping(timeouts, /*rethrow_on_timeout=*/ true);
}

/// Note that a server that went away without closing the connection is not detected here, and
/// neither is a close that has not arrived yet; that only shows up when the connection is used.
/// Pinging the server to find out would add a round trip to every query, and a pong that does
Expand Down Expand Up @@ -372,7 +381,9 @@ class Connection : public IServerConnection
#if USE_SSL
void sendClusterNameAndSalt();
#endif
bool ping(const ConnectionTimeouts & timeouts);
/// `rethrow_on_timeout` lets `SOCKET_TIMEOUT` out. Other failures still return false after
/// `disconnect`, which is what `checkConnected` and the clickhouse-client resync rely on.
bool ping(const ConnectionTimeouts & timeouts, bool rethrow_on_timeout = false);

/// Whether the connection can no longer serve a request, checked without a round trip.
bool isStale();
Expand Down
104 changes: 91 additions & 13 deletions src/Client/ConnectionEstablisher.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -45,8 +45,14 @@ ConnectionEstablisher::ConnectionEstablisher(
const ConnectionTimeouts * timeouts_,
const Settings & settings_,
LoggerPtr log_,
const QualifiedTableName * table_to_check_)
: pool(std::move(pool_)), timeouts(timeouts_), settings(settings_), log(log_), table_to_check(table_to_check_)
const QualifiedTableName * table_to_check_,
bool ping_pooled_connection_)
: pool(std::move(pool_))
, timeouts(timeouts_)
, settings(settings_)
, log(log_)
, table_to_check(table_to_check_)
, ping_pooled_connection(ping_pooled_connection_)
{
}

Expand All @@ -55,24 +61,56 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std::
/// A pooled connection is handed out without pinging it first - a ping would add an unnecessary
/// Ping-Pong round trip when the connection is still alive. A connection that the server has
/// closed while it was idle in the pool is detected by a zero-timeout poll and recovered by
/// reconnecting (see Connection::forceConnected). If a pooled connection nevertheless fails its
/// reconnecting (see `Connection::forceConnected`). If a pooled connection nevertheless fails its
/// very first request (e.g. the server went away without closing it), in the synchronous case we
/// reconnect once and retry, so that a single available replica is not dropped just because its
/// pooled connection went stale. In the asynchronous case we do not retry here - stale
/// connections are handled by retrying the next replica (see ConnectionPoolWithFailover).
/// connections are handled by retrying the next replica (see `ConnectionPoolWithFailover`).
///
/// Cluster reads of a remote shard opt into a ping (`ping_pooled_connection`). The poll cannot
/// see a server that stopped without closing the socket, and the query would then block until
/// `receive_timeout`. The asynchronous path keeps the poll: a ping there would block the fiber.
const bool can_reconnect = !async_callback;
const bool ping_pooled = ping_pooled_connection && can_reconnect;

/// Whether the connection was taken from the pool already established. Only such a connection
/// can be stale; a failure of a freshly established one is a genuine replica problem, and
/// retrying it is left to the caller (with its error accounting).
bool had_pooled_connection = false;
/// Set when `Ping` reached `sync_request_timeout` on this attempt.
bool pooled_ping_timed_out = false;
/// The next attempt is the one reconnect after a ping timeout. Its `Hello` wait is capped
/// by `sync_request_timeout`, including when `handshake_timeout_ms` is `0` (no socket timeout).
bool cap_handshake_after_ping_timeout = false;

auto try_establish = [&]()
{
ProfileEvents::increment(ProfileEvents::DistributedConnectionTries);

ConnectionTimeouts attempt_timeouts = *timeouts;
if (cap_handshake_after_ping_timeout)
{
/// `saturate` keeps a zero timeout: it is smaller than any positive limit. A zero
/// socket timeout never fires, and `handshake_timeout_ms = 0` means no limit, so
/// that value is replaced by `sync_request_timeout`. Any other handshake timeout
/// is the minimum of the two.
const Poco::Timespan & limit = attempt_timeouts.sync_request_timeout;
if (attempt_timeouts.handshake_timeout.totalMicroseconds() == 0)
{
attempt_timeouts.handshake_timeout = limit;
}
else
{
attempt_timeouts.handshake_timeout = ConnectionTimeouts::saturate(attempt_timeouts.handshake_timeout, limit);
}

cap_handshake_after_ping_timeout = false;
LOG_TRACE(log, "Reconnecting after a ping timeout with the handshake capped by `sync_request_timeout`.");
}

had_pooled_connection = false;
result.entry = pool->getUnchecked(*timeouts, settings);
pooled_ping_timed_out = false;
result.entry = pool->getUnchecked(attempt_timeouts, settings);
had_pooled_connection = result.entry->isConnected();

AsyncCallbackSetter<Connection> async_setter(&*result.entry, std::move(async_callback));
Expand All @@ -85,16 +123,50 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std::
});

/// Establish the connection, or revalidate a pooled one, under the async callback: with the
/// callback installed, Connection::connect uses a non-blocking connect and yields on it, so
/// callback installed, `Connection::connect` uses a non-blocking connect and yields on it, so
/// a caller that multiplexes several connection attempts (e.g. hedged connections) can
/// preempt a slow connect or handshake and switch to another replica. `result.entry` is
/// assigned before connecting, so the connection being established is visible to the
/// multi-address timeout handling of ConnectionEstablisherAsync.
result.entry->forceConnected(*timeouts);
/// multi-address timeout handling of `ConnectionEstablisherAsync`.
///
/// A pooled connection that the caller asked to ping is checked with `Ping` instead of the
/// poll. `Ping` fails when the server stopped without closing the socket; the poll reports
/// that socket as idle. A connection that is not established yet is connected, and the
/// handshake is the check. A dead socket is a soft error, so the retry below reconnects
/// once with the usual handshake timeout. A timeout also reconnects once: the old flow may
/// be dead while the server accepts a new connection. That handshake is capped above.
if (ping_pooled && result.entry->isConnected())
{
/// `ping` is private. `checkConnectedThrowOnTimeout` is the wrapper that lets a
/// `SOCKET_TIMEOUT` out. This branch already knows the connection is established.
bool answered = false;
try
{
answered = result.entry->checkConnectedThrowOnTimeout(attempt_timeouts);
}
catch (const Exception & e)
{
if (e.code() == ErrorCodes::SOCKET_TIMEOUT)
{
pooled_ping_timed_out = true;
LOG_TRACE(log, "Pooled connection timed out on ping, will reconnect.");
}
throw;
}
if (!answered)
{
LOG_TRACE(log, "Pooled connection did not answer a ping, will reconnect.");
throw Exception(ErrorCodes::SOCKET_TIMEOUT, "Ping of a pooled connection failed");
}
}
else
{
result.entry->forceConnected(attempt_timeouts);
}

UInt64 server_revision = 0;
if (table_to_check)
server_revision = result.entry->getServerRevision(*timeouts);
server_revision = result.entry->getServerRevision(attempt_timeouts);

if (!table_to_check || server_revision < DBMS_MIN_REVISION_WITH_TABLES_STATUS)
{
Expand All @@ -109,7 +181,7 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std::
TablesStatusRequest status_request;
status_request.tables.emplace(*table_to_check);

TablesStatusResponse status_response = result.entry->getTablesStatus(*timeouts, status_request);
TablesStatusResponse status_response = result.entry->getTablesStatus(attempt_timeouts, status_request);
auto table_status_it = status_response.table_states_by_id.find(*table_to_check);
if (table_status_it == status_response.table_states_by_id.end())
{
Expand Down Expand Up @@ -192,9 +264,14 @@ void ConnectionEstablisher::run(ConnectionEstablisher::TryResult & result, std::

/// Reconnect and retry once if a previously established (pooled) connection went stale.
/// The failed request is safe to repeat: it is either a read-only status request, or the
/// connection was closed before the request reached a live server.
/// connection was closed before the request reached a live server. A ping that timed out
/// is included: a new connection can succeed when the old flow was dropped and the server
/// is up. `cap_handshake_after_ping_timeout` limits that `Hello` wait.
if (can_reconnect && tries == 0 && had_pooled_connection)
{
if (pooled_ping_timed_out)
cap_handshake_after_ping_timeout = true;

ProfileEvents::increment(ProfileEvents::DistributedConnectionReconnectCount);
continue;
}
Expand All @@ -219,9 +296,10 @@ ConnectionEstablisherAsync::ConnectionEstablisherAsync(
const ConnectionTimeouts * timeouts_,
const Settings & settings_,
LoggerPtr log_,
const QualifiedTableName * table_to_check_)
const QualifiedTableName * table_to_check_,
bool ping_pooled_connection_)
: AsyncTaskExecutor(std::make_unique<Task>(*this), "ConnectionEstablisherAsync")
, connection_establisher(std::move(pool_), timeouts_, settings_, log_, table_to_check_)
, connection_establisher(std::move(pool_), timeouts_, settings_, log_, table_to_check_, ping_pooled_connection_)
{
epoll.add(timeout_descriptor.getDescriptor());
}
Expand Down
27 changes: 20 additions & 7 deletions src/Client/ConnectionEstablisher.h
Original file line number Diff line number Diff line change
Expand Up @@ -17,19 +17,30 @@ class ConnectionEstablisher
public:
using TryResult = PoolWithFailoverBase<IConnectionPool>::TryResult;

/// `ping_pooled_connection` sends a `Ping` on a connection taken from the pool already
/// established. A zero-timeout poll cannot see a server that stopped without closing the
/// socket, and a cluster read would then wait for `receive_timeout`. No `Pong` within
/// `sync_request_timeout` reconnects once. That handshake is capped by `sync_request_timeout`,
/// also when `handshake_timeout_ms` is `0`: a live server finishes `Hello` in one round trip,
/// and a stopped server stops at the cap. A socket the server has already closed also
/// reconnects once, with the usual handshake timeout. Callers that can run without this
/// replica (cluster reads of a remote shard) pass true. The default keeps the poll, with no
/// round trip.
ConnectionEstablisher(ConnectionPoolPtr pool_,
const ConnectionTimeouts * timeouts_,
const Settings & settings_,
LoggerPtr log,
const QualifiedTableName * table_to_check = nullptr);
const QualifiedTableName * table_to_check = nullptr,
bool ping_pooled_connection_ = false);

/// Establish connection and save it in result, write possible exception message in fail_message.
/// The connection is returned from the connection pool and it can be stale (the server may have
/// closed it while it was idle). The pooled connection is not pinged before use: pinging it adds
/// an unnecessary Ping-Pong round trip when the connection is still alive. Instead, a stale
/// connection is detected by a zero-timeout poll (see Connection::forceConnected) or, failing
/// that, by retrying - either here (a pooled connection that fails its first request is
/// reconnected once) or at a higher level (ConnectionPoolWithFailover retries the next replica).
/// closed it while it was idle). Unless `ping_pooled_connection` was set, the pooled connection
/// is not pinged before use: pinging it adds an unnecessary Ping-Pong round trip when the
/// connection is still alive. Instead, a stale connection is detected by a zero-timeout poll
/// (see `Connection::forceConnected`) or, failing that, by retrying - either here (a pooled
/// connection that fails its first request is reconnected once) or at a higher level
/// (`ConnectionPoolWithFailover` retries the next replica).
void run(TryResult & result, std::string & fail_message);

/// Set async callback that will be called when reading from socket blocks.
Expand All @@ -41,6 +52,7 @@ class ConnectionEstablisher
const Settings & settings;
LoggerPtr log;
const QualifiedTableName * table_to_check;
bool ping_pooled_connection;

AsyncCallback async_callback = {};
};
Expand All @@ -62,7 +74,8 @@ class ConnectionEstablisherAsync : public AsyncTaskExecutor
const ConnectionTimeouts * timeouts_,
const Settings & settings_,
LoggerPtr log_,
const QualifiedTableName * table_to_check_ = nullptr);
const QualifiedTableName * table_to_check_ = nullptr,
bool ping_pooled_connection_ = false);

/// Get file descriptor that can be added in epoll and be polled,
/// when this fd becomes ready, you call resume establishing connection.
Expand Down
13 changes: 8 additions & 5 deletions src/Client/ConnectionPoolWithFailover.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -134,10 +134,11 @@ std::vector<IConnectionPool::Entry> ConnectionPoolWithFailover::getMany(
PoolMode pool_mode,
AsyncCallback async_callback,
std::optional<bool> skip_unavailable_endpoints,
GetPriorityForLoadBalancing::Func priority_func)
GetPriorityForLoadBalancing::Func priority_func,
bool ping_pooled_connection)
{
TryGetEntryFunc try_get_entry = [&](const NestedPoolPtr & pool, std::string & fail_message)
{ return tryGetEntry(pool, timeouts, fail_message, settings, nullptr, async_callback); };
{ return tryGetEntry(pool, timeouts, fail_message, settings, nullptr, async_callback, ping_pooled_connection); };

std::vector<TryResult> results = getManyImpl(settings, pool_mode, try_get_entry, skip_unavailable_endpoints, priority_func);

Expand Down Expand Up @@ -251,12 +252,14 @@ ConnectionPoolWithFailover::tryGetEntry(
std::string & fail_message,
const Settings & settings,
const QualifiedTableName * table_to_check,
[[maybe_unused]] AsyncCallback async_callback)
[[maybe_unused]] AsyncCallback async_callback,
bool ping_pooled_connection)
{
#if defined(OS_LINUX)
if (async_callback)
{
ConnectionEstablisherAsync connection_establisher_async(pool, &timeouts, settings, log, table_to_check);
ConnectionEstablisherAsync connection_establisher_async(
pool, &timeouts, settings, log, table_to_check, ping_pooled_connection);
while (true)
{
connection_establisher_async.resume();
Expand All @@ -277,7 +280,7 @@ ConnectionPoolWithFailover::tryGetEntry(
}
#endif

ConnectionEstablisher connection_establisher(pool, &timeouts, settings, log, table_to_check);
ConnectionEstablisher connection_establisher(pool, &timeouts, settings, log, table_to_check, ping_pooled_connection);
TryResult result;
connection_establisher.run(result, fail_message);
return result;
Expand Down
Loading
Loading