Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions src/Interpreters/Context.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -8178,6 +8178,16 @@ std::shared_lock<SharedMutex> Context::lockSwarmMode() const
return std::shared_lock(shared->swarm_mode_mutex);
}

bool Context::runWithSwarmTaskAdmission(bool server_cancelled, std::function<void()> send) const
{
auto admission = lockSwarmMode();
if (server_cancelled || !isSwarmModeEnabled())
return false;

send();
return true;
}

Context::ApplicationType Context::getApplicationType() const
{
return shared->application_type;
Expand Down
9 changes: 6 additions & 3 deletions src/Interpreters/Context.h
Original file line number Diff line number Diff line change
Expand Up @@ -1905,10 +1905,13 @@ class Context: public ContextData, public std::enable_shared_from_this<Context>
bool startSwarmMode();
/// Return current swarm mode state.
bool isSwarmModeEnabled() const;
/// Shared lock for one swarm task send. The read-task callback holds it only across the
/// enabled check and `sendReadTaskRequest`. The response read does not. `stopSwarmMode`
/// and `startSwarmMode` take it exclusively.
/// Shared lock for one swarm task send. `runWithSwarmTaskAdmission` holds it across the
/// enabled check and the send. The response read does not. `stopSwarmMode` and
/// `startSwarmMode` take it exclusively.
std::shared_lock<SharedMutex> lockSwarmMode() const;
/// Call `send` while holding the swarm admission lock. Returns false, without calling `send`,
/// when the server is cancelled or swarm mode is disabled. The lock is released before return.
bool runWithSwarmTaskAdmission(bool server_cancelled, std::function<void()> send) const;

bool isInternalQuery() const { return is_internal_query; }
void setInternalQuery(bool internal) { is_internal_query = internal; }
Expand Down
22 changes: 10 additions & 12 deletions src/Server/TCPHandler.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -827,19 +827,17 @@ void TCPHandler::runImpl()

try
{
/// Held across the enabled check and `sendReadTaskRequest`. The block ends
/// before the response read, so `stopSwarmMode` does not wait out
/// `receive_timeout`. Releasing the lock before the send would let stop
/// return and then this callback still send one more request.
/// `runWithSwarmTaskAdmission` holds the admission lock across the check and the
/// send, then releases it. The response read below must not hold the lock.
if (query_state->query_context)
{
const bool sent = query_state->query_context->runWithSwarmTaskAdmission(
server.isCancelled(), [this]() TSA_NO_THREAD_SAFETY_ANALYSIS { sendReadTaskRequest(); });
if (!sent)
return std::make_shared<ClusterFunctionReadTaskResponse>();
}
else
{
std::shared_lock<SharedMutex> admission;
if (query_state->query_context)
{
admission = query_state->query_context->lockSwarmMode();
if (server.isCancelled() || !query_state->query_context->isSwarmModeEnabled())
return std::make_shared<ClusterFunctionReadTaskResponse>();
}

sendReadTaskRequest();
}

Expand Down
169 changes: 33 additions & 136 deletions src/Storages/ObjectStorage/tests/gtest_storage_object_storage_swarm.cpp
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
#include <gtest/gtest.h>

#include <Interpreters/ClusterFunctionReadTask.h>
#include <Storages/ObjectStorage/StorageObjectStorageSource.h>
#include <Common/tests/gtest_global_context.h>

#include <atomic>
Expand All @@ -11,6 +9,9 @@
#include <mutex>
#include <thread>

/// These tests call `Context::runWithSwarmTaskAdmission`, the same function the `TCPHandler`
/// read-task callback uses around `sendReadTaskRequest`.

TEST(StorageObjectStorageSwarm, ConcurrentTaskAdmissionsDoNotSerialize)
{
auto context = DB::Context::createCopy(::getContext().context);
Expand All @@ -21,33 +22,17 @@ TEST(StorageObjectStorageSwarm, ConcurrentTaskAdmissionsDoNotSerialize)
size_t inside = 0;
std::atomic<bool> overlapped{false};

/// Each admission holds the shared lock until the other has entered. An exclusive lock
/// would keep the second admission out until the first returns.
DB::ClusterFunctionReadTaskCallback callback = [&]()
auto send = [&]()
{
auto admission = context->lockSwarmMode();
std::unique_lock lock(mutex);
++inside;
both_inside.notify_all();
if (both_inside.wait_for(lock, std::chrono::seconds(5), [&] { return inside >= 2; }))
overlapped = true;
return std::make_shared<DB::ClusterFunctionReadTaskResponse>();
};

auto run_admission = [&]()
{
DB::StorageObjectStorageSource::ReadTaskIterator iterator(
callback,
/*max_threads_count=*/1,
/*is_archive_=*/false,
/*object_storage_=*/nullptr,
/*table_location_=*/"",
context);
(void)iterator;
};

std::thread first(run_admission);
std::thread second(run_admission);
std::thread first([&] { EXPECT_TRUE(context->runWithSwarmTaskAdmission(/*server_cancelled=*/false, send)); });
std::thread second([&] { EXPECT_TRUE(context->runWithSwarmTaskAdmission(/*server_cancelled=*/false, send)); });
first.join();
second.join();

Expand All @@ -58,161 +43,73 @@ TEST(StorageObjectStorageSwarm, StopBeforeSendDoesNotRequestTask)
{
auto context = DB::Context::createCopy(::getContext().context);
context->startSwarmMode();
ASSERT_TRUE(context->stopSwarmMode());

std::mutex mutex;
std::condition_variable ready;
bool at_send = false;
bool allow_send = false;
std::atomic<int> sends{0};
EXPECT_FALSE(context->runWithSwarmTaskAdmission(/*server_cancelled=*/false, [&] { ++sends; }));

/// Same order as the `TCPHandler` callback: the admission lock covers only the check and the send.
DB::ClusterFunctionReadTaskCallback callback = [&]()
{
{
std::unique_lock lock(mutex);
at_send = true;
ready.notify_all();
ready.wait(lock, [&] { return allow_send; });
}

auto admission = context->lockSwarmMode();
if (!context->isSwarmModeEnabled())
return std::make_shared<DB::ClusterFunctionReadTaskResponse>();

++sends;
return std::make_shared<DB::ClusterFunctionReadTaskResponse>();
};

std::exception_ptr error;
std::thread worker([&]()
{
try
{
DB::StorageObjectStorageSource::ReadTaskIterator iterator(
callback,
/*max_threads_count=*/1,
/*is_archive_=*/false,
/*object_storage_=*/nullptr,
/*table_location_=*/"",
context);
(void)iterator;
}
catch (...)
{
error = std::current_exception();
}
});

bool entered = false;
{
std::unique_lock lock(mutex);
entered = ready.wait_for(lock, std::chrono::seconds(5), [&] { return at_send; });
}
if (!entered)
{
{
std::lock_guard lock(mutex);
allow_send = true;
}
ready.notify_all();
worker.join();
context->startSwarmMode();
FAIL() << "Task callback did not reach the send";
}

EXPECT_TRUE(context->stopSwarmMode());

{
std::lock_guard lock(mutex);
allow_send = true;
}
ready.notify_all();
worker.join();
context->startSwarmMode();
EXPECT_FALSE(context->runWithSwarmTaskAdmission(/*server_cancelled=*/true, [&] { ++sends; }));
context->startSwarmMode();

ASSERT_FALSE(error);
EXPECT_EQ(sends.load(), 0);
}

TEST(StorageObjectStorageSwarm, TaskResponseReadDoesNotBlockStop)
TEST(StorageObjectStorageSwarm, InFlightSendBlocksStopAndNoRequestFollows)
{
auto context = DB::Context::createCopy(::getContext().context);
context->startSwarmMode();

std::mutex mutex;
std::condition_variable ready;
bool in_receive = false;
bool finish_read = false;

DB::ClusterFunctionReadTaskCallback callback = [&]()
{
{
auto admission = context->lockSwarmMode();
if (!context->isSwarmModeEnabled())
return std::make_shared<DB::ClusterFunctionReadTaskResponse>();
}

std::unique_lock lock(mutex);
in_receive = true;
ready.notify_all();
ready.wait(lock, [&] { return finish_read; });
return std::make_shared<DB::ClusterFunctionReadTaskResponse>();
};
bool in_send = false;
bool finish_send = false;
std::atomic<int> sends{0};

std::exception_ptr error;
std::thread worker([&]()
{
try
EXPECT_TRUE(context->runWithSwarmTaskAdmission(/*server_cancelled=*/false, [&]()
{
DB::StorageObjectStorageSource::ReadTaskIterator iterator(
callback,
/*max_threads_count=*/1,
/*is_archive_=*/false,
/*object_storage_=*/nullptr,
/*table_location_=*/"",
context);
(void)iterator;
}
catch (...)
{
error = std::current_exception();
}
++sends;
std::unique_lock lock(mutex);
in_send = true;
ready.notify_all();
ready.wait(lock, [&] { return finish_send; });
}));
});

bool entered = false;
{
std::unique_lock lock(mutex);
entered = ready.wait_for(lock, std::chrono::seconds(5), [&] { return in_receive; });
entered = ready.wait_for(lock, std::chrono::seconds(5), [&] { return in_send; });
}
if (!entered)
{
{
std::lock_guard lock(mutex);
finish_read = true;
finish_send = true;
}
ready.notify_all();
worker.join();
context->startSwarmMode();
FAIL() << "Task callback did not reach the response read";
FAIL() << "Task send did not start";
}

auto stop_result = std::async(std::launch::async, [&] { return context->stopSwarmMode(); });
const bool stop_finished = stop_result.wait_for(std::chrono::seconds(1)) == std::future_status::ready;
bool stopped = false;
if (stop_finished)
stopped = stop_result.get();
EXPECT_EQ(stop_result.wait_for(std::chrono::milliseconds(200)), std::future_status::timeout);

{
std::lock_guard lock(mutex);
finish_read = true;
finish_send = true;
}
ready.notify_all();
if (stop_result.valid())
stop_result.wait();

const auto stop_status = stop_result.wait_for(std::chrono::seconds(5));
worker.join();
context->startSwarmMode();
ASSERT_EQ(stop_status, std::future_status::ready);
EXPECT_TRUE(stop_result.get());

ASSERT_FALSE(error);
ASSERT_TRUE(stop_finished);
EXPECT_TRUE(stopped);
EXPECT_FALSE(context->runWithSwarmTaskAdmission(/*server_cancelled=*/false, [&] { ++sends; }));
context->startSwarmMode();
EXPECT_EQ(sends.load(), 1);
}
Loading