From a3d8286ffcedc5a26718af0a850f195ce9aa7c7e Mon Sep 17 00:00:00 2001 From: gcs-analytics-core-robot Date: Thu, 5 Nov 2026 10:50:19 +0000 Subject: [PATCH] fix(storage): record response state in AsyncWriterConnectionImpl::Close() - Update `AsyncWriterConnectionImpl::OnClose::ConsumeLoop` to record `write_handle`, `persisted_size`, and `resource` returned in intermediate `BidiWriteObjectResponse` messages before EOF via a shared `UpdateFromResponse()` helper (reused across `OnFlush`, `OnClose`, and `OnFinalUpload`). - Guard `MakeRequest()` with `finish_called_` under `mu_` so post-`Finish()` calls to `Write()`, `Flush()`, `Close()`, or `Finalize()` fail immediately with `kCancelled` without issuing RPC operations on a finished stream, mutating `*hash_function_`/`offset_`, or overwriting `persisted_state_`. - Harden `mu_` lock consistency across `AsyncWriterConnectionImpl` state accessors and stream lifecycle operations. - Add unit tests verifying `Close()` state updates and concurrent state queries across `AsyncSequencer`-driven completion-queue callbacks. --- .../internal/async/writer_connection_impl.cc | 108 ++++++--- .../internal/async/writer_connection_impl.h | 8 +- .../async/writer_connection_impl_test.cc | 228 ++++++++++++++++++ 3 files changed, 303 insertions(+), 41 deletions(-) diff --git a/google/cloud/storage/internal/async/writer_connection_impl.cc b/google/cloud/storage/internal/async/writer_connection_impl.cc index 2b6668ab3f0c4..ce2061445ac16 100644 --- a/google/cloud/storage/internal/async/writer_connection_impl.cc +++ b/google/cloud/storage/internal/async/writer_connection_impl.cc @@ -93,11 +93,11 @@ AsyncWriterConnectionImpl::AsyncWriterConnectionImpl( AsyncWriterConnectionImpl::~AsyncWriterConnectionImpl() { // Cancel the streaming RPC. - impl_->Cancel(); + AsyncWriterConnectionImpl::Cancel(); // It is safe to call Finish() because: // (1) this is a no-op if it was already called. - // (2) calls to `Write()`, `Finalize()`, and `Query()` must have completed - // by the time the destructor is called + // (2) calls to `Write()`, `Flush()`, `Close()`, `Finalize()`, and `Query()` + // must have completed by the time the destructor is called Finish(); // We use a local copy of `impl` moved under lock to avoid @@ -114,6 +114,8 @@ AsyncWriterConnectionImpl::~AsyncWriterConnectionImpl() { }); } +void AsyncWriterConnectionImpl::Cancel() { GetImpl()->Cancel(); } + std::string AsyncWriterConnectionImpl::UploadId() const { return request_.upload_id(); } @@ -126,9 +128,13 @@ AsyncWriterConnectionImpl::PersistedState() const { future AsyncWriterConnectionImpl::Write(storage::WritePayload payload) { auto write = MakeRequest(); + if (!write) { + return make_ready_future( + internal::CancelledError("already finished", GCP_ERROR_INFO())); + } auto p = WritePayloadImpl::GetImpl(payload); auto size = p.size(); - auto coro = PartialUpload::Call(impl_, hash_function_, std::move(write), + auto coro = PartialUpload::Call(GetImpl(), hash_function_, *std::move(write), std::move(p), PartialUpload::kNone); return coro->Start().then([coro, size, this](auto f) mutable { @@ -140,7 +146,11 @@ future AsyncWriterConnectionImpl::Write(storage::WritePayload payload) { future> AsyncWriterConnectionImpl::Finalize(storage::WritePayload payload) { auto write = MakeRequest(); - write.set_finish_write(true); + if (!write) { + return make_ready_future(StatusOr( + internal::CancelledError("already finished", GCP_ERROR_INFO()))); + } + write->set_finish_write(true); auto p = WritePayloadImpl::GetImpl(payload); auto size = p.size(); @@ -159,7 +169,7 @@ AsyncWriterConnectionImpl::Finalize(storage::WritePayload payload) { // uploaded in this stream, not the full object. We use `kFinalize` to avoid // sending this partial hash, which would otherwise fail validation. if (merged.has()) { - write.mutable_object_checksums()->set_crc32c( + write->mutable_object_checksums()->set_crc32c( merged.get()); } action = PartialUpload::kFinalize; @@ -171,17 +181,17 @@ AsyncWriterConnectionImpl::Finalize(storage::WritePayload payload) { // so the internal hash function doesn't overwrite it with its own computed // hash. if (current_options.has()) { - write.mutable_object_checksums()->set_crc32c( + write->mutable_object_checksums()->set_crc32c( current_options.get()); } if (current_options.has()) { - write.mutable_object_checksums()->set_md5_hash( + write->mutable_object_checksums()->set_md5_hash( current_options.get()); } action = PartialUpload::kFinalize; } - auto coro = PartialUpload::Call(impl_, hash_function_, std::move(write), + auto coro = PartialUpload::Call(GetImpl(), hash_function_, *std::move(write), std::move(p), std::move(action)); return coro->Start().then([coro, size, this](auto f) mutable { coro.reset(); // breaks the cycle between the completion queue and coro @@ -191,9 +201,13 @@ AsyncWriterConnectionImpl::Finalize(storage::WritePayload payload) { future AsyncWriterConnectionImpl::Flush(storage::WritePayload payload) { auto write = MakeRequest(); + if (!write) { + return make_ready_future( + internal::CancelledError("already finished", GCP_ERROR_INFO())); + } auto p = WritePayloadImpl::GetImpl(payload); auto size = p.size(); - auto coro = PartialUpload::Call(impl_, hash_function_, std::move(write), + auto coro = PartialUpload::Call(GetImpl(), hash_function_, *std::move(write), std::move(p), PartialUpload::kFlush); return coro->Start().then([coro, size, this](auto f) mutable { @@ -204,9 +218,13 @@ future AsyncWriterConnectionImpl::Flush(storage::WritePayload payload) { future AsyncWriterConnectionImpl::Close(storage::WritePayload payload) { auto write = MakeRequest(); + if (!write) { + return make_ready_future( + internal::CancelledError("already finished", GCP_ERROR_INFO())); + } auto p = WritePayloadImpl::GetImpl(payload); auto size = p.size(); - auto coro = PartialUpload::Call(impl_, hash_function_, std::move(write), + auto coro = PartialUpload::Call(GetImpl(), hash_function_, *std::move(write), std::move(p), PartialUpload::kFlushAndClose); return coro->Start().then([coro, size, this](auto f) mutable { @@ -229,12 +247,42 @@ future> AsyncWriterConnectionImpl::Query() { } RpcMetadata AsyncWriterConnectionImpl::GetRequestMetadata() { - return impl_->GetRequestMetadata(); + // `GetRequestMetadata()` may only be called after the stream has finished + // (`grpc::ClientContext::GetServerInitialMetadata()` requires initial + // metadata to have been received, and `GetServerTrailingMetadata()` requires + // stream completion). + return GetImpl()->GetRequestMetadata(); +} + +std::shared_ptr +AsyncWriterConnectionImpl::GetImpl() const { + std::unique_lock lk(mu_); + return impl_; } -google::storage::v2::BidiWriteObjectRequest +bool AsyncWriterConnectionImpl::UpdateFromResponse( + google::storage::v2::BidiWriteObjectResponse response) { + std::unique_lock lk(mu_); + if (response.has_write_handle()) { + latest_write_handle_ = std::move(*response.mutable_write_handle()); + } + bool has_state = false; + if (response.has_persisted_size()) { + persisted_state_ = response.persisted_size(); + has_state = true; + } + if (response.has_resource()) { + persisted_state_ = std::move(*response.mutable_resource()); + has_state = true; + } + return has_state; +} + +std::optional AsyncWriterConnectionImpl::MakeRequest() { - auto request = request_; + std::unique_lock lk(mu_); + if (finish_called_) return std::nullopt; + google::storage::v2::BidiWriteObjectRequest request = request_; if (first_request_) { first_request_ = false; if (latest_write_handle_.has_value() && request.has_append_object_spec()) { @@ -298,19 +346,9 @@ future AsyncWriterConnectionImpl::OnFlush(std::size_t upload_size, return self->Finish().then(HandleFinishAfterError( "Expected error in Finish() after non-ok Read()")); } - std::unique_lock lk(self->mu_); - if (response->has_write_handle()) { - self->latest_write_handle_ = response->write_handle(); - } - if (response->has_persisted_size()) { - self->persisted_state_ = response->persisted_size(); - return make_ready_future(Status{}); - } - if (response->has_resource()) { - self->persisted_state_ = response->resource(); + if (self->UpdateFromResponse(std::move(*response))) { return make_ready_future(Status{}); } - lk.unlock(); return impl->Read().then(std::move(*this)); } }; @@ -332,17 +370,19 @@ future AsyncWriterConnectionImpl::OnClose(std::size_t upload_size, lk.unlock(); struct ConsumeLoop { + AsyncWriterConnectionImpl* self; std::shared_ptr impl; future operator()( future> f) { auto response = std::move(f).get(); if (!response.has_value()) return make_ready_future(); - return impl->Read().then(*this); + self->UpdateFromResponse(std::move(*response)); + return impl->Read().then(std::move(*this)); } }; - return impl->Read().then(ConsumeLoop{impl}).then([this](auto f) { + return impl->Read().then(ConsumeLoop{this, impl}).then([this](auto f) { f.get(); return Finish(); }); @@ -379,18 +419,7 @@ AsyncWriterConnectionImpl::OnFinalUpload(std::size_t upload_size, future> f) { auto response = std::move(f).get(); if (!response.has_value()) return make_ready_future(); - { - std::unique_lock lk(self->mu_); - if (response->has_write_handle()) { - self->latest_write_handle_ = response->write_handle(); - } - if (response->has_persisted_size()) { - self->persisted_state_ = response->persisted_size(); - } - if (response->has_resource()) { - self->persisted_state_ = response->resource(); - } - } + self->UpdateFromResponse(std::move(*response)); return impl->Read().then(std::move(*this)); } }; @@ -401,6 +430,7 @@ AsyncWriterConnectionImpl::OnFinalUpload(std::size_t upload_size, [this](auto f2) -> StatusOr { auto status = f2.get(); if (!status.ok()) return status; + std::unique_lock lk(mu_); if (!std::holds_alternative( persisted_state_)) { return internal::InternalError( diff --git a/google/cloud/storage/internal/async/writer_connection_impl.h b/google/cloud/storage/internal/async/writer_connection_impl.h index 2d15e1c4cabaf..6936de1c41ab9 100644 --- a/google/cloud/storage/internal/async/writer_connection_impl.h +++ b/google/cloud/storage/internal/async/writer_connection_impl.h @@ -25,6 +25,7 @@ #include #include #include +#include namespace google { namespace cloud { @@ -51,7 +52,7 @@ class AsyncWriterConnectionImpl : public storage::AsyncWriterConnection { google::storage::v2::Object metadata, bool first_request = true); ~AsyncWriterConnectionImpl() override; - void Cancel() override { return impl_->Cancel(); } + void Cancel() override; std::string UploadId() const override; std::optional WriteHandle() @@ -81,7 +82,10 @@ class AsyncWriterConnectionImpl : public storage::AsyncWriterConnection { PersistedStateType persisted_state, std::int64_t offset, bool first_request = true); - google::storage::v2::BidiWriteObjectRequest MakeRequest(); + std::optional MakeRequest(); + std::shared_ptr GetImpl() const; + bool UpdateFromResponse( + google::storage::v2::BidiWriteObjectResponse response); future OnPartialUpload(std::size_t upload_size, StatusOr success); diff --git a/google/cloud/storage/internal/async/writer_connection_impl_test.cc b/google/cloud/storage/internal/async/writer_connection_impl_test.cc index 09a48f90b84c5..d379d2fb30c69 100644 --- a/google/cloud/storage/internal/async/writer_connection_impl_test.cc +++ b/google/cloud/storage/internal/async/writer_connection_impl_test.cc @@ -31,6 +31,9 @@ #include "google/cloud/testing_util/is_proto_equal.h" #include "google/cloud/testing_util/status_matchers.h" #include +#include +#include +#include namespace google { namespace cloud { @@ -47,6 +50,7 @@ using ::google::cloud::testing_util::IsProtoEqual; using ::google::cloud::testing_util::StatusIs; using ::testing::_; using ::testing::An; +using ::testing::Optional; using ::testing::Pair; using ::testing::Return; using ::testing::UnorderedElementsAre; @@ -1512,6 +1516,230 @@ TEST(AsyncWriterConnectionTest, ConsecutiveFlushesEachReadOneResponse) { next.first.set_value(true); } +auto SequencedRead(AsyncSequencer& sequencer, char const* step, + std::optional resp) { + return sequencer.PushBack(step).then( + [r = std::move(resp)](future f) mutable { + return f.get() ? std::move(r) : std::optional(); + }); +} + +auto SequencedFinish(AsyncSequencer& sequencer) { + return sequencer.PushBack("Finish").then( + [](future f) { return f.get() ? Status{} : PermanentError(); }); +} + +/// @test Verify that `Close()` records `write_handle` and `persisted_size` +/// returned across multiple intermediate responses before EOF. +TEST(AsyncWriterConnectionTest, CloseUpdatesWriteHandleAndPersistedState) { + AsyncSequencer sequencer; + auto mock = std::make_unique(); + EXPECT_CALL(*mock, Cancel).Times(1); + EXPECT_CALL(*mock, Write) + .WillOnce([&](Request const& request, grpc::WriteOptions wopt) { + EXPECT_EQ(request.write_offset(), 1024); + EXPECT_TRUE(request.flush()); + EXPECT_TRUE(request.state_lookup()); + EXPECT_TRUE(wopt.is_last_message()); + return sequencer.PushBack("Write"); + }); + Response handle_resp; + handle_resp.mutable_write_handle()->set_handle("close-handle-1"); + Response size_resp; + size_resp.set_persisted_size(4096); + EXPECT_CALL(*mock, Read) + .WillOnce([&] { return SequencedRead(sequencer, "Read1", handle_resp); }) + .WillOnce([&] { return SequencedRead(sequencer, "Read2", size_resp); }) + .WillOnce( + [&] { return SequencedRead(sequencer, "Read3", std::nullopt); }); + EXPECT_CALL(*mock, Finish).WillOnce([&] { + return SequencedFinish(sequencer); + }); + auto hash = std::make_shared(); + EXPECT_CALL(*hash, Update(_, An(), _)).Times(1); + EXPECT_CALL(*hash, Finish).Times(0); + auto tested = std::make_unique( + TestOptions(), MakeRequest(), std::move(mock), hash, 1024); + future close = tested->Close(WritePayload(std::string(3072, 'A'))); + for (char const* step : {"Write", "Read1", "Read2", "Read3", "Finish"}) { + std::pair, std::string> next = sequencer.PopFrontWithName(); + ASSERT_THAT(next.second, step); + next.first.set_value(true); + } + EXPECT_THAT(close.get(), IsOk()); + EXPECT_THAT(tested->PersistedState(), VariantWith(4096)); + EXPECT_THAT(tested->Query().get(), IsOkAndHolds(4096)); + EXPECT_THAT(tested->WriteHandle(), + Optional(IsProtoEqual(handle_resp.write_handle()))); + tested.reset(); +} + +/// @test Verify that `Close()` records `resource` metadata when the service +/// returns a finalized object resource before EOF. +TEST(AsyncWriterConnectionTest, CloseUpdatesResourceState) { + AsyncSequencer sequencer; + auto mock = std::make_unique(); + EXPECT_CALL(*mock, Cancel).Times(1); + EXPECT_CALL(*mock, Write).WillOnce([&](Request const&, grpc::WriteOptions) { + return sequencer.PushBack("Write"); + }); + EXPECT_CALL(*mock, Read) + .WillOnce( + [&] { return SequencedRead(sequencer, "Read1", MakeTestResponse()); }) + .WillOnce( + [&] { return SequencedRead(sequencer, "Read2", std::nullopt); }); + EXPECT_CALL(*mock, Finish).WillOnce([&] { + return SequencedFinish(sequencer); + }); + auto hash = std::make_shared(); + EXPECT_CALL(*hash, Update(_, An(), _)).Times(1); + EXPECT_CALL(*hash, Finish).Times(0); + auto tested = std::make_unique( + TestOptions(), MakeRequest(), std::move(mock), hash, 1024); + future close = tested->Close(WritePayload(std::string(1024, 'A'))); + for (char const* step : {"Write", "Read1", "Read2", "Finish"}) { + std::pair, std::string> next = sequencer.PopFrontWithName(); + ASSERT_THAT(next.second, step); + next.first.set_value(true); + } + EXPECT_THAT(close.get(), IsOk()); + EXPECT_THAT( + tested->PersistedState(), + VariantWith(IsProtoEqual(MakeTestObject()))); + EXPECT_THAT(tested->Query().get(), IsOkAndHolds(2048)); + tested.reset(); +} + +/// @test Verify that concurrent state inspections (`PersistedState`, `Query`, +/// `WriteHandle`, `Cancel`) do not data-race with completion-queue callbacks +/// for serialized `Write()`, `Flush()`, and `Finalize()` operations, and that +/// post-`Finalize()` mutating calls fail with `kCancelled` without touching +/// the stream or overwriting `PersistedState()`. +TEST(AsyncWriterConnectionTest, + ConcurrentFlushFinalizeAndStateQueriesAreRaceFree) { + AsyncSequencer sequencer; + auto mock = std::make_unique(); + EXPECT_CALL(*mock, Cancel).Times(::testing::AtLeast(1)); + EXPECT_CALL(*mock, GetRequestMetadata) + .WillOnce(Return(RpcMetadata{{{"hk0", "v0"}}, {{"tk0", "v0"}}})); + EXPECT_CALL(*mock, Write) + .WillOnce([&](Request const& request, grpc::WriteOptions wopt) { + EXPECT_EQ(request.write_offset(), 0); + EXPECT_EQ(request.append_object_spec().write_handle().handle(), + "initial-handle"); + EXPECT_FALSE(request.flush()); + EXPECT_FALSE(request.state_lookup()); + EXPECT_FALSE(request.finish_write()); + EXPECT_FALSE(wopt.is_last_message()); + return sequencer.PushBack("Write1"); + }) + .WillOnce([&](Request const& request, grpc::WriteOptions wopt) { + EXPECT_EQ(request.write_offset(), 1024); + EXPECT_FALSE(request.has_append_object_spec()); + EXPECT_TRUE(request.flush()); + EXPECT_TRUE(request.state_lookup()); + EXPECT_FALSE(request.finish_write()); + EXPECT_FALSE(wopt.is_last_message()); + return sequencer.PushBack("Write2"); + }) + .WillOnce([&](Request const& request, grpc::WriteOptions wopt) { + EXPECT_EQ(request.write_offset(), 2048); + EXPECT_FALSE(request.has_append_object_spec()); + EXPECT_TRUE(request.finish_write()); + EXPECT_TRUE(wopt.is_last_message()); + return sequencer.PushBack("Write3"); + }); + Response handle_resp; + handle_resp.mutable_write_handle()->set_handle("refreshed-handle"); + Response size_resp; + size_resp.set_persisted_size(2048); + EXPECT_CALL(*mock, Read) + .WillOnce([&] { return SequencedRead(sequencer, "Read1", handle_resp); }) + .WillOnce([&] { return SequencedRead(sequencer, "Read2", size_resp); }) + .WillOnce( + [&] { return SequencedRead(sequencer, "Read3", MakeTestResponse()); }) + .WillOnce( + [&] { return SequencedRead(sequencer, "Read4", std::nullopt); }); + EXPECT_CALL(*mock, Finish).WillOnce([&] { + return SequencedFinish(sequencer); + }); + auto hash = std::make_shared(); + EXPECT_CALL(*hash, Update(_, An(), _)).Times(3); + EXPECT_CALL(*hash, Finish).Times(0); + Request req = MakeRequest(); + req.mutable_append_object_spec()->set_bucket( + "projects/_/buckets/test-bucket"); + req.mutable_append_object_spec()->set_object("test-object"); + req.mutable_append_object_spec()->mutable_write_handle()->set_handle( + "initial-handle"); + auto tested = std::make_unique( + TestOptions(), std::move(req), std::move(mock), hash, 0); + + std::thread cq_worker([&] { + for (char const* step : {"Write1", "Write2", "Read1", "Read2", "Write3", + "Read3", "Read4", "Finish"}) { + std::pair, std::string> next = sequencer.PopFrontWithName(); + EXPECT_EQ(next.second, step); + next.first.set_value(true); + } + }); + + std::atomic stop{false}; + promise reader_started; + bool signaled_start = false; + std::thread reader([&] { + while (!stop.load(std::memory_order_acquire)) { + (void)tested->PersistedState(); + (void)tested->Query().get(); + (void)tested->WriteHandle(); + // `MockStream::Cancel` is a no-op stub here, used to probe `GetImpl()` + // synchronization concurrently with completion-queue callbacks. + tested->Cancel(); + if (!signaled_start) { + signaled_start = true; + reader_started.set_value(); + } + std::this_thread::yield(); + } + }); + + reader_started.get_future().get(); + + EXPECT_THAT(tested->Write(WritePayload(std::string(1024, 'A'))).get(), + IsOk()); + EXPECT_THAT(tested->Flush(WritePayload(std::string(1024, 'B'))).get(), + IsOk()); + EXPECT_THAT(tested->Query().get(), IsOkAndHolds(2048)); + EXPECT_THAT(tested->WriteHandle(), + Optional(IsProtoEqual(handle_resp.write_handle()))); + + future> finalized = + tested->Finalize(WritePayload{}); + EXPECT_THAT(finalized.get(), IsOkAndHolds(IsProtoEqual(MakeTestObject()))); + + WritePayload const post_payload(std::string(1024, 'C')); + EXPECT_THAT(tested->Write(post_payload).get(), + StatusIs(StatusCode::kCancelled)); + EXPECT_THAT(tested->Flush(post_payload).get(), + StatusIs(StatusCode::kCancelled)); + EXPECT_THAT(tested->Close(post_payload).get(), + StatusIs(StatusCode::kCancelled)); + EXPECT_THAT(tested->Finalize(post_payload).get(), + StatusIs(StatusCode::kCancelled)); + EXPECT_THAT( + tested->PersistedState(), + VariantWith(IsProtoEqual(MakeTestObject()))); + + RpcMetadata const metadata = tested->GetRequestMetadata(); + EXPECT_THAT(metadata.headers, UnorderedElementsAre(Pair("hk0", "v0"))); + EXPECT_THAT(metadata.trailers, UnorderedElementsAre(Pair("tk0", "v0"))); + + stop.store(true, std::memory_order_release); + reader.join(); + cq_worker.join(); + tested.reset(); +} + GOOGLE_CLOUD_CPP_INLINE_NAMESPACE_END } // namespace storage_internal } // namespace cloud