From 8e2f8cef747dc207eda9f888a8611c3e55a51256 Mon Sep 17 00:00:00 2001 From: tison Date: Wed, 18 Nov 2026 22:35:13 +0800 Subject: [PATCH 1/6] refactor(oneshot): centralize consumption of a published message --- asyncband/src/oneshot/mod.rs | 29 +++++++++++++--------- asyncband/src/oneshot/receiver.rs | 40 +++++-------------------------- 2 files changed, 24 insertions(+), 45 deletions(-) diff --git a/asyncband/src/oneshot/mod.rs b/asyncband/src/oneshot/mod.rs index fbca62b7..7da69ee5 100644 --- a/asyncband/src/oneshot/mod.rs +++ b/asyncband/src/oneshot/mod.rs @@ -183,6 +183,22 @@ impl Channel { } } + /// Disconnects the receiver and takes the published message. + /// + /// # Safety + /// + /// The caller must own the receiving endpoint and have observed `MESSAGE` with a load or + /// failed compare-exchange on `state`. The message must still be unconsumed. + #[inline] + unsafe fn take_published_message(&self) -> T { + // ORDERING: MESSAGE is terminal for the sender, so this receiver-only update publishes + // no data. The fence pairs the caller's observation with the sender's Release publication. + self.state.store(DISCONNECTED, Ordering::Relaxed); + fence(Ordering::Acquire); + // SAFETY: The receiver exclusively owns the message and has synchronized with its write. + unsafe { self.take_message() } + } + /// Initializes the message slot. /// /// # Safety @@ -278,17 +294,8 @@ impl Channel { // state, so it has not accessed the waker. We must drop it. unsafe { self.drop_waker() }; - // ORDERING: The sender has completed, so this receiver-only terminal update does - // not publish data to another thread. - self.state.store(DISCONNECTED, Ordering::Relaxed); - - // ORDERING: The failed CAS read MESSAGE from the sender's Release publication. This - // conditional Acquire makes the initialized message visible before it is taken. - fence(Ordering::Acquire); - - // SAFETY: The MESSAGE state tells us there is a correctly initialized message, - // and the fence above synchronizes with that write. - Poll::Ready(Ok(unsafe { self.take_message() })) + // SAFETY: The receiver observed MESSAGE in the failed compare-exchange. + Poll::Ready(Ok(unsafe { self.take_published_message() })) } // The sender was dropped before sending anything while we prepared to await. Err(DISCONNECTED) => { diff --git a/asyncband/src/oneshot/receiver.rs b/asyncband/src/oneshot/receiver.rs index 921d4ffc..5e12e51b 100644 --- a/asyncband/src/oneshot/receiver.rs +++ b/asyncband/src/oneshot/receiver.rs @@ -28,7 +28,6 @@ use std::mem::ManuallyDrop; use std::pin::Pin; use std::ptr::NonNull; use std::sync::atomic::Ordering; -use std::sync::atomic::fence; use std::task::Context; use std::task::Poll; @@ -145,16 +144,8 @@ impl Receiver { // conditional `Acquire` synchronization below. match channel.state.load(Ordering::Relaxed) { MESSAGE => { - // `MESSAGE` is terminal for the sender, so marking disconnection needs no RMW and - // publishes no data to another thread. - channel.state.store(DISCONNECTED, Ordering::Relaxed); - - // ORDERING: Pairs with the sender's `Release` publication of `MESSAGE` before - // reading the initialized payload. - fence(Ordering::Acquire); - - // SAFETY: `MESSAGE` is uniquely owned by this receiver after synchronization. - Ok(unsafe { channel.take_message() }) + // SAFETY: This receiver's load observed MESSAGE. + Ok(unsafe { channel.take_published_message() }) } EMPTY => Err(TryRecvError::Empty), DISCONNECTED => Err(TryRecvError::Disconnected), @@ -244,17 +235,8 @@ impl Future for Recv { } // The sender sent the message. MESSAGE => { - // ORDERING: The sender has completed, so this receiver-only terminal update does - // not publish data to another thread. - channel.state.store(DISCONNECTED, Ordering::Relaxed); - - // ORDERING: The preceding Relaxed load read MESSAGE from the sender's Release - // publication. This conditional Acquire makes the message visible before it is - // taken. - fence(Ordering::Acquire); - - // SAFETY: we are in the MESSAGE state and have synchronized with the sender. - Poll::Ready(Ok(unsafe { channel.take_message() })) + // SAFETY: This receiver's load observed MESSAGE. + Poll::Ready(Ok(unsafe { channel.take_published_message() })) } // We were polled again while waiting for the sender. Replace the waker unless it // already wakes the polling task. @@ -278,18 +260,8 @@ impl Future for Recv { // We take the message and mark the channel disconnected. // The sender has already taken the waker. Err(MESSAGE) => { - // ORDERING: The sender has completed, so this receiver-only terminal update - // does not publish data to another thread. - channel.state.store(DISCONNECTED, Ordering::Relaxed); - - // ORDERING: The failed CAS read MESSAGE from the sender's Release - // publication. This conditional Acquire makes the message visible before it - // is taken. - fence(Ordering::Acquire); - - // SAFETY: The state tells us the sender has initialized the message, and - // the fence above synchronizes with that write. - Poll::Ready(Ok(unsafe { channel.take_message() })) + // SAFETY: This receiver's failed compare-exchange observed MESSAGE. + Poll::Ready(Ok(unsafe { channel.take_published_message() })) } // The sender started awakening us while we prepared to replace the waker. Err(AWAKING) => { From 1fa50bb4df0e4ccef4cc15f85a554c6d6ea21ba8 Mon Sep 17 00:00:00 2001 From: tison Date: Wed, 18 Nov 2026 22:35:40 +0800 Subject: [PATCH 2/6] refactor: share waker updates for occupied registrations --- asyncband/src/condvar/mod.rs | 6 ++---- asyncband/src/event/auto_reset.rs | 7 ++----- asyncband/src/event/manual_reset.rs | 6 ++---- asyncband/src/internal/mod.rs | 22 +++++++++++++++------- asyncband/src/internal/retry_waiter.rs | 6 ++---- asyncband/src/internal/wakerset.rs | 8 ++------ asyncband/src/mpmc/queue.rs | 6 ++---- asyncband/src/mpsc/bounded/sender.rs | 9 ++++----- 8 files changed, 31 insertions(+), 39 deletions(-) diff --git a/asyncband/src/condvar/mod.rs b/asyncband/src/condvar/mod.rs index f8de04f3..f1fd80a2 100644 --- a/asyncband/src/condvar/mod.rs +++ b/asyncband/src/condvar/mod.rs @@ -66,6 +66,7 @@ use std::task::Poll; use std::task::Waker; use crate::internal::mutex::Mutex; +use crate::internal::update_waker; use crate::internal::waitlist::WaitList; use crate::internal::waitlist::WaiterId; use crate::internal::waker_batch::WakerBatch; @@ -366,12 +367,9 @@ where let index = this.index.expect("wait future polled after completion"); let mut waiters = this.condvar.waiters.lock(); - let mut old_waker = None; let notify_one_baton = match waiters.waiter_mut(index) { WaitState::Waiting(waker) => { - if !waker.will_wake(cx.waker()) { - old_waker = Some(mem::replace(waker, cx.waker().clone())); - } + let old_waker = update_waker(waker, cx.waker()); drop(waiters); drop(old_waker); return Poll::Pending; diff --git a/asyncband/src/event/auto_reset.rs b/asyncband/src/event/auto_reset.rs index 05b7edd7..5ed27137 100644 --- a/asyncband/src/event/auto_reset.rs +++ b/asyncband/src/event/auto_reset.rs @@ -26,6 +26,7 @@ use std::task::Waker; use crate::event::Waiter; use crate::internal::mutex::Mutex; +use crate::internal::update_waker; use crate::internal::waitlist::WaitList; use crate::internal::waitlist::WaiterId; @@ -216,11 +217,7 @@ impl AutoResetEvent { *waiter_id = None; (Poll::Ready(()), None) } - Waiter::Waiting(waker) => { - let retired = (!waker.will_wake(cx.waker())) - .then(|| mem::replace(waker, cx.waker().clone())); - (Poll::Pending, retired) - } + Waiter::Waiting(waker) => (Poll::Pending, update_waker(waker, cx.waker())), }, None if state.is_set => { state.is_set = false; diff --git a/asyncband/src/event/manual_reset.rs b/asyncband/src/event/manual_reset.rs index c73d36df..64368a47 100644 --- a/asyncband/src/event/manual_reset.rs +++ b/asyncband/src/event/manual_reset.rs @@ -17,7 +17,6 @@ use std::fmt; use std::future::Future; -use std::mem; use std::pin::Pin; use std::sync::Arc; use std::sync::atomic::AtomicBool; @@ -27,6 +26,7 @@ use std::task::Poll; use crate::event::Waiter; use crate::internal::mutex::Mutex; +use crate::internal::update_waker; use crate::internal::waitlist::WaitList; use crate::internal::waitlist::WaiterId; use crate::internal::waker_batch::WakerBatch; @@ -234,9 +234,7 @@ impl ManualResetEvent { !self.is_set.load(Ordering::Relaxed), "a linked waiter must belong to an unset event" ); - let retired = (!waker.will_wake(cx.waker())) - .then(|| mem::replace(waker, cx.waker().clone())); - (Poll::Pending, retired) + (Poll::Pending, update_waker(waker, cx.waker())) } }, // A set after the false fast probe must either be observed here or release diff --git a/asyncband/src/internal/mod.rs b/asyncband/src/internal/mod.rs index edc46bbe..215a6e2c 100644 --- a/asyncband/src/internal/mod.rs +++ b/asyncband/src/internal/mod.rs @@ -15,21 +15,29 @@ // specific language governing permissions and limitations // under the License. +use std::mem; use std::task::Waker; +/// Refreshes an occupied registration unless both wakers are known to wake the same task. +#[inline] +#[must_use = "drop the replaced waker after releasing the state lock"] +pub fn update_waker(current: &mut Waker, waker: &Waker) -> Option { + if current.will_wake(waker) { + None + } else { + Some(mem::replace(current, waker.clone())) + } +} + /// Retains the current task waker, returning any replaced registration for unlocked destruction. #[inline] #[must_use = "drop the replaced waker after releasing the state lock"] // Some feature subsets have no primitive that registers a single waker slot. #[allow(dead_code)] pub fn register_waker(slot: &mut Option, waker: &Waker) -> Option { - if slot - .as_ref() - .is_some_and(|current| current.will_wake(waker)) - { - None - } else { - slot.replace(waker.clone()) + match slot { + Some(current) => update_waker(current, waker), + None => slot.replace(waker.clone()), } } diff --git a/asyncband/src/internal/retry_waiter.rs b/asyncband/src/internal/retry_waiter.rs index 6b91f91b..c716aebb 100644 --- a/asyncband/src/internal/retry_waiter.rs +++ b/asyncband/src/internal/retry_waiter.rs @@ -18,6 +18,7 @@ use std::mem; use std::task::Waker; +use crate::internal::update_waker; use crate::internal::waitlist::WaitList; use crate::internal::waitlist::WaiterId; @@ -47,10 +48,7 @@ impl WaitList { pub fn register(&mut self, id: &mut Option, current: &Waker) -> Option { if let Some(queued) = *id { if let RetryWaiter::Waiting(waker) = self.waiter_mut(queued) { - if waker.will_wake(current) { - return None; - } - return Some(mem::replace(waker, current.clone())); + return update_waker(waker, current); } } if let Some(notified) = id.take() { diff --git a/asyncband/src/internal/wakerset.rs b/asyncband/src/internal/wakerset.rs index 211b9c82..5787afad 100644 --- a/asyncband/src/internal/wakerset.rs +++ b/asyncband/src/internal/wakerset.rs @@ -21,11 +21,11 @@ //! their tokens must be cleared rather than passed back to the set. Returned wakers must be woken //! or dropped after releasing the lock. -use std::mem; use std::task::Waker; use crate::internal::arena::Arena; use crate::internal::arena::SlotId; +use crate::internal::update_waker; use crate::internal::waker_batch::WakerBatch; /// An exclusive handle to one waker slot in a [`WakerSet`]. @@ -94,11 +94,7 @@ impl WakerSet { .wakers .get_mut(token.0) .expect("waker token must refer to an occupied slot"); - if current.will_wake(waker) { - None - } else { - Some(mem::replace(current, waker.clone())) - } + update_waker(current, waker) } /// Removes and returns the waker identified by `token`. diff --git a/asyncband/src/mpmc/queue.rs b/asyncband/src/mpmc/queue.rs index dfc5625e..a88d569c 100644 --- a/asyncband/src/mpmc/queue.rs +++ b/asyncband/src/mpmc/queue.rs @@ -25,6 +25,7 @@ use std::task::Waker; use crate::internal::mutex::Mutex; use crate::internal::retry_waiter::RetryWaiter; +use crate::internal::update_waker; use crate::internal::waitlist::WaitList; use crate::internal::waitlist::WaiterId; use crate::mpmc::RecvError; @@ -114,10 +115,7 @@ impl WaitList { let SendWaiter::Waiting(waker) = self.waiter_mut(queued) else { unreachable!("a granted waiter must be claimed before registration"); }; - if waker.will_wake(current) { - return None; - } - return Some(mem::replace(waker, current.clone())); + return update_waker(waker, current); } *id = Some(self.push_back(SendWaiter::Waiting(current.clone()))); None diff --git a/asyncband/src/mpsc/bounded/sender.rs b/asyncband/src/mpsc/bounded/sender.rs index 621f3886..bb48a794 100644 --- a/asyncband/src/mpsc/bounded/sender.rs +++ b/asyncband/src/mpsc/bounded/sender.rs @@ -23,6 +23,7 @@ use std::task::Context; use std::task::Poll; use crate::internal::mutex::Mutex; +use crate::internal::update_waker; use crate::internal::waitlist::WaiterId; use crate::mpsc::SendError; use crate::mpsc::TrySendError; @@ -283,11 +284,9 @@ impl<'a, T> Reserve<'a, T> { if let Some(index) = self.waiter { let waiter = state.send_waiters.waiter_mut(index); if let Waiter::Waiting(waker) = waiter { - if !waker.will_wake(cx.waker()) { - let old = mem::replace(waker, cx.waker().clone()); - drop(state); - drop(old); - } + let old = update_waker(waker, cx.waker()); + drop(state); + drop(old); return Poll::Pending; } state.send_waiters.remove_unlinked_waiter(index); From d6f596a3ed13a9e0f9bc7afa99fe7f3a0c80dd19 Mon Sep 17 00:00:00 2001 From: tison Date: Wed, 18 Nov 2026 22:35:53 +0800 Subject: [PATCH 3/6] refactor(benchmarks): consume single-use channel batches --- benchmarks/benches/ecosystem/mpmc/bounded.rs | 4 ++-- benchmarks/benches/ecosystem/mpmc/unbounded.rs | 4 ++-- benchmarks/benches/ecosystem/spmc/bounded.rs | 2 +- benchmarks/benches/ecosystem/spmc/unbounded.rs | 2 +- benchmarks/src/channels/mpmc.rs | 6 +++--- benchmarks/src/channels/spmc.rs | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/benchmarks/benches/ecosystem/mpmc/bounded.rs b/benchmarks/benches/ecosystem/mpmc/bounded.rs index fe6eb3f2..4d252ed7 100644 --- a/benchmarks/benches/ecosystem/mpmc/bounded.rs +++ b/benchmarks/benches/ecosystem/mpmc/bounded.rs @@ -31,7 +31,7 @@ use benchmarks::implementations; fn thread_batch(bencher: Bencher, topology: Topology) { bencher.bench_local(|| { - let mut batch = ThreadBatch::new_bounded::(BOUNDED_CAPACITY, topology); + let batch = ThreadBatch::new_bounded::(BOUNDED_CAPACITY, topology); batch.run() }); } @@ -39,7 +39,7 @@ fn thread_batch(bencher: Bencher, topology: Topology) { fn task_batch(bencher: Bencher, topology: Topology) { let runtime = runtime(WORKERS); bencher.bench_local(|| { - let mut batch = TaskBatch::new::>(&runtime, topology); + let batch = TaskBatch::new::>(&runtime, topology); runtime.block_on(batch.run()) }); } diff --git a/benchmarks/benches/ecosystem/mpmc/unbounded.rs b/benchmarks/benches/ecosystem/mpmc/unbounded.rs index 688e4293..f47b49dd 100644 --- a/benchmarks/benches/ecosystem/mpmc/unbounded.rs +++ b/benchmarks/benches/ecosystem/mpmc/unbounded.rs @@ -30,7 +30,7 @@ use benchmarks::implementations; fn thread_batch(bencher: Bencher, topology: Topology) { bencher.bench_local(|| { - let mut batch = ThreadBatch::new_unbounded::(topology); + let batch = ThreadBatch::new_unbounded::(topology); batch.run() }); } @@ -38,7 +38,7 @@ fn thread_batch(bencher: Bencher, topology: Topology) { fn task_batch(bencher: Bencher, topology: Topology) { let runtime = runtime(WORKERS); bencher.bench_local(|| { - let mut batch = TaskBatch::new::>(&runtime, topology); + let batch = TaskBatch::new::>(&runtime, topology); runtime.block_on(batch.run()) }); } diff --git a/benchmarks/benches/ecosystem/spmc/bounded.rs b/benchmarks/benches/ecosystem/spmc/bounded.rs index fd11631d..cf95b3b5 100644 --- a/benchmarks/benches/ecosystem/spmc/bounded.rs +++ b/benchmarks/benches/ecosystem/spmc/bounded.rs @@ -33,7 +33,7 @@ where { let runtime = runtime(WORKERS); bencher.bench_local(|| { - let mut batch = TaskBatch::new::>(&runtime, consumers); + let batch = TaskBatch::new::>(&runtime, consumers); runtime.block_on(batch.run()) }); } diff --git a/benchmarks/benches/ecosystem/spmc/unbounded.rs b/benchmarks/benches/ecosystem/spmc/unbounded.rs index 1455becf..d5ef51f6 100644 --- a/benchmarks/benches/ecosystem/spmc/unbounded.rs +++ b/benchmarks/benches/ecosystem/spmc/unbounded.rs @@ -33,7 +33,7 @@ where { let runtime = runtime(WORKERS); bencher.bench_local(|| { - let mut batch = TaskBatch::new::>(&runtime, consumers); + let batch = TaskBatch::new::>(&runtime, consumers); runtime.block_on(batch.run()) }); } diff --git a/benchmarks/src/channels/mpmc.rs b/benchmarks/src/channels/mpmc.rs index 934ba52e..e43c2e67 100644 --- a/benchmarks/src/channels/mpmc.rs +++ b/benchmarks/src/channels/mpmc.rs @@ -93,7 +93,7 @@ impl TaskBatch { Self { start, workers } } - pub async fn run(&mut self) -> (usize, usize) { + pub async fn run(mut self) -> (usize, usize) { self.start.wait().await; let mut count = 0; let mut checksum = 0usize; @@ -178,11 +178,11 @@ impl ThreadBatch { Self { start, workers } } - pub fn run(&mut self) -> (usize, usize) { + pub fn run(self) -> (usize, usize) { self.start.wait(); let mut count = 0; let mut checksum = 0usize; - for worker in self.workers.drain(..) { + for worker in self.workers { let (received, sum) = worker.join().expect("benchmark worker panicked"); count += received; checksum = checksum.wrapping_add(sum); diff --git a/benchmarks/src/channels/spmc.rs b/benchmarks/src/channels/spmc.rs index 64cd158e..1eba08b8 100644 --- a/benchmarks/src/channels/spmc.rs +++ b/benchmarks/src/channels/spmc.rs @@ -69,7 +69,7 @@ impl TaskBatch { Self { start, tasks } } - pub async fn run(&mut self) -> (usize, usize) { + pub async fn run(mut self) -> (usize, usize) { self.start.wait().await; let mut count = 0; let mut checksum = 0usize; From 174500b6d4cfd68ce0ceb9a8c672d3cd31334a04 Mon Sep 17 00:00:00 2001 From: tison Date: Wed, 18 Nov 2026 22:36:05 +0800 Subject: [PATCH 4/6] refactor: delegate infallible keyed computations to their fallible paths --- asyncband/src/once/once_map/mod.rs | 15 +++++++-------- asyncband/src/singleflight/mod.rs | 18 ++++++------------ 2 files changed, 13 insertions(+), 20 deletions(-) diff --git a/asyncband/src/once/once_map/mod.rs b/asyncband/src/once/once_map/mod.rs index 896fd470..2f9adb34 100644 --- a/asyncband/src/once/once_map/mod.rs +++ b/asyncband/src/once/once_map/mod.rs @@ -16,6 +16,7 @@ // under the License. use std::borrow::Borrow; +use std::convert::Infallible; use std::fmt; use std::hash::BuildHasher; use std::hash::Hash; @@ -313,14 +314,12 @@ where where F: AsyncFnOnce() -> V, { - let guard = match self.get_or_insert(key) { - Lookup::Ready(value) => return value, - Lookup::Pending(guard) => guard, - }; - - let result = guard.entry().cell.get_or_init(func).await.clone(); - guard.dismiss(); - result + match self + .try_compute(key, async || Ok::(func().await)) + .await + { + Ok(value) => value, + } } /// Returns a clone of the stored or newly computed value for `key`. diff --git a/asyncband/src/singleflight/mod.rs b/asyncband/src/singleflight/mod.rs index ba9b3352..d274bbfa 100644 --- a/asyncband/src/singleflight/mod.rs +++ b/asyncband/src/singleflight/mod.rs @@ -26,6 +26,7 @@ //! successful values should remain cached by key. use std::borrow::Borrow; +use std::convert::Infallible; use std::fmt; use std::hash::BuildHasher; use std::hash::Hash; @@ -312,19 +313,12 @@ where where F: AsyncFnOnce() -> V, { - let guard = self.get_or_insert(key); - let entry = guard.entry(); - let result = entry - .cell - .get_or_init(async || { - let result = func().await; - self.remove_if_current(entry); - result - }) + match self + .try_work(key, async || Ok::(func().await)) .await - .clone(); - guard.dismiss(); - result + { + Ok(value) => value, + } } /// Returns a clone of the shared computation's value for `key`. From ca794f7ca04724d3923d1e70dff345b93d72afcb Mon Sep 17 00:00:00 2001 From: tison Date: Wed, 18 Nov 2026 22:42:40 +0800 Subject: [PATCH 5/6] refactor(rwlock): acquire read permits through their access token --- asyncband/src/internal/semaphore.rs | 44 ++++++++++------ asyncband/src/rwlock/access.rs | 64 ++++++++++++++++++++++-- asyncband/src/rwlock/owned_read_guard.rs | 13 ++--- asyncband/src/rwlock/read_guard.rs | 13 ++--- asyncband/src/rwlock/upgradable.rs | 24 +++------ 5 files changed, 101 insertions(+), 57 deletions(-) diff --git a/asyncband/src/internal/semaphore.rs b/asyncband/src/internal/semaphore.rs index 020fb4ea..b9c6dd82 100644 --- a/asyncband/src/internal/semaphore.rs +++ b/asyncband/src/internal/semaphore.rs @@ -279,31 +279,43 @@ pub struct Acquire<'a> { impl Drop for Acquire<'_> { fn drop(&mut self) { if let Some(id) = self.waiter_id.take() { - let mut waiters = self.semaphore.waiters.lock(); - let waiter = waiters.remove(id); - let acquired = self.requested_permits - waiter.needed_permits; - if acquired > 0 { - self.semaphore.distribute_permits(acquired, waiters); - } else { - drop(waiters); - } - drop(waiter); + self.semaphore.cancel_acquire(self.requested_permits, id); } } } impl Acquire<'_> { pub fn poll_once(&mut self, waker: &Waker) -> Poll<()> { - let Self { - requested_permits, - waiter_id, - semaphore, - } = self; + self.semaphore + .poll_acquire(self.requested_permits, &mut self.waiter_id, waker) + } +} + +impl Semaphore { + /// Cancels a queued acquisition and returns any permits already assigned to it. + pub fn cancel_acquire(&self, requested: usize, id: WaiterId) { + let mut waiters = self.waiters.lock(); + let waiter = waiters.remove(id); + let acquired = requested - waiter.needed_permits; + if acquired > 0 { + self.distribute_permits(acquired, waiters); + } else { + drop(waiters); + } + drop(waiter); + } + /// Reuse the same count and registration until ready or cancelled. + pub fn poll_acquire( + &self, + requested: usize, + waiter_id: &mut Option, + waker: &Waker, + ) -> Poll<()> { let mut old_waker = None; match waiter_id { Some(id) => { - let mut waiters = semaphore.waiters.lock(); + let mut waiters = self.waiters.lock(); let ready = { let node = waiters.waiter_mut(*id); if node.needed_permits > 0 { @@ -321,7 +333,7 @@ impl Acquire<'_> { } None => { let request = PermitRequest::Acquire { waiter_id, waker }; - if acquire_or_enqueue(semaphore, *requested_permits, request) { + if acquire_or_enqueue(self, requested, request) { return Poll::Ready(()); } } diff --git a/asyncband/src/rwlock/access.rs b/asyncband/src/rwlock/access.rs index 1e92587b..f03136ad 100644 --- a/asyncband/src/rwlock/access.rs +++ b/asyncband/src/rwlock/access.rs @@ -20,10 +20,15 @@ //! Guards hold a token instead of implementing `Drop`, so projecting or downgrading a guard moves //! the token into the new guard. +use std::future::poll_fn; use std::mem; use std::sync::Arc; +use std::task::Poll; +use std::task::Waker; +use std::task::ready; use crate::internal::semaphore::Semaphore; +use crate::internal::waitlist::WaiterId; use crate::rwlock::RwLock; /// Keeps the lock's semaphore alive while a token holds permits from it. @@ -56,9 +61,20 @@ pub struct ReadAccess { } impl ReadAccess { - /// Takes over one permit already acquired from `owner`. - pub fn new(owner: O) -> Self { - Self { owner } + pub fn acquire(owner: O) -> impl Future { + let mut request = ReadRequest { + owner: Some(owner), + waiter_id: None, + }; + poll_fn(move |cx| request.poll(cx.waker())) + } + + pub fn try_acquire(owner: O) -> Option { + if owner.semaphore().try_acquire(1) { + Some(Self { owner }) + } else { + None + } } /// Returns the owner that the permit was acquired from. @@ -67,6 +83,42 @@ impl ReadAccess { } } +// Keep the owner and its queue registration together so waiting does not also need a +// self-referential future borrowing the owner's semaphore. +struct ReadRequest { + owner: Option, + waiter_id: Option, +} + +impl ReadRequest { + fn poll(&mut self, waker: &Waker) -> Poll> { + let owner = self + .owner + .as_ref() + .expect("read acquisition polled after completion"); + ready!( + owner + .semaphore() + .poll_acquire(1, &mut self.waiter_id, waker) + ); + Poll::Ready(ReadAccess { + owner: self.owner.take().unwrap(), + }) + } +} + +impl Drop for ReadRequest { + fn drop(&mut self) { + if let Some(id) = self.waiter_id { + self.owner + .as_ref() + .unwrap() + .semaphore() + .cancel_acquire(1, id); + } + } +} + impl<'a, T: ?Sized> ReadAccess<&'a RwLock> { /// Drops the value type so that mapped guards need not name it. pub fn into_semaphore(self) -> ReadAccess<&'a Semaphore> { @@ -74,7 +126,7 @@ impl<'a, T: ?Sized> ReadAccess<&'a RwLock> { // The new token takes over releasing the permit; forgetting this one avoids releasing it // during the transfer. Its borrowed owner needs no cleanup. mem::forget(self); - ReadAccess::new(&lock.s) + ReadAccess { owner: &lock.s } } } @@ -106,7 +158,9 @@ impl WriteAccess { /// Releases all permits but one, which the returned token keeps. pub fn downgrade(mut self) -> ReadAccess { - let read = ReadAccess::new(self.owner.clone()); + let read = ReadAccess { + owner: self.owner.clone(), + }; self.permits_acquired -= 1; drop(self); read diff --git a/asyncband/src/rwlock/owned_read_guard.rs b/asyncband/src/rwlock/owned_read_guard.rs index 1142679c..4432609b 100644 --- a/asyncband/src/rwlock/owned_read_guard.rs +++ b/asyncband/src/rwlock/owned_read_guard.rs @@ -42,9 +42,8 @@ impl RwLock { /// Dropping a pending future removes the request and returns any permits already assigned to /// it. Retrying joins the back of the queue. The future also releases its owned `Arc`. pub async fn read_owned(self: Arc) -> OwnedRwLockReadGuard { - self.s.acquire(1).await; OwnedRwLockReadGuard { - access: ReadAccess::new(self), + access: ReadAccess::acquire(self).await, } } @@ -52,13 +51,9 @@ impl RwLock { /// /// This consumes the passed `Arc` even when acquisition fails. pub fn try_read_owned(self: Arc) -> Option> { - if self.s.try_acquire(1) { - Some(OwnedRwLockReadGuard { - access: ReadAccess::new(self), - }) - } else { - None - } + Some(OwnedRwLockReadGuard { + access: ReadAccess::try_acquire(self)?, + }) } } diff --git a/asyncband/src/rwlock/read_guard.rs b/asyncband/src/rwlock/read_guard.rs index dc501294..5e0a4395 100644 --- a/asyncband/src/rwlock/read_guard.rs +++ b/asyncband/src/rwlock/read_guard.rs @@ -41,9 +41,8 @@ impl RwLock { /// Dropping a pending future removes the request and returns any permits already assigned to /// it. Retrying joins the back of the queue. pub async fn read(&self) -> RwLockReadGuard<'_, T> { - self.s.acquire(1).await; RwLockReadGuard { - access: ReadAccess::new(self), + access: ReadAccess::acquire(self).await, } } @@ -51,13 +50,9 @@ impl RwLock { /// /// Readers cannot bypass an earlier queued writer. pub fn try_read(&self) -> Option> { - if self.s.try_acquire(1) { - Some(RwLockReadGuard { - access: ReadAccess::new(self), - }) - } else { - None - } + Some(RwLockReadGuard { + access: ReadAccess::try_acquire(self)?, + }) } } diff --git a/asyncband/src/rwlock/upgradable.rs b/asyncband/src/rwlock/upgradable.rs index 1ce3b119..85ae3bff 100644 --- a/asyncband/src/rwlock/upgradable.rs +++ b/asyncband/src/rwlock/upgradable.rs @@ -149,8 +149,7 @@ impl UpgradableRwLock { /// Cancellation releases the reservation and any assigned permits and loses both queue /// positions. pub async fn write(&self) -> RwLockWriteGuard<'_, T> { - self.reservation.acquire(1).await; - let reservation = ReadAccess::new(&self.reservation); + let reservation = ReadAccess::acquire(&self.reservation).await; let write = self.inner.write().await; drop(reservation); write @@ -159,10 +158,7 @@ impl UpgradableRwLock { /// Returns a write guard immediately, or `None` when the reservation or exclusive access is /// unavailable. Failure releases any reservation acquired by this call. pub fn try_write(&self) -> Option> { - if !self.reservation.try_acquire(1) { - return None; - } - let reservation = ReadAccess::new(&self.reservation); + let reservation = ReadAccess::try_acquire(&self.reservation)?; let write = self.inner.try_write(); drop(reservation); write @@ -178,11 +174,9 @@ impl UpgradableRwLock { /// Cancellation releases the reservation and any assigned permit and loses both queue /// positions. pub async fn upgradable_read(&self) -> RwLockUpgradableReadGuard<'_, T> { - self.reservation.acquire(1).await; - let reservation = ReadAccess::new(&self.reservation); - self.inner.s.acquire(1).await; + let reservation = ReadAccess::acquire(&self.reservation).await; RwLockUpgradableReadGuard { - access: ReadAccess::new(&self.inner), + access: ReadAccess::acquire(&self.inner).await, reservation, } } @@ -190,15 +184,9 @@ impl UpgradableRwLock { /// Returns an upgradable guard immediately, or `None` when the reservation or a reader slot /// is unavailable. Failure releases any reservation acquired by this call. pub fn try_upgradable_read(&self) -> Option> { - if !self.reservation.try_acquire(1) { - return None; - } - let reservation = ReadAccess::new(&self.reservation); - if !self.inner.s.try_acquire(1) { - return None; - } + let reservation = ReadAccess::try_acquire(&self.reservation)?; Some(RwLockUpgradableReadGuard { - access: ReadAccess::new(&self.inner), + access: ReadAccess::try_acquire(&self.inner)?, reservation, }) } From 55d4ab99b167a0ebf1d9209cf6898f1d375cb4ce Mon Sep 17 00:00:00 2001 From: tison Date: Wed, 18 Nov 2026 22:43:05 +0800 Subject: [PATCH 6/6] refactor(condvar): model the phases of a wait explicitly --- asyncband/src/condvar/mod.rs | 47 ++++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 21 deletions(-) diff --git a/asyncband/src/condvar/mod.rs b/asyncband/src/condvar/mod.rs index f1fd80a2..3a09e95d 100644 --- a/asyncband/src/condvar/mod.rs +++ b/asyncband/src/condvar/mod.rs @@ -182,8 +182,7 @@ impl Condvar { let mutex = mutex::guard_lock(&guard); let notify_one_baton = Wait { condvar: self, - guard: Some(guard), - index: None, + phase: WaitPhase::Holding(guard), } .await; let guard = mutex.lock().await; @@ -207,8 +206,7 @@ impl Condvar { let mutex = mutex::owned_guard_lock(&guard); let notify_one_baton = Wait { condvar: self, - guard: Some(guard), - index: None, + phase: WaitPhase::Holding(guard), } .await; let guard = mutex.lock_owned().await; @@ -340,8 +338,13 @@ impl Condvar { struct Wait<'a, G> { condvar: &'a Condvar, - guard: Option, - index: Option, + phase: WaitPhase, +} + +enum WaitPhase { + Holding(G), + Registered(WaiterId), + Done, } impl<'a, G> Future for Wait<'a, G> @@ -353,19 +356,21 @@ where fn poll(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let this = self.get_mut(); - if this.guard.is_some() { - let mut waiters = this.condvar.waiters.lock(); - this.index = Some(waiters.push_back(WaitState::Waiting(cx.waker().clone()))); - let guard = this.guard.take().unwrap(); - - // Registration must happen before unlocking the associated mutex. A notifier that - // acquires the mutex after this point will therefore observe this waiter. - drop(waiters); - drop(guard); - return Poll::Pending; - } - - let index = this.index.expect("wait future polled after completion"); + let index = match this.phase { + WaitPhase::Holding(_) => { + let index = this + .condvar + .waiters + .lock() + .push_back(WaitState::Waiting(cx.waker().clone())); + // Replacing Holding unlocks the associated mutex. A notifier that subsequently + // acquires it must already be able to find this registration. + this.phase = WaitPhase::Registered(index); + return Poll::Pending; + } + WaitPhase::Registered(index) => index, + WaitPhase::Done => panic!("wait future polled after completion"), + }; let mut waiters = this.condvar.waiters.lock(); let notify_one_baton = match waiters.waiter_mut(index) { WaitState::Waiting(waker) => { @@ -379,7 +384,7 @@ where }; let waiter = waiters.remove_unlinked_waiter(index); - this.index = None; + this.phase = WaitPhase::Done; drop(waiters); drop(waiter); Poll::Ready(notify_one_baton) @@ -388,7 +393,7 @@ where impl Drop for Wait<'_, G> { fn drop(&mut self) { - let Some(index) = self.index.take() else { + let WaitPhase::Registered(index) = self.phase else { return; };