diff --git a/asyncband/src/event/auto_reset.rs b/asyncband/src/event/auto_reset.rs index 0365fa58..05b7edd7 100644 --- a/asyncband/src/event/auto_reset.rs +++ b/asyncband/src/event/auto_reset.rs @@ -24,6 +24,7 @@ use std::task::Context; use std::task::Poll; use std::task::Waker; +use crate::event::Waiter; use crate::internal::mutex::Mutex; use crate::internal::waitlist::WaitList; use crate::internal::waitlist::WaiterId; @@ -278,10 +279,7 @@ struct State { impl State { fn signal(&mut self) -> Option { if let Some((_, waiter)) = self.waiters.unlink_first_waiter(|_| true) { - let Waiter::Waiting(waker) = mem::replace(waiter, Waiter::Notified) else { - unreachable!("only unselected waits remain queued") - }; - Some(waker) + Some(waiter.mark_notified()) } else { self.is_set = true; None @@ -289,11 +287,6 @@ impl State { } } -enum Waiter { - Waiting(Waker), - Notified, -} - #[must_use = "futures do nothing unless you `.await` or poll them"] struct Wait<'a> { event: &'a AutoResetEvent, diff --git a/asyncband/src/event/manual_reset.rs b/asyncband/src/event/manual_reset.rs index 7a7ec588..c73d36df 100644 --- a/asyncband/src/event/manual_reset.rs +++ b/asyncband/src/event/manual_reset.rs @@ -17,16 +17,16 @@ use std::fmt; use std::future::Future; +use std::mem; use std::pin::Pin; use std::sync::Arc; use std::sync::atomic::AtomicBool; use std::sync::atomic::Ordering; use std::task::Context; use std::task::Poll; -use std::task::Waker; +use crate::event::Waiter; use crate::internal::mutex::Mutex; -use crate::internal::register_waker; use crate::internal::waitlist::WaitList; use crate::internal::waitlist::WaiterId; use crate::internal::waker_batch::WakerBatch; @@ -110,13 +110,8 @@ impl ManualResetEvent { self.is_set.store(true, Ordering::Release); // Detach the complete cohort before invoking any waker. A wake callback may reset the // event and register a new wait, which must belong to the state current at that point. - while let Some((_id, waiter)) = waiters.unlink_first_waiter(|waiter| { - waiter.notified = true; - true - }) { - if let Some(waker) = waiter.waker.take() { - wakers.push(waker); - } + while let Some((_, waiter)) = waiters.unlink_first_waiter(|_| true) { + wakers.push(waiter.mark_notified()); } } wakers.wake_all(); @@ -218,7 +213,7 @@ impl ManualResetEvent { /// /// `set` unlinks every queued waiter and marks it notified, and a wait that starts while the /// event is set never enqueues. A linked waiter therefore always belongs to an unset event, so - /// `notified` alone decides whether a registered waiter is already committed. + /// a notified waiter remains ready even if the event is reset. fn poll_wait(&self, waiter_id: &mut Option, cx: &mut Context<'_>) -> Poll<()> { // Only an unregistered wait can skip the lock: a released waiter still owns a node. if waiter_id.is_none() && self.is_set() { @@ -228,27 +223,27 @@ impl ManualResetEvent { let (poll, retired_waker) = { let mut waiters = self.waiters.lock(); match *waiter_id { - Some(id) if waiters.waiter_mut(id).notified => { - let waiter = waiters.remove_unlinked_waiter(id); - *waiter_id = None; - (Poll::Ready(()), waiter.waker) - } - Some(id) => { - debug_assert!( - !self.is_set.load(Ordering::Relaxed), - "a linked waiter must belong to an unset event" - ); - let waiter = waiters.waiter_mut(id); - (Poll::Pending, register_waker(&mut waiter.waker, cx.waker())) - } + Some(id) => match waiters.waiter_mut(id) { + Waiter::Notified => { + waiters.remove_unlinked_waiter(id); + *waiter_id = None; + (Poll::Ready(()), None) + } + Waiter::Waiting(waker) => { + debug_assert!( + !self.is_set.load(Ordering::Relaxed), + "a linked waiter must belong to an unset event" + ); + let retired = (!waker.will_wake(cx.waker())) + .then(|| mem::replace(waker, cx.waker().clone())); + (Poll::Pending, retired) + } + }, // A set after the false fast probe must either be observed here or release // the node that we register while holding this same lock. None if self.is_set.load(Ordering::Relaxed) => (Poll::Ready(()), None), None => { - *waiter_id = Some(waiters.push_back(Waiter { - notified: false, - waker: Some(cx.waker().clone()), - })); + *waiter_id = Some(waiters.push_back(Waiter::Waiting(cx.waker().clone()))); (Poll::Pending, None) } } @@ -283,12 +278,6 @@ impl fmt::Debug for ManualResetEvent { } } -#[derive(Debug)] -struct Waiter { - notified: bool, - waker: Option, -} - #[must_use = "futures do nothing unless you `.await` or poll them"] struct Wait<'a> { event: &'a ManualResetEvent, diff --git a/asyncband/src/event/mod.rs b/asyncband/src/event/mod.rs index 8f94320f..daf98ed3 100644 --- a/asyncband/src/event/mod.rs +++ b/asyncband/src/event/mod.rs @@ -31,5 +31,22 @@ mod auto_reset; mod manual_reset; +use std::mem; +use std::task::Waker; + pub use self::auto_reset::AutoResetEvent; pub use self::manual_reset::ManualResetEvent; + +enum Waiter { + Waiting(Waker), + Notified, +} + +impl Waiter { + fn mark_notified(&mut self) -> Waker { + let Self::Waiting(waker) = mem::replace(self, Self::Notified) else { + unreachable!("only waiting nodes remain queued"); + }; + waker + } +} diff --git a/asyncband/src/internal/value_cell.rs b/asyncband/src/internal/value_cell.rs index 8c3f63be..78b6178e 100644 --- a/asyncband/src/internal/value_cell.rs +++ b/asyncband/src/internal/value_cell.rs @@ -97,12 +97,16 @@ impl ValueCell { } pub fn write_mut(&mut self, value: T) -> &mut T { - if *self.published.get_mut() { - // SAFETY: `published` indicates an initialized value in `slot`. + let published = self.published.get_mut(); + if *published { + *published = false; + // SAFETY: The slot held a live value. Clear the flag before dropping it so an + // unwinding destructor leaves the cell empty. unsafe { self.slot.get_mut().assume_init_drop() }; } - *self.published.get_mut() = true; - self.slot.get_mut().write(value) + let stored = self.slot.get_mut().write(value); + *published = true; + stored } pub fn take(&mut self) -> Option { @@ -136,6 +140,8 @@ impl Drop for ValueCell { #[cfg(test)] mod tests { use std::cell::Cell; + use std::panic::AssertUnwindSafe; + use std::panic::catch_unwind; use std::thread; use super::*; @@ -217,4 +223,24 @@ mod tests { drop(cell); assert_eq!(drops.get(), 4); } + + #[test] + fn overwrite_leaves_cell_empty_if_dropping_the_old_value_panics() { + struct PanicOnDrop; + + impl Drop for PanicOnDrop { + fn drop(&mut self) { + panic!("old value dropped"); + } + } + + // Only drop the cell after verifying that it no longer claims to own the old value. + let mut cell = ManuallyDrop::new(ValueCell::with_value(Some(PanicOnDrop))); + let result = catch_unwind(AssertUnwindSafe(|| { + cell.write_mut(None); + })); + assert!(result.is_err()); + assert!(!cell.is_published()); + drop(ManuallyDrop::into_inner(cell)); + } } diff --git a/asyncband/src/once/lazy_cell/mod.rs b/asyncband/src/once/lazy_cell/mod.rs index e222ebf7..a4374d10 100644 --- a/asyncband/src/once/lazy_cell/mod.rs +++ b/asyncband/src/once/lazy_cell/mod.rs @@ -408,10 +408,7 @@ where initializer() }; - // SAFETY: The current variant contains no structurally pinned future, and its initializer - // slot is empty. The newly created future is written directly into its final pinned - // location before it is polled. - *unsafe { state.as_mut().get_unchecked_mut() } = State::Attempt(future); + state.set(State::Attempt(future)); } let value = poll_fn(|cx| { @@ -427,11 +424,9 @@ where }) .await; - // Assignment drops the completed future in place before changing the phase. Treat a panic - // from that destructor as an initializer panic as well. + // A panic while dropping the completed future also poisons the cell. let _poison = PoisonOnPanic(poisoned); - // SAFETY: Dropping a pinned value in place is permitted, and no value is moved out. - *unsafe { state.as_mut().get_unchecked_mut() } = State::Complete; + state.set(State::Complete); value } diff --git a/asyncband/src/phaser/mod.rs b/asyncband/src/phaser/mod.rs index 79bf7ecd..a69951aa 100644 --- a/asyncband/src/phaser/mod.rs +++ b/asyncband/src/phaser/mod.rs @@ -419,8 +419,8 @@ impl Drop for PhaserParticipants { #[derive(Debug)] pub struct PhaserParticipant { phaser: Phaser, - // Cleared only after advancement, so a pending current phase also records arrival. - pending: Option, + // Retain the arrival across cancellation so a retry can observe the same phase. + arrived_phase: Option, registered: bool, } @@ -428,7 +428,7 @@ impl PhaserParticipant { fn new(phaser: Phaser) -> Self { Self { phaser, - pending: None, + arrived_phase: None, registered: true, } } @@ -453,10 +453,10 @@ impl PhaserParticipant { return Err(Closed(())); } let phase = state.phase; - if self.pending != Some(phase) { + if self.arrived_phase != Some(phase) { state.unarrived -= 1; } - self.pending = Some(phase); + self.arrived_phase = Some(phase); let mut wakers = state.advance_if_ready(); drop(state); wakers.wake_all(); @@ -480,12 +480,12 @@ impl PhaserParticipant { /// the pending observation, so retrying this method waits for the same phase without /// counting another arrival. An explicit new `arrive` replaces that pending observation. pub async fn wait(&mut self) -> Result { - let observed = match self.pending { + let observed = match self.arrived_phase { Some(phase) => phase, None => self.arrive()?, }; let next = self.phaser.wait(observed).await?; - self.pending = None; + self.arrived_phase = None; Ok(next) } @@ -506,7 +506,7 @@ impl PhaserParticipant { let mut state = self.phaser.state.lock(); self.registered = false; state.registered -= 1; - if self.pending != Some(state.phase) { + if self.arrived_phase != Some(state.phase) { state.unarrived -= 1; } let result = if state.closed { diff --git a/asyncband/src/watch/mod.rs b/asyncband/src/watch/mod.rs index 8a14a958..d68b7b08 100644 --- a/asyncband/src/watch/mod.rs +++ b/asyncband/src/watch/mod.rs @@ -74,6 +74,7 @@ use std::task::Poll; pub use self::error::RecvError; pub use self::error::SendError; use crate::internal::mutex::Mutex; +use crate::internal::waker_batch::WakerBatch; use crate::internal::wakerset::WakerSet; use crate::internal::wakerset::WakerToken; @@ -118,6 +119,19 @@ struct State { waiters: WakerSet, } +impl State { + // Leave the old value with the caller so it can be dropped after unlocking. Borrowing the + // input also keeps its destructor outside the lock if the version counter overflows. + fn publish(&mut self, value: &mut T) -> WakerBatch { + self.version = self + .version + .checked_add(1) + .expect("watch channel version counter overflowed"); + mem::swap(&mut self.value, value); + self.waiters.take_all() + } +} + /// The sending side of a watch channel. pub struct Sender { shared: Arc>, @@ -168,23 +182,15 @@ impl Sender { /// # Panics /// /// Panics if the channel has already published `u64::MAX` updates. - pub fn send(&self, value: T) -> Result<(), SendError> { - let mut state = self.shared.state.lock(); - if state.receivers == 0 { - return Err(SendError::new(value)); - } - let version = state - .version - .checked_add(1) - .expect("watch channel version counter overflowed"); - let replaced = mem::replace(&mut state.value, value); - state.version = version; - let mut wakers = state.waiters.take_all(); - drop(state); - - // Waker callbacks and the replaced value's destructor may reenter this channel. + pub fn send(&self, mut value: T) -> Result<(), SendError> { + let mut wakers = { + let mut state = self.shared.state.lock(); + if state.receivers == 0 { + return Err(SendError::new(value)); + } + state.publish(&mut value) + }; wakers.wake_all(); - drop(replaced); Ok(()) } @@ -196,18 +202,10 @@ impl Sender { /// # Panics /// /// Panics if the channel has already published `u64::MAX` updates. - pub fn send_replace(&self, value: T) -> T { - let mut state = self.shared.state.lock(); - let version = state - .version - .checked_add(1) - .expect("watch channel version counter overflowed"); - let replaced = mem::replace(&mut state.value, value); - state.version = version; - let mut wakers = state.waiters.take_all(); - drop(state); + pub fn send_replace(&self, mut value: T) -> T { + let mut wakers = self.shared.state.lock().publish(&mut value); wakers.wake_all(); - replaced + value } /// Creates a receiver that considers the current value already observed. diff --git a/benchmarks/benches/upgradable_stages.rs b/benchmarks/benches/upgradable_stages.rs index b80f59b5..c41ed584 100644 --- a/benchmarks/benches/upgradable_stages.rs +++ b/benchmarks/benches/upgradable_stages.rs @@ -22,6 +22,7 @@ use std::time::Instant; use asyncband::rwlock::RwLockUpgradableReadGuard; use asyncband::rwlock::UpgradableRwLock; +use benchmarks::harness::summary; use benchmarks::support::bench_context; use benchmarks::support::poll_pending; use benchmarks::support::poll_pinned_ready; @@ -165,17 +166,8 @@ fn measure(rounds: usize, repeat: usize) -> Results { results } -fn report(name: &str, rounds: usize, mut samples: Vec) { - samples.sort_unstable(); - let middle = samples.len() / 2; - let median = if samples.len() % 2 == 0 { - let lower = samples[middle - 1]; - lower + (samples[middle] - lower) / 2 - } else { - samples[middle] - }; - let minimum = samples.first().unwrap(); - let maximum = samples.last().unwrap(); +fn report(name: &str, rounds: usize, samples: &[Duration]) { + let (minimum, median, maximum) = summary(samples); let ns_per_round = median.as_nanos() as f64 / rounds as f64; let runs = samples .iter() @@ -213,16 +205,16 @@ fn main() { report( "empty timing interval", options.rounds, - results.empty_interval, + &results.empty_interval, ); - report("acquisition", options.rounds, results.acquisition); - report("promotion", options.rounds, results.promotion); + report("acquisition", options.rounds, &results.acquisition); + report("promotion", options.rounds, &results.promotion); report( "acquisition + reader held", options.rounds, - results.acquisition_with_reader, + &results.acquisition_with_reader, ); - for (&reader_count, samples) in READER_COUNTS.iter().zip(results.promotion_with_readers) { + for (&reader_count, samples) in READER_COUNTS.iter().zip(&results.promotion_with_readers) { report( &format!("promotion + {reader_count} readers"), options.rounds, @@ -232,11 +224,11 @@ fn main() { report( "promotion + writer queued", options.rounds, - results.promotion_with_waiting_writer, + &results.promotion_with_waiting_writer, ); report( "acquire behind queued writer", options.rounds, - results.acquisition_after_waiting_writer, + &results.acquisition_after_waiting_writer, ); } diff --git a/benchmarks/src/harness.rs b/benchmarks/src/harness.rs index a18cae1f..57e65c34 100644 --- a/benchmarks/src/harness.rs +++ b/benchmarks/src/harness.rs @@ -209,7 +209,8 @@ impl Runner { } } -fn summary(samples: &[Duration]) -> (Duration, Duration, Duration) { +/// Returns the minimum, median, and maximum of a nonempty sample. +pub fn summary(samples: &[Duration]) -> (Duration, Duration, Duration) { let mut sorted = samples.to_vec(); sorted.sort_unstable(); let lo = sorted[(sorted.len() - 1) / 2]; diff --git a/xtask/src/main.rs b/xtask/src/main.rs index bc0419aa..d2751b68 100644 --- a/xtask/src/main.rs +++ b/xtask/src/main.rs @@ -129,37 +129,20 @@ struct CommandMiri; impl CommandMiri { fn run(self) { run_command(make_miri_cmd(PACKAGE_NAME, &["--lib", "--all-features"])); - run_command(make_miri_cmd( - "tests-integration", - &["--test", "oneshot_test"], - )); - run_command(make_miri_cmd( - "tests-integration", - &["--test", "unsafe_paths_test"], - )); - run_command(make_miri_cmd( - "tests-integration", - &["--test", "upgradable_rwlock_test"], - )); - run_command(make_miri_cmd("tests-integration", &["--test", "mpsc_test"])); - run_command(make_miri_cmd("tests-integration", &["--test", "mpmc_test"])); - run_command(make_miri_cmd("tests-integration", &["--test", "spmc_test"])); - run_command(make_miri_cmd( - "tests-integration", - &["--test", "waitgroup_test"], - )); - run_command(make_miri_cmd( - "tests-integration", - &["--test", "broadcast_test"], - )); - run_command(make_miri_cmd( - "tests-integration", - &["--test", "phaser_test"], - )); - run_command(make_miri_cmd( - "tests-integration", - &["--test", "semaphore_ordering_test"], - )); + for target in [ + "oneshot_test", + "unsafe_paths_test", + "upgradable_rwlock_test", + "mpsc_test", + "mpmc_test", + "spmc_test", + "waitgroup_test", + "broadcast_test", + "phaser_test", + "semaphore_ordering_test", + ] { + run_command(make_miri_cmd("tests-integration", &["--test", target])); + } } }