diff --git a/CHANGELOG.md b/CHANGELOG.md index 538df301..931fc205 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,14 +2,18 @@ ## Unreleased -### After-Response Tasks (#476) +### After-Response Tasks (#476, #478) + +- Cloudflare runs gated `waitUntil` tasks after terminal response egress, with fresh + outbound scopes and effective budgets capped at 30 seconds. Workers lifetime limits + still apply; tasks have no persistence or automatic retries. - Add `RequestContext::after_response(budget, task)` and the adapter-facing `edgezero_core::after_response::{AfterResponseScheduler, AfterResponseHandle}`. A task starts after the response attempt ends and is cancelled at its budget. Adapters without support return `501 Not Implemented` at registration. - Add the `after-response-tasks` capability: Axum and Fastly `BoundedCooperative`, - Cloudflare and Spin `Unsupported` (tracked in #478 and #483). + Cloudflare `BoundedCooperative` (#478), and Spin `Unsupported` (tracked in #483). - Axum production, `run_app` and `AxumDevServer::run` shutdown now waits for registered after-response tasks within the existing shutdown grace, after connections drain. Tasks still pending at the grace make the process exit non-zero with `grace exhausted`. diff --git a/crates/edgezero-adapter-cloudflare/src/after_response.rs b/crates/edgezero-adapter-cloudflare/src/after_response.rs new file mode 100644 index 00000000..a450b537 --- /dev/null +++ b/crates/edgezero-adapter-cloudflare/src/after_response.rs @@ -0,0 +1,230 @@ +//! Request-local after-response release and bounded Workers tasks. + +#[cfg(all(feature = "cloudflare", target_arch = "wasm32"))] +mod platform { + use super::{Gate, LocalBoxFuture, ReleaseGuard, run_bounded}; + use std::sync::Arc; + use std::time::Duration; + + use edgezero_core::after_response::{AfterResponseHandle, AfterResponseScheduler}; + use edgezero_core::error::EdgeError; + use edgezero_core::time::MonotonicClock; + use worker::Context; + + use crate::outbound::native_time; + + struct Scheduler { + clock: MonotonicClock, + context: Arc, + gate: Arc, + } + + impl AfterResponseScheduler for Scheduler { + fn schedule( + &self, + budget: Duration, + task: LocalBoxFuture<'static, ()>, + ) -> Result<(), EdgeError> { + let released = self.gate.register()?; + let clock = self.clock.clone(); + self.context.wait_until(async move { + let _released = released.await; + run_bounded(task, budget, native_time(clock)).await; + }); + Ok(()) + } + } + + pub(crate) fn arm( + context: Arc, + clock: MonotonicClock, + ) -> (AfterResponseHandle, ReleaseGuard) { + let release = ReleaseGuard::new(); + let handle = AfterResponseHandle::new(Scheduler { + context, + gate: Arc::clone(&release.0), + clock, + }); + (handle, release) + } +} + +#[cfg(all(feature = "cloudflare", target_arch = "wasm32"))] +pub(crate) use platform::arm; + +use std::panic::AssertUnwindSafe; +use std::sync::{Arc, Mutex, PoisonError}; +use std::time::Duration; + +use edgezero_core::error::EdgeError; +use edgezero_core::outbound::{OutboundScopeOwner, OutboundTime}; +use futures::channel::oneshot; +use futures_util::future::{Either, FutureExt as _, LocalBoxFuture, select}; + +const MAX_BUDGET: Duration = Duration::from_secs(30); + +#[derive(Default)] +struct Gate { + waiters: Mutex>>>, +} + +impl Gate { + fn register(&self) -> Result, EdgeError> { + let mut state = self.waiters.lock().unwrap_or_else(PoisonError::into_inner); + let waiters = state.as_mut().ok_or_else(|| { + EdgeError::internal(anyhow::anyhow!( + "after-response tasks can only be registered while the request is handled" + )) + })?; + let (sender, receiver) = oneshot::channel(); + waiters.push(sender); + Ok(receiver) + } +} + +/// Sole release owner, retained through conversion and the terminal response pump. +pub(crate) struct ReleaseGuard(Arc); + +impl ReleaseGuard { + fn new() -> Self { + Self(Arc::new(Gate { + waiters: Mutex::new(Some(Vec::new())), + })) + } +} + +#[expect( + clippy::missing_trait_methods, + reason = "Drop::pin_drop is unstable and conflicts with drop" +)] +impl Drop for ReleaseGuard { + fn drop(&mut self) { + let waiters = self + .0 + .waiters + .lock() + .unwrap_or_else(PoisonError::into_inner) + .take(); + for sender in waiters.unwrap_or_default() { + let _closed = sender.send(()); + } + } +} + +async fn run_bounded(task: LocalBoxFuture<'static, ()>, budget: Duration, time: OutboundTime) { + let deadline = time.clock().deadline_after(budget.min(MAX_BUDGET)); + let callback = OutboundScopeOwner::callback(); + let result = callback + .run( + AssertUnwindSafe(async move { + match select(time.wait_until(deadline), task).await { + Either::Left((Ok(()), _task)) => { + log::warn!("after-response task exceeded its budget and was cancelled"); + } + Either::Left((Err(_error), _task)) => { + log::warn!("after-response task timer failed; the task was cancelled"); + } + Either::Right(((), _timer)) => {} + } + }) + .catch_unwind(), + ) + .await; + if result.is_err() { + log::error!("after-response task panicked"); + } +} + +#[cfg(test)] +mod tests { + use super::{MAX_BUDGET, ReleaseGuard, run_bounded}; + use std::cell::Cell; + use std::rc::Rc; + use std::sync::Arc; + use std::time::Duration; + + use edgezero_core::error::EdgeError; + use edgezero_core::outbound::OutboundTime; + use edgezero_core::time::MonotonicClock; + use futures::executor::block_on; + use futures::future::{join_all, pending}; + use futures_util::FutureExt as _; + + #[test] + fn release_is_request_local_and_closes_registration() { + let first = ReleaseGuard::new(); + let second = ReleaseGuard::new(); + let retained = Arc::clone(&first.0); + let first_receiver = first.0.register().unwrap(); + let second_receiver = first.0.register().unwrap(); + let mut other = second.0.register().unwrap(); + assert!(other.try_recv().unwrap().is_none()); + drop(first); + assert_eq!(block_on(first_receiver), Ok(())); + assert_eq!(block_on(second_receiver), Ok(())); + retained.register().unwrap_err(); + assert!(other.try_recv().unwrap().is_none()); + drop(second); + assert_eq!(block_on(other), Ok(())); + } + + #[test] + fn panicking_task_does_not_prevent_a_sibling_from_running() { + let ran = Rc::new(Cell::new(false)); + let observed = Rc::clone(&ran); + let time = || OutboundTime::new(MonotonicClock::default(), |_, _| Box::pin(pending())); + let tasks = [ + run_bounded( + async { panic!("task failure") }.boxed_local(), + Duration::from_secs(1), + time(), + ), + run_bounded( + async move { + observed.set(true); + } + .boxed_local(), + Duration::from_secs(1), + time(), + ), + ]; + block_on(join_all(tasks)); + assert!(ran.get()); + } + + #[test] + fn huge_budget_is_capped_and_timer_failure_drops_local_task_unpolled() { + struct Probe(Rc>); + #[expect(clippy::missing_trait_methods, reason = "Drop::pin_drop is unstable")] + impl Drop for Probe { + fn drop(&mut self) { + self.0.set(true); + } + } + let dropped = Rc::new(Cell::new(false)); + let polled = Rc::new(Cell::new(false)); + let observed = Rc::clone(&polled); + let probe = Probe(Rc::clone(&dropped)); + let clock = MonotonicClock::default(); + let start = clock.now(); + let time = OutboundTime::new( + MonotonicClock::new(move || start), + move |deadline, paired_clock| { + assert_eq!(deadline.remaining_at(paired_clock.now()), Some(MAX_BUDGET)); + Box::pin(async { Err(EdgeError::internal(anyhow::anyhow!("timer failure"))) }) + }, + ); + block_on(run_bounded( + async move { + let _probe = probe; + observed.set(true); + pending::<()>().await; + } + .boxed_local(), + Duration::MAX, + time, + )); + assert!(dropped.get()); + assert!(!polled.get()); + } +} diff --git a/crates/edgezero-adapter-cloudflare/src/cli.rs b/crates/edgezero-adapter-cloudflare/src/cli.rs index a4cc2a10..124b7f72 100644 --- a/crates/edgezero-adapter-cloudflare/src/cli.rs +++ b/crates/edgezero-adapter-cloudflare/src/cli.rs @@ -160,8 +160,8 @@ impl Adapter for CloudflareCliAdapter { | Capability::OutboundFlexiblePhaseBudget | Capability::OutboundHttp | Capability::OutboundRedirectFollowing => CapabilitySupport::Native, - Capability::AfterResponseTasks - | Capability::ConfigReadAllocationBounds + Capability::AfterResponseTasks => CapabilitySupport::BoundedCooperative, + Capability::ConfigReadAllocationBounds | Capability::OutboundCompleteResourceAccounting | Capability::RawIngressFramingValidation | Capability::RawIngressHeadLimits @@ -1528,7 +1528,7 @@ mod tests { for (capability, support) in [ ( Capability::AfterResponseTasks, - CapabilitySupport::Unsupported, + CapabilitySupport::BoundedCooperative, ), ( Capability::ConfigReadAllocationBounds, diff --git a/crates/edgezero-adapter-cloudflare/src/lib.rs b/crates/edgezero-adapter-cloudflare/src/lib.rs index a19680b6..76ae6471 100644 --- a/crates/edgezero-adapter-cloudflare/src/lib.rs +++ b/crates/edgezero-adapter-cloudflare/src/lib.rs @@ -3,6 +3,9 @@ #[cfg(all(feature = "cli", not(target_arch = "wasm32")))] pub mod cli; +#[cfg(any(test, all(feature = "cloudflare", target_arch = "wasm32")))] +mod after_response; + // `config_store` compiles on host for its `InMemory` test backend; the // production `Kv` backend is feature-gated internally. #[cfg(any(test, all(feature = "cloudflare", target_arch = "wasm32")))] diff --git a/crates/edgezero-adapter-cloudflare/src/request.rs b/crates/edgezero-adapter-cloudflare/src/request.rs index 6bb0cf6f..ea24202b 100644 --- a/crates/edgezero-adapter-cloudflare/src/request.rs +++ b/crates/edgezero-adapter-cloudflare/src/request.rs @@ -54,6 +54,7 @@ use worker::{ Response as CfResponse, }; +use crate::after_response; use crate::config_store::CloudflareConfigStore; use crate::context::CloudflareRequestContext; use crate::key_value_store::CloudflareKvStore; @@ -618,10 +619,13 @@ pub(crate) async fn dispatch_with_handles( let request_method = CoreMethod::from_bytes(req.inner().method().as_bytes()).unwrap_or(CoreMethod::GET); let context = Arc::new(ctx); - let runtime = CloudflareEgressRuntime::new(signal, Arc::clone(&context)); + let (after_response, release) = + after_response::arm(Arc::clone(&context), app.monotonic_clock()); + let runtime = + CloudflareEgressRuntime::new(signal, Arc::clone(&context)).with_after_response(release); let geo = request_geo(&req); let client_info = request_client_info(&req, stores.direct_ingress); - let head_request = match into_core_request_head(&req, env, context, app.monotonic_clock()) { + let mut head_request = match into_core_request_head(&req, env, context, app.monotonic_clock()) { Ok(request) => request, Err(error) => { drop(req); @@ -630,6 +634,7 @@ pub(crate) async fn dispatch_with_handles( .map_err(|delivery_error| edge_error_to_worker(&delivery_error)); } }; + head_request.extensions_mut().insert(after_response); dispatch_ingress_stream_and( app, head_request, diff --git a/crates/edgezero-adapter-cloudflare/src/response.rs b/crates/edgezero-adapter-cloudflare/src/response.rs index 7cb37114..5f666139 100644 --- a/crates/edgezero-adapter-cloudflare/src/response.rs +++ b/crates/edgezero-adapter-cloudflare/src/response.rs @@ -466,6 +466,7 @@ mod platform { use worker::worker_sys::FixedLengthStream; use worker::{AbortSignal, Context, Delay, EncodeBody, Response as CfResponse}; + use crate::after_response::ReleaseGuard; use crate::outbound::{copy_header_values, native_time}; const fn public_message(error: DeliveryError) -> &'static str { @@ -479,6 +480,7 @@ mod platform { } pub(crate) struct CloudflareEgressRuntime { + release: Option, context: Arc, signal: AbortSignal, } @@ -486,13 +488,20 @@ mod platform { impl CloudflareEgressRuntime { pub(crate) fn new(signal: web_sys::AbortSignal, context: Arc) -> Self { Self { + release: None, context, signal: signal.into(), } } + + pub(crate) fn with_after_response(mut self, release: ReleaseGuard) -> Self { + self.release = Some(release); + self + } } struct WorkersCommitter { + release: Option, context: Arc, signal: AbortSignal, } @@ -561,7 +570,11 @@ mod platform { } fn spawn(&mut self, task: LocalBoxFuture<'static, ()>) { - self.context.wait_until(task); + let release = self.release.take(); + self.context.wait_until(async move { + let _release = release; + task.await; + }); } } @@ -1002,6 +1015,7 @@ mod platform { ) -> Result { let request_method = egress.request_method().clone(); let mut committer = WorkersCommitter { + release: runtime.release, context: runtime.context, signal: runtime.signal, }; @@ -1038,6 +1052,7 @@ mod platform { #[cfg(test)] mod race_tests { + use crate::after_response; use std::cell::Cell; use std::rc::Rc; use std::sync::Mutex; @@ -1131,6 +1146,7 @@ mod platform { .expect("guest waitUntil"); ( WorkersCommitter { + release: None, context: Arc::new(Context::new(context.unchecked_into())), signal: signal.into(), }, @@ -1138,6 +1154,54 @@ mod platform { ) } + fn register_after_response(committer: &mut WorkersCommitter) -> Rc> { + let (handle, release) = + after_response::arm(Arc::clone(&committer.context), MonotonicClock::default()); + let ran = Rc::new(Cell::new(false)); + let observed = Rc::clone(&ran); + handle + .schedule(Duration::from_secs(1), async move { + observed.set(true); + }) + .expect("register task"); + committer.release = Some(release); + ran + } + + #[wasm_bindgen_test] + async fn after_response_releases_after_failed_fallback() { + let _timers = NumericTimeoutGuard::install(); + let controller = web_sys::AbortController::new().expect("controller"); + controller.abort(); + let (mut committer, tasks) = sdk_committer(controller.signal()); + let ran = register_after_response(&mut committer); + let fixture = status_fixture(Response::new(Body::empty()), &Method::HEAD); + let StartOutcome::Precommit { attempt, cause } = start_prepared( + fixture.prepared, + fixture.policy, + fixture.attempt, + fixture.clock.clone(), + EgressKind::Application, + &mut committer, + ) else { + panic!("aborted delivery must fail precommit"); + }; + assert!(!ran.get()); + start_fallback( + &Method::HEAD, + cause, + *attempt, + &fixture.clock, + &mut committer, + ) + .expect_err("aborted fallback"); + drop(committer); + JsFuture::from(Promise::all(&tasks)) + .await + .expect("released after failed fallback"); + assert!(ran.get()); + } + fn sdk_io(signal: web_sys::AbortSignal) -> (WorkersEgressIo, wasm_streams::ReadableStream) { let (mut committer, _) = sdk_committer(signal); let (response, io) = committer @@ -1321,11 +1385,17 @@ mod platform { .call0(&JsValue::UNDEFINED) .expect("guest-only FixedLengthStream shim"); for length in [None, Some(7)] { - assert_pending_source_cancelled(length).await; + assert_pending_source_cancelled(length, false).await; + assert_pending_source_cancelled(length, true).await; } } - async fn assert_pending_source_cancelled(length: Option) { + #[expect( + clippy::cognitive_complexity, + clippy::too_many_lines, + reason = "keep terminal ownership assertions together for both cancellation sources" + )] + async fn assert_pending_source_cancelled(length: Option, abort_request: bool) { let source_drops = Rc::new(Cell::new(0)); let lease = DropCounter(Rc::clone(&source_drops)); let (notify_pending, entered_pending) = oneshot::channel(); @@ -1366,6 +1436,7 @@ mod platform { let deadline = fixture.policy.write_deadline; let controller = web_sys::AbortController::new().expect("controller"); let (mut committer, tasks) = sdk_committer(controller.signal()); + let task_ran = register_after_response(&mut committer); let native_response = match start_prepared( fixture.prepared, fixture.policy, @@ -1391,14 +1462,27 @@ mod platform { assert_eq!(source_drops.get(), 0); assert_eq!(owner_drops.get(), 0); assert!(!controller.signal().aborted()); - reader.cancel().await.expect("cancel readable"); + assert!(!task_ran.get(), "task must wait for terminal delivery"); + if abort_request { + controller.abort(); + } else { + reader.cancel().await.expect("cancel readable"); + } JsFuture::from(Promise::all(&tasks)) .await .expect("egress task"); let reports = fixture.observed.reports(); assert_eq!(reports.len(), 1); assert_eq!(fixture.completed.reports(), reports); - assert_eq!(reports[0].outcome, ResponseEgressOutcome::TransportError); + assert_eq!( + reports[0].outcome, + if abort_request { + ResponseEgressOutcome::RequestCancelled + } else { + ResponseEgressOutcome::TransportError + } + ); + assert!(task_ran.get(), "terminal writer releases task"); assert!(!deadline.is_expired_at(fixture.clock.now())); assert_eq!(reports[0].bytes_written, 6); assert_eq!(reports[0].body_kind, ResponseEgressBodyKind::Application); @@ -1407,7 +1491,7 @@ mod platform { assert_eq!(owner_drops.get(), 1); assert!(retained.upgrade().is_none()); assert!(scope.check().is_err()); - assert!(!controller.signal().aborted()); + assert_eq!(controller.signal().aborted(), abort_request); drop(reader); drop(raw); assert_eq!(fixture.observed.reports(), reports); diff --git a/crates/edgezero-adapter-cloudflare/tests/contract.rs b/crates/edgezero-adapter-cloudflare/tests/contract.rs index e3e48af7..0e237a4d 100644 --- a/crates/edgezero-adapter-cloudflare/tests/contract.rs +++ b/crates/edgezero-adapter-cloudflare/tests/contract.rs @@ -74,12 +74,13 @@ mod tests { use edgezero_core::router::RouterService; use edgezero_core::time::{Deadline, MonotonicClock, MonotonicInstant}; use futures::channel::mpsc::unbounded; - use futures::future::{Either, select}; + use futures::future::{Either, pending, select}; use futures::{StreamExt as _, pin_mut, stream}; use std::io::Error as IoError; use wasm_bindgen_test::wasm_bindgen_test; - use worker::js_sys::{Function, Object, Reflect}; + use worker::js_sys::{Array, Function, Object, Promise, Reflect}; use worker::wasm_bindgen::{JsCast as _, JsValue}; + use worker::wasm_bindgen_futures::JsFuture; use worker::worker_sys::Context as WorkerSysContext; use worker::{Context, Env, Method as CfMethod, Request as CfRequest, RequestInit}; @@ -410,6 +411,321 @@ mod tests { assert!(deadline_body_releases_source_for_test()); } + fn recording_context() -> (Env, Context, Array, NumericTimeoutGuard) { + let (env, _ctx, timers) = test_env_ctx(); + let object = Function::new_no_args( + "return { promises: [], waitUntil(p) { this.promises.push(p); } };", + ) + .call0(&JsValue::UNDEFINED) + .expect("recording context"); + let promises = Reflect::get(&object, &JsValue::from_str("promises")) + .expect("promise list") + .unchecked_into(); + (env, Context::new(object.unchecked_into()), promises, timers) + } + + #[wasm_bindgen_test] + async fn after_response_runs_through_service_and_prebuilt_dispatch() { + static COMPLETED: AtomicUsize = AtomicUsize::new(0); + #[edgezero_core::action] + async fn register(ctx: RequestContext) -> Result { + ctx.after_response(Duration::from_secs(1), async { + COMPLETED.fetch_add(1, Ordering::SeqCst); + })?; + Ok(Response::new(Body::text("delivered"))) + } + struct TestApp; + #[expect( + clippy::missing_trait_methods, + reason = "test uses default application metadata" + )] + impl Hooks for TestApp { + fn routes() -> Result { + Ok(RouterService::builder().get("/task", register).build()) + } + } + let app = App::build::(edgezero_adapter_cloudflare::CLOUDFLARE_PLATFORM) + .expect("application"); + COMPLETED.store(0, Ordering::SeqCst); + for prebuilt in [false, true] { + let (env, ctx, promises, _timers) = recording_context(); + let req = cf_request(CfMethod::Get, "/task", None); + let mut response = if prebuilt { + edgezero_adapter_cloudflare::dispatch_app::(&app, req, env, ctx).await + } else { + CloudflareService::new(&app).dispatch(req, env, ctx).await + } + .expect("response"); + assert_eq!( + response.status_code(), + 200, + "registration must be supported" + ); + assert_eq!(response.text().await.expect("body"), "delivered"); + JsFuture::from(Promise::all(&promises)) + .await + .expect("all waitUntil work completed"); + } + assert_eq!(COMPLETED.load(Ordering::SeqCst), 2); + } + + #[wasm_bindgen_test] + async fn after_response_waits_for_stream_and_budget_starts_at_release() { + use edgezero_core::after_response::AfterResponseHandle; + use std::cell::Cell; + use std::rc::Rc; + use std::sync::Mutex; + + #[derive(Default)] + struct State { + cancelled: AtomicUsize, + completed: AtomicUsize, + produced: AtomicUsize, + retained: Mutex>, + started_early: AtomicUsize, + } + struct DropProbe(Arc); + #[expect(clippy::missing_trait_methods, reason = "Drop::pin_drop is unstable")] + impl Drop for DropProbe { + fn drop(&mut self) { + self.0.cancelled.fetch_add(1, Ordering::SeqCst); + } + } + #[edgezero_core::action] + async fn register(ctx: RequestContext) -> Result { + let state = Arc::clone(ctx.extensions().get::>().expect("state")); + *state.retained.lock().unwrap() = + ctx.extensions().get::().cloned(); + let task_state = Arc::clone(&state); + let local = Rc::new(Cell::new(false)); + ctx.after_response(Duration::from_millis(60), async move { + if task_state.produced.load(Ordering::SeqCst) == 0 { + task_state.started_early.fetch_add(1, Ordering::SeqCst); + } + local.set(true); + worker::Delay::from(Duration::from_millis(25)).await; + assert!(local.get(), "non-Send task ran"); + let handle = task_state.retained.lock().unwrap().clone().unwrap(); + let late_state = Arc::clone(&task_state); + assert!( + handle + .schedule(Duration::from_secs(1), async move { + late_state.started_early.fetch_add(1, Ordering::SeqCst); + }) + .is_err() + ); + task_state.completed.fetch_add(1, Ordering::SeqCst); + })?; + let probe = DropProbe(Arc::clone(&state)); + ctx.after_response(Duration::from_millis(10), async move { + let _probe = probe; + pending::<()>().await; + })?; + Ok(Response::new(Body::stream(stream::once(async move { + worker::Delay::from(Duration::from_millis(100)).await; + state.produced.store(1, Ordering::SeqCst); + Bytes::from_static(b"stream finished") + })))) + } + let state = Arc::new(State::default()); + let app = App::new( + RouterService::builder() + .with_state(Arc::clone(&state)) + .get("/task", register) + .build(), + ); + let (env, ctx, promises, _timers) = recording_context(); + let mut response = CloudflareService::new(&app) + .dispatch(cf_request(CfMethod::Get, "/task", None), env, ctx) + .await + .expect("response"); + worker::Delay::from(Duration::from_millis(20)).await; + assert_eq!(state.completed.load(Ordering::SeqCst), 0); + assert_eq!(state.cancelled.load(Ordering::SeqCst), 0); + assert_eq!(response.text().await.unwrap(), "stream finished"); + JsFuture::from(Promise::all(&promises)) + .await + .expect("waitUntil tasks"); + assert_eq!( + state.completed.load(Ordering::SeqCst), + 1, + "response time must not consume task budget" + ); + assert_eq!( + state.started_early.load(Ordering::SeqCst), + 0, + "task must not start before the stream finishes" + ); + assert_eq!( + state.cancelled.load(Ordering::SeqCst), + 1, + "pending task dropped at budget" + ); + } + + #[wasm_bindgen_test] + async fn after_response_owns_fresh_outbound_scope_and_aborts_timed_out_fetch() { + struct FetchProbe(JsValue); + #[expect(clippy::missing_trait_methods, reason = "Drop::pin_drop is unstable")] + impl Drop for FetchProbe { + fn drop(&mut self) { + let restore = Reflect::get(&self.0, &JsValue::from_str("restore")) + .unwrap() + .unchecked_into::(); + restore.call0(&JsValue::UNDEFINED).unwrap(); + } + } + static COMPLETED: AtomicUsize = AtomicUsize::new(0); + #[edgezero_core::action] + async fn register(ctx: RequestContext) -> Result { + let client = ctx.http_client().unwrap().clone(); + let stale = client.send(OutboundRequest::get("https://upstream.example/stale")?); + let fresh_client = client.clone(); + ctx.after_response(Duration::from_secs(1), async move { + assert!(stale.await.is_err(), "earlier scope stays revoked"); + let response = fresh_client + .send(OutboundRequest::get("https://upstream.example/ready").unwrap()) + .await + .expect("new callback reaches fetch"); + assert_eq!( + response + .into_body() + .into_bytes_bounded(64) + .await + .unwrap() + .as_ref(), + b"ready" + ); + COMPLETED.fetch_add(1, Ordering::SeqCst); + })?; + ctx.after_response(Duration::from_millis(20), async move { + let _response = client + .send(OutboundRequest::get("https://upstream.example/pending").unwrap()) + .await; + COMPLETED.fetch_add(100, Ordering::SeqCst); + })?; + Ok(Response::new(Body::text("delivered"))) + } + let probe = FetchProbe( + Function::new_no_args( + "const original = globalThis.fetch; const probe = {calls:0, aborts:0}; + globalThis.fetch = (request, init) => { + probe.calls++; + if (request.url.endsWith('/pending')) { + init.signal.addEventListener('abort', () => {probe.aborts++;}, {once:true}); + return new Promise(() => {}); + } + return Promise.resolve(new Response('ready')); + }; + probe.restore = () => {globalThis.fetch = original;}; return probe;", + ) + .call0(&JsValue::UNDEFINED) + .unwrap(), + ); + COMPLETED.store(0, Ordering::SeqCst); + let app = App::new(RouterService::builder().get("/task", register).build()); + let (env, ctx, promises, _timers) = recording_context(); + let mut response = CloudflareService::new(&app) + .dispatch(cf_request(CfMethod::Get, "/task", None), env, ctx) + .await + .unwrap(); + response.bytes().await.unwrap(); + JsFuture::from(Promise::all(&promises)) + .await + .expect("bounded work"); + assert_eq!(COMPLETED.load(Ordering::SeqCst), 1); + assert_eq!( + Reflect::get(&probe.0, &JsValue::from_str("calls")) + .unwrap() + .as_f64(), + Some(2.0_f64) + ); + assert_eq!( + Reflect::get(&probe.0, &JsValue::from_str("aborts")) + .unwrap() + .as_f64(), + Some(1.0_f64) + ); + } + + #[wasm_bindgen_test] + async fn after_response_releases_when_dispatch_is_abandoned() { + static COMPLETED: AtomicUsize = AtomicUsize::new(0); + #[edgezero_core::action] + async fn register(ctx: RequestContext) -> Result { + ctx.after_response(Duration::from_secs(1), async { + COMPLETED.fetch_add(1, Ordering::SeqCst); + })?; + pending().await + } + use futures::FutureExt as _; + let app = App::new(RouterService::builder().get("/task", register).build()); + let (env, ctx, promises, _timers) = recording_context(); + COMPLETED.store(0, Ordering::SeqCst); + assert!( + CloudflareService::new(&app) + .dispatch(cf_request(CfMethod::Get, "/task", None), env, ctx) + .now_or_never() + .is_none() + ); + JsFuture::from(Promise::all(&promises)) + .await + .expect("abandoned dispatch task"); + assert_eq!(COMPLETED.load(Ordering::SeqCst), 1); + } + + #[wasm_bindgen_test] + async fn after_response_releases_on_bodyless_fallback_and_source_failure() { + static COMPLETED: AtomicUsize = AtomicUsize::new(0); + #[edgezero_core::action] + async fn register(ctx: RequestContext) -> Result { + ctx.after_response(Duration::from_secs(1), async { + COMPLETED.fetch_add(1, Ordering::SeqCst); + })?; + let mut response = Response::new(Body::empty()); + match ctx.uri().path() { + "/bodyless" => *response.status_mut() = StatusCode::NO_CONTENT, + "/fallback" => *response.status_mut() = StatusCode::from_u16(199).unwrap(), + _ => { + *response.body_mut() = Body::from_stream(stream::once(async { + Err::(EdgeError::internal(IoError::other("source failed"))) + })); + } + } + Ok(response) + } + let app = App::new( + RouterService::builder() + .get("/bodyless", register) + .get("/fallback", register) + .get("/source-error", register) + .build(), + ); + COMPLETED.store(0, Ordering::SeqCst); + for (path, status) in [ + ("/bodyless", 204), + ("/fallback", 500), + ("/source-error", 200), + ] { + let (env, ctx, promises, _timers) = recording_context(); + let mut response = CloudflareService::new(&app) + .dispatch(cf_request(CfMethod::Get, path, None), env, ctx) + .await + .expect("response"); + assert_eq!(response.status_code(), status); + let body = response.bytes().await; + if path == "/source-error" { + body.expect_err("failed source must fail body"); + } else { + body.expect("bodyless or fallback body"); + } + JsFuture::from(Promise::all(&promises)) + .await + .expect("all terminal work"); + } + assert_eq!(COMPLETED.load(Ordering::SeqCst), 3); + } + #[wasm_bindgen_test] async fn dispatch_app_reuses_router_with_fresh_request_bodies() { struct TestApp; diff --git a/docs/guide/adapters/cloudflare.md b/docs/guide/adapters/cloudflare.md index 0a7c95da..e0c23aab 100644 --- a/docs/guide/adapters/cloudflare.md +++ b/docs/guide/adapters/cloudflare.md @@ -546,3 +546,20 @@ captured separately before that conversion. - Learn about [Fastly Compute](/guide/adapters/fastly) as an alternative - Explore the [Axum adapter](/guide/adapters/axum) for local development + +## After-response tasks + +`ctx.after_response(budget, task)` uses Workers `waitUntil` and keeps the task unpolled +until the adapter's response writer terminates, or a bodyless response is handed off. +Errors, cancellation and bounded fallback also release the tasks. This is a terminal +host handoff, not confirmation that the client received the response. + +The effective cooperative budget is `min(budget, 30 seconds)` and starts after release. +Construct outbound requests inside the task so they use its fresh callback scope. +Registration closes at release; retained handles and recursive registrations fail. + +Workers shares up to 30 seconds after response completion or disconnect across all +`waitUntil` work, independently of these task budgets. CPU/memory limits and platform +termination can shorten execution. Blocking code cannot be preempted within a poll. +Tasks are not persistent and have no automatic retries. Use a durable queue for work +that must survive invocation loss. See the [capability matrix](../capabilities.md#after-response-task-matrix). diff --git a/docs/guide/capabilities.md b/docs/guide/capabilities.md index d9f0d94f..9e5d3f04 100644 --- a/docs/guide/capabilities.md +++ b/docs/guide/capabilities.md @@ -347,9 +347,9 @@ not included. These limitations block whole-response-lifetime certification. ## After-Response Task Matrix -| Capability | Axum | Cloudflare | Fastly | Spin | -| ---------------------- | ------------------ | ----------- | ------------------ | ----------- | -| `after-response-tasks` | BoundedCooperative | Unsupported | BoundedCooperative | Unsupported | +| Capability | Axum | Cloudflare | Fastly | Spin | +| ---------------------- | ------------------ | ------------------ | ------------------ | ----------- | +| `after-response-tasks` | BoundedCooperative | BoundedCooperative | BoundedCooperative | Unsupported | `ctx.after_response(budget, task)` registers a task while a request is handled. The adapter starts the task once the response attempt ends at its terminal egress boundary, whatever the outcome. That @@ -358,7 +358,7 @@ response. The task is dropped, and so cancelled, when its budget elapses. Regist the request's tasks are released, so a task cannot register more tasks. Start outbound requests inside the task. -Both supporting adapters are `BoundedCooperative`: the budget cancels a task only when it yields, +All three supporting adapters are `BoundedCooperative`: the budget cancels a task only when it yields, so CPU-bound or blocking work overruns it. - Axum spawns each task on the server's executor and releases it when Hyper drops the response body. @@ -372,8 +372,14 @@ so CPU-bound or blocking work overruns it. the same outbound scope. The budget is checked whenever a task yields to the adapter's progress driver; a blocked hostcall cannot be preempted. The tasks delay the next request on a reused `serve_app` instance, and a caller-sent native replacement after an uncommitted delivery. -- Cloudflare and Spin install no scheduler, so registration returns `501 Not Implemented`. Cloudflare - `waitUntil` support is tracked in [#478](https://github.com/IABTechLab/edgezero/issues/478). +- Cloudflare registers gated tasks with Workers `waitUntil`. Tasks start after the adapter's + response writer terminates (or bodyless native handoff), including delivery failure and + disconnect. Each task gets an effective budget of `min(budget, 30 seconds)` starting at release, + with a fresh outbound callback scope. Workers independently shares an allowance of up to + 30 seconds after response completion or disconnect across all `waitUntil` calls; platform + CPU/memory limits or earlier termination can shorten execution. Tasks have no persistence or + automatic retry. See [Workers context limits](https://developers.cloudflare.com/workers/runtime-apis/context/#waituntil). +- Spin installs no scheduler, so registration returns `501 Not Implemented`. ## Outbound Matrix diff --git a/docs/guide/handlers.md b/docs/guide/handlers.md index 1aa3ad2a..1e4d4cc4 100644 --- a/docs/guide/handlers.md +++ b/docs/guide/handlers.md @@ -350,7 +350,9 @@ async fn page(ctx: RequestContext) -> Result<&'static str, EdgeError> { Registration returns `501 Not Implemented` on adapters without support; see the [after-response task matrix](./capabilities.md#after-response-task-matrix). Declare `after-response-tasks` as a required capability when the work is not optional. Tasks return `()`, so -handle and log their own failures. A task cannot register further tasks. +handle and log their own failures. A task cannot register further tasks. Cloudflare caps the +effective budget at 30 seconds, and the shared Workers lifetime may end sooner; use durable +platform queues when completion and retries must survive an invocation. ### Store Extractors diff --git a/docs/superpowers/plans/2026-11-18-cloudflare-after-response.md b/docs/superpowers/plans/2026-11-18-cloudflare-after-response.md new file mode 100644 index 00000000..defc9bf8 --- /dev/null +++ b/docs/superpowers/plans/2026-11-18-cloudflare-after-response.md @@ -0,0 +1,110 @@ +# Cloudflare after-response tasks implementation plan + +**Goal:** Implement #478 over #509 with request-local terminal-egress gating. + +**Architecture:** A scheduler registers gated local futures with Workers waitUntil. +A sole release guard follows response conversion and the response pump. Reuse the +portable after-response handle and Cloudflare deadline/outbound scope machinery. + +**Tech stack:** Rust 1.98.1, worker 0.8.5, futures oneshot, Workers WASM, Node 24.20.0. + +## Task 1: Scheduler and gate + +Files: create `crates/edgezero-adapter-cloudflare/src/after_response.rs`; modify `src/lib.rs`. + +- [x] Add colocated tests for unpolled registration, release, late registration, + independent gates, multiple receivers and release on drop. Test non-Send futures. + Compile the pure gate under cfg(test) on native; gate SDK code on cloudflare + wasm32. + Add deterministic huge-budget cap, timer-failure drop and sibling survival after panic tests. +- [x] Run `cargo +1.98.1 test -p edgezero-adapter-cloudflare --lib` and observe meaningful + failures before implementing the gate. +- [x] Implement a shared optional sender list with poison-safe drop, plus a sole release + guard. Keep mutexes unlocked while signalling or entering the SDK. +- [x] Implement the platform scheduler using Arc, a signal receiver and + `wait_until`. After release, create a fresh callback owner and clock/deadline; + use `native_time` with effective budget capped at 30 seconds, race task/timer, + drop the loser and log cancellation/timer failure/panic. No unsafe impls or new deps. +- [x] Run crate tests after this code change and target compilation on wasm32-unknown-unknown. + +## Task 2: Production wiring and delivery ownership + +Files: `src/request.rs`, `src/response.rs` in Cloudflare adapter. + +- [x] Add tests that fail for early gate release during a stream and failed preparation + followed by fallback, and for missing production scheduler installation. +- [x] Create scheduler and guard in `dispatch_with_handles`, install handle before app + dispatch, attach guard to the runtime. Preserve all other request extensions. +- [x] Add optional release ownership to CloudflareEgressRuntime and WorkersCommitter; + take the guard into the spawned writer future and keep it until the writer terminates. + Synchronous no-body/error paths release when committer drops; dispatch cancellation + drops the retained runtime. Existing assembly errors need no scheduler. +- [x] Run Cloudflare crate tests after changes; compile WASM and run existing egress tests. + +## Task 3: Contract and runtime proof + +Files: Cloudflare adapter colocated tests and `tests/contract.rs` as appropriate. + +- [x] Capture test Context waitUntil promises and await them deterministically. Exercise + real CloudflareService and registry-based dispatch, gated stream, pending task drop, + deadline starts after release, stale/recursive handles and independent requests. +- [x] Include no-body response, fallback, failed writer/cancellation, and an outbound + request constructed inside an after-response task. Contrast with a retained original-scope + request failing after delivery, and observe outbound cancellation on task timeout. Keep existing contract checks. +- [x] Run cargo tests after test changes, plus Cloudflare guest suites through the + project's available WASM harness. Record fixture limitations separately. +- [x] Build an owned disposable consumer using the public API and normal Worker entrypoint, + start local Wrangler/workerd with isolated configuration, and test delayed task completion, + response-first timing, streaming, timeout cancellation, repeated requests and outbound + transport. Prove timing discriminates using an inline-delay control. Run a missing-gate + negative control in an isolated source copy. Stop only owned processes after evidence capture. + +## Task 4: Capability and documentation + +Files: Cloudflare `src/cli.rs`, `docs/guide/capabilities.md`, `docs/guide/handlers.md`, +`docs/guide/adapters/cloudflare.md`, `CHANGELOG.md`, `scripts/check_outbound_docs_contract.mjs`. + +- [x] Change Cloudflare AfterResponseTasks to BoundedCooperative and its matrix test. +- [x] Document terminal boundary, cooperative effective min(budget,30s), shared Workers + lifetime, outbound scope rule, no persistence/retries and unchanged Spin support. +- [x] Update docs contract expected matrix and current changelog statements. +- [x] Run cargo tests after code changes and the docs contract script, formatting/lint/build. + +## Task 5: Verify and publish + +- [x] Run independent code review, resolve all supported blocking and optional findings, + then rerun affected evidence. +- [x] Run cargo +1.98.1 fmt --all -- --check; strict workspace all-feature Clippy; + cargo +1.98.1 test --workspace --all-targets; workspace check with fastly cloudflare spin; + Spin wasm32-wasip2 check. Run Cloudflare target Clippy for cloudflare and + cloudflare,test-utils features so native all-feature unification cannot hide errors. +- [x] Reconcile changed documentation and checks with current source. Retain unavailable + qualification honestly; do not substitute native tests for Workers execution. +- [ ] Explain the verified changes, sign commits and verify signatures locally. Push and + create an assigned draft PR against feature/476-after-response-tasks, using the project + template and Closes #478. Confirm GitHub Verified on new commits. +- [ ] Independently self-review the complete PR against its merge base; resolve findings + under the implementation authorization, rerun affected checks, and refresh the draft. + +## Local verification + +Local checks use Rust 1.98.1, Node 24.20.0 and lock-matched wasm-bindgen 0.2.128. +Workspace tests passed (2,826 passed, two ignored); final focused Cloudflare host +checks cover subsequent test refinements. App-demo passed 68 tests. Workspace strict +Clippy, formatting, all-adapter feature compilation and Spin WASI compilation passed. +Cloudflare WASM library and test-utils contracts passed 142 and 35 tests; Chrome +browser ingress passed five tests. Standalone cloudflare contracts and Clippy also run +without test-utils so feature unification cannot hide errors. + +A public `run_app` consumer built with worker-build 0.8.3 ran in Wrangler 4.83.0's local +workerd with compatibility date 2025-05-05. Response-first timing was about 4 ms versus +303 ms for an inline 300 ms control. A roughly 609 ms stream finished before the +follow-up task began; timeout dropped a pending task. Real outbound HTTP worked in a +fresh task scope while an earlier-scope future remained revoked. Bodyless responses, +bounded fallback and overlapping request gates passed. The final rebuilt artifact was +probed again. These are local runtime observations, not hosted Cloudflare qualification. + +In an isolated negative-control copy, releasing the guard at dispatch return instead +of retaining it in the writer made the streaming contract fail: a task was cancelled +before the stream finished. The unmodified consumer passes. The parent revision also +failed the public registration test with 501, and the old capability matrix failed its +new Cloudflare support assertion. Host panic isolation is an unwind-only check. diff --git a/docs/superpowers/specs/2026-11-18-cloudflare-after-response-design.md b/docs/superpowers/specs/2026-11-18-cloudflare-after-response-design.md new file mode 100644 index 00000000..38c853bc --- /dev/null +++ b/docs/superpowers/specs/2026-11-18-cloudflare-after-response-design.md @@ -0,0 +1,85 @@ +# Cloudflare after-response tasks + +Issue: #478. Base: #509 (`feature/476-after-response-tasks`, `c3ba7e56`). + +## Goal and scope + +Enable the existing `RequestContext::after_response(budget, task)` API on Cloudflare +Workers without delaying response delivery. Preserve #476's Fastly and Axum behavior +and Spin's unsupported capability. This delivers reusable adapter infrastructure; +Trusted Server integration and deployed-provider qualification are separate work. + +## Contract + +A registered local, non-Send future remains unpolled until the request's terminal +egress boundary. For streamed responses this is completion or termination of the +adapter-owned writer, including cancellation, source errors and transport failure. +For bodyless responses it is native response handoff. This boundary does not certify +client receipt. Conversion failure retains the gate through the bounded fallback; +failed fallback and abandoned dispatch also release it. Registration closes at release, +and retained handles or recursive registration fail without polling the supplied task. + +Each accepted task is retained with the request's Workers `Context::wait_until` before +dispatch returns, but awaits a request-local signal first. A fresh outbound callback +scope and the task deadline are created only after release. The task must construct +outbound requests inside its future; futures made earlier keep their original scope. +The effective cooperative budget is `min(requested budget, 30 seconds)`. A pending +future is dropped when this expires; a single blocking poll cannot be preempted. +Zero budgets retain the core API's existing rejection. Very large durations cannot +wrap JavaScript timer integers. Tasks on separate requests never release each other. + +Workers independently limits all waitUntil work to a shared allowance of up to 30 +seconds after response completion or disconnect, and may terminate work earlier. +This is bounded best-effort execution, without persistence, retries or a guarantee +that a task finishes. Platform CPU and memory limits also continue to apply. + +## Architecture and alternatives + +Use a request-local scheduler containing the existing SDK `Arc` and a +mutex-protected optional list of oneshot senders. The SDK implements Send/Sync for +Context; local futures live only in the waitUntil closures. No unsafe trait impls, +ambient task queue, new dependencies or core API changes are needed. + +Scheduling adds a release receiver while registration is open, then calls waitUntil +outside the mutex. The queued future awaits release and races its task against the +adapter's existing deadline timer in a fresh callback scope. Log timeout, timer failure +and recoverable task panic without changing the already-delivered response. Panic +recovery requires an unwinding build; ordinary aborting WASM panics may terminate +the invocation. A host unwind test does not qualify Workers panic recovery. + +One release guard owns closure of registration and signalling all accepted tasks. +It lives in `CloudflareEgressRuntime`, transfers to `WorkersCommitter`, and moves into +the actual response pump on spawn. Drop covers successful completion, caught panic, +cancellation, bodyless handoff, synchronous errors and dispatch abandonment. Fallback +uses the same committer so failed preparation cannot release tasks prematurely. + +Installing the handle in the converged real dispatch path covers CloudflareService, +run_app and dispatch_app. Standalone request conversion has no delivery ownership and +continues without a scheduler. Capability reporting becomes BoundedCooperative, +matching Fastly/Axum, with Cloudflare's effective cap and platform limits documented. + +Calling waitUntil directly at registration would permit work before delivery. Keeping +an ambient queue would complicate overlapping requests and non-Send ownership. The +release-signal design reuses existing egress ownership and supports concurrent requests. + +## Acceptance and preservation + +| Scenario | Required observation | +| --------------------------------------- | ------------------------------------------------------------------------- | +| Normal registration | Response succeeds and task completes after terminal egress | +| Slow streaming response | Task stays unpolled while writer is active; full budget starts at release | +| Pending task | Future is dropped at its effective budget | +| Bodyless/error/fallback/cancel/drop | Gate releases exactly once after terminal attempt | +| Late/recursive registration | Typed error; supplied future remains unpolled | +| Multiple tasks and overlapping requests | Every accepted task runs; gates remain independent | +| Outbound inside task | Real adapter HTTP client works after original callback scope closes | +| Required capability | CLI accepts Cloudflare after-response-tasks; Spin remains unsupported | +| Existing egress | Framing, observations, cancellation and bounded fallback remain correct | + +Use colocated gate/scheduler tests, Cloudflare WASM fixture tests and the existing +contract/egress suites. Fixture tests must await accepted promises rather than only +counting them. Run the actual Worker through local Wrangler/workerd using a disposable +consumer of the public API: response timing, delayed marker, timeout cancellation, +streaming, repeated requests and outbound side effects. Compare an inline-delay control +and a base or deliberately broken gate in an isolated copy. Keep fixture results separate +from workerd evidence and from unexecuted hosted Cloudflare deployment qualification. diff --git a/scripts/check_outbound_docs_contract.mjs b/scripts/check_outbound_docs_contract.mjs index ed100ece..07b56641 100644 --- a/scripts/check_outbound_docs_contract.mjs +++ b/scripts/check_outbound_docs_contract.mjs @@ -236,7 +236,7 @@ const expectedAfterResponseRows = [ [ 'after-response-tasks', 'BoundedCooperative', - 'Unsupported', + 'BoundedCooperative', 'BoundedCooperative', 'Unsupported', ],