diff --git a/CHANGELOG.md b/CHANGELOG.md index 7770881f..587faa8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,18 @@ and listed with their assets on the ## [Unreleased] +### Fixed + +- **A static service's `forward` keeps its connections to the target** + (#556). It opened a new connection for every request and closed it, so + each left a socket waiting two minutes; an end-to-end suite of about 1000 + tests drained the ephemeral ports on Windows, and pages and API calls + failed with `ERR_ADDRESS_IN_USE` or `502`. Each target now keeps up to 32 + idle connections and reuses them, as a reverse proxy does; server-sent + events and WebSockets keep a connection of their own. The hub keeps its + connections to the daemons the same way, so a dashboard that stays open + no longer leaves a socket behind for every call. + ## [0.49.1] - 2026-11-17 A static service forwards paths to another server, as a reverse proxy in diff --git a/crates/devstack-daemon/src/staticsrv.rs b/crates/devstack-daemon/src/staticsrv.rs index 68f12932..2c8caf45 100644 --- a/crates/devstack-daemon/src/staticsrv.rs +++ b/crates/devstack-daemon/src/staticsrv.rs @@ -91,11 +91,32 @@ fn percent_decode(s: &str) -> String { String::from_utf8_lossy(&out).into_owned() } -/// Where a forward sends its requests: `http://host:port`. -#[derive(Debug, Clone, PartialEq)] +use devstack_node::connpool::{GiveBack, IDLE_MAX, Pool}; + +/// Where a forward sends its requests: `http://host:port`, with the +/// connections to it that are idle now. They are reused, as a reverse proxy +/// does: a connection per request would leave each one in `TIME_WAIT` on +/// this side, and an end-to-end suite drains the ephemeral ports (#556). +#[derive(Debug, Clone)] pub struct Target { host: String, port: u16, + pool: Arc>, +} + +impl Target { + /// A new connection. + async fn connect(&self) -> Result, String> { + let stream = tokio::net::TcpStream::connect((self.host.as_str(), self.port)) + .await + .map_err(|e| e.to_string())?; + let _ = stream.set_nodelay(true); + let (sender, conn) = hyper::client::conn::http1::handshake(TokioIo::new(stream)) + .await + .map_err(|e| e.to_string())?; + tokio::spawn(conn.with_upgrades()); + Ok(sender) + } } /// The `--forward PREFIX=URL` arguments, checked as `devstack check` does. @@ -109,7 +130,14 @@ pub fn forwards(args: &[String]) -> Result, String> { forward::prefix(prefix).map_err(|e| format!("--forward `{prefix}`: {e}"))?; let (host, port) = forward::origin(url).map_err(|e| format!("--forward {prefix}: `{url}`: {e}"))?; - Ok((prefix.to_string(), Target { host, port })) + Ok(( + prefix.to_string(), + Target { + host, + port, + pool: Pool::new(IDLE_MAX), + }, + )) }) .collect() } @@ -210,20 +238,9 @@ async fn pass(to: &Target, peer: SocketAddr, mut req: Request) -> Response *req.uri_mut() = path.parse().unwrap_or_else(|_| Uri::from_static("/")); *req.version_mut() = Version::HTTP_11; - let stream = match tokio::net::TcpStream::connect((to.host.as_str(), to.port)).await { - Ok(s) => s, - Err(e) => return fail(e.to_string()), - }; - let _ = stream.set_nodelay(true); - let (mut sender, conn) = match hyper::client::conn::http1::handshake(TokioIo::new(stream)).await - { - Ok(c) => c, - Err(e) => return fail(e.to_string()), - }; - tokio::spawn(conn.with_upgrades()); - let mut resp = match sender.send_request(req).await { + let (sender, mut resp) = match to.pool.send(&(), req, || to.connect()).await { Ok(r) => r, - Err(e) => return fail(e.to_string()), + Err(e) => return fail(e), }; let upgraded = resp.status() == StatusCode::SWITCHING_PROTOCOLS && client_upgrade.is_some(); crate::proxy::strip_hop_by_hop(resp.headers_mut(), upgraded); @@ -235,11 +252,14 @@ async fn pass(to: &Target, peer: SocketAddr, mut req: Request) -> Response let _ = tokio::io::copy_bidirectional(&mut c, &mut t).await; } }); + // The upgraded connection is the WebSocket's now: not given back. let (parts, _) = resp.into_parts(); return Response::from_parts(parts, Body::empty()); } - let (parts, body) = resp.into_parts(); - Response::from_parts(parts, Body::new(body)) + // Read to its end, the answer's connection takes the next request. One + // the browser leaves early (or a stream that does not end, SSE) keeps or + // drops its connection, never back in the pool. + GiveBack::wrap(to.pool.clone(), (), sender, resp).map(Body::new) } /// Whether a file's name carries a hash of its content (`main-3f2a1b9c.js`, @@ -532,6 +552,93 @@ mod tests { ); } + /// A target that counts the connections it accepts (#556). + async fn counting_target() -> (u16, Arc) { + use std::sync::atomic::Ordering; + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = l.local_addr().unwrap().port(); + let accepted = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let count = accepted.clone(); + tokio::spawn(async move { + while let Ok((s, _)) = l.accept().await { + count.fetch_add(1, Ordering::SeqCst); + tokio::spawn(async move { + let svc = hyper::service::service_fn(|_req| async { + Ok::<_, std::convert::Infallible>(Response::new(http_body_util::Full::new( + bytes::Bytes::from("pong"), + ))) + }); + let _ = hyper::server::conn::http1::Builder::new() + .serve_connection(TokioIo::new(s), svc) + .await; + }); + } + }); + (port, accepted) + } + + #[tokio::test] + async fn a_forward_reuses_its_connections_to_the_target() { + let d = tempfile::tempdir().unwrap(); + std::fs::write(d.path().join("index.html"), "").unwrap(); + let (target, accepted) = counting_target().await; + let port = site(d.path(), &[format!("/api=http://127.0.0.1:{target}")]).await; + // One browser connection per request, closed after it, as a test + // suite's many pages do: the target's side must not follow suit. + for i in 0..30 { + let (_, r) = talk( + port, + format!("GET /api/ping/{i} HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n") + .as_bytes(), + "\0", + ) + .await; + assert!(r.starts_with("HTTP/1.1 200") && r.ends_with("pong"), "{r}"); + } + let n = accepted.load(std::sync::atomic::Ordering::SeqCst); + assert!( + n <= 2, + "30 requests one after another opened {n} connections to the target" + ); + // Four at a time, as four test workers send them: about four. + for round in 0..10 { + let all = (0..4).map(|w| { + talk( + port, + format!( + "GET /api/{round}/{w} HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n" + ) + .leak() + .as_bytes(), + "\0", + ) + }); + for (_, r) in futures_join(all).await { + assert!(r.ends_with("pong"), "{r}"); + } + } + let n = accepted.load(std::sync::atomic::Ordering::SeqCst); + assert!( + n <= 8, + "70 requests, four at a time, opened {n} connections to the target" + ); + } + + /// Run the futures side by side, their results in order. + async fn futures_join( + fs: impl Iterator, + ) -> Vec + where + F::Output: Send + 'static, + { + let handles: Vec<_> = fs.map(tokio::spawn).collect(); + let mut out = Vec::new(); + for h in handles { + out.push(h.await.unwrap()); + } + out + } + #[test] fn a_forward_argument_is_checked_at_the_start() { assert!(forwards(&["/api=http://127.0.0.1:5000".into()]).is_ok()); diff --git a/crates/devstack-hub/src/lib.rs b/crates/devstack-hub/src/lib.rs index 654811f3..14eb949a 100644 --- a/crates/devstack-hub/src/lib.rs +++ b/crates/devstack-hub/src/lib.rs @@ -3615,7 +3615,7 @@ async fn daemon_get( s: &AppState, id: &str, path: &str, -) -> Result, DaemonError> { +) -> Result, DaemonError> { let req = hyper::Request::get(path) .header(header::AUTHORIZATION, format!("Bearer {}", s.token)) .header(header::ACCEPT, "text/event-stream") @@ -3635,10 +3635,23 @@ async fn daemon_get( /// one the request went to, the answer is `503` with `Retry-After` and /// what the stack does; only a daemon that still serves and broke the /// exchange is a `502`. +/// An answer of a daemon; read to its end, its connection takes the hub's +/// next request to that daemon (#556: a connection per request leaves each +/// in `TIME_WAIT`, and a dashboard that polls adds them up). +type DaemonBody = devstack_node::connpool::GiveBack; + +/// The hub's idle connections to its daemons, by port. +fn daemon_pool() -> Arc> { + static POOL: std::sync::OnceLock>> = + std::sync::OnceLock::new(); + POOL.get_or_init(|| devstack_node::connpool::Pool::new(devstack_node::connpool::IDLE_MAX)) + .clone() +} + async fn daemon_send( resolve: impl Fn() -> DaemonNow, mut req: hyper::Request, -) -> Result, DaemonError> { +) -> Result, DaemonError> { use hyper::body::Body as _; let t0 = Instant::now(); // GET, HEAD, OPTIONS without a body: safe to repeat whatever happened. @@ -3661,33 +3674,55 @@ async fn daemon_send( HeaderValue::from_str(&format!("127.0.0.1:{port}")).unwrap(), ); let in_window = || t0.elapsed() < RESEND_WINDOW; - let stream = match connect_daemon(port).await { - Ok(s) => s, - Err(e) => { - // Nothing was sent. A daemon that replaced this one is - // worth another try; otherwise it is not running. - let then = resolve(); - if then != now && in_window() { - continue; - } - return Err(match then { - DaemonNow::Serves { .. } if then == now => { - DaemonError::unavailable(DaemonNow::Absent { starting: false }, Some(e)) + let kept = daemon_pool().kept(&port); + let reused = kept.is_some(); + let mut sender = match kept { + Some(s) => s, + None => { + let stream = match connect_daemon(port).await { + Ok(s) => s, + Err(e) => { + // Nothing was sent. A daemon that replaced this one is + // worth another try; otherwise it is not running. + let then = resolve(); + if then != now && in_window() { + continue; + } + return Err(match then { + DaemonNow::Serves { .. } if then == now => DaemonError::unavailable( + DaemonNow::Absent { starting: false }, + Some(e), + ), + then => DaemonError::unavailable(then, Some(e)), + }); } - then => DaemonError::unavailable(then, Some(e)), - }); + }; + let (sender, conn) = hyper::client::conn::http1::handshake(TokioIo::new(stream)) + .await + .map_err(|e| DaemonError::new(StatusCode::BAD_GATEWAY, e.to_string()))?; + tokio::spawn(conn); + sender } }; - let (mut sender, conn) = hyper::client::conn::http1::handshake(TokioIo::new(stream)) - .await - .map_err(|e| DaemonError::new(StatusCode::BAD_GATEWAY, e.to_string()))?; - tokio::spawn(conn); let copy = repeatable.then(|| again(&req)); match sender.try_send_request(req).await { - Ok(resp) => return Ok(resp), + Ok(resp) => { + return Ok(devstack_node::connpool::GiveBack::wrap( + daemon_pool(), + port, + sender, + resp, + )); + } Err(mut e) => { - let then = resolve(); let unsent = e.take_message(); + // A kept connection the daemon closed while it was idle hands + // the request back unsent: it goes out on a new one. + if reused && let Some(r) = unsent { + req = r; + continue; + } + let then = resolve(); let sent = unsent.is_none(); let error = e.into_error().to_string(); diag::warn( @@ -4465,6 +4500,41 @@ mod daemon_send_tests { assert_eq!(n.load(Ordering::SeqCst), 1); } + #[tokio::test] + async fn the_hub_reuses_its_connections_to_a_daemon() { + // A daemon that keeps its connections open and counts them (#556). + let l = tokio::net::TcpListener::bind(("127.0.0.1", 0)) + .await + .unwrap(); + let port = l.local_addr().unwrap().port(); + let accepted = Arc::new(AtomicUsize::new(0)); + let count = accepted.clone(); + tokio::spawn(async move { + while let Ok((c, _)) = l.accept().await { + count.fetch_add(1, Ordering::SeqCst); + tokio::spawn(async move { + let svc = hyper::service::service_fn(|_req| async { + Ok::<_, std::convert::Infallible>(hyper::Response::new( + http_body_util::Full::new(bytes::Bytes::from("{}")), + )) + }); + let _ = hyper::server::conn::http1::Builder::new() + .serve_connection(TokioIo::new(c), svc) + .await; + }); + } + }); + use http_body_util::BodyExt as _; + for _ in 0..20 { + let r = daemon_send(looks(vec![serves(port)]), get()).await.unwrap(); + assert_eq!(r.status(), StatusCode::OK); + let body = r.into_body().collect().await.unwrap().to_bytes(); + assert_eq!(&body[..], b"{}"); + } + let n = accepted.load(Ordering::SeqCst); + assert!(n <= 2, "20 requests opened {n} connections to the daemon"); + } + #[tokio::test] async fn a_daemon_that_serves_and_breaks_the_exchange_is_a_502() { let port = cutting().await; diff --git a/crates/devstack-node/src/connpool.rs b/crates/devstack-node/src/connpool.rs new file mode 100644 index 00000000..38047b24 --- /dev/null +++ b/crates/devstack-node/src/connpool.rs @@ -0,0 +1,197 @@ +//! Kept HTTP/1.1 connections to upstreams, per key (#556). A proxy that +//! opens a connection per request and closes it first leaves each one in +//! `TIME_WAIT` on its side for two minutes; under the load of a test suite +//! that drains the ephemeral ports (Windows has 16384). A connection goes +//! back once its answer is read to its end ([`GiveBack`]); one the browser +//! leaves early, or an upgraded one, does not. +//! +//! Used by the static server's forwards and the hub's way to a daemon. The +//! service proxy (`devstack-daemon`, `proxy.rs`) keeps its own, tied to the +//! drain of an instance it switches away from. + +use bytes::Bytes; +use hyper::body::{Body, Frame, Incoming, SizeHint}; +use hyper::client::conn::http1::SendRequest; +use hyper::{Request, Response}; +use std::collections::HashMap; +use std::hash::Hash; +use std::pin::Pin; +use std::sync::{Arc, Mutex}; +use std::task::{Context, Poll}; + +/// Idle connections kept per key, unless a pool says otherwise. +pub const IDLE_MAX: usize = 32; + +pub struct Pool { + idle: Mutex>>>, + max: usize, +} + +impl std::fmt::Debug for Pool { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("Pool").field("max", &self.max).finish() + } +} + +impl Pool +where + K: Eq + Hash + Clone + Send + 'static, + B: Body + Send + 'static, +{ + pub fn new(max: usize) -> Arc { + Arc::new(Self { + idle: Mutex::new(HashMap::new()), + max, + }) + } + + /// An idle connection for `key` that can take a request, if there is one. + pub fn kept(&self, key: &K) -> Option> { + let mut idle = self.idle.lock().unwrap(); + let list = idle.get_mut(key)?; + while let Some(s) = list.pop() { + if s.is_ready() { + return Some(s); + } + // closed meanwhile: next + } + None + } + + /// Keep `sender` for `key` once its connection is idle again, unless it + /// closes or `max` are kept already. + pub async fn give_back(&self, key: K, mut sender: SendRequest) { + if sender.ready().await.is_err() { + return; + } + let mut idle = self.idle.lock().unwrap(); + let list = idle.entry(key).or_default(); + if list.len() < self.max { + list.push(sender); + } + } + + /// How many idle connections `key` has (for tests and views). + pub fn idle(&self, key: &K) -> usize { + self.idle.lock().unwrap().get(key).map_or(0, Vec::len) + } + + /// Send `req` on a kept connection for `key`, else on one `connect` + /// opens. A kept one the upstream closed while it was idle hands the + /// request back unsent; it goes out on a new one. + pub async fn send( + &self, + key: &K, + req: Request, + connect: F, + ) -> Result<(SendRequest, Response), String> + where + F: Fn() -> Fut, + Fut: std::future::Future, String>>, + { + let mut req = Some(req); + let mut pooled = self.kept(key); + loop { + let fresh = pooled.is_none(); + let mut s = match pooled.take() { + Some(s) => s, + None => connect().await?, + }; + match s + .try_send_request(req.take().expect("a request to send")) + .await + { + Ok(r) => return Ok((s, r)), + Err(mut e) => match e.take_message() { + Some(back) if !fresh => req = Some(back), + _ => return Err(e.into_error().to_string()), + }, + } + } + } +} + +/// The body of an answer that gives its connection back to the pool once it +/// is read to its end. Dropped before that (the browser went away, a stream +/// that does not end), the connection goes with it. +pub struct GiveBack { + body: Incoming, + back: Option>, +} + +/// Where a connection goes at the end of its answer. +type Back = (Arc>, K, SendRequest); + +impl std::fmt::Debug for GiveBack { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("GiveBack") + .field("body", &self.body) + .field("pending", &self.back.is_some()) + .finish() + } +} + +impl GiveBack +where + K: Eq + Hash + Clone + Send + 'static, + B: Body + Send + 'static, +{ + /// `resp`'s body, giving `sender` back to `pool` under `key` at its end. + pub fn wrap( + pool: Arc>, + key: K, + sender: SendRequest, + resp: Response, + ) -> Response { + let (parts, body) = resp.into_parts(); + let mut wrapped = Self { + body, + back: Some((pool, key, sender)), + }; + // An empty answer (a 204, a HEAD) may never be polled. + if wrapped.body.is_end_stream() { + wrapped.done(); + } + Response::from_parts(parts, wrapped) + } + + fn done(&mut self) { + if let Some((pool, key, sender)) = self.back.take() { + tokio::spawn(async move { pool.give_back(key, sender).await }); + } + } +} + +impl Body for GiveBack +where + K: Eq + Hash + Clone + Send + Unpin + 'static, + B: Body + Send + 'static, +{ + type Data = Bytes; + type Error = hyper::Error; + + fn poll_frame( + self: Pin<&mut Self>, + cx: &mut Context<'_>, + ) -> Poll, hyper::Error>>> { + let this = self.get_mut(); + let r = Pin::new(&mut this.body).poll_frame(cx); + match &r { + // A body of known length is not polled after its last frame: + // its end is `is_end_stream` then, not a `None`. + Poll::Ready(None) => this.done(), + Poll::Ready(Some(Ok(_))) if this.body.is_end_stream() => this.done(), + Poll::Ready(Some(Err(_))) => this.back = None, // broken: not reused + _ => {} + } + r + } + + fn is_end_stream(&self) -> bool { + self.body.is_end_stream() + } + + fn size_hint(&self) -> SizeHint { + self.body.size_hint() + } +} diff --git a/crates/devstack-node/src/lib.rs b/crates/devstack-node/src/lib.rs index a2b9f4fb..4c4c5f03 100644 --- a/crates/devstack-node/src/lib.rs +++ b/crates/devstack-node/src/lib.rs @@ -2,6 +2,7 @@ //! (ADR-0025, ADR-0031). pub mod confine; +pub mod connpool; pub mod coordinator; pub mod egress; pub mod events; diff --git a/docs/adr/0110-a-static-service-forwards-paths-to-another-server.md b/docs/adr/0110-a-static-service-forwards-paths-to-another-server.md index c930acf7..6fc848ae 100644 --- a/docs/adr/0110-a-static-service-forwards-paths-to-another-server.md +++ b/docs/adr/0110-a-static-service-forwards-paths-to-another-server.md @@ -1,6 +1,6 @@ # ADR-0110: A static service forwards paths to another server, as a reverse proxy in front of it would -- **Status:** Accepted (card in the pull request); built +- **Status:** Accepted (card in the pull request); built; amended 2026-11-17 (connections to the target are kept, #556) - **Date:** 2026-11-17 - **Deciders:** project owner - **Related:** [ADR-0041](0041-a-profile-says-how-a-lane-runs-for-a-purpose.md) (profiles), [ADR-0023](0023-stack-file-compatibility.md) (a new key is optional), [ADR-0010](0010-promotion-mechanism.md) (the service proxy whose header handling it shares), [ADR-0058](0058-a-profile-can-leave-out-containers-and-services.md) (`skip`), [ADR-0090](0090-a-service-can-get-a-public-link-by-hand.md) (public links), issue #551 @@ -50,8 +50,9 @@ As built: appends one per entry to the static service's generated `run` (`with_static_servers`), so its placeholders get their values like the rest of `run`, per lane. -3. **Passing a request** (`staticsrv::pass`): a new HTTP/1.1 connection to - the target per request, the request target in origin form, the client's +3. **Passing a request** (`staticsrv::pass`): an HTTP/1.1 connection to + the target from its idle pool, or a new one (amendment below), the + request target in origin form, the client's `Host` kept (or taken from an HTTP/2 authority), `X-Forwarded-For` appended to, `X-Forwarded-Host` and `-Proto` set where a proxy in front (a tunnel) has not set them; `proxy::strip_hop_by_hop` both ways; the @@ -62,8 +63,8 @@ Decided while building, not on the card: - The forward runs inside the static service's own process, not in the daemon: it starts, stops and moves with the instance. -- No connection pool to the target: one connection per request is enough - on loopback and keeps the server small. +- ~~No connection pool to the target: one connection per request is enough + on loopback and keeps the server small.~~ Wrong, see the amendment. - A target without a port is port 80; an IPv6 host is written in brackets. - Prefixes take letters, digits and `-._~` only, so they need no quoting in the generated command line. @@ -91,3 +92,31 @@ Decided while building, not on the card: it does not see that. - HTTPS targets and path rewriting are not there; both can be added without breaking a file that works now. + +## Amendment 2026-11-17: connections to the target are kept (#556) + +One connection per request was wrong. devstack closed each one first, so +each left a socket in `TIME_WAIT` on this machine for two minutes; an +end-to-end suite (four workers, about 1000 tests in 11 minutes) left about +7000 and drained Windows' ephemeral ports, and the browser and the forward +failed with `ERR_ADDRESS_IN_USE` and `502`. A reverse proxy keeps its +upstream connections, and so does a dev server's proxy. + +Each target now keeps up to 32 idle connections. A request takes a ready +one or opens a new one; one the target closed while idle is sent again on a +new connection, as the service proxy does (ADR-0010). A connection goes back +once its answer is read to its end (for a body of known length: its last +frame, which hyper does not poll past); an upgraded one (WebSocket) is the +upgrade's, and a stream the browser leaves early, or one that does not end +(server-sent events), keeps or drops its connection. 2000 requests four at +a time over new browser connections now open 4 connections to the target +instead of 2001; a unit test bounds the count. No point of the card +changes: the pool was decided while building. + +The pool is `devstack_node::connpool` (`Pool`, and `GiveBack`, the body that +returns the connection), shared with the hub's way to its daemons, which +had the same flaw. Two places keep their own way: the service proxy's pool +is tied to the drain of an instance it switches away from, and the public +link router (`linkrouter.rs`) still opens a connection per request, since +its `Dial` is also its check that a host is still open; a kept connection +would pass a link its owner closed. diff --git a/docs/adr/README.md b/docs/adr/README.md index e1a70b4c..0a02728e 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -192,5 +192,5 @@ stay. | [0107](0107-the-start-page-has-a-search-and-a-state-filter.md) | The start page has a search by name or folder and a filter by state | Accepted (card in the pull request); built | | [0108](0108-the-start-page-has-large-and-small-tiles-and-a-list.md) | The start page shows the stacks as large tiles, small tiles or a sortable list | Accepted (card in the pull request); built | | [0109](0109-the-order-of-the-stacks-is-one-choice-for-the-sidebar-and-the-start-page.md) | The order of the stacks is one choice for the sidebar and the start page | Accepted (card in the pull request); built | -| [0110](0110-a-static-service-forwards-paths-to-another-server.md) | A static service forwards paths to another server, as a reverse proxy in front of it would | Accepted (card in the pull request); built | +| [0110](0110-a-static-service-forwards-paths-to-another-server.md) | A static service forwards paths to another server, as a reverse proxy in front of it would | Accepted (card in the pull request); built; amended 2026-11-17 (connections to the target kept, #556) | | [0111](0111-the-dashboards-advisory-gate-counts-what-the-bundle-holds.md) | The dashboard's advisory gate counts what the bundle holds, not npm's runtime tree | Accepted (card in the pull request); built |