Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,18 @@ and listed with their assets on the

## [Unreleased]

### Fixed

- **A static service's `forward` keeps its connections to the target**
(#556). It opened a new connection for every request and closed it, so
each left a socket waiting two minutes; an end-to-end suite of about 1000
tests drained the ephemeral ports on Windows, and pages and API calls
failed with `ERR_ADDRESS_IN_USE` or `502`. Each target now keeps up to 32
idle connections and reuses them, as a reverse proxy does; server-sent
events and WebSockets keep a connection of their own. The hub keeps its
connections to the daemons the same way, so a dashboard that stays open
no longer leaves a socket behind for every call.

## [0.49.1] - 2026-11-17

A static service forwards paths to another server, as a reverse proxy in
Expand Down
143 changes: 125 additions & 18 deletions crates/devstack-daemon/src/staticsrv.rs
Original file line number Diff line number Diff line change
Expand Up @@ -91,11 +91,32 @@ fn percent_decode(s: &str) -> String {
String::from_utf8_lossy(&out).into_owned()
}

/// Where a forward sends its requests: `http://host:port`.
#[derive(Debug, Clone, PartialEq)]
use devstack_node::connpool::{GiveBack, IDLE_MAX, Pool};

/// Where a forward sends its requests: `http://host:port`, with the
/// connections to it that are idle now. They are reused, as a reverse proxy
/// does: a connection per request would leave each one in `TIME_WAIT` on
/// this side, and an end-to-end suite drains the ephemeral ports (#556).
#[derive(Debug, Clone)]
pub struct Target {
host: String,
port: u16,
pool: Arc<Pool<(), Body>>,
}

impl Target {
/// A new connection.
async fn connect(&self) -> Result<hyper::client::conn::http1::SendRequest<Body>, String> {
let stream = tokio::net::TcpStream::connect((self.host.as_str(), self.port))
.await
.map_err(|e| e.to_string())?;
let _ = stream.set_nodelay(true);
let (sender, conn) = hyper::client::conn::http1::handshake(TokioIo::new(stream))
.await
.map_err(|e| e.to_string())?;
tokio::spawn(conn.with_upgrades());
Ok(sender)
}
}

/// The `--forward PREFIX=URL` arguments, checked as `devstack check` does.
Expand All @@ -109,7 +130,14 @@ pub fn forwards(args: &[String]) -> Result<Vec<(String, Target)>, String> {
forward::prefix(prefix).map_err(|e| format!("--forward `{prefix}`: {e}"))?;
let (host, port) =
forward::origin(url).map_err(|e| format!("--forward {prefix}: `{url}`: {e}"))?;
Ok((prefix.to_string(), Target { host, port }))
Ok((
prefix.to_string(),
Target {
host,
port,
pool: Pool::new(IDLE_MAX),
},
))
})
.collect()
}
Expand Down Expand Up @@ -210,20 +238,9 @@ async fn pass(to: &Target, peer: SocketAddr, mut req: Request) -> Response<Body>
*req.uri_mut() = path.parse().unwrap_or_else(|_| Uri::from_static("/"));
*req.version_mut() = Version::HTTP_11;

let stream = match tokio::net::TcpStream::connect((to.host.as_str(), to.port)).await {
Ok(s) => s,
Err(e) => return fail(e.to_string()),
};
let _ = stream.set_nodelay(true);
let (mut sender, conn) = match hyper::client::conn::http1::handshake(TokioIo::new(stream)).await
{
Ok(c) => c,
Err(e) => return fail(e.to_string()),
};
tokio::spawn(conn.with_upgrades());
let mut resp = match sender.send_request(req).await {
let (sender, mut resp) = match to.pool.send(&(), req, || to.connect()).await {
Ok(r) => r,
Err(e) => return fail(e.to_string()),
Err(e) => return fail(e),
};
let upgraded = resp.status() == StatusCode::SWITCHING_PROTOCOLS && client_upgrade.is_some();
crate::proxy::strip_hop_by_hop(resp.headers_mut(), upgraded);
Expand All @@ -235,11 +252,14 @@ async fn pass(to: &Target, peer: SocketAddr, mut req: Request) -> Response<Body>
let _ = tokio::io::copy_bidirectional(&mut c, &mut t).await;
}
});
// The upgraded connection is the WebSocket's now: not given back.
let (parts, _) = resp.into_parts();
return Response::from_parts(parts, Body::empty());
}
let (parts, body) = resp.into_parts();
Response::from_parts(parts, Body::new(body))
// Read to its end, the answer's connection takes the next request. One
// the browser leaves early (or a stream that does not end, SSE) keeps or
// drops its connection, never back in the pool.
GiveBack::wrap(to.pool.clone(), (), sender, resp).map(Body::new)
}

/// Whether a file's name carries a hash of its content (`main-3f2a1b9c.js`,
Expand Down Expand Up @@ -532,6 +552,93 @@ mod tests {
);
}

/// A target that counts the connections it accepts (#556).
async fn counting_target() -> (u16, Arc<std::sync::atomic::AtomicUsize>) {
use std::sync::atomic::Ordering;
let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
let port = l.local_addr().unwrap().port();
let accepted = Arc::new(std::sync::atomic::AtomicUsize::new(0));
let count = accepted.clone();
tokio::spawn(async move {
while let Ok((s, _)) = l.accept().await {
count.fetch_add(1, Ordering::SeqCst);
tokio::spawn(async move {
let svc = hyper::service::service_fn(|_req| async {
Ok::<_, std::convert::Infallible>(Response::new(http_body_util::Full::new(
bytes::Bytes::from("pong"),
)))
});
let _ = hyper::server::conn::http1::Builder::new()
.serve_connection(TokioIo::new(s), svc)
.await;
});
}
});
(port, accepted)
}

#[tokio::test]
async fn a_forward_reuses_its_connections_to_the_target() {
let d = tempfile::tempdir().unwrap();
std::fs::write(d.path().join("index.html"), "<app>").unwrap();
let (target, accepted) = counting_target().await;
let port = site(d.path(), &[format!("/api=http://127.0.0.1:{target}")]).await;
// One browser connection per request, closed after it, as a test
// suite's many pages do: the target's side must not follow suit.
for i in 0..30 {
let (_, r) = talk(
port,
format!("GET /api/ping/{i} HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n")
.as_bytes(),
"\0",
)
.await;
assert!(r.starts_with("HTTP/1.1 200") && r.ends_with("pong"), "{r}");
}
let n = accepted.load(std::sync::atomic::Ordering::SeqCst);
assert!(
n <= 2,
"30 requests one after another opened {n} connections to the target"
);
// Four at a time, as four test workers send them: about four.
for round in 0..10 {
let all = (0..4).map(|w| {
talk(
port,
format!(
"GET /api/{round}/{w} HTTP/1.1\r\nHost: x\r\nConnection: close\r\n\r\n"
)
.leak()
.as_bytes(),
"\0",
)
});
for (_, r) in futures_join(all).await {
assert!(r.ends_with("pong"), "{r}");
}
}
let n = accepted.load(std::sync::atomic::Ordering::SeqCst);
assert!(
n <= 8,
"70 requests, four at a time, opened {n} connections to the target"
);
}

/// Run the futures side by side, their results in order.
async fn futures_join<F: std::future::Future + Send + 'static>(
fs: impl Iterator<Item = F>,
) -> Vec<F::Output>
where
F::Output: Send + 'static,
{
let handles: Vec<_> = fs.map(tokio::spawn).collect();
let mut out = Vec::new();
for h in handles {
out.push(h.await.unwrap());
}
out
}

#[test]
fn a_forward_argument_is_checked_at_the_start() {
assert!(forwards(&["/api=http://127.0.0.1:5000".into()]).is_ok());
Expand Down
114 changes: 92 additions & 22 deletions crates/devstack-hub/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3615,7 +3615,7 @@ async fn daemon_get(
s: &AppState,
id: &str,
path: &str,
) -> Result<hyper::Response<hyper::body::Incoming>, DaemonError> {
) -> Result<hyper::Response<DaemonBody>, DaemonError> {
let req = hyper::Request::get(path)
.header(header::AUTHORIZATION, format!("Bearer {}", s.token))
.header(header::ACCEPT, "text/event-stream")
Expand All @@ -3635,10 +3635,23 @@ async fn daemon_get(
/// one the request went to, the answer is `503` with `Retry-After` and
/// what the stack does; only a daemon that still serves and broke the
/// exchange is a `502`.
/// An answer of a daemon; read to its end, its connection takes the hub's
/// next request to that daemon (#556: a connection per request leaves each
/// in `TIME_WAIT`, and a dashboard that polls adds them up).
type DaemonBody = devstack_node::connpool::GiveBack<u16, Body>;

/// The hub's idle connections to its daemons, by port.
fn daemon_pool() -> Arc<devstack_node::connpool::Pool<u16, Body>> {
static POOL: std::sync::OnceLock<Arc<devstack_node::connpool::Pool<u16, Body>>> =
std::sync::OnceLock::new();
POOL.get_or_init(|| devstack_node::connpool::Pool::new(devstack_node::connpool::IDLE_MAX))
.clone()
}

async fn daemon_send(
resolve: impl Fn() -> DaemonNow,
mut req: hyper::Request<Body>,
) -> Result<hyper::Response<hyper::body::Incoming>, DaemonError> {
) -> Result<hyper::Response<DaemonBody>, DaemonError> {
use hyper::body::Body as _;
let t0 = Instant::now();
// GET, HEAD, OPTIONS without a body: safe to repeat whatever happened.
Expand All @@ -3661,33 +3674,55 @@ async fn daemon_send(
HeaderValue::from_str(&format!("127.0.0.1:{port}")).unwrap(),
);
let in_window = || t0.elapsed() < RESEND_WINDOW;
let stream = match connect_daemon(port).await {
Ok(s) => s,
Err(e) => {
// Nothing was sent. A daemon that replaced this one is
// worth another try; otherwise it is not running.
let then = resolve();
if then != now && in_window() {
continue;
}
return Err(match then {
DaemonNow::Serves { .. } if then == now => {
DaemonError::unavailable(DaemonNow::Absent { starting: false }, Some(e))
let kept = daemon_pool().kept(&port);
let reused = kept.is_some();
let mut sender = match kept {
Some(s) => s,
None => {
let stream = match connect_daemon(port).await {
Ok(s) => s,
Err(e) => {
// Nothing was sent. A daemon that replaced this one is
// worth another try; otherwise it is not running.
let then = resolve();
if then != now && in_window() {
continue;
}
return Err(match then {
DaemonNow::Serves { .. } if then == now => DaemonError::unavailable(
DaemonNow::Absent { starting: false },
Some(e),
),
then => DaemonError::unavailable(then, Some(e)),
});
}
then => DaemonError::unavailable(then, Some(e)),
});
};
let (sender, conn) = hyper::client::conn::http1::handshake(TokioIo::new(stream))
.await
.map_err(|e| DaemonError::new(StatusCode::BAD_GATEWAY, e.to_string()))?;
tokio::spawn(conn);
sender
}
};
let (mut sender, conn) = hyper::client::conn::http1::handshake(TokioIo::new(stream))
.await
.map_err(|e| DaemonError::new(StatusCode::BAD_GATEWAY, e.to_string()))?;
tokio::spawn(conn);
let copy = repeatable.then(|| again(&req));
match sender.try_send_request(req).await {
Ok(resp) => return Ok(resp),
Ok(resp) => {
return Ok(devstack_node::connpool::GiveBack::wrap(
daemon_pool(),
port,
sender,
resp,
));
}
Err(mut e) => {
let then = resolve();
let unsent = e.take_message();
// A kept connection the daemon closed while it was idle hands
// the request back unsent: it goes out on a new one.
if reused && let Some(r) = unsent {
req = r;
continue;
}
let then = resolve();
let sent = unsent.is_none();
let error = e.into_error().to_string();
diag::warn(
Expand Down Expand Up @@ -4465,6 +4500,41 @@ mod daemon_send_tests {
assert_eq!(n.load(Ordering::SeqCst), 1);
}

#[tokio::test]
async fn the_hub_reuses_its_connections_to_a_daemon() {
// A daemon that keeps its connections open and counts them (#556).
let l = tokio::net::TcpListener::bind(("127.0.0.1", 0))
.await
.unwrap();
let port = l.local_addr().unwrap().port();
let accepted = Arc::new(AtomicUsize::new(0));
let count = accepted.clone();
tokio::spawn(async move {
while let Ok((c, _)) = l.accept().await {
count.fetch_add(1, Ordering::SeqCst);
tokio::spawn(async move {
let svc = hyper::service::service_fn(|_req| async {
Ok::<_, std::convert::Infallible>(hyper::Response::new(
http_body_util::Full::new(bytes::Bytes::from("{}")),
))
});
let _ = hyper::server::conn::http1::Builder::new()
.serve_connection(TokioIo::new(c), svc)
.await;
});
}
});
use http_body_util::BodyExt as _;
for _ in 0..20 {
let r = daemon_send(looks(vec![serves(port)]), get()).await.unwrap();
assert_eq!(r.status(), StatusCode::OK);
let body = r.into_body().collect().await.unwrap().to_bytes();
assert_eq!(&body[..], b"{}");
}
let n = accepted.load(Ordering::SeqCst);
assert!(n <= 2, "20 requests opened {n} connections to the daemon");
}

#[tokio::test]
async fn a_daemon_that_serves_and_breaks_the_exchange_is_a_502() {
let port = cutting().await;
Expand Down
Loading
Loading