diff --git a/CHANGELOG.md b/CHANGELOG.md index 2accf48a..35505fb6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,15 @@ and listed with their assets on the ### Added +- **A static service forwards paths to another server.** `forward = { + "/api" = "{service.api.url}" }` on a `kind = "static"` service, also in a + profile, sends requests below a prefix on as a reverse proxy in front of + the bundle would: the path unchanged, `Host` kept, `X-Forwarded-*` added, + answers streamed (server-sent events), WebSocket upgrades passed. A + single-page app that calls its API on its own origin now runs as a static + build in an acceptance profile without a proxy of its own (#551, + ADR-0110). Needs `requires = "0.49.0"`. + - **An order menu on the start page.** The same menu as the sidebar's (by name, last opened first, last edited first; favourites always first) sits in the filter row and orders the tiles and the list. The order is one diff --git a/crates/devstack-config/src/forward.rs b/crates/devstack-config/src/forward.rs new file mode 100644 index 00000000..eabd8641 --- /dev/null +++ b/crates/devstack-config/src/forward.rs @@ -0,0 +1,154 @@ +//! `forward` of a `kind = "static"` service (ADR-0110): which requests go to +//! another server, and where. The check and the static server share these +//! rules, so a file `devstack check` passes is one the server takes. + +/// A prefix such as `/api` or `/api/v2`: a path of plain segments, without a +/// trailing `/` (it matches at a segment boundary anyway). +pub fn prefix(p: &str) -> Result<(), String> { + let Some(rest) = p.strip_prefix('/') else { + return Err("a path prefix starts with `/`, e.g. `/api`".into()); + }; + if rest.is_empty() { + return Err( + "`/` would forward every request, the bundle's files too: name a prefix below it" + .into(), + ); + } + for seg in rest.split('/') { + if seg.is_empty() || seg == "." || seg == ".." { + return Err( + "plain path segments only, without `//`, `.`, `..` or a trailing `/`".into(), + ); + } + if let Some(c) = seg + .chars() + .find(|c| !(c.is_ascii_alphanumeric() || "-._~".contains(*c))) + { + return Err(format!("`{c}`: a prefix takes letters, digits and `-._~`")); + } + } + Ok(()) +} + +/// Whether request path `path` lies below `prefix`: the prefix itself, or +/// the prefix followed by `/` (`/api` takes `/api/x`, not `/apix`). +pub fn matches(prefix: &str, path: &str) -> bool { + path.strip_prefix(prefix) + .is_some_and(|rest| rest.is_empty() || rest.starts_with('/')) +} + +/// The forward whose prefix takes `path`: the longest one. +pub fn pick<'a, T>(forwards: &'a [(String, T)], path: &str) -> Option<&'a (String, T)> { + forwards + .iter() + .filter(|(p, _)| matches(p, path)) + .max_by_key(|(p, _)| p.len()) +} + +/// A target: an origin `http://host[:port]` (one trailing `/` allowed), +/// as host and port. +pub fn origin(t: &str) -> Result<(String, u16), String> { + if t.starts_with("https://") { + return Err( + "devstack forwards to plain HTTP only: give the target's `http://` port (a service with `health.insecure_tls` speaks HTTPS)".into(), + ); + } + let Some(rest) = t.strip_prefix("http://") else { + return Err("a target is an origin, `http://host:port`, e.g. `{service.api.url}`".into()); + }; + let authority = rest.strip_suffix('/').unwrap_or(rest); + if authority.is_empty() { + return Err("names no host: `http://host:port`".into()); + } + if authority.contains(['/', '?', '#', '@']) { + return Err( + "a target is an origin without a path: the request's path goes on unchanged".into(), + ); + } + let (host, port) = match authority.strip_prefix('[') { + // An IPv6 address, `[::1]:8080`. + Some(v6) => { + let (h, after) = v6 + .split_once(']') + .ok_or_else(|| "an unclosed `[` in the host".to_string())?; + (h, after.strip_prefix(':')) + } + None => match authority.rsplit_once(':') { + Some((h, p)) => (h, Some(p)), + None => (authority, None), + }, + }; + if host.is_empty() { + return Err("names no host: `http://host:port`".into()); + } + let port = match port { + None => 80, + Some(p) => p + .parse::() + .ok() + .filter(|p| *p > 0) + .ok_or_else(|| format!("`{p}` is not a port"))?, + }; + Ok((host.to_string(), port)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn prefixes() { + for ok in ["/api", "/api/v2", "/file", "/otel", "/a-b_c.d~e"] { + assert_eq!(prefix(ok), Ok(()), "{ok}"); + } + for bad in [ + "api", "/", "/api/", "//api", "/a/../b", "/a b", "/a?x", "/a%20", + ] { + assert!(prefix(bad).is_err(), "{bad}"); + } + } + + #[test] + fn the_longest_prefix_wins_at_a_segment_boundary() { + let f = vec![ + ("/api".to_string(), 1), + ("/api/files".to_string(), 2), + ("/file".to_string(), 3), + ]; + let to = |p: &str| pick(&f, p).map(|(_, n)| *n); + assert_eq!(to("/api"), Some(1)); + assert_eq!(to("/api/users/7"), Some(1)); + assert_eq!(to("/api/files"), Some(2)); + assert_eq!(to("/api/files/a.pdf"), Some(2)); + assert_eq!(to("/api/filesystem"), Some(1)); + assert_eq!(to("/apix"), None, "not at a segment boundary"); + assert_eq!(to("/files"), None); + assert_eq!(to("/"), None); + assert_eq!(to("/main.js"), None); + } + + #[test] + fn origins() { + assert_eq!( + origin("http://127.0.0.1:5000"), + Ok(("127.0.0.1".into(), 5000)) + ); + assert_eq!( + origin("http://127.0.0.1:5000/"), + Ok(("127.0.0.1".into(), 5000)) + ); + assert_eq!(origin("http://localhost"), Ok(("localhost".into(), 80))); + assert_eq!(origin("http://[::1]:8080"), Ok(("::1".into(), 8080))); + for bad in [ + "https://127.0.0.1:5001", + "127.0.0.1:5000", + "http://127.0.0.1:5000/api", + "http://127.0.0.1:x", + "http://:5000", + "http://", + "http://user@host:1", + ] { + assert!(origin(bad).is_err(), "{bad}"); + } + } +} diff --git a/crates/devstack-config/src/lib.rs b/crates/devstack-config/src/lib.rs index 3f3eb746..66b3ee81 100644 --- a/crates/devstack-config/src/lib.rs +++ b/crates/devstack-config/src/lib.rs @@ -2,6 +2,7 @@ //! interpolation. See ADR-0017 and plan §4. pub mod compat; +pub mod forward; pub mod interpolate; pub mod model; pub mod validate; @@ -79,6 +80,10 @@ pub const LINK_SINCE: &str = "0.44.0"; /// sets it wants `requires` at least this. pub const SHORTCUT_SINCE: &str = "0.48.0"; +/// The first devstack that knows a static service's `forward` (ADR-0110): a +/// file that sets it wants `requires` at least this. +pub const FORWARD_SINCE: &str = "0.49.0"; + /// `"1.2.3"`, `"v1.2"`, `"1"` as numbers; `None` when malformed. A /// pre-release or build suffix (`1.2.3-rc1`, `1.2.3+abc`) is ignored here; /// [`version_order`] takes it into account. diff --git a/crates/devstack-config/src/model.rs b/crates/devstack-config/src/model.rs index bf94bd40..febccf12 100644 --- a/crates/devstack-config/src/model.rs +++ b/crates/devstack-config/src/model.rs @@ -150,6 +150,9 @@ pub struct ServiceProfile { /// What ends what `run` started outside its process tree (ADR-0059). pub stop: Option, pub spa: Option, + /// Added to the service's `forward` (ADR-0110). + #[serde(default)] + pub forward: BTreeMap, pub cwd: Option, pub health: Option, /// Added to the service's `env`, over the profile's. @@ -286,6 +289,7 @@ impl Stack { } if k != Kind::Static { svc.spa = None; + svc.forward.clear(); } if k != Kind::Service { svc.hot_reload = None; @@ -313,6 +317,9 @@ impl Stack { if o.spa.is_some() { svc.spa = o.spa; } + for (k, v) in &o.forward { + svc.forward.insert(k.clone(), v.clone()); + } if o.cwd.is_some() { svc.cwd = o.cwd.clone(); } @@ -916,6 +923,13 @@ pub struct Service { /// `kind = "static"`: answer paths without a file extension with /// `index.html` (client-side routes of a single-page app). Default `true`. pub spa: Option, + /// `kind = "static"`: requests below a path prefix go to another server, + /// as a reverse proxy in front of the bundle would send them (ADR-0110): + /// `{ "/api" = "{service.api.url}" }`. The longest prefix wins, at a + /// segment boundary; the path goes on unchanged; a target is an origin + /// (`http://host:port`). + #[serde(default)] + pub forward: BTreeMap, /// Working directory relative to the repository. pub cwd: Option, /// Paths whose change starts a candidate build (ADR-0003). diff --git a/crates/devstack-config/src/validate.rs b/crates/devstack-config/src/validate.rs index 5e42f069..81bf27b7 100644 --- a/crates/devstack-config/src/validate.rs +++ b/crates/devstack-config/src/validate.rs @@ -209,6 +209,49 @@ impl Check<'_> { } } + /// A `forward` target (ADR-0110): its placeholders as in `env`, and an + /// origin `http://host:port` once they have a value. A `${VAR}` has one + /// only at the start, where the static server checks it. + fn forward_target(&mut self, path: &str, s: &str) { + self.text( + path, + s, + Scope::Service { + port: true, + artifact: true, + }, + ); + let Ok(segs) = interpolate::parse(s) else { + return; // reported above + }; + let mut probe = String::new(); + for seg in &segs { + match seg { + Segment::Literal(l) => probe.push_str(l), + Segment::Env { .. } => return, + Segment::Placeholder(ph) => probe.push_str(&match ph { + Placeholder::ServiceUrl(n) => match self.stack.services.get(n) { + Some(svc) => format!("{}://127.0.0.1:1", svc.scheme()), + None => return, // reported above + }, + // `https://` with a named tunnel: not a local origin. + Placeholder::ServicePublicUrl(_) => "https://public".into(), + Placeholder::ComposePort { part, .. } => part.of("127.0.0.1:1"), + Placeholder::ServicePort(_) | Placeholder::Port => "1".into(), + Placeholder::IsMain => "true".into(), + Placeholder::LaneSuffix => String::new(), + Placeholder::Repo + | Placeholder::Artifact + | Placeholder::Lane + | Placeholder::ComposeProject(_) => "/path".into(), + }), + } + } + if let Err(why) = crate::forward::origin(&probe) { + self.err(path, format!("`{s}`: {why}")); + } + } + fn compose_service(&mut self, path: &str, what: &str, group: &str, service: &str) { match self.stack.compose.get(group) { None => self.err(path, format!("`{what}`: no compose group `{group}`")), @@ -267,6 +310,11 @@ fn skipped_needs(c: &mut Check, stack: &Stack) { .enumerate() .map(|(i, d)| (format!("{p}.ensure_dirs[{i}]"), d.as_str())), ); + texts.extend( + svc.forward + .iter() + .map(|(k, t)| (format!("{p}.forward.\"{k}\""), t.as_str())), + ); for (at, t) in texts { let Ok(segs) = interpolate::parse(t) else { continue; // reported where the text is checked @@ -654,6 +702,19 @@ pub fn validate(stack: &Stack) -> Vec { if svc.spa.is_some() && svc.kind != Kind::Static { c.err(format!("{p}.spa"), "`spa` belongs to `kind = \"static\"`"); } + if !svc.forward.is_empty() && svc.kind != Kind::Static { + c.err( + format!("{p}.forward"), + "`forward` belongs to `kind = \"static\"`: another kind's own server forwards (a dev server's proxy config)", // ADR-0110 + ); + } + for (prefix, target) in &svc.forward { + let at = format!("{p}.forward.\"{prefix}\""); + if let Err(why) = crate::forward::prefix(prefix) { + c.err(&at, format!("`{prefix}`: {why}")); + } + c.forward_target(&at, target); + } if svc.hot_reload.is_some() && (svc.kind != Kind::Service || svc.build.is_none()) { c.err( format!("{p}.hot_reload"), @@ -1334,6 +1395,36 @@ pub fn warnings(stack: &Stack) -> Vec { }); } } + let forward = crate::version_order(crate::FORWARD_SINCE); + if stack + .stack + .requires + .as_deref() + .is_none_or(|r| crate::version_order(r) < forward) + { + let profiled = stack.profiles.iter().flat_map(|(p, prof)| { + prof.services + .iter() + .filter(|(_, s)| !s.forward.is_empty()) + .map(move |(n, _)| format!("profiles.{p}.services.{n}.forward")) + }); + let paths: Vec = stack + .services + .iter() + .filter(|(_, s)| !s.forward.is_empty()) + .map(|(n, _)| format!("services.{n}.forward")) + .chain(profiled) + .collect(); + for path in paths { + out.push(Issue { + path, + message: format!( + "a devstack before {v} does not know this key and fails on it with \"unknown field\": `requires = \"{v}\"` makes it ask to be updated instead", + v = crate::FORWARD_SINCE + ), + }); + } + } if step == 0 { return out; } diff --git a/crates/devstack-config/tests/profiles.rs b/crates/devstack-config/tests/profiles.rs index 2969b309..04a3a690 100644 --- a/crates/devstack-config/tests/profiles.rs +++ b/crates/devstack-config/tests/profiles.rs @@ -319,3 +319,110 @@ output = "dist" assert_eq!(a.services["ocr"].stop, None); assert!(s.services["mailer"].stop.is_none()); } + +#[test] +fn a_static_profile_forwards_paths_to_the_api() { + use devstack_config::{FORWARD_SINCE, validate::warnings}; + let s = load( + r#" +[compose.app] +project = "shop" +files = ["compose.yml"] +services = ["otel"] + +[profiles.acceptance.services.web] +kind = "static" +build = "npm run build" +output = "dist/web" +forward = { "/api" = "{service.api.url}", "/file" = "{service.api.url}", "/otel" = "{compose.app.otel.4318}" } +"#, + ) + .unwrap(); + let a = s.with_profile("acceptance").unwrap(); + let web = &a.services["web"]; + assert_eq!(web.forward.len(), 3); + assert_eq!(web.forward["/api"], "{service.api.url}"); + assert!( + s.services["web"].forward.is_empty(), + "the dev server keeps its own proxy" + ); + let w = warnings(&s); + assert!( + w.iter() + .any(|i| i.path == "profiles.acceptance.services.web.forward" + && i.message.contains(FORWARD_SINCE)), + "{w:?}" + ); +} + +#[test] +fn a_forward_is_checked() { + let static_web = |forward: &str| { + load(&format!( + r#" +[compose.app] +project = "shop" +files = ["compose.yml"] +services = ["otel"] + +[services.secure] +run = "secure --port {{port}}" +port = 5443 +health = {{ http = "/health", timeout = "30s", insecure_tls = true }} + +[profiles.x] +skip = ["compose.app.otel"] + +[profiles.x.services.web] +kind = "static" +build = "npm run build" +output = "dist/web" +forward = {forward} +"# + )) + .map(|_| ()) + .map_err(|e| e.to_string()) + }; + assert_eq!(static_web(r#"{ "/api" = "{service.api.url}" }"#), Ok(())); + assert_eq!( + static_web(r#"{ "/api" = "http://127.0.0.1:5000/" }"#), + Ok(()) + ); + assert_eq!( + static_web(r#"{ "/api" = "${API_URL}" }"#), + Ok(()), + "known at the start only" + ); + for (forward, says) in [ + (r#"{ "api" = "{service.api.url}" }"#, "starts with `/`"), + (r#"{ "/" = "{service.api.url}" }"#, "every request"), + (r#"{ "/api/" = "{service.api.url}" }"#, "trailing"), + (r#"{ "/api" = "{service.api.url}/v1" }"#, "without a path"), + (r#"{ "/api" = "{service.secure.url}" }"#, "plain HTTP"), + ( + r#"{ "/api" = "{service.ghost.url}" }"#, + "no service `ghost`", + ), + (r#"{ "/api" = "{service.mailer.url}" }"#, "has no port"), + ( + r#"{ "/otel" = "{compose.app.otel.4318}" }"#, + "leaves `compose.app.otel` out", + ), + ] { + let e = static_web(forward).unwrap_err(); + assert!(e.contains("forward") && e.contains(says), "{forward}: {e}"); + } + // Only a static service forwards. + let e = load( + r#" +[profiles.x.services.web] +forward = { "/api" = "{service.api.url}" } +"#, + ) + .unwrap_err() + .to_string(); + assert!( + e.contains("services.web.forward") && e.contains("belongs to `kind = \"static\"`"), + "{e}" + ); +} diff --git a/crates/devstack-daemon/src/staticsrv.rs b/crates/devstack-daemon/src/staticsrv.rs index ee33b0c3..68f12932 100644 --- a/crates/devstack-daemon/src/staticsrv.rs +++ b/crates/devstack-daemon/src/staticsrv.rs @@ -3,10 +3,19 @@ //! the service's process (`devstack serve-static --port

`), so the //! candidate pipeline, the proxy and the health check treat it like any //! service, and an end-to-end run pins a build id. +//! +//! Its `forward`s (ADR-0110) send requests below a path prefix to another +//! server, as a reverse proxy in front of the bundle would in production: +//! a single-page app that calls its API at a relative `/api` finds it on its +//! own origin. use axum::body::Body; -use axum::http::{HeaderMap, HeaderValue, Response, StatusCode, Uri, header}; +use axum::extract::{ConnectInfo, Request}; +use axum::http::{HeaderValue, Response, StatusCode, Uri, Version, header}; +use hyper_util::rt::TokioIo; +use std::net::SocketAddr; use std::path::{Component, Path, PathBuf}; +use std::sync::Arc; fn content_type(path: &Path) -> &'static str { match path @@ -82,28 +91,155 @@ fn percent_decode(s: &str) -> String { String::from_utf8_lossy(&out).into_owned() } -/// Serve `root` on 127.0.0.1:`port` until the process ends. -pub fn serve(root: PathBuf, port: u16, spa: bool) -> Result<(), String> { +/// Where a forward sends its requests: `http://host:port`. +#[derive(Debug, Clone, PartialEq)] +pub struct Target { + host: String, + port: u16, +} + +/// The `--forward PREFIX=URL` arguments, checked as `devstack check` does. +pub fn forwards(args: &[String]) -> Result, String> { + use devstack_config::forward; + args.iter() + .map(|a| { + let (prefix, url) = a + .split_once('=') + .ok_or_else(|| format!("--forward `{a}`: takes PREFIX=URL"))?; + forward::prefix(prefix).map_err(|e| format!("--forward `{prefix}`: {e}"))?; + let (host, port) = + forward::origin(url).map_err(|e| format!("--forward {prefix}: `{url}`: {e}"))?; + Ok((prefix.to_string(), Target { host, port })) + }) + .collect() +} + +/// Serve `root` on 127.0.0.1:`port` until the process ends; requests below +/// a prefix of `forward` (`PREFIX=URL`) go to its target. +pub fn serve(root: PathBuf, port: u16, spa: bool, forward: &[String]) -> Result<(), String> { if !root.join("index.html").is_file() && spa { return Err(format!("{} has no index.html", root.display())); } + let forwards = forwards(forward)?; let rt = tokio::runtime::Runtime::new().map_err(|e| e.to_string())?; rt.block_on(async move { let listener = tokio::net::TcpListener::bind(("127.0.0.1", port)) .await .map_err(|e| format!("port {port}: {e}"))?; - let app = axum::Router::new().fallback(move |uri: Uri, headers: HeaderMap| { + axum::serve( + listener, + app(root, spa, forwards).into_make_service_with_connect_info::(), + ) + .await + .map_err(|e| e.to_string()) + }) +} + +fn app(root: PathBuf, spa: bool, forwards: Vec<(String, Target)>) -> axum::Router { + let forwards = Arc::new(forwards); + axum::Router::new().fallback( + move |ConnectInfo(peer): ConnectInfo, req: Request| { let root = root.clone(); + let forwards = forwards.clone(); async move { - let tag = headers + // A forwarded prefix never falls back to a file. + if let Some((_, to)) = devstack_config::forward::pick(&forwards, req.uri().path()) { + return pass(to, peer, req).await; + } + let tag = req + .headers() .get(header::IF_NONE_MATCH) .and_then(|v| v.to_str().ok()) .map(str::to_string); - file(&root, uri.path(), spa, tag.as_deref()).await + file(&root, req.uri().path(), spa, tag.as_deref()).await + } + }, + ) +} + +/// Send `req` on to `to`, as a reverse proxy does: the path unchanged, +/// `Host` as the browser sent it, `X-Forwarded-*` saying where it came from +/// (kept where a proxy in front, a tunnel, set them), hop-by-hop headers on +/// their hop, the answer streamed (SSE, a download) and an upgrade +/// (WebSocket) joined. An unreachable target is a 502 at once: nothing here +/// knows whether it is starting. +async fn pass(to: &Target, peer: SocketAddr, mut req: Request) -> Response { + let fail = |e: String| { + Response::builder() + .status(StatusCode::BAD_GATEWAY) + .header(header::CONTENT_TYPE, "text/plain; charset=utf-8") + .body(Body::from(format!( + "devstack: forward to http://{}:{}: {e}\n", + to.host, to.port + ))) + .unwrap() + }; + let client_upgrade = req + .headers() + .contains_key(header::UPGRADE) + .then(|| hyper::upgrade::on(&mut req)); + let host = req + .headers() + .get(header::HOST) + .cloned() + .or_else(|| req.uri().authority().and_then(|a| a.as_str().parse().ok())); + let h = req.headers_mut(); + crate::proxy::strip_hop_by_hop(h, client_upgrade.is_some()); + let from = match h.get("x-forwarded-for").and_then(|v| v.to_str().ok()) { + Some(before) => format!("{before}, {}", peer.ip()), + None => peer.ip().to_string(), + }; + if let Ok(v) = from.parse() { + h.insert("x-forwarded-for", v); + } + if !h.contains_key("x-forwarded-proto") { + h.insert("x-forwarded-proto", HeaderValue::from_static("http")); + } + if let Some(host) = host { + if !h.contains_key("x-forwarded-host") { + h.insert("x-forwarded-host", host.clone()); + } + h.insert(header::HOST, host); + } + // HTTP/1.1 to the target, the request target in origin form. + let path = req + .uri() + .path_and_query() + .map_or("/", |p| p.as_str()) + .to_string(); + *req.uri_mut() = path.parse().unwrap_or_else(|_| Uri::from_static("/")); + *req.version_mut() = Version::HTTP_11; + + let stream = match tokio::net::TcpStream::connect((to.host.as_str(), to.port)).await { + Ok(s) => s, + Err(e) => return fail(e.to_string()), + }; + let _ = stream.set_nodelay(true); + let (mut sender, conn) = match hyper::client::conn::http1::handshake(TokioIo::new(stream)).await + { + Ok(c) => c, + Err(e) => return fail(e.to_string()), + }; + tokio::spawn(conn.with_upgrades()); + let mut resp = match sender.send_request(req).await { + Ok(r) => r, + Err(e) => return fail(e.to_string()), + }; + let upgraded = resp.status() == StatusCode::SWITCHING_PROTOCOLS && client_upgrade.is_some(); + crate::proxy::strip_hop_by_hop(resp.headers_mut(), upgraded); + if upgraded && let Some(client_upgrade) = client_upgrade { + let target_upgrade = hyper::upgrade::on(&mut resp); + tokio::spawn(async move { + if let (Ok(c), Ok(t)) = (client_upgrade.await, target_upgrade.await) { + let (mut c, mut t) = (TokioIo::new(c), TokioIo::new(t)); + let _ = tokio::io::copy_bidirectional(&mut c, &mut t).await; } }); - axum::serve(listener, app).await.map_err(|e| e.to_string()) - }) + let (parts, _) = resp.into_parts(); + return Response::from_parts(parts, Body::empty()); + } + let (parts, body) = resp.into_parts(); + Response::from_parts(parts, Body::new(body)) } /// Whether a file's name carries a hash of its content (`main-3f2a1b9c.js`, @@ -244,6 +380,171 @@ mod tests { assert!(!matches_etag("\"a\"", "\"b\"")); } + /// The API behind a forward: echoes what it got, streams an event that + /// never ends, and echoes on a WebSocket. + async fn api() -> u16 { + use axum::extract::ws::{Message, WebSocketUpgrade}; + use axum::routing::{any, get}; + let echo = |req: Request| async move { + let h = |n: &str| { + req.headers() + .get(n) + .and_then(|v| v.to_str().ok()) + .unwrap_or("-") + .to_string() + }; + format!( + "{} {} host={} xff={} xfh={} xfp={}", + req.method(), + req.uri(), + h("host"), + h("x-forwarded-for"), + h("x-forwarded-host"), + h("x-forwarded-proto") + ) + }; + let sse = || async { + use tokio_stream::StreamExt; + let first = + tokio_stream::once(Ok::<_, std::io::Error>(bytes::Bytes::from("data: one\n\n"))); + Response::builder() + .header(header::CONTENT_TYPE, "text/event-stream") + .body(Body::from_stream(first.chain(tokio_stream::pending()))) + .unwrap() + }; + let ws = |up: WebSocketUpgrade| async { + up.on_upgrade(|mut s| async move { + while let Some(Ok(Message::Text(t))) = s.recv().await { + let _ = s.send(Message::Text(t)).await; + } + }) + }; + let app = axum::Router::new() + .route("/api/sse", get(sse)) + .route("/api/ws", get(ws)) + .fallback(any(echo)); + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = l.local_addr().unwrap().port(); + tokio::spawn(async move { axum::serve(l, app).await }); + port + } + + async fn site(root: &Path, forward: &[String]) -> u16 { + let app = app(root.to_path_buf(), true, forwards(forward).unwrap()); + let l = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = l.local_addr().unwrap().port(); + tokio::spawn(async move { + axum::serve(l, app.into_make_service_with_connect_info::()).await + }); + port + } + + /// Send `raw` and read until `until` shows up (or the connection ends). + async fn talk(port: u16, raw: &[u8], until: &str) -> (tokio::net::TcpStream, String) { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let mut c = tokio::net::TcpStream::connect(("127.0.0.1", port)) + .await + .unwrap(); + c.write_all(raw).await.unwrap(); + let mut got = Vec::new(); + let mut buf = [0u8; 4096]; + let read = async { + while !String::from_utf8_lossy(&got).contains(until) { + match c.read(&mut buf).await.unwrap() { + 0 => break, + n => got.extend_from_slice(&buf[..n]), + } + } + }; + tokio::time::timeout(std::time::Duration::from_secs(10), read) + .await + .expect("an answer in time"); + let text = String::from_utf8_lossy(&got).into_owned(); + (c, text) + } + + #[tokio::test] + async fn a_forward_reaches_the_api_on_the_sites_origin() { + let d = tempfile::tempdir().unwrap(); + std::fs::write(d.path().join("index.html"), "").unwrap(); + let api = api().await; + let closed = { + let l = std::net::TcpListener::bind("127.0.0.1:0").unwrap(); + l.local_addr().unwrap().port() + }; + let port = site( + d.path(), + &[ + format!("/api=http://127.0.0.1:{api}"), + format!("/gone=http://127.0.0.1:{closed}/"), + ], + ) + .await; + let get = |path: &str| { + format!("GET {path} HTTP/1.1\r\nHost: shop.test:{port}\r\nConnection: close\r\n\r\n") + }; + + // The path and query go on unchanged, Host as sent, X-Forwarded-*. + let (_, r) = talk(port, get("/api/users/7?q=a%20b").as_bytes(), "\0").await; + assert!(r.starts_with("HTTP/1.1 200"), "{r}"); + assert!( + r.contains(&format!( + "GET /api/users/7?q=a%20b host=shop.test:{port} xff=127.0.0.1 xfh=shop.test:{port} xfp=http" + )), + "{r}" + ); + // Not at a segment boundary, or not below a prefix: the app. + for path in ["/apix", "/customers/42", "/"] { + let (_, r) = talk(port, get(path).as_bytes(), "\0").await; + assert!(r.contains(""), "{path}: {r}"); + } + // A server-sent event arrives while its stream is still open. + let (_, r) = talk(port, get("/api/sse").as_bytes(), "data: one").await; + assert!( + r.contains("text/event-stream") && r.contains("data: one"), + "{r}" + ); + // A WebSocket is joined through. + let upgrade = format!( + "GET /api/ws HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nConnection: Upgrade\r\nUpgrade: websocket\r\nSec-WebSocket-Version: 13\r\nSec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n\r\n" + ); + let (mut c, r) = talk(port, upgrade.as_bytes(), "\r\n\r\n").await; + assert!(r.starts_with("HTTP/1.1 101"), "{r}"); + { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let mask = [1u8, 2, 3, 4]; + let mut frame = vec![0x81, 0x80 | 2]; + frame.extend_from_slice(&mask); + frame.extend(b"hi".iter().zip(mask).map(|(b, m)| b ^ m)); + c.write_all(&frame).await.unwrap(); + let mut back = [0u8; 4]; + tokio::time::timeout(std::time::Duration::from_secs(10), c.read_exact(&mut back)) + .await + .expect("the echo in time") + .unwrap(); + assert_eq!(back, [0x81, 2, b'h', b'i']); + } + // A target that does not answer: 502 at once, naming it. + let (_, r) = talk(port, get("/gone/x").as_bytes(), "\0").await; + assert!( + r.starts_with("HTTP/1.1 502") && r.contains(&format!("127.0.0.1:{closed}")), + "{r}" + ); + } + + #[test] + fn a_forward_argument_is_checked_at_the_start() { + assert!(forwards(&["/api=http://127.0.0.1:5000".into()]).is_ok()); + for bad in [ + "/api", + "api=http://127.0.0.1:5000", + "/api=https://127.0.0.1:5001", + "/api=http://127.0.0.1:5000/v1", + ] { + assert!(forwards(&[bad.into()]).is_err(), "{bad}"); + } + } + #[tokio::test] async fn an_unchanged_file_is_not_sent_again() { let d = tempfile::tempdir().unwrap(); diff --git a/crates/devstack-daemon/src/supervisor/mod.rs b/crates/devstack-daemon/src/supervisor/mod.rs index a02c38a7..556377dd 100644 --- a/crates/devstack-daemon/src/supervisor/mod.rs +++ b/crates/devstack-daemon/src/supervisor/mod.rs @@ -1305,7 +1305,8 @@ impl Supervisor { /// `kind = "static"` services run devstack's own file server on their /// artifact (plan §3.4); with that command they are ordinary services for the -/// pipeline, the proxy and the health check. +/// pipeline, the proxy and the health check. Its forwards (ADR-0110) go in +/// as written: their placeholders get their values like the rest of `run`. fn with_static_servers(mut stack: Stack) -> Stack { let exe = std::env::current_exe() .map(|p| p.to_string_lossy().into_owned()) @@ -1320,6 +1321,10 @@ fn with_static_servers(mut stack: Stack) -> Stack { "" } ); + for (prefix, target) in &svc.forward { + svc.run + .push_str(&format!(" --forward \"{prefix}={target}\"")); + } } } stack diff --git a/crates/devstack/src/dispatch/internal.rs b/crates/devstack/src/dispatch/internal.rs index 953346ec..15f0e38e 100644 --- a/crates/devstack/src/dispatch/internal.rs +++ b/crates/devstack/src/dispatch/internal.rs @@ -8,9 +8,13 @@ use std::path::Path; pub(crate) fn run(file: &Path, cmd: Cmd) -> Result<()> { match cmd { Cmd::Daemon => devstack_daemon::serve(file).map_err(|e| anyhow::anyhow!(e)), - Cmd::ServeStatic { dir, port, no_spa } => { - devstack_daemon::staticsrv::serve(dir, port, !no_spa).map_err(|e| anyhow::anyhow!(e)) - } + Cmd::ServeStatic { + dir, + port, + no_spa, + forward, + } => devstack_daemon::staticsrv::serve(dir, port, !no_spa, &forward) + .map_err(|e| anyhow::anyhow!(e)), Cmd::Shim { args } => { #[cfg(unix)] devstack_proc::shim_main(&args); diff --git a/crates/devstack/src/main.rs b/crates/devstack/src/main.rs index 2211d36b..a1209d14 100644 --- a/crates/devstack/src/main.rs +++ b/crates/devstack/src/main.rs @@ -237,6 +237,10 @@ enum Cmd { /// Without the single-page-app fallback to index.html. #[arg(long)] no_spa: bool, + /// `PREFIX=http://host:port`: requests below the prefix go there + /// (a `forward` of the service); repeatable. + #[arg(long, value_name = "PREFIX=URL")] + forward: Vec, }, /// The builds of a service, newest first: which went live, failed, were /// replaced by a newer change or wait for promote. @@ -1426,6 +1430,15 @@ mod tests { .. } )); + assert!(matches!( + parse(&[ + "serve-static", "dist", "--port", "8080", + "--forward", "/api=http://127.0.0.1:5000", + "--forward", "/file=http://127.0.0.1:5000", + ]) + .cmd, + Cmd::ServeStatic { forward, .. } if forward.len() == 2 + )); assert!(matches!( parse(&["shim", "run", "--flag", "-x"]).cmd, Cmd::Shim { args } if args == ["run", "--flag", "-x"] diff --git a/crates/devstack/tests/daemon_lifecycle.rs b/crates/devstack/tests/daemon_lifecycle.rs index a663b771..ae75c969 100644 --- a/crates/devstack/tests/daemon_lifecycle.rs +++ b/crates/devstack/tests/daemon_lifecycle.rs @@ -139,6 +139,7 @@ build = "git --version" output = "src/Site" watch = ["src/Site/**"] port = {site_port} +forward = {{ "/info" = "{{service.api.url}}" }} [commands.ping] description = "health" @@ -597,6 +598,9 @@ fn lifecycle(with_compose: bool) { get_on(s.site_port, "/assets/missing.js").is_none(), "a missing file is a 404" ); + // Its `forward` (ADR-0110): the API on the site's own origin. + let info = get_on(s.site_port, "/info").unwrap_or_default(); + assert!(info.contains("\"pid\""), "the API through the site: {info}"); let status = s.ok(&["status"]); assert!(status.contains("healthy"), "{status}"); assert!( diff --git a/docs/adr/0110-a-static-service-forwards-paths-to-another-server.md b/docs/adr/0110-a-static-service-forwards-paths-to-another-server.md new file mode 100644 index 00000000..c930acf7 --- /dev/null +++ b/docs/adr/0110-a-static-service-forwards-paths-to-another-server.md @@ -0,0 +1,93 @@ +# ADR-0110: A static service forwards paths to another server, as a reverse proxy in front of it would + +- **Status:** Accepted (card in the pull request); built +- **Date:** 2026-11-17 +- **Deciders:** project owner +- **Related:** [ADR-0041](0041-a-profile-says-how-a-lane-runs-for-a-purpose.md) (profiles), [ADR-0023](0023-stack-file-compatibility.md) (a new key is optional), [ADR-0010](0010-promotion-mechanism.md) (the service proxy whose header handling it shares), [ADR-0058](0058-a-profile-can-leave-out-containers-and-services.md) (`skip`), [ADR-0090](0090-a-service-can-get-a-public-link-by-hand.md) (public links), issue #551 + +## Context + +Profiles (ADR-0041) are meant to replace a hand-written acceptance setup. +A single-page app that calls its API on its own origin (a relative `/api`, +as in production behind a reverse proxy) did not fit: `kind = "static"` +served the bundle's files only, so every `/api/...` request got the app's +`index.html` or a 404. In development the dev server's proxy config covers +this (`ng serve --proxy-config`, Vite `server.proxy`); a profile that turns +that dev server into `kind = "static"` lost the proxy, and the stack file +had nothing to put in its place. A downstream stack kept a separate Compose +stack with a reverse proxy in front only for this. + +## Decision + +The owner approved the review card in the conversation on **2026-11-17**, +all six points: + +| # | Category | Point | Beats | Reversing costs | +|---|---|---|---|---| +| 1 | contract | A new optional key `forward` maps a path prefix to a target (`{ "/api" = "{service.api.url}" }`); only for `kind = "static"`, also in `[profiles.

.services.]`; elsewhere a check error; `FORWARD_SINCE` with its warning for an older `requires`. | A proxy service of its own per repository (Caddy, Node); an absolute API URL plus CORS | A published key goes only as a deprecated alias (ADR-0023) | +| 2 | concept | The longest prefix wins, at a segment boundary only (`/api` takes `/api`, `/api/…`, `/api?…`, not `/apix`); path and query go on unchanged; a forwarded path never falls back to a file or `index.html`. | A plain string prefix that would take `/apix` routes | Low while no one uses it | +| 3 | contract | A target is an origin `http://host:port`, without a path or https; it takes the placeholders of `env` and gets their values when the instance starts; `check` refuses a path, a service with `insecure_tls` and a container the profile `skip`s. | Path rewriting (`strip`) and TLS to the target now | Loosening later is compatible; tightening would be a break | +| 4 | trust | As a reverse proxy: `Host` stays as the browser sent it, `X-Forwarded-For`/`-Host`/`-Proto` are added, hop-by-hop headers stay on their hop, answers are streamed (SSE, chunked), a WebSocket upgrade passes; as the service proxy (ADR-0010). | Rewriting `Host` to the target (Vite's `changeOrigin`) | A header change that targets checking `Host` notice | +| 5 | concept | An unreachable target is a `502` at once, naming the target; no holding. The static service's health check stays on its own files, not the targets. | Holding requests until the target is ready, as the service proxy does | Adding holding later is compatible | +| 6 | trust | Forwards are reachable wherever the static port is: the lane's port, and a public link (`devstack tunnel`) to the static service reaches the API too, as in production behind a reverse proxy. | Limiting the tunnel to the files, which breaks the app there | Switching forwards off in the tunnel: a visible change for whoever shares that way | + +As built: + +1. **The key** (`crates/devstack-config`): `Service::forward` and + `ServiceProfile::forward` (`BTreeMap`); a profile's + entries are added to the service's, and a profile that changes the kind + away from `static` drops them, as it drops `spa`. The rules live in + `forward.rs` (`prefix`, `matches`, `pick`, `origin`), shared by the check + and the server. `validate` checks each prefix, checks each target's + placeholders as `env`'s, and, where no `${VAR}` leaves it open, its form + with a probe value per placeholder (`{service.x.url}` with the service's + scheme, so one with `insecure_tls` is refused). A skipped container's + `{compose.…}` in a target is an error, as in `env`. + `FORWARD_SINCE = "0.49.0"`, with its warning for the service's and each + profile's `forward`. +2. **The server.** `devstack serve-static` takes `--forward PREFIX=URL` + (repeatable) and checks it with the same rules at the start. The daemon + appends one per entry to the static service's generated `run` + (`with_static_servers`), so its placeholders get their values like the + rest of `run`, per lane. +3. **Passing a request** (`staticsrv::pass`): a new HTTP/1.1 connection to + the target per request, the request target in origin form, the client's + `Host` kept (or taken from an HTTP/2 authority), `X-Forwarded-For` + appended to, `X-Forwarded-Host` and `-Proto` set where a proxy in front + (a tunnel) has not set them; `proxy::strip_hop_by_hop` both ways; the + answer's body passed as a stream; a `101` joins both upgraded + connections. + +Decided while building, not on the card: + +- The forward runs inside the static service's own process, not in the + daemon: it starts, stops and moves with the instance. +- No connection pool to the target: one connection per request is enough + on loopback and keeps the server small. +- A target without a port is port 80; an IPv6 host is written in brackets. +- Prefixes take letters, digits and `-._~` only, so they need no quoting in + the generated command line. + +## Alternatives considered + +- **A proxy service per repository** (`kind = "service"` running Caddy or a + small Node server in front of the build output): works today, but every + repository writes the same proxy again, and the stack file no longer says + what the service is. +- **An absolute API URL in the build plus CORS on the API:** changes the + artifact under test and the API's configuration; the run no longer tests + what ships. +- **Forwarding in the daemon's service proxy** (ADR-0010) instead of the + static server: the proxy sits only in front of a service with `proxy` + on, and the forwards belong to the build's way of being served, not to + the switch between instances. + +## Consequences + +- A profile can turn a dev server with a proxy config into a static build + without a second stack for the reverse proxy. +- A target that is still starting answers `502` until it is up; an + end-to-end suite started with `--run` waits for the tree to be ready, so + it does not see that. +- HTTPS targets and path rewriting are not there; both can be added without + breaking a file that works now. diff --git a/docs/adr/README.md b/docs/adr/README.md index f4bbef48..52760625 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -192,3 +192,4 @@ stay. | [0107](0107-the-start-page-has-a-search-and-a-state-filter.md) | The start page has a search by name or folder and a filter by state | Accepted (card in the pull request); built | | [0108](0108-the-start-page-has-large-and-small-tiles-and-a-list.md) | The start page shows the stacks as large tiles, small tiles or a sortable list | Accepted (card in the pull request); built | | [0109](0109-the-order-of-the-stacks-is-one-choice-for-the-sidebar-and-the-start-page.md) | The order of the stacks is one choice for the sidebar and the start page | Accepted (card in the pull request); built | +| [0110](0110-a-static-service-forwards-paths-to-another-server.md) | A static service forwards paths to another server, as a reverse proxy in front of it would | Accepted (card in the pull request); built | diff --git a/docs/stack-file.md b/docs/stack-file.md index 8b4227de..07a1a019 100644 --- a/docs/stack-file.md +++ b/docs/stack-file.md @@ -239,8 +239,8 @@ Rules `devstack check` enforces: - `service`, `job`: `build` and `output` go together (both or neither). - `devserver`, `tool`: no `build`, no `output`. -- `static`: needs `build`, `output` and `port`, has no `run`; `spa` belongs - to `static` only. +- `static`: needs `build`, `output` and `port`, has no `run`; `spa` and + `forward` belong to `static` only. - Every other kind needs `run`. - `job`: no `port`, no `health`. `tool`: no `health`. - `health` needs `port`; `proxy = true` only for a `service`, `static` or @@ -275,9 +275,41 @@ file extension get `index.html` (client-side routes) unless `spa = false`; a file whose name carries its content hash (`index-Bd3x9_kQ.js`, `main.3f2a1b9c.css`) may be cached for a year, every other file (`index.html`, `config.json`, a manifest) is sent with `no-cache` and an -`ETag`, so a promoted build shows at the next load. The API the bundle -calls must be reachable from the browser as the bundle expects it (an -absolute URL, or the same origin through your own proxy setup). +`ETag`, so a promoted build shows at the next load. + +A bundle that calls its API on its own origin (a relative `/api`, as behind +a reverse proxy in production) names where those paths go with `forward` +([ADR-0110](adr/0110-a-static-service-forwards-paths-to-another-server.md), +devstack 0.49.0 and later): + +```toml +forward = { "/api" = "{service.api.url}", "/file" = "{service.api.url}", "/otel" = "{compose.app.otel-collector.4318}" } +``` + +- A request whose path is a prefix or lies below it (`/api`, `/api/…`, + `/api?…`; not `/apix`) goes to the target; the longest prefix wins. The + path and query go on unchanged, and a forwarded path never gets a file or + `index.html`. +- A prefix is a path of plain segments (letters, digits, `-._~`) without a + trailing `/`; `/` alone is not one. +- A target is an origin, `http://host:port`, without a path; it takes the + placeholders of `env` (`{service..url}`, `{compose.…}`, `${VAR}`) + and gets their values when the site starts. HTTPS targets (a service with + `health.insecure_tls`) are not supported; `devstack check` says so, and + refuses a container the profile `skip`s. +- As a reverse proxy: `Host` stays as the browser sent it, `X-Forwarded-For`, + `-Host` and `-Proto` are added (kept where a proxy in front set them), the + answer is streamed (server-sent events, downloads) and a WebSocket upgrade + passes. A target that does not answer is a `502` naming it, at once. +- Wherever the site's port is reachable, so are its forwards: a public link + to the site (`devstack tunnel`) reaches the API through it too. + +`forward` belongs to `kind = "static"` (a dev server has its own proxy +config) and can be set in a profile +(`[profiles..services.]`), where it adds to the service's. +Without `forward`, the API must be reachable from the browser as the bundle +expects it (an absolute URL, or the same origin through your own proxy +setup). ### Builds and artifacts ([ADR-0002](adr/0002-processes-run-from-immutable-artifacts.md)) @@ -1050,6 +1082,7 @@ need = { memory = "8G" } # a lane with it needs at least this, o kind = "static" # a build served as files instead of a dev server build = "npm run build -- --configuration e2e" output = "dist/web/browser" +forward = { "/api" = "{service.api.url}" } # the relative /api the dev server's proxy served [profiles.lite] description = "daily work without the heavy containers" @@ -1067,9 +1100,10 @@ devstack lane start --profile acceptance --run # from the main tree `{is_main}`), `autostart`, `skip` (below), `command` (one of `[commands]`), `need` (as `[lanes.need]`, each part the larger of the two; a lane with the profile is measured on its own, its runs are the profile's), and - per service `kind`, `build`, `output`, `run`, `stop`, `spa`, `cwd`, - `health` and `env`. A service it does not name runs as the file says. Its `env` is - added to every service's, a service's own `env` in the profile on top. + per service `kind`, `build`, `output`, `run`, `stop`, `spa`, `forward`, + `cwd`, `health` and `env`. A service it does not name runs as the file says. Its `env` is + added to every service's, a service's own `env` in the profile on top; + a service's `forward` in the profile adds to its own. `watch`, `port` and the other keys stay as the file says. - **`skip`** ([ADR-0058](adr/0058-a-profile-can-leave-out-containers-and-services.md)) leaves out what the profile does not need: a container @@ -1081,7 +1115,7 @@ devstack lane start --profile acceptance --run # from the main tree compose.app.cache` starts it on demand, and a port `main_ports` holds for it stays held and refuses connections while it is down. A service that starts and needs a skipped container's address (a `{compose.…}` - placeholder in its `run`, `env`, `workers.env` or `ensure_dirs`, the + placeholder in its `run`, `env`, `workers.env`, `ensure_dirs` or `forward`, the profile's `env` included, or a `rewrite_hosts` entry) is an error, since its start would fail: skip it too, or keep the container. One that only waits for it (`after`) starts without waiting. On a node a skipped @@ -1090,7 +1124,7 @@ devstack lane start --profile acceptance --run # from the main tree - **Another `kind` drops what only the old one takes:** a dev server made `static` loses its `run` and `stop` (unless the profile gives them), `rebuild` and `restart_on`; a `devserver` or `tool` loses `build` and `output`; `spa` - stays with `static` only, `hot_reload` with `service`, `timeout` with + and `forward` stay with `static` only, `hot_reload` with `service`, `timeout` with `job`; a `job` or `tool` loses `health` and `proxy`. What the profile sets itself comes on top. - **Its name** uses letters, digits, `-` and `_`; `description` is optional, diff --git a/skills/devstack-config/SKILL.md b/skills/devstack-config/SKILL.md index ab73068b..cc7ebce2 100644 --- a/skills/devstack-config/SKILL.md +++ b/skills/devstack-config/SKILL.md @@ -97,7 +97,10 @@ without a stack file starts from `devstack init` (a draft with TODOs). `[profiles.]` changes how the stack runs for an acceptance run or a light review: `workers`, `data = "fresh"`, `env` with the tree's addresses, `autostart`, per service `kind = "static"` with `build` and - `output`, and the suite as `command`. Each profile is checked as the stack + `output`, and the suite as `command`. A static bundle that calls its API + on its own origin (relative `/api`, the dev server's proxy config) gets + `forward = { "/api" = "{service.api.url}" }` in place of that proxy + (ADR-0110; `requires = "0.49.0"`), not a Caddy or Node proxy of its own. Each profile is checked as the stack it makes; add `requires = "0.11.0"` to `[stack]`. A light profile for daily work leaves heavy containers out with `skip` (ADR-0058): `skip = ["compose.search", "compose.app.scanner", "indexer"]` (a whole