From dda6139a507c4e405d24312542bc63bc13c3d9cf Mon Sep 17 00:00:00 2001 From: Seth Hoenig Date: Tue, 20 Oct 2026 12:45:38 -0600 Subject: [PATCH] useragent: better parsing of from values --- Justfile | 2 +- README.md | 44 +++++++++ go.mod | 2 +- useragent.go | 104 ++++++++++++++++++-- useragent_test.go | 246 ++++++++++++++++++++++++++++++++++++++++++---- 5 files changed, 370 insertions(+), 28 deletions(-) diff --git a/Justfile b/Justfile index 76e5bad..5f724d4 100644 --- a/Justfile +++ b/Justfile @@ -42,4 +42,4 @@ lint: vet # locally install build dependencies [group('build')] init: - go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.11.4 + go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.14.0 diff --git a/README.md b/README.md index 096c74e..2691bec 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,50 @@ fmt.Println(ua.Anchor()) # e.g. example.org/path/to/page fmt.Println(ua.From()) # e.g. 10.0.0.1 ``` +##### Origin headers + +The `From()` method returns the originating client IP. The address is +parsed from HTTP headers at construction time (`Origins()`) using the +following precedence: + +1. `Forwarded` (RFC 7239) — the leftmost usable `for=` parameter. Surrounding + double-quotes are stripped, and any port is stripped. Redacted tokens + (`unknown`, `_hidden`, and any value beginning with `_`) are treated as + missing and cause a fall-through. +2. `X-Forwarded-For` — the leftmost comma-separated entry with any port + stripped. +3. `X-Real-IP` — common nginx convention, with any port stripped. +4. `RemoteAddr` — the host portion of the TCP source address, via + `net.SplitHostPort`. + +Regardless of which source produced the value: + +- Ports are always stripped (IPv4 `:port`, bracketed IPv6 `[::1]:port`, + bare hostnames `:port`). +- The leftmost entry is always returned. Each proxy along the chain appends + to the right, so the leftmost value is the one nearest the originating + client. +- Redacted `Forwarded` tokens (`for=unknown`, `for=_hidden`) are silently + dropped so callers transparently fall back to the next source rather + than logging an opaque placeholder. + +In all three header layouts below `From()` returns the same value for the +same originating client: + +```go +// 1. RFC 7239 Forwarded header +r.Header.Set("Forwarded", "for=192.0.2.43;proto=https") +// From() -> 192.0.2.43 + +// 2. Legacy X-Forwarded-For (most common in practice) +r.Header.Set("X-Forwarded-For", "192.0.2.43, 10.0.0.5, proxy.example.com") +// From() -> 192.0.2.43 (leftmost = originating client) + +// 3. nginx-style X-Real-IP, or nothing at all +r.Header.Set("X-Real-IP", "192.0.2.43") +// From() -> 192.0.2.43 +``` + #### package webtools/htmx Provides utilities for building HTMX-compatible responses. The package includes an `Interface` for rendering HTMX responses, along with support for plain text, raw HTML, and templated content. diff --git a/go.mod b/go.mod index 512723c..60da93b 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module cattlecloud.net/go/webtools -go 1.26 +go 1.27 require ( cattlecloud.net/go/forms v1.1.0 diff --git a/useragent.go b/useragent.go index e8fb913..2d56c33 100644 --- a/useragent.go +++ b/useragent.go @@ -1,6 +1,7 @@ package webtools import ( + "net" "net/http" "net/url" "strings" @@ -16,13 +17,23 @@ type Origin struct { UserAgent useragent.UserAgent // about the requesting entity - Remote string + // + // Forward is the originating client as derived from the X-Forwarded-For, + // RFC 7239 Forwarded, or X-Real-IP headers (consulted in that order). + // It is the leftmost usable entry with any port stripped. The value + // is a bare host with no port. + // + // Remote is the host portion of the TCP source address (r.RemoteAddr) + // with any port stripped. Forward string + Remote string Reference string } -// From returns the value of the X-Forwarded-For if set, otherwise defaulting -// to the Remote address. +// From returns the originating client IP. If available, this is the +// leftmost usable value from the Forwarded (RFC 7239), X-Forwarded-For, +// or X-Real-IP headers, in that order. Otherwise it falls back to the +// host portion of RemoteAddr. Ports are always stripped. func (o *Origin) From() string { if o.Forward != "" { return o.Forward @@ -65,14 +76,31 @@ func (o *Origin) String() string { // // - Method // - Host -// - Forwarder // - Referer // - User-Agent +// +// The originating client IP (Forward/Remote) is resolved from headers in this +// order: +// +// 1. Forwarded (RFC 7239) — leftmost usable "for=" value, with quotes and +// port stripped; redacted values ("unknown", "_hidden", any value +// starting with "_") are treated as missing. +// 2. X-Forwarded-For — leftmost comma-separated entry, port stripped. +// 3. X-Real-IP — common nginx convention, port stripped. +// 4. RemoteAddr — host portion of the TCP source address, port stripped. +// +// Ports are always stripped regardless of source. func Origins(r *http.Request) *Origin { method := strings.ToUpper(r.Method) host := r.Host - remote := r.RemoteAddr - forward := r.Header.Get("X-Forwarded-For") + remote := sanitize(r.RemoteAddr) + forward := forwarded(r.Header.Get("Forwarded")) + if forward == "" { + forward = sanitize(leftmost(r.Header.Get("X-Forwarded-For"))) + } + if forward == "" { + forward = sanitize(r.Header.Get("X-Real-IP")) + } reference := r.Header.Get("Referer") agent := r.Header.Get("User-Agent") ua := useragent.Parse(agent) @@ -85,3 +113,67 @@ func Origins(r *http.Request) *Origin { Reference: reference, } } + +// leftmost returns the first comma-separated token of a header value with +// surrounding whitespace stripped, or "" when the input is empty. +func leftmost(header string) string { + if i := strings.Index(header, ","); i >= 0 { + header = header[:i] + } + return strings.TrimSpace(header) +} + +// forwarded parses an RFC 7239 Forwarded header and returns the leftmost +// usable "for=" value with surrounding quotes and any port stripped. It +// returns "" when the header is absent, malformed, or contains only +// redacted values (the literal "unknown" or any value beginning with "_"). +func forwarded(header string) string { + if header == "" { + return "" + } + for _, entry := range strings.Split(header, ",") { + entry = strings.TrimSpace(entry) + var value string + var found bool + for _, param := range strings.Split(entry, ";") { + param = strings.TrimSpace(param) + if !strings.HasPrefix(param, "for=") { + continue + } + value = strings.TrimPrefix(param, "for=") + found = true + break + } + if !found { + continue + } + // Strip surrounding double quotes, per RFC 7239. + if len(value) >= 2 && value[0] == '"' && value[len(value)-1] == '"' { + value = value[1 : len(value)-1] + } + // Treat redacted tokens as missing so callers transparently fall + // through to the next source (X-Forwarded-For, X-Real-IP, RemoteAddr). + if value == "unknown" || strings.HasPrefix(value, "_") { + continue + } + return sanitize(value) + } + return "" +} + +// sanitize returns the host portion of an address, stripping any trailing +// port. It accepts "host:port", "[ipv6]:port", and bare bracketed IPv6 +// addresses. If no port can be detected the input is returned unchanged. +func sanitize(address string) string { + if address == "" { + return "" + } + if host, _, err := net.SplitHostPort(address); err == nil { + return host + } + // Bare bracketed IPv6, e.g. "[2001:db8::1]" with no port. + if len(address) >= 2 && address[0] == '[' && address[len(address)-1] == ']' { + return address[1 : len(address)-1] + } + return address +} diff --git a/useragent_test.go b/useragent_test.go index 1c3d152..70bb787 100644 --- a/useragent_test.go +++ b/useragent_test.go @@ -12,26 +12,32 @@ import ( func TestOrigin_From(t *testing.T) { t.Parallel() - t.Run("forward set", func(t *testing.T) { - o := &Origin{ - Host: "example.com", - Forward: "10.0.0.1", - Remote: "100.100.1.1", - } - - s := o.From() - must.Eq(t, "10.0.0.1", s) - }) - - t.Run("not set", func(t *testing.T) { - o := &Origin{ - Host: "example.com", - Remote: "100.100.1.1", - } - - s := o.From() - must.Eq(t, "100.100.1.1", s) - }) + cases := []struct { + name string + forward string + remote string + exp string + }{ + // From() is intentionally a simple "Forward ? Remote" pick; the + // heavy lifting (port stripping, header cascade) lives in Origins() + // and is exercised by TestOrigins_HeaderPrecedence. + {"forward set", "10.0.0.1", "100.100.1.1", "10.0.0.1"}, + {"forward empty falls back to remote", "", "100.100.1.1", "100.100.1.1"}, + {"both empty", "", "", ""}, + {"forward already pre-parsed (chain)", "10.0.0.1", "100.100.1.1", "10.0.0.1"}, + {"forward already port-stripped", "10.0.0.1", "100.100.1.1:80", "10.0.0.1"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + o := &Origin{ + Host: "example.com", + Forward: tc.forward, + Remote: tc.remote, + } + must.Eq(t, tc.exp, o.From()) + }) + } } func TestOrigin_Anchor(t *testing.T) { @@ -79,6 +85,86 @@ func TestOrigin_String(t *testing.T) { } } +func TestLeftmost(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + input string + expect string + }{ + {"empty", "", ""}, + {"single value", "10.0.0.1", "10.0.0.1"}, + {"single value with surrounding whitespace", " 10.0.0.1 ", "10.0.0.1"}, + {"chain", "10.0.0.1, 192.168.1.1, proxy.example.com", "10.0.0.1"}, + {"chain with whitespace", " 10.0.0.1 , 192.168.1.1", "10.0.0.1"}, + {"only commas", ",,,", ""}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + must.Eq(t, tc.expect, leftmost(tc.input)) + }) + } +} + +func TestForwarded(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + input string + expect string + }{ + {"empty", "", ""}, + {"simple for=", "for=192.0.2.43", "192.0.2.43"}, + {"for with by and proto", "by=203.0.113.7;for=192.0.2.43;proto=https", "192.0.2.43"}, + {"proto before for", "proto=https;for=192.0.2.43", "192.0.2.43"}, + {"bracketed IPv6 quoted", `for="[2001:db8::1]:443";proto=https`, "2001:db8::1"}, + {"bracketed IPv6 quoted no port", `for="[2001:db8::1]"`, "2001:db8::1"}, + {"non-quoted v4 with port stripped", "for=192.0.2.43:443", "192.0.2.43"}, + {"multi-hop chain uses leftmost", "for=192.0.2.43, for=198.51.100.17", "192.0.2.43"}, + {"multi-entry leftmost wins", "for=192.0.2.43, by=10.0.0.5;for=198.51.100.17", "192.0.2.43"}, + {"redacted unknown falls through", "for=unknown", ""}, + {"redacted _hidden falls through", "for=_hidden", ""}, + {"redacted underscore-prefixed falls through", "for=_anything", ""}, + {"unknown falls through even with X-Real-IP style", "for=unknown ; proto=https", ""}, + {"for absent in entry", "by=203.0.113.7;proto=https", ""}, + {"only quotes stripped when no port", `for="10.0.0.1"`, "10.0.0.1"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + must.Eq(t, tc.expect, forwarded(tc.input)) + }) + } +} + +func TestSanitize(t *testing.T) { + t.Parallel() + + cases := []struct { + name string + input string + expect string + }{ + {"empty", "", ""}, + {"IPv4 with port", "198.51.100.7:54321", "198.51.100.7"}, + {"IPv4 without port", "198.51.100.7", "198.51.100.7"}, + {"bracketed IPv6 with port", "[2001:db8::1]:443", "2001:db8::1"}, + {"bracketed IPv6 without port", "[2001:db8::1]", "2001:db8::1"}, + {"bare IPv6 no port passes through", "::1", "::1"}, + {"hostname with port", "proxy.example.com:8443", "proxy.example.com"}, + {"hostname without port", "proxy.example.com", "proxy.example.com"}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + must.Eq(t, tc.expect, sanitize(tc.input)) + }) + } +} + func TestOrigins(t *testing.T) { t.Parallel() @@ -96,3 +182,123 @@ func TestOrigins(t *testing.T) { must.Eq(t, "https://dashboard.example.org/home", origin.Reference) must.Eq(t, "Chrome", origin.UserAgent.Name) } + +// TestOrigins_HeaderPrecedence exercises the Forwarded -> X-Forwarded-For -> +// X-Real-IP -> RemoteAddr cascade against realistic header scenarios. +func TestOrigins_HeaderPrecedence(t *testing.T) { + t.Parallel() + + type header struct { + name, value string + } + + cases := []struct { + name string + headers []header + remoteAddr string + wantForward string + wantRemote string + wantFrom string + }{ + { + name: "Forwarded wins over XFF", + headers: []header{ + {"Forwarded", "for=192.0.2.43"}, + {"X-Forwarded-For", "10.0.0.1"}, + }, + remoteAddr: "198.51.100.7:54321", + wantForward: "192.0.2.43", + wantRemote: "198.51.100.7", + wantFrom: "192.0.2.43", + }, + { + name: "Forwarded with bracket-quoted IPv6 wins", + headers: []header{ + {"Forwarded", `for="[2001:db8::1]:443";proto=https`}, + {"X-Forwarded-For", "10.0.0.1"}, + }, + remoteAddr: "[::1]:443", + wantForward: "2001:db8::1", + wantRemote: "::1", + wantFrom: "2001:db8::1", + }, + { + name: "Forwarded redacted falls through to XFF", + headers: []header{ + {"Forwarded", "for=_hidden"}, + {"X-Forwarded-For", "10.0.0.1, 192.168.1.1"}, + }, + remoteAddr: "198.51.100.7:54321", + wantForward: "10.0.0.1", + wantRemote: "198.51.100.7", + wantFrom: "10.0.0.1", + }, + { + name: "Forwarded unknown falls through to XFF", + headers: []header{ + {"Forwarded", "for=unknown"}, + {"X-Forwarded-For", "10.0.0.1:8080"}, + }, + remoteAddr: "198.51.100.7:54321", + wantForward: "10.0.0.1", + wantRemote: "198.51.100.7", + wantFrom: "10.0.0.1", + }, + { + name: "Forwarded empty falls through to XFF", + headers: []header{ + {"X-Forwarded-For", "10.0.0.1, 192.168.1.1, proxy.example.com"}, + }, + remoteAddr: "198.51.100.7:54321", + wantForward: "10.0.0.1", + wantRemote: "198.51.100.7", + wantFrom: "10.0.0.1", + }, + { + name: "XFF absent, X-Real-IP used", + headers: []header{{"X-Real-IP", "10.42.42.42:8080"}}, + remoteAddr: "198.51.100.7:54321", + wantForward: "10.42.42.42", + wantRemote: "198.51.100.7", + wantFrom: "10.42.42.42", + }, + { + name: "all headers absent, RemoteAddr used", + headers: nil, + remoteAddr: "198.51.100.7:54321", + wantForward: "", + wantRemote: "198.51.100.7", + wantFrom: "198.51.100.7", + }, + { + name: "all empty, bare IPv6 remote", + headers: nil, + remoteAddr: "::1", + wantForward: "", + wantRemote: "::1", + wantFrom: "::1", + }, + { + name: "Forwarded malformed: only by/proto", + headers: []header{{"Forwarded", "by=203.0.113.7;proto=https"}}, + remoteAddr: "198.51.100.7:54321", + wantForward: "", + wantRemote: "198.51.100.7", + wantFrom: "198.51.100.7", + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + r := httptest.NewRequestWithContext(t.Context(), http.MethodGet, "https://example.org/", nil) + r.RemoteAddr = tc.remoteAddr + for _, h := range tc.headers { + r.Header.Set(h.name, h.value) + } + o := Origins(r) + must.Eq(t, tc.wantForward, o.Forward) + must.Eq(t, tc.wantRemote, o.Remote) + must.Eq(t, tc.wantFrom, o.From()) + }) + } +}