From 28fe8244d228526f2f2ffc524063f01ae9fc6c20 Mon Sep 17 00:00:00 2001 From: enowdev Date: Wed, 11 Nov 2026 20:01:42 +0700 Subject: [PATCH 1/2] Add an end-to-end channel between a member and a shared host Members will reach shared hosts through the cloud's relay, which must not read what passes. The member's app opens a channel to the key the host announced in its team's sealed space: fresh X25519 keys both ways, a proof the host holds the announced key, and ChaCha20-Poly1305 records under a counter, so a relay can't read, change, drop or replay them. Co-Authored-By: Claude Opus 5.5 --- cloud/relay/e2e.go | 276 ++++++++++++++++++++++++++++++++++++++++ cloud/relay/e2e_test.go | 75 +++++++++++ 2 files changed, 351 insertions(+) create mode 100644 cloud/relay/e2e.go create mode 100644 cloud/relay/e2e_test.go diff --git a/cloud/relay/e2e.go b/cloud/relay/e2e.go new file mode 100644 index 00000000..2dd4dd27 --- /dev/null +++ b/cloud/relay/e2e.go @@ -0,0 +1,276 @@ +package relay + +import ( + "crypto/cipher" + "crypto/ecdh" + "crypto/hkdf" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/binary" + "errors" + "fmt" + "io" + "net" + "os" + "sync" + "time" + + "golang.org/x/crypto/chacha20poly1305" +) + +// End to end between a member and the host they visit, so the relay passes +// bytes it can't read. The host has a key of its own (X25519), whose public +// half it announces in its team's space, which the cloud keeps sealed and +// can't forge. The member's app opens a channel to that key alone: +// +// 1. the member sends a fresh key e; +// 2. the host answers with a fresh key e2 and a proof that it holds the +// announced key (a box only DH(e, host key) opens); +// 3. both derive a key each way from DH(e, host key) and DH(e, e2). +// +// The relay sees two fresh public keys and a proof; without the host's key +// it can neither read the records that follow nor stand in for the host, +// and the fresh keys mean a key stolen later can't read what was recorded. +// Who the member is, the host learns inside the channel, from their ticket. + +const ( + magic = "ENXR1" + recordMax = 16 << 10 // plaintext of a record + handshakeT = 15 * time.Second +) + +// ErrHostKey is a host that couldn't prove it holds the key its team +// announced: not the host, or a relay standing in for it. +var ErrHostKey = errors.New("relay: the host didn't prove it holds the key its team announced") + +// Key is a host's own key. +type Key struct{ priv *ecdh.PrivateKey } + +// LoadKey reads the host's key from file, making one the first time. +func LoadKey(file string) (*Key, error) { + if data, err := os.ReadFile(file); err == nil { + priv, err := ecdh.X25519().NewPrivateKey(data) + if err != nil { + return nil, fmt.Errorf("relay: the host key in %s: %w", file, err) + } + return &Key{priv}, nil + } + priv, err := ecdh.X25519().GenerateKey(rand.Reader) + if err != nil { + return nil, err + } + if err := os.WriteFile(file, priv.Bytes(), 0o600); err != nil { + return nil, fmt.Errorf("relay: write the host key: %w", err) + } + return &Key{priv}, nil +} + +// NewKey is a key kept in memory alone (tests). +func NewKey() *Key { + priv, _ := ecdh.X25519().GenerateKey(rand.Reader) + return &Key{priv} +} + +// Public is the key's public half, as the host announces it. +func (k *Key) Public() string { return base64.StdEncoding.EncodeToString(k.priv.PublicKey().Bytes()) } + +func transcript(parts ...[]byte) []byte { + h := sha256.New() + h.Write([]byte("enowx-relay-e2e-v1")) + for _, p := range parts { + h.Write(p) + } + return h.Sum(nil) +} + +func derive(secret, salt []byte, info string, n int) ([]byte, error) { + return hkdf.Key(sha256.New, secret, salt, info, n) +} + +// Client opens the channel to the host whose announced public key is +// hostKey, over nc (the member's end). +func Client(nc net.Conn, hostKey string) (net.Conn, error) { + raw, err := base64.StdEncoding.DecodeString(hostKey) + if err != nil { + return nil, fmt.Errorf("relay: the host's announced key: %w", err) + } + S, err := ecdh.X25519().NewPublicKey(raw) + if err != nil { + return nil, fmt.Errorf("relay: the host's announced key: %w", err) + } + e, err := ecdh.X25519().GenerateKey(rand.Reader) + if err != nil { + return nil, err + } + _ = nc.SetDeadline(time.Now().Add(handshakeT)) + defer nc.SetDeadline(time.Time{}) + ePub := e.PublicKey().Bytes() + if _, err := nc.Write(append([]byte(magic), ePub...)); err != nil { + return nil, err + } + var answer [32 + 2 + chacha20poly1305.Overhead]byte + if _, err := io.ReadFull(nc, answer[:]); err != nil { + return nil, fmt.Errorf("relay: the host's answer: %w", err) + } + e2Pub := answer[:32] + dh1, err := e.ECDH(S) + if err != nil { + return nil, ErrHostKey + } + h := transcript(raw, ePub) + h2 := transcript(h, e2Pub) + k1, err := derive(dh1, h, "host proof", chacha20poly1305.KeySize) + if err != nil { + return nil, err + } + box, _ := chacha20poly1305.New(k1) + if ok, err := box.Open(nil, make([]byte, chacha20poly1305.NonceSize), answer[32:], h2); err != nil || string(ok) != "ok" { + return nil, ErrHostKey + } + E2, err := ecdh.X25519().NewPublicKey(e2Pub) + if err != nil { + return nil, ErrHostKey + } + dh2, err := e.ECDH(E2) + if err != nil { + return nil, ErrHostKey + } + return newSecure(nc, dh1, dh2, h2, true) +} + +// Server answers a member's channel over nc (the host's end). +func (k *Key) Server(nc net.Conn) (net.Conn, error) { + _ = nc.SetDeadline(time.Now().Add(handshakeT)) + defer nc.SetDeadline(time.Time{}) + var hello [len(magic) + 32]byte + if _, err := io.ReadFull(nc, hello[:]); err != nil { + return nil, fmt.Errorf("relay: the member's hello: %w", err) + } + if string(hello[:len(magic)]) != magic { + return nil, errors.New("relay: not a member's channel") + } + ePub := hello[len(magic):] + E, err := ecdh.X25519().NewPublicKey(ePub) + if err != nil { + return nil, err + } + dh1, err := k.priv.ECDH(E) + if err != nil { + return nil, err + } + e2, err := ecdh.X25519().GenerateKey(rand.Reader) + if err != nil { + return nil, err + } + dh2, err := e2.ECDH(E) + if err != nil { + return nil, err + } + e2Pub := e2.PublicKey().Bytes() + h := transcript(k.priv.PublicKey().Bytes(), ePub) + h2 := transcript(h, e2Pub) + k1, err := derive(dh1, h, "host proof", chacha20poly1305.KeySize) + if err != nil { + return nil, err + } + box, _ := chacha20poly1305.New(k1) + proof := box.Seal(nil, make([]byte, chacha20poly1305.NonceSize), []byte("ok"), h2) + if _, err := nc.Write(append(e2Pub, proof...)); err != nil { + return nil, err + } + return newSecure(nc, dh1, dh2, h2, false) +} + +// secure is a channel's records: a length, then a box sealed with the +// direction's key and a counter, so a record dropped, repeated or moved by +// the relay fails to open and ends the channel. +type secure struct { + net.Conn + rmu sync.Mutex // the reading side: in, rbuf, rn + in cipher.AEAD + rn uint64 + rbuf []byte + wmu sync.Mutex // the writing side: out, wn + out cipher.AEAD + wn uint64 + readErr error + writeErr error +} + +func newSecure(nc net.Conn, dh1, dh2, salt []byte, client bool) (net.Conn, error) { + keys, err := derive(append(append([]byte{}, dh1...), dh2...), salt, "enowx-relay keys", 2*chacha20poly1305.KeySize) + if err != nil { + return nil, err + } + toHost, _ := chacha20poly1305.New(keys[:32]) + toMember, _ := chacha20poly1305.New(keys[32:]) + s := &secure{Conn: nc, in: toMember, out: toHost} + if !client { + s.in, s.out = toHost, toMember + } + return s, nil +} + +func nonce(n uint64) []byte { + var b [chacha20poly1305.NonceSize]byte + binary.LittleEndian.PutUint64(b[4:], n) + return b[:] +} + +func (s *secure) Read(p []byte) (int, error) { + s.rmu.Lock() + defer s.rmu.Unlock() + for len(s.rbuf) == 0 { + if s.readErr != nil { + return 0, s.readErr + } + var hdr [2]byte + if _, err := io.ReadFull(s.Conn, hdr[:]); err != nil { + s.readErr = err + return 0, err + } + n := int(binary.BigEndian.Uint16(hdr[:])) + if n < chacha20poly1305.Overhead || n > recordMax+chacha20poly1305.Overhead { + s.readErr = errors.New("relay: a record of a wrong size") + return 0, s.readErr + } + rec := make([]byte, n) + if _, err := io.ReadFull(s.Conn, rec); err != nil { + s.readErr = err + return 0, err + } + plain, err := s.in.Open(rec[:0], nonce(s.rn), rec, nil) + if err != nil { + s.readErr = errors.New("relay: a record that doesn't open: changed on its way") + return 0, s.readErr + } + s.rn++ + s.rbuf = plain + } + n := copy(p, s.rbuf) + s.rbuf = s.rbuf[n:] + return n, nil +} + +func (s *secure) Write(p []byte) (int, error) { + s.wmu.Lock() + defer s.wmu.Unlock() + if s.writeErr != nil { + return 0, s.writeErr + } + done := 0 + for done < len(p) { + chunk := p[done:min(len(p), done+recordMax)] + rec := make([]byte, 2, 2+len(chunk)+chacha20poly1305.Overhead) + rec = s.out.Seal(rec, nonce(s.wn), chunk, nil) + binary.BigEndian.PutUint16(rec, uint16(len(rec)-2)) + if _, err := s.Conn.Write(rec); err != nil { + s.writeErr = err + return done, err + } + s.wn++ + done += len(chunk) + } + return done, nil +} diff --git a/cloud/relay/e2e_test.go b/cloud/relay/e2e_test.go new file mode 100644 index 00000000..e4149451 --- /dev/null +++ b/cloud/relay/e2e_test.go @@ -0,0 +1,75 @@ +package relay + +import ( + "errors" + "io" + "net" + "testing" +) + +// A member's channel opens to the host holding the announced key, carries +// bytes both ways, and refuses one that doesn't hold it or a record changed +// on its way. +func TestChannel(t *testing.T) { + host := NewKey() + a, b := net.Pipe() + got := make(chan net.Conn, 1) + go func() { + sc, err := host.Server(b) + if err != nil { + t.Error(err) + } + got <- sc + }() + mc, err := Client(a, host.Public()) + if err != nil { + t.Fatal(err) + } + hc := <-got + go mc.Write(make([]byte, 40000)) // past one record + buf := make([]byte, 40000) + if _, err := io.ReadFull(hc, buf); err != nil { + t.Fatal(err) + } + go hc.Write([]byte("back")) + small := make([]byte, 4) + if _, err := io.ReadFull(mc, small); err != nil || string(small) != "back" { + t.Fatalf("%q %v", small, err) + } + + a, b = net.Pipe() + go NewKey().Server(b) // another host than the one announced + if _, err := Client(a, host.Public()); !errors.Is(err, ErrHostKey) { + t.Fatalf("a host without the announced key: %v", err) + } +} + +func TestChangedRecordFails(t *testing.T) { + host := NewKey() + a, b := net.Pipe() + mid1, mid2 := net.Pipe() // what passes from member to host, through here + go func() { + sc, _ := host.Server(mid2) + buf := make([]byte, 8) + if _, err := sc.Read(buf); err == nil { + t.Error("a changed record opened") + } + }() + go func() { // the handshake both ways, then a record with a bit flipped + hello := make([]byte, 37) + io.ReadFull(b, hello) + mid1.Write(hello) + answer := make([]byte, 50) + io.ReadFull(mid1, answer) + b.Write(answer) + rec := make([]byte, 2+5+16) + io.ReadFull(b, rec) + rec[4] ^= 1 + mid1.Write(rec) + }() + mc, err := Client(a, host.Public()) + if err != nil { + t.Fatal(err) + } + mc.Write([]byte("hello")) +} From 693d5b7aef08054620f7cbb5d2ac99e9c8cf5c16 Mon Sep 17 00:00:00 2001 From: enowdev Date: Wed, 11 Nov 2026 20:01:42 +0700 Subject: [PATCH 2/2] Reach shared hosts through the cloud's relay, without Tailscale A member reached another's shared host only over Tailscale, so teammates outside the owner's tailnet couldn't. The host now keeps a connection to the cloud's relay while it shares anything; a member connects with their ticket, the relay checks it without using it, pairs the two and passes bytes, end to end encrypted to the host's announced key. The host redeems the ticket and applies roles, the 2-minute recheck, per-address limits and the first-frame rules as before, as relay connections feed the same team listener. The relay holds each team to a number of connections and a byte rate. Hosts with a Tailscale address keep listening there for members of an older enowX, which knows no relay; the app tries the relay first. Co-Authored-By: Claude Opus 5.5 --- README.md | 2 +- cloud/api/tickets.go | 26 +++ cloud/relay/relay.go | 229 +++++++++++++++++++++++++ cloud/server/items.go | 1 + cloud/server/relay.go | 332 +++++++++++++++++++++++++++++++++++++ cloud/server/relay_test.go | 223 +++++++++++++++++++++++++ cloud/server/server.go | 10 ++ cloud/store/tickets.go | 23 +++ cmd/enowx-cloud/main.go | 13 ++ daemon/cloud.go | 3 +- daemon/hostshare.go | 82 +++++---- daemon/teamrelay.go | 103 ++++++++++++ daemon/teamrelay_test.go | 98 +++++++++++ docs/cli.md | 2 +- docs/cloud.md | 5 +- go.mod | 2 +- proto/hostshare.go | 21 ++- proto/testdata/api.txt | 2 + ui/teamhosts.go | 51 +++++- 19 files changed, 1178 insertions(+), 50 deletions(-) create mode 100644 cloud/relay/relay.go create mode 100644 cloud/server/relay.go create mode 100644 cloud/server/relay_test.go create mode 100644 daemon/teamrelay.go create mode 100644 daemon/teamrelay_test.go diff --git a/README.md b/README.md index f5fa8a95..dd92b9a4 100644 --- a/README.md +++ b/README.md @@ -252,7 +252,7 @@ enowX Cloud, opt-in, puts a team together around the same projects ([cloud.md](d - **Teams with GitHub sign-in**, owners, admins, members and viewers. An invite is a link that works in the browser too: someone without the app signs in and joins there, then downloads enowX and finds the team. - **A team's space:** its notes and canvases, its projects as Git links (cloned in one click, found when you have them already), its kit of skills, commands, subagents, pipelines, automations and templates, and each project's rules for the agents. -- **Shared hosts** over Tailscale: a member sees the sessions of a host shared with the team, watches one live, asks to take it over, or starts their own there with their own agent account, never the host owner's credentials. +- **Shared hosts** through the cloud's relay, end to end encrypted, no Tailscale needed: a member sees the sessions of a host shared with the team, watches one live, asks to take it over, or starts their own there with their own agent account, never the host owner's credentials. - **Team chat** in the side panel, with `@` to mention members, notes, canvases, projects and files; messages edit and delete. - **End-to-end encrypted:** what a team shares is sealed with the team's key on its members' devices, so the cloud holds ciphertext only; credentials never go to it. An audit log and a recovery key for the team's keys come with it. diff --git a/cloud/api/tickets.go b/cloud/api/tickets.go index 5c9545d0..28be265a 100644 --- a/cloud/api/tickets.go +++ b/cloud/api/tickets.go @@ -25,6 +25,9 @@ type HostTicketParams struct { type HostTicket struct { Ticket string `json:"ticket"` Expires time.Time `json:"expires"` + // Relay is where the member connects to reach the host through the + // cloud's relay (wss://…/v1/relay/connect), "" for a cloud without one. + Relay string `json:"relay,omitempty"` } // RedeemParams redeems a ticket handed to the host. @@ -40,3 +43,26 @@ type Visitor struct { Role string `json:"role"` Device string `json:"device"` } + +// The relay (docs/cloud.md, Hosts and sessions): a host sharing projects +// keeps a connection open to it (PathRelayHost, its own token), so it needs +// no address members can reach. A member connects with their ticket +// (PathRelayConnect, the ticket in HeaderTicket); the relay tells the host +// (a RelayCall), the host opens a connection for it (PathRelayAccept), and +// the relay passes bytes between the two, which are end to end encrypted +// (cloud/relay): it sees who connects to whom and how much, not what. +// All three are WebSocket connections. +const ( + PathRelayHost = "/v1/relay/host" // the host's, its token: RelayCall messages + PathRelayAccept = "/v1/relay/accept/{id}" // the host's, its token: a call's bytes + PathRelayConnect = "/v1/relay/connect" // the member's, HeaderTicket: the host's bytes + HeaderTicket = "X-Enowx-Ticket" +) + +// RelayCall tells the host a member waits on the relay: it opens +// PathRelayAccept with ID. From is the address the member came from, for +// the host's limits by address. +type RelayCall struct { + ID string `json:"id"` + From string `json:"from"` +} diff --git a/cloud/relay/relay.go b/cloud/relay/relay.go new file mode 100644 index 00000000..0be2d0f9 --- /dev/null +++ b/cloud/relay/relay.go @@ -0,0 +1,229 @@ +// Package relay connects a team's members to a shared host through the +// cloud's relay, end to end encrypted (e2e.go): Host is the host's side, a +// listener of members' connections; Dial the member's. +package relay + +import ( + "context" + "encoding/json" + "fmt" + "net" + "net/http" + "strings" + "sync" + "time" + + "github.com/coder/websocket" + "github.com/enowx/enowx/cloud/api" +) + +const readLimit = 1 << 20 + +// Dial reaches the host whose announced key is hostKey through the relay at +// relayURL (the ticket's), handing the relay ticket so it pairs the two. +// The connection it gives is the channel to the host. +func Dial(ctx context.Context, relayURL, ticket, hostKey string) (net.Conn, error) { + ws, resp, err := websocket.Dial(ctx, relayURL, &websocket.DialOptions{HTTPHeader: http.Header{api.HeaderTicket: {ticket}}}) + if err != nil { + return nil, refusal(resp, err) + } + ws.SetReadLimit(readLimit) + nc := websocket.NetConn(context.Background(), ws, websocket.MessageBinary) + sc, err := Client(nc, hostKey) + if err != nil { + nc.Close() + return nil, err + } + return sc, nil +} + +// refusal is why the relay said no, in its words when it gave them. +func refusal(resp *http.Response, err error) error { + if resp == nil || resp.Body == nil { + return err + } + var e api.Error + if json.NewDecoder(resp.Body).Decode(&e) == nil && e.Error != "" { + return &Refused{Status: resp.StatusCode, Message: e.Error} + } + return fmt.Errorf("relay: %s", resp.Status) +} + +// Refused is the relay saying no. +type Refused struct { + Status int + Message string +} + +func (e *Refused) Error() string { return "relay: " + e.Message } + +// Host keeps the host's connection to the relay at URL (the cloud's +// address) with Token, the host's own, and gives members' channels, opened +// to Key, as a listener's connections. +type Host struct { + URL, Token string + Key *Key + // Retry is the first wait before connecting again; it doubles to + // RetryMax. Zero: a second, and five minutes. + Retry, RetryMax time.Duration + // Log notes what goes wrong; nil, nothing. + Log func(msg string, args ...any) + + mu sync.Mutex // guards ws, err, up + ws *websocket.Conn + err string + up bool + conns chan net.Conn + ctx context.Context + stop context.CancelFunc + stopOnce sync.Once +} + +// Start connects and keeps connecting until Close. +func (h *Host) Start() { + h.conns = make(chan net.Conn) + h.ctx, h.stop = context.WithCancel(context.Background()) + go h.run() +} + +// Status says whether the host is connected to the relay, and why not. +func (h *Host) Status() (bool, string) { + h.mu.Lock() + defer h.mu.Unlock() + return h.up, h.err +} + +func (h *Host) wsURL(path string) string { + base := strings.TrimSuffix(h.URL, "/") + base = strings.Replace(strings.Replace(base, "https://", "wss://", 1), "http://", "ws://", 1) + return base + path +} + +func (h *Host) auth() *websocket.DialOptions { + return &websocket.DialOptions{HTTPHeader: http.Header{"Authorization": {"Bearer " + h.Token}}} +} + +func (h *Host) run() { + wait := h.Retry + if wait <= 0 { + wait = time.Second + } + most := h.RetryMax + if most <= 0 { + most = 5 * time.Minute + } + first := wait + for h.ctx.Err() == nil { + began := time.Now() + err := h.serve() + if h.ctx.Err() != nil { + return + } + h.mu.Lock() + h.up, h.err = false, err.Error() + h.mu.Unlock() + if h.Log != nil { + h.Log("relay: connect again", "err", err, "in", wait) + } + if time.Since(began) > time.Minute { + wait = first // it stayed up a while: the drop is news, not a pattern + } + select { + case <-h.ctx.Done(): + return + case <-time.After(wait): + } + wait = min(wait*2, most) + } +} + +// serve holds one connection to the relay, answering its calls. +func (h *Host) serve() error { + ctx, cancel := context.WithTimeout(h.ctx, 20*time.Second) + ws, resp, err := websocket.Dial(ctx, h.wsURL(api.PathRelayHost), h.auth()) + cancel() + if err != nil { + return refusal(resp, err) + } + h.mu.Lock() + h.ws, h.up, h.err = ws, true, "" + h.mu.Unlock() + defer ws.CloseNow() + for { + _, data, err := ws.Read(h.ctx) + if err != nil { + return fmt.Errorf("relay: the connection dropped: %w", err) + } + var call api.RelayCall + if json.Unmarshal(data, &call) != nil || call.ID == "" { + continue + } + go h.answer(call) + } +} + +// answer opens the call's connection and the member's channel over it. +func (h *Host) answer(call api.RelayCall) { + ctx, cancel := context.WithTimeout(h.ctx, 15*time.Second) + defer cancel() + ws, _, err := websocket.Dial(ctx, h.wsURL(strings.Replace(api.PathRelayAccept, "{id}", call.ID, 1)), h.auth()) + if err != nil { + return + } + ws.SetReadLimit(readLimit) + nc := websocket.NetConn(h.ctx, ws, websocket.MessageBinary) + sc, err := h.Key.Server(nc) + if err != nil { + nc.Close() + return + } + select { + case h.conns <- &member{Conn: sc, from: call.From}: + case <-ctx.Done(): + sc.Close() + } +} + +// member is a member's channel, from the address the relay saw. +type member struct { + net.Conn + from string +} + +func (m *member) RemoteAddr() net.Addr { return Addr(m.from) } + +// Addr is where the relay saw a member come from. +type Addr string + +func (a Addr) Network() string { return "relay" } +func (a Addr) String() string { return string(a) } + +// Accept waits for a member's channel. +func (h *Host) Accept() (net.Conn, error) { + select { + case c := <-h.conns: + return c, nil + case <-h.ctx.Done(): + return nil, net.ErrClosed + } +} + +// Close stops connecting, and drops the connection to the relay. +func (h *Host) Close() error { + h.stopOnce.Do(func() { + h.stop() + h.mu.Lock() + if h.ws != nil { + h.ws.CloseNow() + } + h.up = false + h.mu.Unlock() + }) + return nil +} + +// Addr is the relay's, as a listener's. +func (h *Host) Addr() net.Addr { return Addr("relay") } + + +var _ net.Listener = (*Host)(nil) diff --git a/cloud/server/items.go b/cloud/server/items.go index 36e3d044..657d477f 100644 --- a/cloud/server/items.go +++ b/cloud/server/items.go @@ -61,6 +61,7 @@ func (s *Server) issueTicket(w http.ResponseWriter, r *http.Request) { return } t, err := s.db.IssueTicket(r.Context(), r.PathValue("team"), me.Account.ID, me.Device.ID, p.Host) + t.Relay = s.relayURL() s.answer(w, t, err) } diff --git a/cloud/server/relay.go b/cloud/server/relay.go new file mode 100644 index 00000000..45f6ba57 --- /dev/null +++ b/cloud/server/relay.go @@ -0,0 +1,332 @@ +package server + +import ( + "context" + "crypto/rand" + "crypto/sha256" + "encoding/base64" + "encoding/json" + "errors" + "io" + "net/http" + "strings" + "sync" + "time" + + "github.com/coder/websocket" + "github.com/enowx/enowx/cloud/api" + "github.com/enowx/enowx/cloud/store" +) + +// The relay (api.PathRelayHost): hosts sharing projects keep a connection +// here, members connect with a ticket, and the relay pairs them and passes +// bytes it can't read (they are end to end encrypted, cloud/relay). It +// checks the ticket without using it (the host redeems it, as over any +// other path), lets a ticket pair once, and holds each team to a number of +// connections and a rate of bytes. State is this server's memory: one +// relay per cloud. + +// RelayLimits are a team's share of the relay; zero takes the default. +type RelayLimits struct { + // Pairs is how many members' connections a team has at once. + Pairs int + // BytesPerSecond is a team's rate, both ways together, all its + // connections together; Burst is how much may go at once above it. + BytesPerSecond, Burst int +} + +const ( + relayPairs = 32 + relayRate = 4 << 20 // bytes a second + relayBurst = 16 << 20 + relayAnswer = 15 * time.Second // for the host to open a call + relayPing = 30 * time.Second + relayConnects = 30 // a minute, from one address + relayReadLimit = 1 << 20 +) + +// relay is the hosts connected and the calls waiting; mu guards all of it. +type relay struct { + mu sync.Mutex + hosts map[string]*relayHost // by device + calls map[string]*relayCall // by ID + tickets map[string]time.Time // hashes of tickets paired, until they expire + teams map[string]*teamShare +} + +type relayHost struct { + ws *websocket.Conn + wmu sync.Mutex // writes to ws +} + +type relayCall struct { + host string + conn chan *websocket.Conn + done chan struct{} + start time.Time +} + +// teamShare is what a team uses of the relay now. +type teamShare struct { + pairs int + tokens float64 + last time.Time +} + +func (s *Server) relayLimits() RelayLimits { + l := s.cfg.Relay + if l.Pairs <= 0 { + l.Pairs = relayPairs + } + if l.BytesPerSecond <= 0 { + l.BytesPerSecond = relayRate + } + if l.Burst <= 0 { + l.Burst = relayBurst + } + return l +} + +// relayURL is where members connect, "" when the server has no address. +func (s *Server) relayURL() string { + base := strings.TrimSuffix(s.cfg.URL, "/") + switch { + case strings.HasPrefix(base, "https://"): + return "wss://" + strings.TrimPrefix(base, "https://") + api.PathRelayConnect + case strings.HasPrefix(base, "http://"): + return "ws://" + strings.TrimPrefix(base, "http://") + api.PathRelayConnect + } + return "" +} + +func randomID() string { + var b [18]byte + rand.Read(b[:]) + return base64.RawURLEncoding.EncodeToString(b[:]) +} + +// relayHostConn is a host's standing connection: calls go out on it. +func (s *Server) relayHostConn(w http.ResponseWriter, r *http.Request) { + dev := asker(r).Device.ID + ws, err := websocket.Accept(w, r, nil) + if err != nil { + return + } + h := &relayHost{ws: ws} + s.relay.mu.Lock() + if s.relay.hosts == nil { + s.relay.hosts = map[string]*relayHost{} + } + old := s.relay.hosts[dev] + s.relay.hosts[dev] = h + s.relay.mu.Unlock() + if old != nil { + old.ws.Close(websocket.StatusPolicyViolation, "this host connected again") + } + s.log.Info("relay host", "device", dev, "state", "connected") + defer func() { + s.relay.mu.Lock() + if s.relay.hosts[dev] == h { + delete(s.relay.hosts, dev) + } + s.relay.mu.Unlock() + ws.CloseNow() + s.log.Info("relay host", "device", dev, "state", "gone") + }() + ctx := ws.CloseRead(r.Context()) // the host sends nothing on it + t := time.NewTicker(relayPing) + defer t.Stop() + for { + select { + case <-ctx.Done(): + return + case <-t.C: + pctx, cancel := context.WithTimeout(ctx, 10*time.Second) + h.wmu.Lock() + err := ws.Ping(pctx) + h.wmu.Unlock() + cancel() + if err != nil { + return + } + } + } +} + +// relayAccept is the host's connection for one call. +func (s *Server) relayAccept(w http.ResponseWriter, r *http.Request) { + s.relay.mu.Lock() + call := s.relay.calls[r.PathValue("id")] + s.relay.mu.Unlock() + if call == nil || call.host != asker(r).Device.ID { + s.fail(w, http.StatusNotFound, "no such call for this host") + return + } + ws, err := websocket.Accept(w, r, nil) + if err != nil { + return + } + select { + case call.conn <- ws: + <-call.done // the member's side passes the bytes, then lets go + default: + ws.Close(websocket.StatusPolicyViolation, "this call is answered") + } +} + +// relayConnect is a member's connection, with their ticket: paired with +// the host's for the ticket's host. +func (s *Server) relayConnect(w http.ResponseWriter, r *http.Request) { + from := clientAddr(r) + if !s.limits.allow("relay", from, relayConnects) { + w.Header().Set("Retry-After", "60") + s.fail(w, http.StatusTooManyRequests, "too many tries from here: wait a minute") + return + } + ticket := r.Header.Get(api.HeaderTicket) + team, account, host, err := s.db.PeekTicket(r.Context(), ticket) + if errors.Is(err, store.ErrTicket) || ticket == "" { + s.fail(w, http.StatusForbidden, "the ticket is no good: used, expired, or of one no longer in the team") + return + } + if err != nil { + s.internal(w, err) + return + } + limits := s.relayLimits() + s.relay.mu.Lock() + sum := sha256.Sum256([]byte(ticket)) + hash := string(sum[:]) + now := time.Now() + for k, exp := range s.relay.tickets { + if now.After(exp) { + delete(s.relay.tickets, k) + } + } + if _, seen := s.relay.tickets[hash]; seen { + s.relay.mu.Unlock() + s.fail(w, http.StatusForbidden, "this ticket was used on the relay already: ask for another") + return + } + hc := s.relay.hosts[host] + if hc == nil { + s.relay.mu.Unlock() + s.fail(w, http.StatusServiceUnavailable, "the host isn't connected to the relay: it may be off, or not sharing") + return + } + if s.relay.teams == nil { + s.relay.teams = map[string]*teamShare{} + s.relay.calls = map[string]*relayCall{} + s.relay.tickets = map[string]time.Time{} + } + ts := s.relay.teams[team] + if ts == nil { + ts = &teamShare{tokens: float64(limits.Burst), last: now} + s.relay.teams[team] = ts + } + if ts.pairs >= limits.Pairs { + s.relay.mu.Unlock() + s.fail(w, http.StatusTooManyRequests, "your team has as many connections through the relay as it may: close one") + return + } + ts.pairs++ + s.relay.tickets[hash] = now.Add(api.TicketLife) + call := &relayCall{host: host, conn: make(chan *websocket.Conn, 1), done: make(chan struct{}), start: now} + id := randomID() + s.relay.calls[id] = call + s.relay.mu.Unlock() + defer func() { + s.relay.mu.Lock() + delete(s.relay.calls, id) + ts.pairs-- + s.relay.mu.Unlock() + close(call.done) + }() + + msg, _ := json.Marshal(api.RelayCall{ID: id, From: from}) + wctx, cancel := context.WithTimeout(r.Context(), 10*time.Second) + hc.wmu.Lock() + err = hc.ws.Write(wctx, websocket.MessageText, msg) + hc.wmu.Unlock() + cancel() + if err != nil { + s.fail(w, http.StatusServiceUnavailable, "the host isn't connected to the relay") + return + } + var hostWS *websocket.Conn + select { + case hostWS = <-call.conn: + case <-time.After(relayAnswer): + s.fail(w, http.StatusGatewayTimeout, "the host didn't answer") + return + case <-r.Context().Done(): + return + } + defer hostWS.CloseNow() + memberWS, err := websocket.Accept(w, r, nil) + if err != nil { + return + } + defer memberWS.CloseNow() + memberWS.SetReadLimit(relayReadLimit) + hostWS.SetReadLimit(relayReadLimit) + ctx, stop := context.WithCancel(context.Background()) + defer stop() + m := websocket.NetConn(ctx, memberWS, websocket.MessageBinary) + h := websocket.NetConn(ctx, hostWS, websocket.MessageBinary) + var up, down int64 + done := make(chan struct{}, 2) + go func() { up = s.relayCopy(h, m, ts); stop(); done <- struct{}{} }() + go func() { down = s.relayCopy(m, h, ts); stop(); done <- struct{}{} }() + <-done + m.Close() + h.Close() + <-done + // Who reached whom and how much: all the relay knows of a visit. + s.log.Info("relay visit", "team", team, "account", account, "host", host, "up", up, "down", down, "s", int(time.Since(call.start).Seconds())) +} + +// relayTap sees what the relay passes, for tests. +var relayTap func([]byte) + +// relayCopy passes bytes from src to dst within the team's rate. +func (s *Server) relayCopy(dst io.Writer, src io.Reader, ts *teamShare) int64 { + buf := make([]byte, 32<<10) + var total int64 + for { + n, err := src.Read(buf) + if n > 0 { + s.relayWait(ts, n) + if relayTap != nil { + relayTap(buf[:n]) + } + if _, werr := dst.Write(buf[:n]); werr != nil { + return total + } + total += int64(n) + } + if err != nil { + return total + } + } +} + +// relayWait takes n bytes from the team's allowance, waiting for it. +func (s *Server) relayWait(ts *teamShare, n int) { + l := s.relayLimits() + n = min(n, l.Burst) + for { + s.relay.mu.Lock() + now := time.Now() + ts.tokens = min(float64(l.Burst), ts.tokens+now.Sub(ts.last).Seconds()*float64(l.BytesPerSecond)) + ts.last = now + if ts.tokens >= float64(n) { + ts.tokens -= float64(n) + s.relay.mu.Unlock() + return + } + need := (float64(n) - ts.tokens) / float64(l.BytesPerSecond) + s.relay.mu.Unlock() + time.Sleep(time.Duration(need*float64(time.Second)) + time.Millisecond) + } +} diff --git a/cloud/server/relay_test.go b/cloud/server/relay_test.go new file mode 100644 index 00000000..064acd2f --- /dev/null +++ b/cloud/server/relay_test.go @@ -0,0 +1,223 @@ +package server + +import ( + "bytes" + "context" + "errors" + "io" + "log/slog" + "net" + "net/http" + "net/http/httptest" + "sync" + "testing" + "time" + + "github.com/enowx/enowx/cloud/api" + relaypkg "github.com/enowx/enowx/cloud/relay" + "github.com/enowx/enowx/cloud/store" + "github.com/enowx/enowx/cloud/store/storetest" +) + +type relayTeam struct { + srv *httptest.Server + api *Server + db *store.DB + team string + adaHost, adaTok string + bobTok string + key *relaypkg.Key + host *relaypkg.Host +} + +// startRelayTeam is a cloud with Ada's host on its relay, echoing what a +// member sends, and Bob a member of her team. +func startRelayTeam(t *testing.T, cfg Config) *relayTeam { + ctx := context.Background() + cfg.URL = "http://cloud.test" // the relay's address comes from the request in tests + db := storetest.Open(t) + cs := New(cfg, db, slog.New(slog.NewTextHandler(io.Discard, nil))) + srv := httptest.NewServer(cs) + t.Cleanup(srv.Close) + device := func(gh int64, login string) (string, string) { + acc, _ := db.GitHubAccount(ctx, gh, login, login, "", "") + d, tok, _ := db.AddDevice(ctx, acc.ID, login+"'s", api.DeviceHost, make([]byte, 32)) + return d.ID, tok + } + r := &relayTeam{srv: srv, api: cs, db: db, key: relaypkg.NewKey()} + r.adaHost, r.adaTok = device(1, "ada") + _, r.bobTok = device(2, "bob") + name := "Design" + var team api.Team + send(t, "POST", srv.URL+api.PathTeams, r.adaTok, api.TeamParams{Name: &name}, &team) + var inv api.Invite + send(t, "POST", srv.URL+"/v1/teams/"+team.ID+"/invites", r.adaTok, api.InviteParams{Role: api.RoleMember}, &inv) + send(t, "POST", srv.URL+api.PathJoin, r.bobTok, api.JoinParams{Token: inv.Token}, nil) + r.team = team.ID + r.host = &relaypkg.Host{URL: srv.URL, Token: r.adaTok, Key: r.key, Retry: 50 * time.Millisecond} + r.host.Start() + t.Cleanup(func() { r.host.Close() }) + go func() { + for { + c, err := r.host.Accept() + if err != nil { + return + } + go io.Copy(c, c) + } + }() + r.waitUp(t) + return r +} + +func (r *relayTeam) waitUp(t *testing.T) { + t.Helper() + deadline := time.Now().Add(5 * time.Second) + for { + if up, _ := r.host.Status(); up { + return + } + if time.Now().After(deadline) { + _, why := r.host.Status() + t.Fatalf("the host never reached the relay: %s", why) + } + time.Sleep(10 * time.Millisecond) + } +} + +func (r *relayTeam) ticket(t *testing.T) string { + t.Helper() + var tk api.HostTicket + if code, msg := send(t, "POST", r.srv.URL+"/v1/teams/"+r.team+"/tickets", r.bobTok, api.HostTicketParams{Host: r.adaHost}, &tk); code != http.StatusOK { + t.Fatalf("a ticket: %d %s", code, msg) + } + if tk.Relay == "" { + t.Fatal("a ticket without the relay's address") + } + return tk.Ticket +} + +func (r *relayTeam) dial(ticket string) (net.Conn, error) { + ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + defer cancel() + return relaypkg.Dial(ctx, "ws"+r.srv.URL[len("http"):]+api.PathRelayConnect, ticket, r.key.Public()) +} + +// A member with a ticket is paired with the host; what passes between them +// is sealed, so the relay can't read it; a ticket pairs once. +func TestRelayPairsAndCantRead(t *testing.T) { + r := startRelayTeam(t, Config{}) + var mu sync.Mutex + var seen bytes.Buffer + relayTap = func(b []byte) { mu.Lock(); seen.Write(b); mu.Unlock() } + t.Cleanup(func() { relayTap = nil }) + + tk := r.ticket(t) + c, err := r.dial(tk) + if err != nil { + t.Fatal(err) + } + defer c.Close() + secret := []byte("cat ~/projects/launch-plan.md: the terminal's screen") + c.Write(secret) + back := make([]byte, len(secret)) + if _, err := io.ReadFull(c, back); err != nil || !bytes.Equal(back, secret) { + t.Fatalf("echoed through the relay: %q %v", back, err) + } + mu.Lock() + n := seen.Len() + leaked := bytes.Contains(seen.Bytes(), []byte("launch-plan")) || bytes.Contains(seen.Bytes(), []byte("terminal")) + mu.Unlock() + if n < 2*len(secret) { + t.Fatalf("the relay passed %d bytes, less than the visit", n) + } + if leaked { + t.Fatal("the relay read what a member and the host said") + } + if _, err := r.dial(tk); !refusedWith(err, http.StatusForbidden) { + t.Fatalf("a ticket paired twice: %v", err) + } +} + +// A forged ticket, or one of someone no longer in the team, pairs nobody. +func TestRelayRefusesTickets(t *testing.T) { + r := startRelayTeam(t, Config{}) + if _, err := r.dial("enxt_forged"); !refusedWith(err, http.StatusForbidden) { + t.Fatalf("a forged ticket: %v", err) + } + tk := r.ticket(t) + ctx := context.Background() + ada, _ := r.db.GitHubAccount(ctx, 1, "ada", "ada", "", "") + bob, _ := r.db.GitHubAccount(ctx, 2, "bob", "bob", "", "") + if err := r.db.RemoveMember(ctx, r.team, ada.ID, bob.ID); err != nil { + t.Fatal(err) + } + if _, err := r.dial(tk); !refusedWith(err, http.StatusForbidden) { + t.Fatalf("one no longer in the team was paired: %v", err) + } +} + +// A team has so many connections at once through the relay. +func TestRelayTeamLimit(t *testing.T) { + r := startRelayTeam(t, Config{Relay: RelayLimits{Pairs: 1}}) + c, err := r.dial(r.ticket(t)) + if err != nil { + t.Fatal(err) + } + if _, err := r.dial(r.ticket(t)); !refusedWith(err, http.StatusTooManyRequests) { + t.Fatalf("past the team's connections: %v", err) + } + c.Close() + deadline := time.Now().Add(3 * time.Second) + for { + c, err := r.dial(r.ticket(t)) + if err == nil { + c.Close() + break + } + if time.Now().After(deadline) { + t.Fatalf("a connection closed left its room taken: %v", err) + } + time.Sleep(20 * time.Millisecond) + } +} + +// A host whose connection to the relay drops connects again, and members +// reach it as before. +func TestRelayHostReconnects(t *testing.T) { + r := startRelayTeam(t, Config{}) + r.api.relay.mu.Lock() + h := r.api.relay.hosts[r.adaHost] + r.api.relay.mu.Unlock() + h.ws.CloseNow() // as a proxy or the network would + deadline := time.Now().Add(3 * time.Second) + for up, _ := r.host.Status(); up; up, _ = r.host.Status() { + if time.Now().After(deadline) { + t.Fatal("the host didn't notice its connection dropped") + } + time.Sleep(5 * time.Millisecond) + } + r.waitUp(t) + deadline = time.Now().Add(5 * time.Second) + for { + c, err := r.dial(r.ticket(t)) + if err == nil { + c.Write([]byte("hi")) + b := make([]byte, 2) + if _, err := io.ReadFull(c, b); err != nil { + t.Fatal(err) + } + c.Close() + return + } + if time.Now().After(deadline) { + t.Fatalf("after the host's connection dropped: %v", err) + } + time.Sleep(50 * time.Millisecond) + } +} + +func refusedWith(err error, status int) bool { + var r *relaypkg.Refused + return errors.As(err, &r) && r.Status == status +} diff --git a/cloud/server/server.go b/cloud/server/server.go index 2477b057..fa23eeea 100644 --- a/cloud/server/server.go +++ b/cloud/server/server.go @@ -44,6 +44,8 @@ type Config struct { Contributor ContributorConfig // Library is enowX UI's setup; nil or without a store, there's none. Library *library.Config + // Relay is a team's share of the relay (relay.go). + Relay RelayLimits } // Server answers the cloud's API. Its handlers run on net/http's @@ -65,6 +67,8 @@ type Server struct { contributor *contributorRole // library is enowX UI, nil when it's off. library *library.Service + // relay pairs members with the hosts they visit (relay.go). + relay relay } // New is a server over db. @@ -106,6 +110,9 @@ func New(cfg Config, db *store.DB, log *slog.Logger) *Server { s.mux.HandleFunc("GET "+api.PathAudit, s.signedIn(s.audit)) s.mux.HandleFunc("POST "+api.PathHostTickets, s.signedIn(s.issueTicket)) s.mux.HandleFunc("POST "+api.PathTicketRedeem, s.signedIn(s.redeemTicket)) + s.mux.HandleFunc("GET "+api.PathRelayHost, s.signedIn(s.relayHostConn)) + s.mux.HandleFunc("GET "+api.PathRelayAccept, s.signedIn(s.relayAccept)) + s.mux.HandleFunc("GET "+api.PathRelayConnect, s.relayConnect) s.mux.HandleFunc("GET "+api.PathRecoveryChallenge, s.signedIn(s.recoveryChallenge)) s.mux.HandleFunc("POST "+api.PathRecovery, s.signedIn(s.setRecovery)) s.mux.HandleFunc("GET "+api.PathTeamRecovery, s.signedIn(s.recoveryKeys)) @@ -209,6 +216,9 @@ type statusWriter struct { status int } +// Unwrap lets the relay's WebSockets take the connection over. +func (w *statusWriter) Unwrap() http.ResponseWriter { return w.ResponseWriter } + func (w *statusWriter) WriteHeader(code int) { w.status = code w.ResponseWriter.WriteHeader(code) diff --git a/cloud/store/tickets.go b/cloud/store/tickets.go index 91c355e5..bed29041 100644 --- a/cloud/store/tickets.go +++ b/cloud/store/tickets.go @@ -83,3 +83,26 @@ func (s *DB) RedeemTicket(ctx context.Context, ticket, hostDevice string) (api.V tx.ExecContext(ctx, `DELETE FROM host_tickets WHERE expires < $1`, stamp(now().Add(-time.Hour))) return v, tx.Commit() } + +// PeekTicket says which host and team ticket leads to, without using it: +// the relay's check before it pairs a member with a host. It is no good +// used, past its time, or of one no longer in the team. +func (s *DB) PeekTicket(ctx context.Context, ticket string) (teamID, accountID, hostDevice string, err error) { + var expires string + var used int + err = s.QueryRowContext(ctx, `SELECT team_id, account_id, host_device, expires, used FROM host_tickets WHERE hash = $1`, hashToken(ticket)). + Scan(&teamID, &accountID, &hostDevice, &expires, &used) + if errors.Is(err, sql.ErrNoRows) { + return "", "", "", ErrTicket + } + if err != nil { + return "", "", "", err + } + if used != 0 || time.Now().After(parseStamp(expires)) { + return "", "", "", ErrTicket + } + if _, err := role(ctx, s.DB, teamID, accountID); err != nil { + return "", "", "", ErrTicket + } + return teamID, accountID, hostDevice, nil +} diff --git a/cmd/enowx-cloud/main.go b/cmd/enowx-cloud/main.go index 599ba6d0..44c9787e 100644 --- a/cmd/enowx-cloud/main.go +++ b/cmd/enowx-cloud/main.go @@ -61,6 +61,13 @@ func run(log *slog.Logger) error { LinkSecret: env("ENOWX_CLOUD_CONTRIBUTOR_LINK_SECRET", ""), Repo: env("ENOWX_CLOUD_CONTRIBUTOR_REPO", ""), } + // The relay's share for each team: connections at once, bytes a second + // and a burst above it; unset, the server's defaults. + cfg.Relay = server.RelayLimits{ + Pairs: envInt("ENOWX_CLOUD_RELAY_TEAM_CONNS"), + BytesPerSecond: envInt("ENOWX_CLOUD_RELAY_TEAM_RATE"), + Burst: envInt("ENOWX_CLOUD_RELAY_TEAM_BURST"), + } addr, data := env("ENOWX_CLOUD_ADDR", ":8080"), env("ENOWX_CLOUD_DATA", "data") if err := os.MkdirAll(data, 0o700); err != nil { return err @@ -129,3 +136,9 @@ func run(log *slog.Logger) error { } return nil } + +// envInt is a whole number from the environment, 0 when unset or not one. +func envInt(name string) int { + n, _ := strconv.Atoi(os.Getenv(name)) + return n +} diff --git a/daemon/cloud.go b/daemon/cloud.go index 92674e58..ea8977e7 100644 --- a/daemon/cloud.go +++ b/daemon/cloud.go @@ -52,7 +52,8 @@ func (s *Server) cloudChanged() { if st, err := s.cloudStatus(); err == nil { s.broadcast(proto.EventCloudChanged, st) } - s.spacesChanged() // signed in or out: the account's teams' spaces + s.spacesChanged() // signed in or out: the account's teams' spaces + go s.restartSharing() // the relay is the cloud's, with the host's token } // cloudSetURL sets the cloud's address, checking it is one; another diff --git a/daemon/hostshare.go b/daemon/hostshare.go index 3500d2cf..b25af933 100644 --- a/daemon/hostshare.go +++ b/daemon/hostshare.go @@ -15,6 +15,7 @@ import ( "time" "github.com/enowx/enowx/cloud/api" + "github.com/enowx/enowx/cloud/relay" "github.com/enowx/enowx/core/projectctx" "github.com/enowx/enowx/core/tailnet" "github.com/enowx/enowx/proto" @@ -22,8 +23,9 @@ import ( // A host shared with its teams (docs/cloud.md, Hosts and sessions): the // projects its user picks for each team, in share.json. While any is -// shared, the team listener serves the teams' members on the host's -// Tailscale address, and the host announces itself in each team's space +// shared, the team listener serves the teams' members through the cloud's +// relay (and the host's Tailscale address, when it has one), and the host +// announces itself in each team's space // (hosts/.json, sealed and synced with it), so the members' // hosts list it. A visitor sees the shared projects of their team alone. @@ -33,12 +35,14 @@ var listenTeam func(*Server, net.Listener) error func init() { listenTeam = (*Server).ListenTeam } -// hostSharing is the share and the team listener; its mutex guards both. +// hostSharing is the share and the team listener; its mutex guards all. type hostSharing struct { - mu sync.Mutex - ln net.Listener - addr string - err string + mu sync.Mutex + ln net.Listener // the team listener: the relay's and Tailscale's (teamrelay.go) + relay *relay.Host // nil without a cloud signed in to + key string // the public key members' channels open to + addr string // the Tailscale address, "" without one + err string } type hostShareConfig struct { @@ -81,6 +85,13 @@ func (s *Server) hostShare() proto.HostShare { } s.share.mu.Lock() out.Addr, out.Err = s.share.addr, s.share.err + if s.share.relay != nil { + up, why := s.share.relay.Status() + out.Relay = up + if !up && why != "" && out.Err == "" { + out.Err = "the relay: " + why + } + } s.share.mu.Unlock() return out } @@ -140,27 +151,47 @@ func (s *Server) startSharing() { s.share.mu.Lock() switch { case shared && s.share.ln == nil: + fan := newFanIn() + var why []string + h, key, err := s.startRelay() + if err != nil { + why = append(why, "the relay: "+err.Error()) + } else if h != nil { + fan.add(h) + } + // Tailscale too, when the host has it: members of an older enowX + // know no other way. addr := os.Getenv(proto.EnvTeamListen) if addr == "" { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) - ip, err := tailnet.SelfIP(ctx) + if ip, err := tailnet.SelfIP(ctx); err == nil && ip != "" { + addr = net.JoinHostPort(ip, strconv.Itoa(proto.TeamPort)) + } cancel() - if err != nil || ip == "" { - s.share.err = "members reach a shared host over Tailscale: " + errText(err, "this host has no Tailscale address") - break + } + var ln net.Listener + if addr != "" { + if ln, err = net.Listen("tcp", addr); err != nil { + why = append(why, "the team listener: "+err.Error()) + } else { + fan.add(ln) } - addr = net.JoinHostPort(ip, strconv.Itoa(proto.TeamPort)) } - ln, err := net.Listen("tcp", addr) - if err != nil { - s.share.err = "the team listener: " + err.Error() + if h == nil && ln == nil { + if len(why) == 0 { + why = append(why, "sign this host in to the team's cloud: members reach it through the cloud's relay") + } + s.share.err = strings.Join(why, "; ") break } - s.share.ln, s.share.addr, s.share.err = ln, ln.Addr().String(), "" - go listenTeam(s, ln) + s.share.ln, s.share.relay, s.share.key, s.share.err = fan, h, key, strings.Join(why, "; ") + if ln != nil { + s.share.addr = ln.Addr().String() + } + go listenTeam(s, fan) case !shared && s.share.ln != nil: s.share.ln.Close() - s.share.ln, s.share.addr, s.share.err = nil, "", "" + s.share.ln, s.share.relay, s.share.key, s.share.addr, s.share.err = nil, nil, "", "", "" case !shared: s.share.err = "" } @@ -168,21 +199,14 @@ func (s *Server) startSharing() { s.announceHost() } -func errText(err error, none string) string { - if err != nil { - return err.Error() - } - return none -} - // stopSharing closes the team listener, as the daemon stops. func (s *Server) stopSharing() { s.share.mu.Lock() defer s.share.mu.Unlock() if s.share.ln != nil { s.share.ln.Close() - s.share.ln = nil } + s.share.ln, s.share.relay, s.share.key, s.share.addr = nil, nil, "", "" } // announceHost writes, or takes back, the host's entry in each of its @@ -194,18 +218,18 @@ func (s *Server) announceHost() { } f := s.readShare() s.share.mu.Lock() - addr := s.share.addr + addr, key := s.share.addr, s.share.key s.share.mu.Unlock() for _, sp := range s.spaces() { file := filepath.Join(s.teamsDir(), sp.Team, "hosts", st.DeviceID+".json") list := f.Teams[sp.Team] - if len(list) == 0 || addr == "" { + if len(list) == 0 || addr == "" && key == "" { if os.Remove(file) == nil { s.teamSync.kick() } continue } - h := proto.TeamHost{Team: sp.Team, Device: st.DeviceID, Name: s.userdata.Name, Owner: st.Account, Addr: addr, Projects: []proto.TeamHostProject{}} + h := proto.TeamHost{Team: sp.Team, Device: st.DeviceID, Name: s.userdata.Name, Owner: st.Account, Addr: addr, Key: key, Projects: []proto.TeamHostProject{}} for _, path := range list { h.Projects = append(h.Projects, proto.TeamHostProject{Name: filepath.Base(path), Path: path, Ref: s.remoteRef(path)}) } diff --git a/daemon/teamrelay.go b/daemon/teamrelay.go new file mode 100644 index 00000000..d29eca71 --- /dev/null +++ b/daemon/teamrelay.go @@ -0,0 +1,103 @@ +package daemon + +import ( + "net" + "path/filepath" + "sync" + "time" + + "github.com/enowx/enowx/cloud/relay" +) + +// The ways members reach a shared host (docs/cloud.md, Hosts and sessions): +// the cloud's relay, which needs no address of the host's, and the host's +// Tailscale address when it has one (members of an older enowX know only +// that). Both feed one listener, the team listener, whose checks are the +// same whichever way a member came. + +// relayRetry is the first wait before the host connects to the relay again. +var relayRetry = time.Second + +// relayKeyFile is the host's key members' channels open to (cloud/relay). +func (s *Server) relayKeyFile() string { return filepath.Join(s.dataDir, "team-relay.key") } + +// startRelay connects the host to its cloud's relay, signed in: nil +// without a cloud or a sign-in. +func (s *Server) startRelay() (*relay.Host, string, error) { + st, err := s.cloud.store.Get() + if err != nil || !st.SignedIn() || st.URL == "" { + return nil, "", nil + } + key, err := relay.LoadKey(s.relayKeyFile()) + if err != nil { + return nil, "", err + } + h := &relay.Host{URL: st.URL, Token: st.Token, Key: key, Retry: relayRetry} + if s.Log != nil { + h.Log = s.Log.Warn + } + h.Start() + return h, key.Public(), nil +} + +// restartSharing opens the team listener again, as the cloud the host +// signs in to changed. +func (s *Server) restartSharing() { + s.stopSharing() + s.startSharing() +} + +// fanIn is a listener of several: it gives what any of them accepts, and +// closes them all. Its mutex guards closed. +type fanIn struct { + conns chan net.Conn + done chan struct{} + mu sync.Mutex + closed bool + srcs []net.Listener +} + +func newFanIn() *fanIn { return &fanIn{conns: make(chan net.Conn), done: make(chan struct{})} } + +// add takes l's connections in, until it closes. +func (f *fanIn) add(l net.Listener) { + f.srcs = append(f.srcs, l) + go func() { + for { + c, err := l.Accept() + if err != nil { + return + } + select { + case f.conns <- c: + case <-f.done: + c.Close() + return + } + } + }() +} + +func (f *fanIn) Accept() (net.Conn, error) { + select { + case c := <-f.conns: + return c, nil + case <-f.done: + return nil, net.ErrClosed + } +} + +func (f *fanIn) Close() error { + f.mu.Lock() + defer f.mu.Unlock() + if !f.closed { + f.closed = true + close(f.done) + for _, l := range f.srcs { + l.Close() + } + } + return nil +} + +func (f *fanIn) Addr() net.Addr { return relay.Addr("team") } diff --git a/daemon/teamrelay_test.go b/daemon/teamrelay_test.go new file mode 100644 index 00000000..fc32b99c --- /dev/null +++ b/daemon/teamrelay_test.go @@ -0,0 +1,98 @@ +package daemon + +import ( + "context" + "encoding/json" + "io" + "log/slog" + "net/http/httptest" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/enowx/enowx/client" + "github.com/enowx/enowx/cloud/api" + "github.com/enowx/enowx/cloud/relay" + "github.com/enowx/enowx/cloud/server" + "github.com/enowx/enowx/cloud/store/storetest" + "github.com/enowx/enowx/proto" +) + +// A member reaches a host shared with their team through the cloud's relay, +// with no address of the host's: the host signs them in with their ticket +// as over any path, and lets go of them once they leave the team. +func TestVisitorThroughTheRelay(t *testing.T) { + ctx := context.Background() + db := storetest.Open(t) + cloud := httptest.NewServer(server.New(server.Config{URL: "http://cloud.test"}, db, slog.New(slog.NewTextHandler(io.Discard, nil)))) + t.Cleanup(cloud.Close) + ada := signInHost(t, db, cloud.URL, 1, "ada") + bob := signInHost(t, db, cloud.URL, 2, "bob") + team, _ := db.CreateTeam(ctx, ada.acc, "Design") + inv, _ := db.CreateInvite(ctx, team.ID, ada.acc, api.InviteParams{Role: api.RoleMember}) + db.AcceptInvite(ctx, inv.Token, bob.acc) + ada.srv.learnTeams(ada.acc, []api.Team{{ID: team.ID, Name: "Design", Role: api.RoleOwner}}) + project := gitRepo(t) + ada.c.AddProject(project) + t.Setenv(proto.EnvTeamListen, "127.0.0.1:0") // not this machine's Tailscale + if _, err := ada.c.SetHostShare(team.ID, []string{project}); err != nil { + t.Fatal(err) + } + deadline := time.Now().Add(5 * time.Second) + for { + sh, err := ada.c.HostShare() + if err == nil && sh.Relay { + break + } + if time.Now().After(deadline) { + t.Fatalf("the host never reached the relay: %+v %v", sh, err) + } + time.Sleep(20 * time.Millisecond) + } + adaHost, _ := ada.srv.cloud.store.Get() + var announced proto.TeamHost + data, _ := os.ReadFile(filepath.Join(ada.srv.teamsDir(), team.ID, "hosts", adaHost.DeviceID+".json")) + if json.Unmarshal(data, &announced) != nil || announced.Key == "" { + t.Fatalf("the host announced no key for members' channels: %s", data) + } + + bobDev, _ := bob.srv.cloud.store.Get() + tk, err := db.IssueTicket(ctx, team.ID, bob.acc, bobDev.DeviceID, adaHost.DeviceID) + if err != nil { + t.Fatal(err) + } + dctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + nc, err := relay.Dial(dctx, "ws"+strings.TrimPrefix(cloud.URL, "http")+api.PathRelayConnect, tk.Ticket, announced.Key) + if err != nil { + t.Fatal(err) + } + v, err := client.New(nc, nil) + if err != nil { + t.Fatal(err) + } + defer v.Close() + if me, err := v.TeamAuth(tk.Ticket); err != nil || me.GitHub != "bob" { + t.Fatalf("signed in through the relay: %+v %v", me, err) + } + if list, err := v.Projects(); err != nil || len(list) != 1 || list[0].Path != project { + t.Fatalf("the shared project through the relay: %+v %v", list, err) + } + + if err := db.RemoveMember(ctx, team.ID, ada.acc, bob.acc); err != nil { + t.Fatal(err) + } + ada.srv.recheckVisitors("") + deadline = time.Now().Add(5 * time.Second) + for { + if _, err := v.Projects(); err != nil { + break + } + if time.Now().After(deadline) { + t.Fatal("a member removed from the team stayed connected through the relay") + } + time.Sleep(20 * time.Millisecond) + } +} diff --git a/docs/cli.md b/docs/cli.md index 8ad6f019..676a0199 100644 --- a/docs/cli.md +++ b/docs/cli.md @@ -37,7 +37,7 @@ | `features [list \| on NAME \| off NAME \| reset NAME \| lazy on\|off \| preset P] [--global\|--host\|--project P\|--session]` | Which features agents are told of and get tools for; from a session, `on` asks the user ([features.md](features.md)). `help NAME` prints a feature's guide. | | `notify [status \| sound [done\|input\|error] [--text T] \| set … \| mute ID \| unmute ID]` | The sounds the user's enowX app plays when agents finish, ask or fail: how they are set, one played now on the user's computer from any host, or changed (`--on`/`--off`, `--volume 0-100`, `--sound KIND=NAME`, `--quiet 22:00-07:00\|off`, `--watched`, `--banner`, `--badge on\|off`); an agent or a session muted ([settings.md](settings.md)). | | `cloud [status \| whoami \| url URL\|off \| login [--name N] \| logout \| devices \| approve CODE \| recovery new\|use KEY]` | The host's sign-in to a team cloud ([cloud.md](cloud.md)): its address (off by default), signing in with a code approved in a browser, the app, or `cloud approve` on a host signed in, which waits until it is; the account's devices; the account's recovery key, made and shown once, or used on a new device to get the teams' keys back. The token and the device's key stay on the host, in `~/.enowx-app/cloud/`, its user's alone. | -| `team [list \| create NAME \| members TEAM \| invite TEAM [--role R] [--email E] \| invites TEAM \| join LINK\|TOKEN \| role TEAM MEMBER ROLE \| remove TEAM MEMBER \| leave TEAM \| sync [TEAM]]` | The account's teams in the cloud, TEAM by name or ID and MEMBER by GitHub login, as the host's device; `sync` syncs a team's space (its notes and canvases) with the cloud now; `projects [TEAM]` lists the teams' projects and where each is on the host, `project add TEAM LINK [--name N]` and `project rm TEAM PROJECT` change them (admins), `project clone TEAM PROJECT [--dir D]` clones one here unless it is here already; `audit TEAM` prints the team's log (admins); `chat TEAM [--last N]` reads the team's chat and `chat TEAM --send TEXT` writes in it; `hosts` lists the hosts the teams' members share, `share [TEAM [PROJECT…]]` shares this host's projects with a team (none unshares); `kit [TEAM]`, `kit add TEAM skill|command|subagent NAME`, `kit rm TEAM KIND NAME` share the host's agents' items with the team's projects' sessions. | +| `team [list \| create NAME \| members TEAM \| invite TEAM [--role R] [--email E] \| invites TEAM \| join LINK\|TOKEN \| role TEAM MEMBER ROLE \| remove TEAM MEMBER \| leave TEAM \| sync [TEAM]]` | The account's teams in the cloud, TEAM by name or ID and MEMBER by GitHub login, as the host's device; `sync` syncs a team's space (its notes and canvases) with the cloud now; `projects [TEAM]` lists the teams' projects and where each is on the host, `project add TEAM LINK [--name N]` and `project rm TEAM PROJECT` change them (admins), `project clone TEAM PROJECT [--dir D]` clones one here unless it is here already; `audit TEAM` prints the team's log (admins); `chat TEAM [--last N]` reads the team's chat and `chat TEAM --send TEXT` writes in it; `hosts` lists the hosts the teams' members share, `share [TEAM [PROJECT…]]` shares this host's projects with a team (none unshares), which members then reach through the cloud's relay, end to end encrypted ([cloud.md](cloud.md#hosts-and-sessions)); `kit [TEAM]`, `kit add TEAM skill|command|subagent NAME`, `kit rm TEAM KIND NAME` share the host's agents' items with the team's projects' sessions. | | `icons [search QUERY [--lib L] [--limit N] \| libs]` | The SVG icons notes and groups take, found by name and keyword: `lib:name` each, best first, with its keywords; needs no daemon. | | `notes import FILE\|DIR [--from F] [--group G] [--project P] [--mode M] [--rows-as-notes] [--dry-run] [--no-wait]` | Notes from other apps: a Notion export, an Obsidian vault or folder of Markdown, .md/.txt, Evernote's .enex, Keep's Takeout, HTML, an enowX export or backup (`--from notion\|obsidian\|md\|enex\|keep\|html\|enowx`, found out when left out); `--dry-run` lists what it would make; notes imported before are skipped, or `--mode overwrite` / `duplicate`. `--from notion-api [--token-env VAR]` imports every page a Notion integration sees, its token read from `$VAR` (`NOTION_TOKEN`). Waits, showing its progress, then says what it made, skipped and couldn't convert. | | `notes import-status [ID] [--wait]`, `notes import-cancel ID` | How an import goes (none: the imports lately); stop one, the notes it made taken back. | diff --git a/docs/cloud.md b/docs/cloud.md index d9c6e9e7..e333762d 100644 --- a/docs/cloud.md +++ b/docs/cloud.md @@ -88,9 +88,9 @@ Tokens are random, kept hashed on the server, one per device; signing out or rem ## Hosts and sessions -Two kinds of hosts serve a team: a **team host**, paired to the team by an admin, which stays when members come and go, and a member's **personal host shared** with the team, project by project, until they take it back. Members reach them as today, over SSH or the tailnet; the cloud only says which hosts and projects a team has, and the daemon checks a member's token and role before letting them in. Sessions in a team's project show to the team (status, watching) unless marked private; taking over one needs its owner's approval. A session runs with the member's own agent account and git identity, lent from their vault for that session. An agent reads another session only when the user asks it to (`enowx-app peek`, docs/cli.md): its last messages and project, live from its host, never kept in the cloud; the user's own hosts, and the team's shared ones within what the member sees there. +Two kinds of hosts serve a team: a **team host**, paired to the team by an admin, which stays when members come and go, and a member's **personal host shared** with the team, project by project, until they take it back. Members reach them through the cloud's relay (below); the cloud only says which hosts and projects a team has, and the daemon checks a member's token and role before letting them in. Sessions in a team's project show to the team (status, watching) unless marked private; taking over one needs its owner's approval. A session runs with the member's own agent account and git identity, lent from their vault for that session. An agent reads another session only when the user asks it to (`enowx-app peek`, docs/cli.md): its last messages and project, live from its host, never kept in the cloud; the user's own hosts, and the team's shared ones within what the member sees there. -A member reaches another's host over Tailscale, the host's node shared with their Tailscale account (no relay of enowX's). The host's daemon serves them on its team listener, apart from its own socket: a connection there gets nothing, not even events, until it hands a ticket the cloud gave the member for that host (`team.auth`): before it, a connection has 10 seconds, frames of 64 KiB at most and one request (anything but its ticket, or a ticket refused, closes it), and the listener takes a few connections and tickets a minute from one address; the host redeems the ticket with its own token, so no member's token ever leaves their device, and serves the visitor as their role allows. It asks the cloud again every 2 minutes, and before a visitor starts a session, whether each is still the team's member and in which role: a member removed is let go, one made a viewer keeps what a viewer may. A visitor sees no more of the host than the shared projects: not its home, name or process, not how a session was started or what the vault lent it, and a session they don't see is as missing as one that isn't there. The host's own app lists who is connected (`host.visitors`). Its user shares a host from a team's Hosts tab (or `enowx-app team share TEAM PROJECT…`), picking which of its projects the team sees: the team listener then opens on the host's Tailscale address (port 7178), and the host is announced in the team's space (`hosts/.json`, sealed and synced like its other items). The members' apps list it in the rail, under TEAM (`enowx-app team hosts`); opening it gets a ticket from the cloud through the member's own host, connects, and shows the shared projects and their sessions. A visitor's lists hold the shared projects alone, and their files read only within them, a link that leads out of a shared project refused. Taking a project back closes visitors' views of its sessions and ends their control of them; sharing nothing more with a team, or leaving it, closes its visitors' connections. A visitor **watches** a shared session live, its terminal read-only: what they type and their window's size reach it only once they **take over**, which they ask from the watching view (one ask a session at a time, a few a minute); the host's user is asked (Let them type, or Not now), sees in the session's header who watches it and who has control, and takes it back any time; a visitor who leaves gives back what they took over. A member (not a viewer) also **starts sessions of their own** in a shared project, from its page, once the host's user lets them: **Members start sessions here** in the team's Hosts tab, off until turned on (`allowSessions` of `host.share.set`). Such a session runs as the host's OS user, so an agent there can do what that user can, the daemon's socket included: turn it on only for people trusted with the machine. It is Claude Code or Codex, run as an agent account of theirs on the host (`Team · @login` in its Agent accounts, kept by their cloud account, signed in by them the first time), its home their own with nothing linked from the host's user's (no sign-in, settings, skills or conversations); with their git identity and a GitHub CLI sign-in of their own (`GH_CONFIG_DIR` in that account's home); and none of the host's credentials: not the variables its user set for sessions, not its model API, nothing the visitor passes, no project setup script; the vault lends nothing to it. The session is theirs to type into, and shows as theirs in the host's list. The host's user keeps a session of a shared project from the team from its menu (**Keep from the team**): it leaves the team's lists, watchers' views close, and a takeover of it ends. A member's agent asked to read another session (`enowx-app peek`) reaches the team's shared hosts too, seeing only what the member sees there, and each read goes in the team's log (`session.read`). +A member reaches another's host through the cloud's **relay**, with no address of the host's and no port open on it: the host keeps a connection to the relay (`/v1/relay/host`, its own token) while it shares anything; a member's app connects with their ticket (`/v1/relay/connect`), the relay checks the ticket without using it (one pairing a ticket), tells the host, the host opens a connection for that member (`/v1/relay/accept/{id}`), and the relay passes bytes between the two. What passes is **end to end encrypted** (`cloud/relay`): the host has a key of its own (`team-relay.key` in its data folder), whose public half it announces with itself in the team's space, sealed, so the cloud can't put another there. The member's app opens a channel to that key alone: its fresh key, the host's fresh key and a proof that the host holds the announced one (X25519, HKDF-SHA256), then records sealed with ChaCha20-Poly1305 each way under a counter, so a record read, changed, dropped or replayed by the relay fails. The relay sees who connects to which host, when, and how many bytes; not a terminal, a file or a ticket's answer. It holds each team to a number of connections at once and a rate of bytes (32, and 4 MiB a second with bursts of 16 MiB, unless the server sets others), and an address to 30 connections a minute. A host that also has a Tailscale address keeps its team listener there too (port 7178), for members of an older enowX, which knows no relay; a member's app tries the relay first and that address after. The host's daemon serves them on its team listener, apart from its own socket: a connection there gets nothing, not even events, until it hands a ticket the cloud gave the member for that host (`team.auth`): before it, a connection has 10 seconds, frames of 64 KiB at most and one request (anything but its ticket, or a ticket refused, closes it), and the listener takes a few connections and tickets a minute from one address; the host redeems the ticket with its own token, so no member's token ever leaves their device, and serves the visitor as their role allows. It asks the cloud again every 2 minutes, and before a visitor starts a session, whether each is still the team's member and in which role: a member removed is let go, one made a viewer keeps what a viewer may. A visitor sees no more of the host than the shared projects: not its home, name or process, not how a session was started or what the vault lent it, and a session they don't see is as missing as one that isn't there. The host's own app lists who is connected (`host.visitors`). Its user shares a host from a team's Hosts tab (or `enowx-app team share TEAM PROJECT…`), picking which of its projects the team sees: the host then connects to the relay (and opens its team listener on its Tailscale address, port 7178, when it has one), and the host is announced in the team's space (`hosts/.json`, sealed and synced like its other items). The members' apps list it in the rail, under TEAM (`enowx-app team hosts`); opening it gets a ticket from the cloud through the member's own host, connects, and shows the shared projects and their sessions. A visitor's lists hold the shared projects alone, and their files read only within them, a link that leads out of a shared project refused. Taking a project back closes visitors' views of its sessions and ends their control of them; sharing nothing more with a team, or leaving it, closes its visitors' connections. A visitor **watches** a shared session live, its terminal read-only: what they type and their window's size reach it only once they **take over**, which they ask from the watching view (one ask a session at a time, a few a minute); the host's user is asked (Let them type, or Not now), sees in the session's header who watches it and who has control, and takes it back any time; a visitor who leaves gives back what they took over. A member (not a viewer) also **starts sessions of their own** in a shared project, from its page, once the host's user lets them: **Members start sessions here** in the team's Hosts tab, off until turned on (`allowSessions` of `host.share.set`). Such a session runs as the host's OS user, so an agent there can do what that user can, the daemon's socket included: turn it on only for people trusted with the machine. It is Claude Code or Codex, run as an agent account of theirs on the host (`Team · @login` in its Agent accounts, kept by their cloud account, signed in by them the first time), its home their own with nothing linked from the host's user's (no sign-in, settings, skills or conversations); with their git identity and a GitHub CLI sign-in of their own (`GH_CONFIG_DIR` in that account's home); and none of the host's credentials: not the variables its user set for sessions, not its model API, nothing the visitor passes, no project setup script; the vault lends nothing to it. The session is theirs to type into, and shows as theirs in the host's list. The host's user keeps a session of a shared project from the team from its menu (**Keep from the team**): it leaves the team's lists, watchers' views close, and a takeover of it ends. A member's agent asked to read another session (`enowx-app peek`) reaches the team's shared hosts too, seeing only what the member sees there, and each read goes in the team's log (`session.read`). ## API @@ -160,6 +160,7 @@ docker run -p 8080:8080 -e ENOWX_CLOUD_URL=https://cloud.example.com -e ENOWX_CL | `ENOWX_CLOUD_R2_BUCKET` | a Cloudflare R2 bucket keeping teams' files (a chat's images, sealed); unset, they're kept in the data folder's `blobs` | | `ENOWX_CLOUD_R2_ACCOUNT_ID`, `ENOWX_CLOUD_R2_ACCESS_KEY_ID`, `ENOWX_CLOUD_R2_SECRET_ACCESS_KEY` | the bucket's account and an R2 API token's keys | | `ENOWX_CLOUD_DEFAULT_TEAM`, `ENOWX_CLOUD_DEFAULT_ROLE` | a team's ID everyone joins as they sign in, and the role they get (`viewer` by default); optional | +| `ENOWX_CLOUD_RELAY_TEAM_CONNS`, `ENOWX_CLOUD_RELAY_TEAM_RATE`, `ENOWX_CLOUD_RELAY_TEAM_BURST` | each team's share of the relay: connections at once (32), bytes a second (4 MiB) and a burst above it (16 MiB); optional. The relay's WebSockets need a proxy that passes `Upgrade` and keeps a connection open past a minute (the relay pings every 30 seconds) | | `ENOWX_CLOUD_SMTP_URL` | `smtp://user:pass@host:587`, to send invites by email; optional | It speaks plain HTTP: put it behind a proxy that terminates TLS (Coolify's, Caddy, nginx). diff --git a/go.mod b/go.mod index 0dcd1ac1..e5262d09 100644 --- a/go.mod +++ b/go.mod @@ -7,6 +7,7 @@ require ( github.com/alecthomas/chroma/v2 v2.27.0 github.com/andybalholm/brotli v1.2.7 github.com/anthropics/anthropic-sdk-go/v2 v2.5.0 + github.com/coder/websocket v1.8.15 github.com/egoist/mygo v0.6.2 github.com/gen2brain/webp v0.6.4 github.com/jackc/pgx/v5 v5.11.0 @@ -49,7 +50,6 @@ require ( github.com/bradfitz/reco v0.0.0-20260929154613-b883fbd17e3f // indirect github.com/buger/jsonparser v1.1.2 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/coder/websocket v1.8.15 // indirect github.com/coreos/go-iptables v0.8.0 // indirect github.com/creachadair/msync v0.10.1 // indirect github.com/creack/pty v1.1.24 // indirect diff --git a/proto/hostshare.go b/proto/hostshare.go index c640b8d0..8375e9c5 100644 --- a/proto/hostshare.go +++ b/proto/hostshare.go @@ -4,8 +4,9 @@ import "time" // A host shared with a team (docs/cloud.md, Hosts and sessions): its user // picks which of its projects each team sees. The host then serves the -// team's members on its team listener, on its Tailscale address, and -// announces itself in the team's space; the members' hosts list it. +// team's members on its team listener, through the cloud's relay (and its +// Tailscale address when it has one), and announces itself in the team's +// space; the members' hosts list it. const ( MethodHostShare = "host.share" // → HostShare (the host's own connections) MethodHostShareSet = "host.share.set" // HostShareTeam (no Projects: unshared) → HostShare @@ -31,6 +32,9 @@ type HostShare struct { // why it doesn't when something is shared. Addr string `json:"addr,omitempty"` Err string `json:"err,omitempty"` + // Relay says the host is connected to its cloud's relay, through which + // members reach it with no address of its own. + Relay bool `json:"relay,omitempty"` } // HostShareTeam is the projects (their folders) the host shares with Team. @@ -46,11 +50,14 @@ type HostShareTeam struct { // TeamHost is a host a member shares with a team, as its space says. type TeamHost struct { - Team string `json:"team"` - Device string `json:"device"` // the host's device in the cloud - Name string `json:"name"` - Owner string `json:"owner"` // the member sharing it - Addr string `json:"addr"` + Team string `json:"team"` + Device string `json:"device"` // the host's device in the cloud + Name string `json:"name"` + Owner string `json:"owner"` // the member sharing it + Addr string `json:"addr"` // its Tailscale address, "" without one + // Key is the host's public key, which a member's channel through the + // cloud's relay opens to (cloud/relay); "" for a host of an older enowX. + Key string `json:"key,omitempty"` Projects []TeamHostProject `json:"projects"` Updated time.Time `json:"updated"` } diff --git a/proto/testdata/api.txt b/proto/testdata/api.txt index 3304d96c..61ef5052 100644 --- a/proto/testdata/api.txt +++ b/proto/testdata/api.txt @@ -1430,6 +1430,7 @@ field HostSessions.hostName string field HostSessions.sessions []SessionInfo field HostShare.addr string field HostShare.err string +field HostShare.relay bool field HostShare.teams []HostShareTeam field HostShareTeam.allowSessions *bool field HostShareTeam.projects []string @@ -2492,6 +2493,7 @@ field TeamChatParams.last int field TeamChatParams.team string field TeamHost.addr string field TeamHost.device string +field TeamHost.key string field TeamHost.name string field TeamHost.owner string field TeamHost.projects []TeamHostProject diff --git a/ui/teamhosts.go b/ui/teamhosts.go index 81071fbb..c3b094a2 100644 --- a/ui/teamhosts.go +++ b/ui/teamhosts.go @@ -14,6 +14,7 @@ import ( "github.com/enowx/enowx/client" "github.com/enowx/enowx/cloud/api" + "github.com/enowx/enowx/cloud/relay" "github.com/enowx/enowx/proto" "github.com/enowx/enowx/ui/icon" "github.com/enowx/enowx/ui/textfmt" @@ -23,9 +24,10 @@ import ( // Hosts shared with a team (docs/cloud.md, Hosts and sessions): the // connected host's share, picked in the team's Hosts tab, and the hosts -// the team's members share, in the rail. Opening one connects to it over -// Tailscale as a visitor, with a ticket the cloud gives this member for -// it, and shows its shared projects and their sessions. +// the team's members share, in the rail. Opening one connects to it as a +// visitor, through the cloud's relay (or its Tailscale address, for a host +// of an older enowX), with a ticket the cloud gives this member for it, and +// shows its shared projects and their sessions. // pageTeamHost is a team's shared host, as a visitor sees it. const pageTeamHost = "teamhost" @@ -139,10 +141,12 @@ func (a *App) hostShareBody(c *ui.Context, team api.Team) { shared, allow = t.Projects, t.AllowSessions } } - widget.CardNote(c, "The projects picked show to the team: their sessions and files. Members reach "+a.host.Name+" over Tailscale: share its node with their Tailscale accounts.", 0) + widget.CardNote(c, "The projects picked show to the team: their sessions and files. Members reach "+a.host.Name+" through the cloud's relay, end to end encrypted: the relay can't read what passes.", 0) switch { case th.share.Err != "" && len(shared) > 0: widget.Notice(c, icon.Alert, theme.Warning, th.share.Err, "", nil) + case th.share.Relay && len(shared) > 0: + widget.Running(c.Key("addr"), "Connected to the relay: members can reach it", widget.RunStyle{Size: theme.TextLabel, Color: theme.Done}) case th.share.Addr != "" && len(shared) > 0: widget.Running(c.Key("addr"), "Members reach it at "+th.share.Addr, widget.RunStyle{Size: theme.TextLabel, Color: theme.Done}) } @@ -185,7 +189,7 @@ func (a *App) shareHost(team, name string, projects []string) { return } a.confirm("Share "+a.host.Name+" with "+name+"?", - "The team's members see the projects you pick here, their sessions and files, reaching "+a.host.Name+" over Tailscale. Each signs in with a ticket the cloud gives them; take a project back here any time.", + "The team's members see the projects you pick here, their sessions and files, reaching "+a.host.Name+" through the cloud's relay, which can't read what passes. Each signs in with a ticket the cloud gives them; take a project back here any time.", "Share", do) } @@ -269,11 +273,11 @@ func visitHost(cl *client.Client, h proto.TeamHost, onEvent func(proto.Event)) ( if err := cl.CloudCall("POST", teamPath(api.PathHostTickets, h.Team), api.HostTicketParams{Host: h.Device}, &tk); err != nil { return nil, proto.Visitor{}, fmt.Errorf("a ticket into %s: %s", h.Name, client.Explain(err)) } - ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) + ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() - nc, err := (&net.Dialer{}).DialContext(ctx, "tcp", h.Addr) + nc, err := reachHost(ctx, h, tk) if err != nil { - return nil, proto.Visitor{}, fmt.Errorf("can't reach %s at %s: is its node shared with your Tailscale account, and Tailscale on here? (%v)", h.Name, h.Addr, err) + return nil, proto.Visitor{}, err } vc, err := client.New(nc, onEvent) if err != nil { @@ -288,6 +292,37 @@ func visitHost(cl *client.Client, h proto.TeamHost, onEvent func(proto.Event)) ( return vc, me, nil } +// reachHost connects to h: through the relay when both it and the cloud +// have one, else at its Tailscale address (a host of an older enowX). +func reachHost(ctx context.Context, h proto.TeamHost, tk api.HostTicket) (net.Conn, error) { + var viaRelay error + if h.Key != "" && tk.Relay != "" { + nc, err := relay.Dial(ctx, tk.Relay, tk.Ticket, h.Key) + if err == nil { + return nc, nil + } + viaRelay = err + if errors.Is(err, relay.ErrHostKey) { + // Not the host its team announced: no other way is safer. + return nil, fmt.Errorf("can't reach %s: %v", h.Name, err) + } + } + if h.Addr == "" { + if viaRelay == nil { + return nil, fmt.Errorf("can't reach %s: it shares no way in yet; ask %s to update enowX", h.Name, h.Owner) + } + return nil, fmt.Errorf("can't reach %s through the relay: %v", h.Name, viaRelay) + } + nc, err := (&net.Dialer{}).DialContext(ctx, "tcp", h.Addr) + if err != nil { + if viaRelay != nil { + return nil, fmt.Errorf("can't reach %s through the relay (%v), nor at %s (%v)", h.Name, viaRelay, h.Addr, err) + } + return nil, fmt.Errorf("can't reach %s at %s: it may be off, or offline (%v)", h.Name, h.Addr, err) + } + return nc, nil +} + // refreshVisit reads the shared host's projects and sessions again. func (a *App) refreshVisit(v *teamVisit) { if v.cl == nil {