diff --git a/contributing/PROXY.md b/contributing/PROXY.md index d053b43d88..cc5c2f6831 100644 --- a/contributing/PROXY.md +++ b/contributing/PROXY.md @@ -64,6 +64,8 @@ The in-server proxy opens an SSH connection when it needs to forward the first r The gateway opens SSH connections when each replica is registered. +When a replica is unregistered, the gateway first reloads Nginx so that new requests stop going to the replica, and only then closes its SSH connection. The connection is closed in the background so that requests already sent through it can finish: after a short grace period for Nginx workers that have not picked up the reload yet, once nothing is connected to the replica's socket (as listed in `/proc/net/unix`), or after 5 minutes at most. The `dstack` server stops the replica's container independently, so it can still cut requests that outlive the container. + ### Communication with the `dstack` server The in-server proxy is part of the `dstack` server, so no network communication takes place between them. diff --git a/src/dstack/_internal/proxy/gateway/services/registry.py b/src/dstack/_internal/proxy/gateway/services/registry.py index 3db3f04903..c4711cf6ef 100644 --- a/src/dstack/_internal/proxy/gateway/services/registry.py +++ b/src/dstack/_internal/proxy/gateway/services/registry.py @@ -122,11 +122,13 @@ async def unregister_service( logger.debug("Unregistering service %s", service.fmt()) + await nginx.unregister(service) + # Only after the reload, so that Nginx stops routing requests to the replicas await stop_replica_connections( ids=(r.id for r in service.replicas), service_conn_pool=service_conn_pool, + drain=True, ) - await nginx.unregister(service) await repo.delete_models_by_run(project_name, run_name) await repo.delete_service(project_name, run_name) @@ -344,18 +346,16 @@ async def apply_service( nginx: Nginx, service_conn_pool: ServiceConnectionPool, ) -> dict[models.Replica, BaseException]: + removed_replica_ids = [] if old_service is not None: if service.domain != old_service.domain: raise UnexpectedProxyError( f"Did not expect service {service.fmt()}" f" domain name to change ({old_service.domain} -> {service.domain})" ) - await stop_replica_connections( - ids=( - replica.id for replica in old_service.replicas if replica not in service.replicas - ), - service_conn_pool=service_conn_pool, - ) + removed_replica_ids = [ + replica.id for replica in old_service.replicas if replica not in service.replicas + ] replica_conns, replica_failures = await get_or_add_replica_connections( service, repo, service_conn_pool ) @@ -370,6 +370,10 @@ async def apply_service( ] service_config = await get_nginx_service_config(service, replica_configs) await nginx.register(service_config, (await repo.get_config()).acme_settings) + # Only after the reload, so that Nginx stops routing requests to the removed replicas + await stop_replica_connections( + ids=removed_replica_ids, service_conn_pool=service_conn_pool, drain=True + ) return replica_failures @@ -402,10 +406,17 @@ async def get_or_add_replica_connections( async def stop_replica_connections( - ids: Iterable[str], service_conn_pool: ServiceConnectionPool + ids: Iterable[str], service_conn_pool: ServiceConnectionPool, drain: bool = False ) -> None: - tasks = map(service_conn_pool.remove, ids) - results = await asyncio.gather(*tasks, return_exceptions=True) + """ + With `drain`, return right away and close each connection once requests already sent + through it finish. See `ServiceConnectionPool.remove`. + """ + ids = list(ids) + results = await asyncio.gather( + *(service_conn_pool.remove(replica_id, drain=drain) for replica_id in ids), + return_exceptions=True, + ) for replica_id, exc in zip(ids, results): if isinstance(exc, Exception): logger.error("Error stopping connection to replica %s: %s", replica_id, exc) diff --git a/src/dstack/_internal/proxy/gateway/testing/common.py b/src/dstack/_internal/proxy/gateway/testing/common.py index 2718ec6bae..ba859a40ba 100644 --- a/src/dstack/_internal/proxy/gateway/testing/common.py +++ b/src/dstack/_internal/proxy/gateway/testing/common.py @@ -11,3 +11,4 @@ class Mocks: run_certbot: AnyMock open_conn: AnyMock close_conn: AnyMock + wait_until_idle: AnyMock diff --git a/src/dstack/_internal/proxy/lib/services/service_connection.py b/src/dstack/_internal/proxy/lib/services/service_connection.py index 1cfd9472bb..d40c6e0d59 100644 --- a/src/dstack/_internal/proxy/lib/services/service_connection.py +++ b/src/dstack/_internal/proxy/lib/services/service_connection.py @@ -18,7 +18,7 @@ from dstack._internal.proxy.lib.errors import UnexpectedProxyError from dstack._internal.proxy.lib.models import Project, Replica, Service from dstack._internal.proxy.lib.repo import BaseProxyRepo -from dstack._internal.utils.common import get_or_error +from dstack._internal.utils.common import get_or_error, run_async from dstack._internal.utils.logging import get_logger from dstack._internal.utils.path import FileContent @@ -27,6 +27,13 @@ # Bound SSH startup work during a shared outage, without limiting service requests. MAX_CONCURRENT_TUNNEL_RECONNECTS = 8 MAX_TUNNEL_RECONNECT_DELAY = 30 +# A drained connection stays open for at least this long, since Nginx workers may still route +# requests to a removed replica for a short while after a reload. +REPLICA_DRAIN_GRACE_PERIOD = 5 +REPLICA_DRAIN_POLL_INTERVAL = 1 +# Upper bound for draining, e.g. if a WebSocket connection keeps the replica busy. +# Matches the default `stop_duration`, the time a job is given to stop gracefully. +REPLICA_DRAIN_TIMEOUT = 300 class ServiceClient(httpx.AsyncClient): @@ -202,6 +209,7 @@ class ServiceConnectionPool: def __init__(self) -> None: # TODO(#2238): remove connections to stopped replicas in-server self.connections: Dict[str, ServiceConnection] = {} + self._draining: set[asyncio.Task] = set() self._reconnect_semaphore = asyncio.Semaphore(MAX_CONCURRENT_TUNNEL_RECONNECTS) async def get(self, replica_id: str) -> Optional[ServiceConnection]: @@ -239,12 +247,30 @@ async def get_or_add( raise return connection - async def remove(self, replica_id: str) -> None: + async def remove(self, replica_id: str, drain: bool = False) -> None: + """ + Remove the replica's connection from the pool and close it. + + With `drain`, return right away and close the connection in the background once nothing + is connected to its socket, so that requests already sent to the replica can finish. + The caller must stop routing requests to the socket first, e.g. by reloading Nginx. + """ connection = self.connections.pop(replica_id, None) - if connection is not None: + if connection is None: + return + if not drain: await connection.close() + return + task = asyncio.create_task(_close_after_drain(connection)) + # The event loop only keeps weak references to tasks + self._draining.add(task) + task.add_done_callback(self._draining.discard) async def remove_all(self) -> None: + draining = list(self._draining) + for task in draining: + task.cancel() # closes the connection without waiting for the drain + await asyncio.gather(*draining, return_exceptions=True) replica_ids = list(self.connections) results = await asyncio.gather( *(self.remove(replica_id) for replica_id in replica_ids), return_exceptions=True @@ -280,3 +306,65 @@ async def get_service_replica_client( ) connection = await service_conn_pool.get_or_add(project, service, replica) return await connection.client() + + +async def _close_after_drain(connection: ServiceConnection) -> None: + replica_id = connection.replica.id + try: + if await _wait_until_idle(connection.app_socket_path): + logger.debug("Connection to replica %s is drained, closing", replica_id) + else: + logger.warning( + "Connection to replica %s is still in use after %ss, closing", + replica_id, + REPLICA_DRAIN_TIMEOUT, + ) + finally: + try: + await connection.close() + except Exception: + logger.exception("Error closing drained connection to replica %s", replica_id) + + +async def _wait_until_idle(socket_path: Path) -> bool: + """ + Wait until nothing is connected to the Unix socket, for at most `REPLICA_DRAIN_TIMEOUT`. + + Returns whether the socket became idle in time. + """ + await asyncio.sleep(REPLICA_DRAIN_GRACE_PERIOD) + waited = REPLICA_DRAIN_GRACE_PERIOD + # `None` means the connections cannot be counted, so wait for the timeout + while await _count_socket_connections(socket_path) != 0: + if waited >= REPLICA_DRAIN_TIMEOUT: + return False + await asyncio.sleep(REPLICA_DRAIN_POLL_INTERVAL) + waited += REPLICA_DRAIN_POLL_INTERVAL + return True + + +# `SS_CONNECTED`, the state of a connected socket in `/proc/net/unix` +_SS_CONNECTED = "03" + + +async def _count_socket_connections(socket_path: Path) -> Optional[int]: + """ + Count the connections accepted on a listening Unix socket, or return `None` if they cannot + be counted. + + Accepted connections are listed in `/proc/net/unix` with the listening socket's path + and the connected state. Linux-only, like the gateway. Not `psutil.net_connections()`: + it reports no state for Unix sockets, so it cannot tell connections from the listener. + """ + try: + table = await run_async(Path("/proc/net/unix").read_text) + except OSError: + return None + path = str(socket_path) + count = 0 + # Columns: Num RefCount Protocol Flags Type St Inode [Path] + for line in table.splitlines()[1:]: + fields = line.split(maxsplit=7) + if len(fields) == 8 and fields[7] == path and fields[5] == _SS_CONNECTED: + count += 1 + return count diff --git a/src/tests/_internal/proxy/gateway/conftest.py b/src/tests/_internal/proxy/gateway/conftest.py index 008bf9460d..4657e3f630 100644 --- a/src/tests/_internal/proxy/gateway/conftest.py +++ b/src/tests/_internal/proxy/gateway/conftest.py @@ -16,11 +16,15 @@ def system_mocks() -> Generator[Mocks, None, None]: patch(f"{nginx}.Nginx.run_certbot") as run_certbot, patch(f"{connection}.ServiceConnection.open") as open_conn, patch(f"{connection}.ServiceConnection.close") as close_conn, + patch(f"{connection}._wait_until_idle") as wait_until_idle, ): sudo.return_value = [] + # Drains finish right away unless a test holds them + wait_until_idle.return_value = True yield Mocks( reload_nginx=reload_nginx, run_certbot=run_certbot, open_conn=open_conn, close_conn=close_conn, + wait_until_idle=wait_until_idle, ) diff --git a/src/tests/_internal/proxy/gateway/routers/test_registry.py b/src/tests/_internal/proxy/gateway/routers/test_registry.py index eeca8c0546..854c26062a 100644 --- a/src/tests/_internal/proxy/gateway/routers/test_registry.py +++ b/src/tests/_internal/proxy/gateway/routers/test_registry.py @@ -1,3 +1,4 @@ +import asyncio import re import uuid from datetime import datetime, timezone @@ -95,6 +96,28 @@ def sample_model_options(name: str = "test-model") -> dict: } +def hold_drains(mocks: Mocks) -> asyncio.Event: + """Keep the connections of removed replicas draining until the returned event is set.""" + drained = asyncio.Event() + + async def wait_until_idle(socket_path: Path) -> bool: + await drained.wait() + return True + + mocks.wait_until_idle.side_effect = wait_until_idle + return drained + + +async def wait_for_closed_connections(mocks: Mocks, count: int) -> None: + """Wait for drained connections, which are closed in the background.""" + + async def closed() -> None: + while mocks.close_conn.await_count < count: + await asyncio.sleep(0) + + await asyncio.wait_for(closed(), timeout=5) + + @pytest.mark.asyncio class TestUpdateReplicaProxyJumpKeys: async def test_saves_key_without_opening_a_missing_connection(self, tmp_path, system_mocks): @@ -656,6 +679,7 @@ async def test_unregister_not_registered_error( assert system_mocks.reload_nginx.call_count == 0 async def test_unregister_with_replicas(self, tmp_path: Path, system_mocks: Mocks) -> None: + drained = hold_drains(system_mocks) client = make_client(tmp_path) # register service resp = await client.post( @@ -676,7 +700,10 @@ async def test_unregister_with_replicas(self, tmp_path: Path, system_mocks: Mock assert resp.status_code == 200 assert not (tmp_path / "sites-enabled" / "443-test-run.gtw.test.conf").exists() assert system_mocks.reload_nginx.call_count == 4 - assert system_mocks.close_conn.call_count == 2 + # Requests already sent to the replicas can still finish + system_mocks.close_conn.assert_not_awaited() + drained.set() + await wait_for_closed_connections(system_mocks, 2) async def test_unregister_with_model(self, tmp_path: Path, system_mocks: Mocks) -> None: repo = GatewayProxyRepo() @@ -698,6 +725,7 @@ async def test_unregister_with_model(self, tmp_path: Path, system_mocks: Mocks) @pytest.mark.asyncio class TestUnregisterReplica: async def test_unregister(self, tmp_path: Path, system_mocks: Mocks) -> None: + drained = hold_drains(system_mocks) client = make_client(tmp_path) # register service resp = await client.post( @@ -725,7 +753,11 @@ async def test_unregister(self, tmp_path: Path, system_mocks: Mocks) -> None: assert "replica xxx-xxx" in conf assert "replica yyy-yyy" not in conf assert system_mocks.reload_nginx.call_count == 4 - assert system_mocks.close_conn.call_count == 1 + # Requests already sent to the replica can still finish + system_mocks.close_conn.assert_not_awaited() + drained.set() + await wait_for_closed_connections(system_mocks, 1) + system_mocks.wait_until_idle.assert_awaited_once() async def test_unregister_no_replica_error(self, tmp_path: Path, system_mocks: Mocks) -> None: client = make_client(tmp_path) diff --git a/src/tests/_internal/proxy/lib/services/test_service_connection.py b/src/tests/_internal/proxy/lib/services/test_service_connection.py index 3e891b3f44..c511658b0c 100644 --- a/src/tests/_internal/proxy/lib/services/test_service_connection.py +++ b/src/tests/_internal/proxy/lib/services/test_service_connection.py @@ -1,6 +1,10 @@ import asyncio +import socket +import sys +from pathlib import Path +from tempfile import TemporaryDirectory from typing import AsyncIterator -from unittest.mock import create_autospec +from unittest.mock import AsyncMock, create_autospec import pytest import pytest_asyncio @@ -379,6 +383,99 @@ async def reconnect(): assert not pool.connections assert sum(t.aopen.await_count - 1 for t in tunnels) == peak + async def test_drained_removal_closes_connection_once_socket_is_idle( + self, project, service, pool, tunnel_factory, monitor_clock, monkeypatch + ): + tunnel = tunnel_factory.return_value + closed = asyncio.Event() + tunnel.aclose.side_effect = closed.set + count = AsyncMock(side_effect=[2, 0]) + monkeypatch.setattr(service_connection, "_count_socket_connections", count) + replica = service.replicas[0] + connection = await pool.get_or_add(project, service, replica) + + await pool.remove(replica.id, drain=True) + + # Gone from the pool right away, but the tunnel stays open while it is in use + assert await pool.get(replica.id) is None + assert await monitor_clock.tick() == service_connection.REPLICA_DRAIN_GRACE_PERIOD + tunnel.aclose.assert_not_awaited() + assert await monitor_clock.tick() == service_connection.REPLICA_DRAIN_POLL_INTERVAL + await asyncio.wait_for(closed.wait(), timeout=1) + count.assert_awaited_with(connection.app_socket_path) + assert count.await_count == 2 + + @pytest.mark.parametrize("connections", [1, None], ids=["in-use", "unknown"]) + async def test_drained_removal_closes_connection_after_timeout( + self, project, service, pool, tunnel_factory, monitor_clock, monkeypatch, connections + ): + monkeypatch.setattr(service_connection, "REPLICA_DRAIN_GRACE_PERIOD", 5) + monkeypatch.setattr(service_connection, "REPLICA_DRAIN_POLL_INTERVAL", 1) + monkeypatch.setattr(service_connection, "REPLICA_DRAIN_TIMEOUT", 7) + monkeypatch.setattr( + service_connection, + "_count_socket_connections", + AsyncMock(return_value=connections), + ) + tunnel = tunnel_factory.return_value + closed = asyncio.Event() + tunnel.aclose.side_effect = closed.set + replica = service.replicas[0] + await pool.get_or_add(project, service, replica) + + await pool.remove(replica.id, drain=True) + + assert [await monitor_clock.tick() for _ in range(2)] == [5, 1] + tunnel.aclose.assert_not_awaited() + assert await monitor_clock.tick() == 1 + await asyncio.wait_for(closed.wait(), timeout=1) + assert monitor_clock.pending.empty() + + async def test_remove_all_closes_draining_connections_without_waiting( + self, project, service, pool, tunnel_factory, monitor_clock, monkeypatch + ): + monkeypatch.setattr( + service_connection, "_count_socket_connections", AsyncMock(return_value=1) + ) + tunnel = tunnel_factory.return_value + replica = service.replicas[0] + await pool.get_or_add(project, service, replica) + await pool.remove(replica.id, drain=True) + await _yield_to_event_loop() + + await pool.remove_all() + + tunnel.aclose.assert_awaited_once() + + +@pytest.mark.skipif(sys.platform != "linux", reason="reads /proc/net/unix") +@pytest.mark.asyncio +class TestCountSocketConnections: + async def test_counts_connections_accepted_on_the_socket(self): + # A real socket, since the /proc/net/unix format is what this test is about. + # Pyright doesn't know the test is skipped on Windows, which has no AF_UNIX. + af_unix = socket.AF_UNIX # pyright: ignore[reportAttributeAccessIssue] + with TemporaryDirectory() as tmp_dir, socket.socket(af_unix) as server: + path = Path(tmp_dir) / "replica.sock" + server.bind(str(path)) + server.listen() + assert await service_connection._count_socket_connections(path) == 0 + + clients = [socket.socket(af_unix) for _ in range(2)] + accepted = [] + try: + for client in clients: + client.connect(str(path)) + accepted.append(server.accept()[0]) + assert await service_connection._count_socket_connections(path) == 2 + finally: + for sock in clients + accepted: + sock.close() + assert await service_connection._count_socket_connections(path) == 0 + + async def test_missing_socket_has_no_connections(self, tmp_path): + assert await service_connection._count_socket_connections(tmp_path / "missing") == 0 + class _MonitorClock: """Wake monitor timers explicitly, without waiting for wall-clock time."""