diff --git a/hyperliquid/websocket_manager.py b/hyperliquid/websocket_manager.py index 4c73a68..9175a6d 100644 --- a/hyperliquid/websocket_manager.py +++ b/hyperliquid/websocket_manager.py @@ -7,7 +7,15 @@ from hyperliquid.utils.types import Any, Callable, Dict, List, NamedTuple, Optional, Subscription, Tuple, WsMsg -ActiveSubscription = NamedTuple("ActiveSubscription", [("callback", Callable[[Any], None]), ("subscription_id", int)]) +ActiveSubscription = NamedTuple( + "ActiveSubscription", + [("callback", Callable[[Any], None]), ("subscription_id", int), ("subscription", Subscription)], +) + +# Delay in seconds between websocket reconnect attempts. The server +# disconnects periodically and without announcement, so the manager must +# re-establish the connection on its own instead of going dark. +RECONNECT_DELAY_SECONDS = 5 def subscription_to_identifier(subscription: Subscription) -> str: @@ -82,20 +90,27 @@ def __init__(self, base_url): self.queued_subscriptions: List[Tuple[Subscription, ActiveSubscription]] = [] self.active_subscriptions: Dict[str, List[ActiveSubscription]] = defaultdict(list) ws_url = "ws" + base_url[len("http") :] + "/ws" - self.ws = websocket.WebSocketApp(ws_url, on_message=self.on_message, on_open=self.on_open) + self.ws = websocket.WebSocketApp( + ws_url, on_message=self.on_message, on_open=self.on_open, on_close=self.on_close, on_error=self.on_error + ) self.ping_sender = threading.Thread(target=self.send_ping) self.stop_event = threading.Event() def run(self): self.ping_sender.start() - self.ws.run_forever() + self.ws.run_forever(reconnect=RECONNECT_DELAY_SECONDS) def send_ping(self): while not self.stop_event.wait(50): if not self.ws.keep_running: break logging.debug("Websocket sending ping") - self.ws.send(json.dumps({"method": "ping"})) + try: + self.ws.send(json.dumps({"method": "ping"})) + except websocket.WebSocketConnectionClosedException: + # The socket is down while run_forever reconnects; keep the + # ping thread alive so pings resume once it is back. + logging.debug("Websocket ping failed, connection is reconnecting") logging.debug("Websocket ping sender stopped") def stop(self): @@ -127,9 +142,31 @@ def on_message(self, _ws, message): def on_open(self, _ws): logging.debug("on_open") self.ws_ready = True - for subscription, active_subscription in self.queued_subscriptions: + # Take the queue before replaying it: subscribe() consults this list, + # and leaving entries in place would replay them again on the next + # on_open (e.g. after a reconnect). + queued_subscriptions, self.queued_subscriptions = self.queued_subscriptions, [] + # Subscriptions that were already active before a reconnect are not in + # the queue, but the server forgot them with the old connection, so + # re-send them first. Snapshot before replaying the queue: replaying + # appends to active_subscriptions, and those must only be sent once. + active_subscriptions = [s for subs in self.active_subscriptions.values() for s in subs] + for active_subscription in active_subscriptions: + try: + self.ws.send(json.dumps({"method": "subscribe", "subscription": active_subscription.subscription})) + except websocket.WebSocketConnectionClosedException: + logging.debug("Websocket send failed while replaying subscriptions on reconnect") + for subscription, active_subscription in queued_subscriptions: self.subscribe(subscription, active_subscription.callback, active_subscription.subscription_id) + def on_close(self, _ws, close_status_code, close_msg): + logging.debug(f"Websocket closed: {close_status_code} {close_msg}") + self.ws_ready = False + + def on_error(self, _ws, error): + logging.debug(f"Websocket error: {error}") + self.ws_ready = False + def subscribe( self, subscription: Subscription, callback: Callable[[Any], None], subscription_id: Optional[int] = None ) -> int: @@ -138,7 +175,9 @@ def subscribe( subscription_id = self.subscription_id_counter if not self.ws_ready: logging.debug("enqueueing subscription") - self.queued_subscriptions.append((subscription, ActiveSubscription(callback, subscription_id))) + self.queued_subscriptions.append( + (subscription, ActiveSubscription(callback, subscription_id, subscription)) + ) else: logging.debug("subscribing") identifier = subscription_to_identifier(subscription) @@ -146,8 +185,20 @@ def subscribe( # TODO: ideally the userEvent and orderUpdates messages would include the user so that we can multiplex if len(self.active_subscriptions[identifier]) != 0: raise NotImplementedError(f"Cannot subscribe to {identifier} multiple times") - self.active_subscriptions[identifier].append(ActiveSubscription(callback, subscription_id)) - self.ws.send(json.dumps({"method": "subscribe", "subscription": subscription})) + self.active_subscriptions[identifier].append(ActiveSubscription(callback, subscription_id, subscription)) + try: + self.ws.send(json.dumps({"method": "subscribe", "subscription": subscription})) + except websocket.WebSocketConnectionClosedException: + # The socket dropped between the ws_ready check and the send + # (e.g. mid-reconnect). Queue the subscription instead of + # raising: it is re-sent on the next on_open. + logging.debug("Websocket send failed, queueing subscription") + self.active_subscriptions[identifier] = [ + x for x in self.active_subscriptions[identifier] if x.subscription_id != subscription_id + ] + self.queued_subscriptions.append( + (subscription, ActiveSubscription(callback, subscription_id, subscription)) + ) return subscription_id def unsubscribe(self, subscription: Subscription, subscription_id: int) -> bool: diff --git a/tests/websocket_manager_reconnect_test.py b/tests/websocket_manager_reconnect_test.py new file mode 100644 index 0000000..831b7e7 --- /dev/null +++ b/tests/websocket_manager_reconnect_test.py @@ -0,0 +1,141 @@ +import json + +import pytest +import websocket + +from hyperliquid.websocket_manager import RECONNECT_DELAY_SECONDS, WebsocketManager + + +class FakeWebSocketApp: + """Stub for websocket.WebSocketApp: records sends, never connects.""" + + def __init__(self, url, on_message=None, on_open=None, on_close=None, on_error=None): + self.url = url + self.on_message = on_message + self.on_open = on_open + self.on_close = on_close + self.on_error = on_error + self.sent = [] + self.keep_running = True + self.fail_send = False + self.run_forever_kwargs = None + + def send(self, data): + if self.fail_send or not self.keep_running: + raise websocket.WebSocketConnectionClosedException("Connection is already closed.") + self.sent.append(data) + + def close(self): + self.keep_running = False + + def run_forever(self, **kwargs): + self.run_forever_kwargs = kwargs + return False + + +@pytest.fixture +def manager(monkeypatch): + monkeypatch.setattr("hyperliquid.websocket_manager.websocket.WebSocketApp", FakeWebSocketApp) + return WebsocketManager("http://test") + + +def subscribe_frames(ws): + return [s for s in ws.sent if '"subscribe"' in s] + + +def test_reconnect_replays_active_subscriptions_once(manager): + manager.subscribe({"type": "allMids"}, lambda m: None) + manager.ws.on_open(manager.ws) + assert manager.ws_ready + assert subscribe_frames(manager.ws) == ['{"method": "subscribe", "subscription": {"type": "allMids"}}'] + manager.subscribe({"type": "userEvents"}, lambda m: None) + + sent_before_reconnect = len(manager.ws.sent) + manager.ws.on_open(manager.ws) # simulated reconnect + + replayed = subscribe_frames(manager.ws)[sent_before_reconnect:] + assert replayed == [ + '{"method": "subscribe", "subscription": {"type": "allMids"}}', + '{"method": "subscribe", "subscription": {"type": "userEvents"}}', + ] + assert manager.queued_subscriptions == [] + + +def test_reconnect_does_not_duplicate_callbacks(manager): + received = [] + manager.subscribe({"type": "allMids"}, lambda m: received.append(m)) + manager.ws.on_open(manager.ws) + manager.ws.on_open(manager.ws) # simulated reconnect + manager.ws.on_open(manager.ws) # second reconnect + manager.on_message(manager.ws, json.dumps({"channel": "allMids", "data": {"BTC": "1"}})) + assert len(received) == 1 + + +def test_queued_subscriptions_not_replayed_twice(manager): + manager.subscribe({"type": "allMids"}, lambda m: None) + manager.ws.on_open(manager.ws) + assert manager.queued_subscriptions == [] + manager.ws.on_open(manager.ws) # simulated reconnect: active replay, not queue replay + assert subscribe_frames(manager.ws) == [ + '{"method": "subscribe", "subscription": {"type": "allMids"}}', + '{"method": "subscribe", "subscription": {"type": "allMids"}}', + ] + + +def test_on_close_and_on_error_clear_ws_ready(manager): + manager.subscribe({"type": "allMids"}, lambda m: None) + manager.ws.on_open(manager.ws) + assert manager.ws_ready + + manager.ws.on_close(manager.ws, 1006, "") + assert not manager.ws_ready + # Subscribing while disconnected queues instead of sending. + manager.subscribe({"type": "userEvents"}, lambda m: None) + assert len(subscribe_frames(manager.ws)) == 1 + assert len(manager.queued_subscriptions) == 1 + + manager.ws.on_open(manager.ws) + assert manager.ws_ready + assert subscribe_frames(manager.ws) == [ + '{"method": "subscribe", "subscription": {"type": "allMids"}}', + # Reconnect replay of the still-active allMids subscription... + '{"method": "subscribe", "subscription": {"type": "allMids"}}', + # ...followed by the queued userEvents subscription. + '{"method": "subscribe", "subscription": {"type": "userEvents"}}', + ] + + manager.ws.on_error(manager.ws, Exception("boom")) + assert not manager.ws_ready + + +def test_subscribe_while_socket_down_queues_instead_of_raising(manager): + manager.ws.on_open(manager.ws) + manager.ws.fail_send = True + subscription_id = manager.subscribe({"type": "allMids"}, lambda m: None) + assert isinstance(subscription_id, int) + assert len(manager.queued_subscriptions) == 1 + + manager.ws.fail_send = False + manager.ws.on_open(manager.ws) + assert subscribe_frames(manager.ws) == ['{"method": "subscribe", "subscription": {"type": "allMids"}}'] + + +def test_run_passes_reconnect_delay(manager): + manager.run() + try: + assert manager.ws.run_forever_kwargs == {"reconnect": RECONNECT_DELAY_SECONDS} + assert RECONNECT_DELAY_SECONDS == 5 + finally: + manager.stop() + + +def test_ping_thread_survives_send_failure(manager): + from unittest.mock import MagicMock + + manager.ws.fail_send = True + manager.stop_event = MagicMock() + manager.stop_event.wait.side_effect = [False, True] + # Would raise WebSocketConnectionClosedException without the guard, killing + # the ping thread mid-reconnect. + manager.send_ping() + assert manager.stop_event.wait.call_count == 2