Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 59 additions & 8 deletions hyperliquid/websocket_manager.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,15 @@

from hyperliquid.utils.types import Any, Callable, Dict, List, NamedTuple, Optional, Subscription, Tuple, WsMsg

ActiveSubscription = NamedTuple("ActiveSubscription", [("callback", Callable[[Any], None]), ("subscription_id", int)])
ActiveSubscription = NamedTuple(
"ActiveSubscription",
[("callback", Callable[[Any], None]), ("subscription_id", int), ("subscription", Subscription)],
)

# Delay in seconds between websocket reconnect attempts. The server
# disconnects periodically and without announcement, so the manager must
# re-establish the connection on its own instead of going dark.
RECONNECT_DELAY_SECONDS = 5


def subscription_to_identifier(subscription: Subscription) -> str:
Expand Down Expand Up @@ -82,20 +90,27 @@ def __init__(self, base_url):
self.queued_subscriptions: List[Tuple[Subscription, ActiveSubscription]] = []
self.active_subscriptions: Dict[str, List[ActiveSubscription]] = defaultdict(list)
ws_url = "ws" + base_url[len("http") :] + "/ws"
self.ws = websocket.WebSocketApp(ws_url, on_message=self.on_message, on_open=self.on_open)
self.ws = websocket.WebSocketApp(
ws_url, on_message=self.on_message, on_open=self.on_open, on_close=self.on_close, on_error=self.on_error
)
self.ping_sender = threading.Thread(target=self.send_ping)
self.stop_event = threading.Event()

def run(self):
self.ping_sender.start()
self.ws.run_forever()
self.ws.run_forever(reconnect=RECONNECT_DELAY_SECONDS)

def send_ping(self):
while not self.stop_event.wait(50):
if not self.ws.keep_running:
break
logging.debug("Websocket sending ping")
self.ws.send(json.dumps({"method": "ping"}))
try:
self.ws.send(json.dumps({"method": "ping"}))
except websocket.WebSocketConnectionClosedException:
# The socket is down while run_forever reconnects; keep the
# ping thread alive so pings resume once it is back.
logging.debug("Websocket ping failed, connection is reconnecting")
logging.debug("Websocket ping sender stopped")

def stop(self):
Expand Down Expand Up @@ -127,9 +142,31 @@ def on_message(self, _ws, message):
def on_open(self, _ws):
logging.debug("on_open")
self.ws_ready = True
for subscription, active_subscription in self.queued_subscriptions:
# Take the queue before replaying it: subscribe() consults this list,
# and leaving entries in place would replay them again on the next
# on_open (e.g. after a reconnect).
queued_subscriptions, self.queued_subscriptions = self.queued_subscriptions, []
# Subscriptions that were already active before a reconnect are not in
# the queue, but the server forgot them with the old connection, so
# re-send them first. Snapshot before replaying the queue: replaying
# appends to active_subscriptions, and those must only be sent once.
active_subscriptions = [s for subs in self.active_subscriptions.values() for s in subs]
for active_subscription in active_subscriptions:
try:
self.ws.send(json.dumps({"method": "subscribe", "subscription": active_subscription.subscription}))
except websocket.WebSocketConnectionClosedException:
logging.debug("Websocket send failed while replaying subscriptions on reconnect")
for subscription, active_subscription in queued_subscriptions:
self.subscribe(subscription, active_subscription.callback, active_subscription.subscription_id)

def on_close(self, _ws, close_status_code, close_msg):
logging.debug(f"Websocket closed: {close_status_code} {close_msg}")
self.ws_ready = False

def on_error(self, _ws, error):
logging.debug(f"Websocket error: {error}")
self.ws_ready = False

def subscribe(
self, subscription: Subscription, callback: Callable[[Any], None], subscription_id: Optional[int] = None
) -> int:
Expand All @@ -138,16 +175,30 @@ def subscribe(
subscription_id = self.subscription_id_counter
if not self.ws_ready:
logging.debug("enqueueing subscription")
self.queued_subscriptions.append((subscription, ActiveSubscription(callback, subscription_id)))
self.queued_subscriptions.append(
(subscription, ActiveSubscription(callback, subscription_id, subscription))
)
else:
logging.debug("subscribing")
identifier = subscription_to_identifier(subscription)
if identifier == "userEvents" or identifier == "orderUpdates":
# TODO: ideally the userEvent and orderUpdates messages would include the user so that we can multiplex
if len(self.active_subscriptions[identifier]) != 0:
raise NotImplementedError(f"Cannot subscribe to {identifier} multiple times")
self.active_subscriptions[identifier].append(ActiveSubscription(callback, subscription_id))
self.ws.send(json.dumps({"method": "subscribe", "subscription": subscription}))
self.active_subscriptions[identifier].append(ActiveSubscription(callback, subscription_id, subscription))
try:
self.ws.send(json.dumps({"method": "subscribe", "subscription": subscription}))
except websocket.WebSocketConnectionClosedException:
# The socket dropped between the ws_ready check and the send
# (e.g. mid-reconnect). Queue the subscription instead of
# raising: it is re-sent on the next on_open.
logging.debug("Websocket send failed, queueing subscription")
self.active_subscriptions[identifier] = [
x for x in self.active_subscriptions[identifier] if x.subscription_id != subscription_id
]
self.queued_subscriptions.append(
(subscription, ActiveSubscription(callback, subscription_id, subscription))
)
return subscription_id

def unsubscribe(self, subscription: Subscription, subscription_id: int) -> bool:
Expand Down
141 changes: 141 additions & 0 deletions tests/websocket_manager_reconnect_test.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,141 @@
import json

import pytest
import websocket

from hyperliquid.websocket_manager import RECONNECT_DELAY_SECONDS, WebsocketManager


class FakeWebSocketApp:
"""Stub for websocket.WebSocketApp: records sends, never connects."""

def __init__(self, url, on_message=None, on_open=None, on_close=None, on_error=None):
self.url = url
self.on_message = on_message
self.on_open = on_open
self.on_close = on_close
self.on_error = on_error
self.sent = []
self.keep_running = True
self.fail_send = False
self.run_forever_kwargs = None

def send(self, data):
if self.fail_send or not self.keep_running:
raise websocket.WebSocketConnectionClosedException("Connection is already closed.")
self.sent.append(data)

def close(self):
self.keep_running = False

def run_forever(self, **kwargs):
self.run_forever_kwargs = kwargs
return False


@pytest.fixture
def manager(monkeypatch):
monkeypatch.setattr("hyperliquid.websocket_manager.websocket.WebSocketApp", FakeWebSocketApp)
return WebsocketManager("http://test")


def subscribe_frames(ws):
return [s for s in ws.sent if '"subscribe"' in s]


def test_reconnect_replays_active_subscriptions_once(manager):
manager.subscribe({"type": "allMids"}, lambda m: None)
manager.ws.on_open(manager.ws)
assert manager.ws_ready
assert subscribe_frames(manager.ws) == ['{"method": "subscribe", "subscription": {"type": "allMids"}}']
manager.subscribe({"type": "userEvents"}, lambda m: None)

sent_before_reconnect = len(manager.ws.sent)
manager.ws.on_open(manager.ws) # simulated reconnect

replayed = subscribe_frames(manager.ws)[sent_before_reconnect:]
assert replayed == [
'{"method": "subscribe", "subscription": {"type": "allMids"}}',
'{"method": "subscribe", "subscription": {"type": "userEvents"}}',
]
assert manager.queued_subscriptions == []


def test_reconnect_does_not_duplicate_callbacks(manager):
received = []
manager.subscribe({"type": "allMids"}, lambda m: received.append(m))
manager.ws.on_open(manager.ws)
manager.ws.on_open(manager.ws) # simulated reconnect
manager.ws.on_open(manager.ws) # second reconnect
manager.on_message(manager.ws, json.dumps({"channel": "allMids", "data": {"BTC": "1"}}))
assert len(received) == 1


def test_queued_subscriptions_not_replayed_twice(manager):
manager.subscribe({"type": "allMids"}, lambda m: None)
manager.ws.on_open(manager.ws)
assert manager.queued_subscriptions == []
manager.ws.on_open(manager.ws) # simulated reconnect: active replay, not queue replay
assert subscribe_frames(manager.ws) == [
'{"method": "subscribe", "subscription": {"type": "allMids"}}',
'{"method": "subscribe", "subscription": {"type": "allMids"}}',
]


def test_on_close_and_on_error_clear_ws_ready(manager):
manager.subscribe({"type": "allMids"}, lambda m: None)
manager.ws.on_open(manager.ws)
assert manager.ws_ready

manager.ws.on_close(manager.ws, 1006, "")
assert not manager.ws_ready
# Subscribing while disconnected queues instead of sending.
manager.subscribe({"type": "userEvents"}, lambda m: None)
assert len(subscribe_frames(manager.ws)) == 1
assert len(manager.queued_subscriptions) == 1

manager.ws.on_open(manager.ws)
assert manager.ws_ready
assert subscribe_frames(manager.ws) == [
'{"method": "subscribe", "subscription": {"type": "allMids"}}',
# Reconnect replay of the still-active allMids subscription...
'{"method": "subscribe", "subscription": {"type": "allMids"}}',
# ...followed by the queued userEvents subscription.
'{"method": "subscribe", "subscription": {"type": "userEvents"}}',
]

manager.ws.on_error(manager.ws, Exception("boom"))
assert not manager.ws_ready


def test_subscribe_while_socket_down_queues_instead_of_raising(manager):
manager.ws.on_open(manager.ws)
manager.ws.fail_send = True
subscription_id = manager.subscribe({"type": "allMids"}, lambda m: None)
assert isinstance(subscription_id, int)
assert len(manager.queued_subscriptions) == 1

manager.ws.fail_send = False
manager.ws.on_open(manager.ws)
assert subscribe_frames(manager.ws) == ['{"method": "subscribe", "subscription": {"type": "allMids"}}']


def test_run_passes_reconnect_delay(manager):
manager.run()
try:
assert manager.ws.run_forever_kwargs == {"reconnect": RECONNECT_DELAY_SECONDS}
assert RECONNECT_DELAY_SECONDS == 5
finally:
manager.stop()


def test_ping_thread_survives_send_failure(manager):
from unittest.mock import MagicMock

manager.ws.fail_send = True
manager.stop_event = MagicMock()
manager.stop_event.wait.side_effect = [False, True]
# Would raise WebSocketConnectionClosedException without the guard, killing
# the ping thread mid-reconnect.
manager.send_ping()
assert manager.stop_event.wait.call_count == 2