""" test_dolphin_tui_reconnect.py Verifies the HZ reconnect loop behavior of DolphinDataFetcher. Tests: - test_hz_connected_flag_set_on_connect - test_hz_disconnected_flag_on_failure - test_reconnect_within_10s - test_backoff_resets_on_success - test_fetch_returns_none_fields_when_disconnected All tests are self-contained and do NOT require a live Hazelcast instance. Backoff delays are patched to 0.05 s so the suite runs in seconds. """ from __future__ import annotations import asyncio import sys import os import time import unittest from unittest.mock import AsyncMock, MagicMock, patch, call # --------------------------------------------------------------------------- # Make sure the TUI module is importable from this directory # --------------------------------------------------------------------------- sys.path.insert(0, os.path.dirname(__file__)) # Provide a stub for hazelcast so the import succeeds even without the package import types _hz_stub = types.ModuleType("hazelcast") _hz_stub.HazelcastClient = MagicMock() sys.modules.setdefault("hazelcast", _hz_stub) # Provide stubs for textual and httpx so dolphin_tui imports cleanly for _mod in [ "textual", "textual.app", "textual.containers", "textual.widgets", ]: if _mod not in sys.modules: _stub = types.ModuleType(_mod) sys.modules[_mod] = _stub # Minimal textual stubs _textual_app = sys.modules["textual.app"] _textual_app.App = object _textual_app.ComposeResult = object _textual_containers = sys.modules["textual.containers"] _textual_containers.Horizontal = object _textual_widgets = sys.modules["textual.widgets"] _textual_widgets.Static = object _textual_widgets.VerticalScroll = object if "httpx" not in sys.modules: _httpx_stub = types.ModuleType("httpx") _httpx_stub.AsyncClient = MagicMock() sys.modules["httpx"] = _httpx_stub from dolphin_tui import ( # noqa: E402 DolphinDataFetcher, DataSnapshot, RECONNECT_INIT_S, RECONNECT_MULT, RECONNECT_MAX_S, ) # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- FAST_BACKOFF = 0.05 # seconds — replaces 5 s initial delay in tests def _make_mock_client() -> MagicMock: """Return a minimal mock that looks like a HazelcastClient.""" client = MagicMock() client.shutdown = MagicMock() return client def _run(coro): """Run a coroutine in a fresh event loop.""" return asyncio.get_event_loop().run_until_complete(coro) # --------------------------------------------------------------------------- # Tests # --------------------------------------------------------------------------- class TestHZConnectedFlagOnConnect(unittest.IsolatedAsyncioTestCase): """test_hz_connected_flag_set_on_connect A successful connect_hz() call must set hz_connected=True and store the client handle. """ async def test_hz_connected_flag_set_on_connect(self): mock_client = _make_mock_client() fetcher = DolphinDataFetcher(hz_host="localhost", hz_port=5701) with patch("hazelcast.HazelcastClient", return_value=mock_client): result = await fetcher.connect_hz() self.assertTrue(result, "connect_hz() should return True on success") self.assertTrue(fetcher.hz_connected, "hz_connected must be True after successful connect") self.assertIs(fetcher.hz_client, mock_client, "hz_client must be the returned client") # Clean up any background task fetcher._running = False if fetcher._reconnect_task and not fetcher._reconnect_task.done(): fetcher._reconnect_task.cancel() class TestHZDisconnectedFlagOnFailure(unittest.IsolatedAsyncioTestCase): """test_hz_disconnected_flag_on_failure When HazelcastClient() raises, connect_hz() must return False and hz_connected must be False. """ async def test_hz_disconnected_flag_on_failure(self): fetcher = DolphinDataFetcher(hz_host="localhost", hz_port=5701) with patch("hazelcast.HazelcastClient", side_effect=Exception("Connection refused")): result = await fetcher.connect_hz() self.assertFalse(result, "connect_hz() should return False on failure") self.assertFalse(fetcher.hz_connected, "hz_connected must be False after failed connect") self.assertIsNone(fetcher.hz_client, "hz_client must remain None after failed connect") # Stop the reconnect loop that was started by connect_hz on failure fetcher._running = False if fetcher._reconnect_task and not fetcher._reconnect_task.done(): fetcher._reconnect_task.cancel() try: await fetcher._reconnect_task except (asyncio.CancelledError, Exception): pass class TestReconnectWithin10s(unittest.IsolatedAsyncioTestCase): """test_reconnect_within_10s Scenario: 1. Initial connect fails → hz_connected=False, reconnect loop starts. 2. After ~0.1 s the mock is switched to succeed. 3. hz_connected must become True within 10 s of the mock being restored. Backoff is patched to FAST_BACKOFF (0.05 s) so the test completes quickly. """ async def test_reconnect_within_10s(self): mock_client = _make_mock_client() # State shared between the mock and the test should_succeed = False def hz_client_factory(**kwargs): if should_succeed: return mock_client raise Exception("Connection refused") fetcher = DolphinDataFetcher(hz_host="localhost", hz_port=5701) # Patch backoff to be very short so the test is fast fetcher._reconnect_backoff_initial = FAST_BACKOFF fetcher._reconnect_backoff = FAST_BACKOFF fetcher._reconnect_backoff_max = FAST_BACKOFF * 3 with patch("hazelcast.HazelcastClient", side_effect=hz_client_factory): # Start the reconnect loop manually (simulates connect_hz failing) fetcher._start_reconnect() # Let the loop spin for a moment while HZ is "down" await asyncio.sleep(FAST_BACKOFF * 2) self.assertFalse(fetcher.hz_connected, "Should still be disconnected while HZ is down") # "Restart" HZ should_succeed = True t0 = time.monotonic() # Wait up to 10 s for reconnect deadline = 10.0 while not fetcher.hz_connected and (time.monotonic() - t0) < deadline: await asyncio.sleep(0.05) elapsed = time.monotonic() - t0 self.assertTrue( fetcher.hz_connected, f"hz_connected must be True within 10 s of HZ restart (elapsed: {elapsed:.2f}s)", ) self.assertLess(elapsed, 10.0, f"Reconnect took too long: {elapsed:.2f}s") # Cleanup await fetcher.disconnect_hz() class TestBackoffResetsOnSuccess(unittest.IsolatedAsyncioTestCase): """test_backoff_resets_on_success After a successful reconnect the backoff delay must be reset to the initial value (RECONNECT_INIT_S / patched FAST_BACKOFF). """ async def test_backoff_resets_on_success(self): mock_client = _make_mock_client() call_count = 0 def hz_client_factory(**kwargs): nonlocal call_count call_count += 1 if call_count == 1: raise Exception("First attempt fails") return mock_client fetcher = DolphinDataFetcher(hz_host="localhost", hz_port=5701) fetcher._reconnect_backoff_initial = FAST_BACKOFF fetcher._reconnect_backoff = FAST_BACKOFF fetcher._reconnect_backoff_max = FAST_BACKOFF * 10 with patch("hazelcast.HazelcastClient", side_effect=hz_client_factory): fetcher._start_reconnect() # Wait for reconnect to succeed deadline = 5.0 t0 = time.monotonic() while not fetcher.hz_connected and (time.monotonic() - t0) < deadline: await asyncio.sleep(0.05) self.assertTrue(fetcher.hz_connected, "Should have reconnected") self.assertAlmostEqual( fetcher._reconnect_backoff, FAST_BACKOFF, delta=1e-9, msg="Backoff must reset to initial value after successful reconnect", ) await fetcher.disconnect_hz() class TestFetchReturnsNoneFieldsWhenDisconnected(unittest.IsolatedAsyncioTestCase): """test_fetch_returns_none_fields_when_disconnected When hz_client is None (disconnected), fetch() must return a DataSnapshot with hz_connected=False and all HZ-derived fields as None. """ async def test_fetch_returns_none_fields_when_disconnected(self): fetcher = DolphinDataFetcher(hz_host="localhost", hz_port=5701) # Ensure no client is set fetcher.hz_client = None fetcher.hz_connected = False # Patch fetch_prefect so we don't need a live Prefect server fetcher.fetch_prefect = AsyncMock(return_value=(False, [])) # Patch tail_log so we don't need a real log file fetcher.tail_log = MagicMock(return_value=[]) # Prevent reconnect loop from starting during fetch fetcher._start_reconnect = MagicMock() snap = await fetcher.fetch() self.assertIsInstance(snap, DataSnapshot) self.assertFalse(snap.hz_connected, "hz_connected must be False when disconnected") # All HZ-derived numeric/string fields must be None hz_fields = [ "scan_number", "vel_div", "w50_velocity", "w750_velocity", "instability_50", "scan_bridge_ts", "scan_age_s", "acb_boost", "acb_beta", "funding_btc", "dvol_btc", "fng", "taker", "vix", "ls_btc", "acb_ready", "acb_present", "exf_age_s", "moon_phase", "mercury_retro", "liquidity_session", "market_cycle_pos", "esof_age_s", "posture", "rm", "cat1", "cat2", "cat3", "cat4", "cat5", "capital", "drawdown", "peak_capital", "pnl", "trades", "nautilus_capital", "nautilus_pnl", "nautilus_trades", "nautilus_posture", "nautilus_param_hash", "heartbeat_ts", "heartbeat_phase", "heartbeat_flow", "heartbeat_age_s", "meta_rm", "meta_status", "m1_proc", "m2_heartbeat", "m3_data", "m4_cp", "m5_coh", ] for field_name in hz_fields: value = getattr(snap, field_name) self.assertIsNone( value, f"Field '{field_name}' must be None when disconnected, got {value!r}", ) # Collection fields must be empty self.assertEqual(snap.asset_prices, {}, "asset_prices must be empty dict when disconnected") self.assertEqual(snap.obf_top, [], "obf_top must be empty list when disconnected") # --------------------------------------------------------------------------- # Backoff constants sanity check (not a reconnect test, but validates spec) # --------------------------------------------------------------------------- class TestBackoffConstants(unittest.TestCase): """Verify the module-level backoff constants match the spec.""" def test_reconnect_init_s(self): self.assertEqual(RECONNECT_INIT_S, 5.0, "Initial backoff must be 5 s per spec") def test_reconnect_multiplier(self): self.assertEqual(RECONNECT_MULT, 1.5, "Backoff multiplier must be 1.5x per spec") def test_reconnect_max_s(self): self.assertEqual(RECONNECT_MAX_S, 60.0, "Max backoff must be 60 s per spec") def test_backoff_sequence(self): """Verify the exponential sequence: 5 → 7.5 → 11.25 → ... capped at 60.""" backoff = RECONNECT_INIT_S sequence = [backoff] for _ in range(10): backoff = min(backoff * RECONNECT_MULT, RECONNECT_MAX_S) sequence.append(backoff) self.assertAlmostEqual(sequence[0], 5.0) self.assertAlmostEqual(sequence[1], 7.5) self.assertAlmostEqual(sequence[2], 11.25) self.assertTrue(all(v <= RECONNECT_MAX_S for v in sequence)) self.assertEqual(sequence[-1], RECONNECT_MAX_S) if __name__ == "__main__": unittest.main()