DOLPHIN/Observability/TUI/test_dolphin_tui_reconnect.py

"""
test_dolphin_tui_reconnect.py

Verifies the HZ reconnect loop behavior of DolphinDataFetcher.

Tests:
  - test_hz_connected_flag_set_on_connect
  - test_hz_disconnected_flag_on_failure
  - test_reconnect_within_10s
  - test_backoff_resets_on_success
  - test_fetch_returns_none_fields_when_disconnected

All tests are self-contained and do NOT require a live Hazelcast instance.
Backoff delays are patched to 0.05 s so the suite runs in seconds.
"""

from __future__ import annotations

import asyncio
import sys
import os
import time
import unittest
from unittest.mock import AsyncMock, MagicMock, patch, call

# ---------------------------------------------------------------------------
# Make sure the TUI module is importable from this directory
# ---------------------------------------------------------------------------
sys.path.insert(0, os.path.dirname(__file__))

# Provide a stub for hazelcast so the import succeeds even without the package
import types

_hz_stub = types.ModuleType("hazelcast")
_hz_stub.HazelcastClient = MagicMock()
sys.modules.setdefault("hazelcast", _hz_stub)

# Provide stubs for textual and httpx so dolphin_tui imports cleanly
for _mod in [
    "textual",
    "textual.app",
    "textual.containers",
    "textual.widgets",
]:
    if _mod not in sys.modules:
        _stub = types.ModuleType(_mod)
        sys.modules[_mod] = _stub

# Minimal textual stubs
_textual_app = sys.modules["textual.app"]
_textual_app.App = object
_textual_app.ComposeResult = object

_textual_containers = sys.modules["textual.containers"]
_textual_containers.Horizontal = object

_textual_widgets = sys.modules["textual.widgets"]
_textual_widgets.Static = object
_textual_widgets.VerticalScroll = object

if "httpx" not in sys.modules:
    _httpx_stub = types.ModuleType("httpx")
    _httpx_stub.AsyncClient = MagicMock()
    sys.modules["httpx"] = _httpx_stub

from dolphin_tui import (  # noqa: E402
    DolphinDataFetcher,
    DataSnapshot,
    RECONNECT_INIT_S,
    RECONNECT_MULT,
    RECONNECT_MAX_S,
)

# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------

FAST_BACKOFF = 0.05   # seconds — replaces 5 s initial delay in tests


def _make_mock_client() -> MagicMock:
    """Return a minimal mock that looks like a HazelcastClient."""
    client = MagicMock()
    client.shutdown = MagicMock()
    return client


def _run(coro):
    """Run a coroutine in a fresh event loop."""
    return asyncio.get_event_loop().run_until_complete(coro)


# ---------------------------------------------------------------------------
# Tests
# ---------------------------------------------------------------------------

class TestHZConnectedFlagOnConnect(unittest.IsolatedAsyncioTestCase):
    """test_hz_connected_flag_set_on_connect

    A successful connect_hz() call must set hz_connected=True and store
    the client handle.
    """

    async def test_hz_connected_flag_set_on_connect(self):
        mock_client = _make_mock_client()

        fetcher = DolphinDataFetcher(hz_host="localhost", hz_port=5701)

        with patch("hazelcast.HazelcastClient", return_value=mock_client):
            result = await fetcher.connect_hz()

        self.assertTrue(result, "connect_hz() should return True on success")
        self.assertTrue(fetcher.hz_connected, "hz_connected must be True after successful connect")
        self.assertIs(fetcher.hz_client, mock_client, "hz_client must be the returned client")

        # Clean up any background task
        fetcher._running = False
        if fetcher._reconnect_task and not fetcher._reconnect_task.done():
            fetcher._reconnect_task.cancel()


class TestHZDisconnectedFlagOnFailure(unittest.IsolatedAsyncioTestCase):
    """test_hz_disconnected_flag_on_failure

    When HazelcastClient() raises, connect_hz() must return False and
    hz_connected must be False.
    """

    async def test_hz_disconnected_flag_on_failure(self):
        fetcher = DolphinDataFetcher(hz_host="localhost", hz_port=5701)

        with patch("hazelcast.HazelcastClient", side_effect=Exception("Connection refused")):
            result = await fetcher.connect_hz()

        self.assertFalse(result, "connect_hz() should return False on failure")
        self.assertFalse(fetcher.hz_connected, "hz_connected must be False after failed connect")
        self.assertIsNone(fetcher.hz_client, "hz_client must remain None after failed connect")

        # Stop the reconnect loop that was started by connect_hz on failure
        fetcher._running = False
        if fetcher._reconnect_task and not fetcher._reconnect_task.done():
            fetcher._reconnect_task.cancel()
            try:
                await fetcher._reconnect_task
            except (asyncio.CancelledError, Exception):
                pass


class TestReconnectWithin10s(unittest.IsolatedAsyncioTestCase):
    """test_reconnect_within_10s

    Scenario:
      1. Initial connect fails → hz_connected=False, reconnect loop starts.
      2. After ~0.1 s the mock is switched to succeed.
      3. hz_connected must become True within 10 s of the mock being restored.

    Backoff is patched to FAST_BACKOFF (0.05 s) so the test completes quickly.
    """

    async def test_reconnect_within_10s(self):
        mock_client = _make_mock_client()

        # State shared between the mock and the test
        should_succeed = False

        def hz_client_factory(**kwargs):
            if should_succeed:
                return mock_client
            raise Exception("Connection refused")

        fetcher = DolphinDataFetcher(hz_host="localhost", hz_port=5701)
        # Patch backoff to be very short so the test is fast
        fetcher._reconnect_backoff_initial = FAST_BACKOFF
        fetcher._reconnect_backoff = FAST_BACKOFF
        fetcher._reconnect_backoff_max = FAST_BACKOFF * 3

        with patch("hazelcast.HazelcastClient", side_effect=hz_client_factory):
            # Start the reconnect loop manually (simulates connect_hz failing)
            fetcher._start_reconnect()

            # Let the loop spin for a moment while HZ is "down"
            await asyncio.sleep(FAST_BACKOFF * 2)
            self.assertFalse(fetcher.hz_connected, "Should still be disconnected while HZ is down")

            # "Restart" HZ
            should_succeed = True
            t0 = time.monotonic()

            # Wait up to 10 s for reconnect
            deadline = 10.0
            while not fetcher.hz_connected and (time.monotonic() - t0) < deadline:
                await asyncio.sleep(0.05)

        elapsed = time.monotonic() - t0
        self.assertTrue(
            fetcher.hz_connected,
            f"hz_connected must be True within 10 s of HZ restart (elapsed: {elapsed:.2f}s)",
        )
        self.assertLess(elapsed, 10.0, f"Reconnect took too long: {elapsed:.2f}s")

        # Cleanup
        await fetcher.disconnect_hz()


class TestBackoffResetsOnSuccess(unittest.IsolatedAsyncioTestCase):
    """test_backoff_resets_on_success

    After a successful reconnect the backoff delay must be reset to the
    initial value (RECONNECT_INIT_S / patched FAST_BACKOFF).
    """

    async def test_backoff_resets_on_success(self):
        mock_client = _make_mock_client()

        call_count = 0

        def hz_client_factory(**kwargs):
            nonlocal call_count
            call_count += 1
            if call_count == 1:
                raise Exception("First attempt fails")
            return mock_client

        fetcher = DolphinDataFetcher(hz_host="localhost", hz_port=5701)
        fetcher._reconnect_backoff_initial = FAST_BACKOFF
        fetcher._reconnect_backoff = FAST_BACKOFF
        fetcher._reconnect_backoff_max = FAST_BACKOFF * 10

        with patch("hazelcast.HazelcastClient", side_effect=hz_client_factory):
            fetcher._start_reconnect()

            # Wait for reconnect to succeed
            deadline = 5.0
            t0 = time.monotonic()
            while not fetcher.hz_connected and (time.monotonic() - t0) < deadline:
                await asyncio.sleep(0.05)

        self.assertTrue(fetcher.hz_connected, "Should have reconnected")
        self.assertAlmostEqual(
            fetcher._reconnect_backoff,
            FAST_BACKOFF,
            delta=1e-9,
            msg="Backoff must reset to initial value after successful reconnect",
        )

        await fetcher.disconnect_hz()


class TestFetchReturnsNoneFieldsWhenDisconnected(unittest.IsolatedAsyncioTestCase):
    """test_fetch_returns_none_fields_when_disconnected

    When hz_client is None (disconnected), fetch() must return a DataSnapshot
    with hz_connected=False and all HZ-derived fields as None.
    """

    async def test_fetch_returns_none_fields_when_disconnected(self):
        fetcher = DolphinDataFetcher(hz_host="localhost", hz_port=5701)
        # Ensure no client is set
        fetcher.hz_client = None
        fetcher.hz_connected = False

        # Patch fetch_prefect so we don't need a live Prefect server
        fetcher.fetch_prefect = AsyncMock(return_value=(False, []))
        # Patch tail_log so we don't need a real log file
        fetcher.tail_log = MagicMock(return_value=[])
        # Prevent reconnect loop from starting during fetch
        fetcher._start_reconnect = MagicMock()

        snap = await fetcher.fetch()

        self.assertIsInstance(snap, DataSnapshot)
        self.assertFalse(snap.hz_connected, "hz_connected must be False when disconnected")

        # All HZ-derived numeric/string fields must be None
        hz_fields = [
            "scan_number", "vel_div", "w50_velocity", "w750_velocity",
            "instability_50", "scan_bridge_ts", "scan_age_s",
            "acb_boost", "acb_beta",
            "funding_btc", "dvol_btc", "fng", "taker", "vix", "ls_btc",
            "acb_ready", "acb_present", "exf_age_s",
            "moon_phase", "mercury_retro", "liquidity_session",
            "market_cycle_pos", "esof_age_s",
            "posture", "rm", "cat1", "cat2", "cat3", "cat4", "cat5",
            "capital", "drawdown", "peak_capital", "pnl", "trades",
            "nautilus_capital", "nautilus_pnl", "nautilus_trades",
            "nautilus_posture", "nautilus_param_hash",
            "heartbeat_ts", "heartbeat_phase", "heartbeat_flow", "heartbeat_age_s",
            "meta_rm", "meta_status",
            "m1_proc", "m2_heartbeat", "m3_data", "m4_cp", "m5_coh",
        ]
        for field_name in hz_fields:
            value = getattr(snap, field_name)
            self.assertIsNone(
                value,
                f"Field '{field_name}' must be None when disconnected, got {value!r}",
            )

        # Collection fields must be empty
        self.assertEqual(snap.asset_prices, {}, "asset_prices must be empty dict when disconnected")
        self.assertEqual(snap.obf_top, [], "obf_top must be empty list when disconnected")


# ---------------------------------------------------------------------------
# Backoff constants sanity check (not a reconnect test, but validates spec)
# ---------------------------------------------------------------------------

class TestBackoffConstants(unittest.TestCase):
    """Verify the module-level backoff constants match the spec."""

    def test_reconnect_init_s(self):
        self.assertEqual(RECONNECT_INIT_S, 5.0, "Initial backoff must be 5 s per spec")

    def test_reconnect_multiplier(self):
        self.assertEqual(RECONNECT_MULT, 1.5, "Backoff multiplier must be 1.5x per spec")

    def test_reconnect_max_s(self):
        self.assertEqual(RECONNECT_MAX_S, 60.0, "Max backoff must be 60 s per spec")

    def test_backoff_sequence(self):
        """Verify the exponential sequence: 5 → 7.5 → 11.25 → ... capped at 60."""
        backoff = RECONNECT_INIT_S
        sequence = [backoff]
        for _ in range(10):
            backoff = min(backoff * RECONNECT_MULT, RECONNECT_MAX_S)
            sequence.append(backoff)

        self.assertAlmostEqual(sequence[0], 5.0)
        self.assertAlmostEqual(sequence[1], 7.5)
        self.assertAlmostEqual(sequence[2], 11.25)
        self.assertTrue(all(v <= RECONNECT_MAX_S for v in sequence))
        self.assertEqual(sequence[-1], RECONNECT_MAX_S)


if __name__ == "__main__":
    unittest.main()