""" Lightweight, dependency-free decision seam for the scan-flow watchdog. Why a separate, dependency-free module: ``nautilus_event_trader.py`` drags in the engine / Hazelcast / CH-writer stack at *import* time (the module-level ``from nautilus_dolphin.nautilus.proxy_boost_engine import create_d_liq_engine`` and companion imports connect to infra that only exists under supervisord; outside that environment the import blocks). The watchdog's restart decision therefore can't be unit-tested in isolation there. This module holds the **only** new logic for the 2026-09-16 04:10:40 ghost-subscription wedge recovery โ€” so it has zero dependencies and imports instantly. ``nautilus_event_trader.py`` imports ``UPSTREAM_DARK_RESTART_S`` and ``upstream_dark_restart`` from here and consults the predicate in the previously log-only ``"NO SCANS ... UNMANAGED"`` branch, promoting a long-frozen HZ ``latest_eigen_scan`` key (ghost-subscription after a WS reconnect) from a reminder print to a self-restart via the existing ``_watchdog_restart`` -> ``os._exit(WATCHDOG_EXIT_CODE=86)`` -> supervisord respawn path. Cadence invariants (enforced by ``prod/tests/test_watchdog_decision.py``): SCAN_STALL_S (120s) < UPSTREAM_DARK_LOG_EVERY_S (300s) < UPSTREAM_DARK_RESTART_S (900s) [this module] <= warm-up-gated (uptime_ok checked before the predicate in _scan_watchdog_loop) See prod/docs/SYSTEM_BIBLE_v7.md ยง38.10 and the r27 py-spy report (pid 3506857, 2026-09-16 04:10:40). """ from __future__ import annotations import math # --------------------------------------------------------------------------- # (b) 2026-09-16 ghost-subscription self-heal threshold. # # The scan watchdog only *logs* "upstream dark / UNMANAGED" while the HZ # latest_eigen_scan key is frozen (probe == last_probe_num, i.e. NOT None โ€” a # None probe is the "HZ client dead" case handled by the 3x-failure restart # path in _scan_watchdog_loop). A frozen-but-not-None key is the hallmark of a # *ghost subscription*: the reconnect re-subscribed + ACKed but the server never # resumed the event stream, so the WS reader blocks in poll()/recv() (no # liveness watchdog) and the key never advances. After this much # accepted-scan staleness we stop nagging and self-restart, because the engine # is starved and -- per the r27 soak -- the venue is FLAT (zero fills, capital # intact), so a restart is free of position side-effects. # # Tunable: 900s (15 min) >> SCAN_STALL_S (120) and UPSTREAM_DARK_LOG_EVERY_S # (300) so restarts only fire on a *confirmed* long dark window, never on a # quiet market or a warm-up probe miss. Raise for calmer pairs/markets; lower # only behind the r27 HL-testnet WS stability fix. UPSTREAM_DARK_RESTART_S = 900.0 # Sentinel "no probe read" returned by _probe_latest_scan_number when the HZ # key is missing/empty/corrupt. Kept here (vs the loop) so the seam is the # single authority for the (b) restart condition. _PROBE_MISSING = object() def upstream_dark_restart( acc_age_s: float, uptime_ok: bool, scan_number_probe: object, ) -> bool: """(b) Ghost-subscription recovery decision (pure โ€” no I/O, no self state). Returns True iff the watchdog should self-restart for the "upstream dark (HZ key frozen)" case: * ``scan_number_probe`` is a real number (the HZ ``latest_eigen_scan`` probe SUCCEEDED and returned an int). This is the *frozen-key* ghost-subscription case (probe == last_probe_num). A *None / falsy* probe means the HZ client itself is dead/unreachable; that is owned by the separate 3x-failure restart path in ``_scan_watchdog_loop``, so this predicate MUST return False for it (avoids a double-restart / racing two restart paths). * ``uptime_ok`` -- warm-up window elapsed; never self-restart during the first ``WATCHDOG_RESTART_MIN_UPTIME_S`` to dodge boot-strap flakes. * ``acc_age_s >= UPSTREAM_DARK_RESTART_S`` -- no scan ACCEPTED for at least the dark-restart threshold (the engine has been starved long enough that "it will come back" is an assumption, not evidence). Poison inputs are handled defensively (never raises): * NaN acc_age -> False (corrupt clock; `nan >= x` is False) * negative acc_age -> False (clock skew backward) * +inf acc_age -> True (definitely dead) * non-numeric probe (str/dict/etc.) -> treated as None (safe: the loop still owns a None probe; (b) stays off) """ # (1) Probe must be a real scan number (frozen-key case). None / falsy / # non-numeric probe is owned by the probe-None-3x path -> do NOT fire. if scan_number_probe is None or scan_number_probe is _PROBE_MISSING: return False if not isinstance(scan_number_probe, (int, float)): return False if math.isnan(scan_number_probe) or math.isinf(scan_number_probe): # A scan number that is NaN/inf is a corrupt probe, not a frozen key; # let the probe-None-3x path handle it. (b) stays off. return False # (2) Warm-up: never self-restart during boot. if not uptime_ok: return False # (3) Accepted-scan staleness past the ghost-subscription threshold. # NaN acc_age -> `nan >= x` is False -> no spurious restart on a # corrupt acc_age clock. -inf -> False. +inf -> True (== definitely # dead). negative -> False (clock skew). try: return acc_age_s >= UPSTREAM_DARK_RESTART_S except TypeError: # Non-numeric acc_age (str/dict) -> don't crash the watchdog; treat as # "not stale enough" and keep logging dark instead. return False def scan_watchdog_dark_restart( acc_age_s: float, uptime_ok: bool, scan_number_probe: object, ev_age_s: float = 0.0, ) -> str | None: """(b) Seam the live ``_scan_watchdog_loop`` calls for the ghost-subscription restart decision. Pure (no I/O / no kernel state). Returns the canonical restart-reason string iff :func:`upstream_dark_restart` says restart, else ``None``. Centralising the reason text here (instead of building it inline in the heavy ``nautilus_event_trader`` module) keeps the entire (b) branch contract -- predicate + reason wording -- unit-testable without importing ``nautilus_event_trader`` (whose module-level engine/HZ import blocks outside the live supervisord environment). """ if not upstream_dark_restart(acc_age_s, uptime_ok, scan_number_probe): return None return ( f"upstream dark: HZ latest_eigen_scan frozen at {scan_number_probe} " f"for {acc_age_s:.0f}s (>= {UPSTREAM_DARK_RESTART_S}s) -- " "ghost-subscription after WS reconnect (no reader liveness " f"watchdog); acc_age={acc_age_s:.0f}s ev_age={float(ev_age_s):.0f}s" )