sentiment-engine/prod/watchdog_decision.py

"""
Lightweight, dependency-free decision seam for the scan-flow watchdog.

Why a separate, dependency-free module:
    ``nautilus_event_trader.py`` drags in the engine / Hazelcast / CH-writer
    stack at *import* time (the module-level
    ``from nautilus_dolphin.nautilus.proxy_boost_engine import create_d_liq_engine``
    and companion imports connect to infra that only exists under supervisord;
    outside that environment the import blocks).  The watchdog's restart
    decision therefore can't be unit-tested in isolation there.  This module
    holds the **only** new logic for the 2026-09-16 04:10:40 ghost-subscription
    wedge recovery — so it has zero dependencies and imports instantly.

``nautilus_event_trader.py`` imports ``UPSTREAM_DARK_RESTART_S`` and
``upstream_dark_restart`` from here and consults the predicate in the previously
log-only ``"NO SCANS ... UNMANAGED"`` branch, promoting a long-frozen HZ
``latest_eigen_scan`` key (ghost-subscription after a WS reconnect) from a
reminder print to a self-restart via the existing ``_watchdog_restart`` ->
``os._exit(WATCHDOG_EXIT_CODE=86)`` -> supervisord respawn path.

Cadence invariants (enforced by ``prod/tests/test_watchdog_decision.py``):

    SCAN_STALL_S              (120s)  <  UPSTREAM_DARK_LOG_EVERY_S (300s)
                                  <  UPSTREAM_DARK_RESTART_S   (900s)  [this module]
                                  <= warm-up-gated (uptime_ok checked before the
                                     predicate in _scan_watchdog_loop)

See prod/docs/SYSTEM_BIBLE_v7.md §38.10 and the r27 py-spy report (pid 3506857,
2026-09-16 04:10:40).
"""
from __future__ import annotations

import math

# ---------------------------------------------------------------------------
# (b) 2026-09-16 ghost-subscription self-heal threshold.
#
# The scan watchdog only *logs* "upstream dark / UNMANAGED" while the HZ
# latest_eigen_scan key is frozen (probe == last_probe_num, i.e. NOT None — a
# None probe is the "HZ client dead" case handled by the 3x-failure restart
# path in _scan_watchdog_loop).  A frozen-but-not-None key is the hallmark of a
# *ghost subscription*: the reconnect re-subscribed + ACKed but the server never
# resumed the event stream, so the WS reader blocks in poll()/recv() (no
# liveness watchdog) and the key never advances.  After this much
# accepted-scan staleness we stop nagging and self-restart, because the engine
# is starved and -- per the r27 soak -- the venue is FLAT (zero fills, capital
# intact), so a restart is free of position side-effects.
#
# Tunable: 900s (15 min) >> SCAN_STALL_S (120) and UPSTREAM_DARK_LOG_EVERY_S
# (300) so restarts only fire on a *confirmed* long dark window, never on a
# quiet market or a warm-up probe miss.  Raise for calmer pairs/markets; lower
# only behind the r27 HL-testnet WS stability fix.
UPSTREAM_DARK_RESTART_S = 900.0

# Sentinel "no probe read" returned by _probe_latest_scan_number when the HZ
# key is missing/empty/corrupt.  Kept here (vs the loop) so the seam is the
# single authority for the (b) restart condition.
_PROBE_MISSING = object()


def upstream_dark_restart(
    acc_age_s: float,
    uptime_ok: bool,
    scan_number_probe: object,
) -> bool:
    """(b) Ghost-subscription recovery decision (pure — no I/O, no self state).

    Returns True iff the watchdog should self-restart for the
    "upstream dark (HZ key frozen)" case:

      * ``scan_number_probe`` is a real number (the HZ ``latest_eigen_scan``
        probe SUCCEEDED and returned an int).  This is the *frozen-key*
        ghost-subscription case (probe == last_probe_num).  A *None / falsy*
        probe means the HZ client itself is dead/unreachable; that is owned by
        the separate 3x-failure restart path in ``_scan_watchdog_loop``, so
        this predicate MUST return False for it (avoids a double-restart /
        racing two restart paths).
      * ``uptime_ok`` -- warm-up window elapsed; never self-restart during the
        first ``WATCHDOG_RESTART_MIN_UPTIME_S`` to dodge boot-strap flakes.
      * ``acc_age_s >= UPSTREAM_DARK_RESTART_S`` -- no scan ACCEPTED for at
        least the dark-restart threshold (the engine has been starved long
        enough that "it will come back" is an assumption, not evidence).

    Poison inputs are handled defensively (never raises):
      * NaN acc_age -> False  (corrupt clock; `nan >= x` is False)
      * negative acc_age -> False (clock skew backward)
      * +inf acc_age -> True  (definitely dead)
      * non-numeric probe (str/dict/etc.) -> treated as None (safe: the loop
        still owns a None probe; (b) stays off)
    """
    # (1) Probe must be a real scan number (frozen-key case).  None / falsy /
    #     non-numeric probe is owned by the probe-None-3x path -> do NOT fire.
    if scan_number_probe is None or scan_number_probe is _PROBE_MISSING:
        return False
    if not isinstance(scan_number_probe, (int, float)):
        return False
    if math.isnan(scan_number_probe) or math.isinf(scan_number_probe):
        # A scan number that is NaN/inf is a corrupt probe, not a frozen key;
        # let the probe-None-3x path handle it.  (b) stays off.
        return False

    # (2) Warm-up: never self-restart during boot.
    if not uptime_ok:
        return False

    # (3) Accepted-scan staleness past the ghost-subscription threshold.
    #     NaN acc_age -> `nan >= x` is False -> no spurious restart on a
    #     corrupt acc_age clock.  -inf -> False.  +inf -> True (== definitely
    #     dead).  negative -> False (clock skew).
    try:
        return acc_age_s >= UPSTREAM_DARK_RESTART_S
    except TypeError:
        # Non-numeric acc_age (str/dict) -> don't crash the watchdog; treat as
        # "not stale enough" and keep logging dark instead.
        return False

def scan_watchdog_dark_restart(
    acc_age_s: float,
    uptime_ok: bool,
    scan_number_probe: object,
    ev_age_s: float = 0.0,
) -> str | None:
    """(b) Seam the live ``_scan_watchdog_loop`` calls for the ghost-subscription
    restart decision.

    Pure (no I/O / no kernel state).  Returns the canonical restart-reason
    string iff :func:`upstream_dark_restart` says restart, else ``None``.
    Centralising the reason text here (instead of building it inline in the
    heavy ``nautilus_event_trader`` module) keeps the entire (b) branch
    contract -- predicate + reason wording -- unit-testable without importing
    ``nautilus_event_trader`` (whose module-level engine/HZ import blocks
    outside the live supervisord environment).
    """
    if not upstream_dark_restart(acc_age_s, uptime_ok, scan_number_probe):
        return None
    return (
        f"upstream dark: HZ latest_eigen_scan frozen at {scan_number_probe} "
        f"for {acc_age_s:.0f}s (>= {UPSTREAM_DARK_RESTART_S}s) -- "
        "ghost-subscription after WS reconnect (no reader liveness "
        f"watchdog); acc_age={acc_age_s:.0f}s ev_age={float(ev_age_s):.0f}s"
    )