siloqy/prod/clean_arch/violet/live_blue_source.py

"""VIOLET V3.4c: read BLUE-published live organs from Hazelcast.

This is VIOLET-only. BLUE is untouched.

The adapter reads the published BLUE surfaces that already exist in HZ and
translates them into ``SizingFactors`` for the shadow path:
  - ``posture`` from ``DOLPHIN_STATE_BLUE.latest_nautilus`` / ``engine_snapshot``
  - ``esof_score`` from ``DOLPHIN_FEATURES.esof_latest`` or ``esof_advisor_latest`` via
    BLUE's own ``parse_esof_payload`` / ``esof_score_from_payload``
  - ``boost`` / ``beta`` RECOMPUTED via ``AdaptiveCircuitBreaker.get_dynamic_boost_from_hz``
    over ``DOLPHIN_FEATURES.exf_latest`` + ``latest_eigen_scan.w750_velocity`` — IDENTICAL
    to the trader's on_exf_update path (NOT the published ``acb_boost`` scalar)
  - ``mc_scale`` from ``DOLPHIN_FEATURES.mc_forewarner_latest`` via ``_derive_mc_scale``
    (begin_day's cat/env thresholds, not the MC service's status label)
  - OB market consensus from the live ``asset_*_ob`` maps via BLUE's own ``HZOBProvider`` +
    ``OBFeatureEngine``
  - ``dc_status`` via BLUE's ``AlphaSignalGenerator`` (params pinned to BLUE's ENGINE_KWARGS)
    over the replayed scan price-history

PARITY (see prod/docs/VIOLET_BLUE_PARITY_STRUCTURAL_DIVERGENCE.md): this module reconstructs
BLUE's factors in a DIFFERENT file/scope structure than BLUE's monolithic NDAlphaEngine.
Every kernel is now WRAPPED, not copied (ACB, OBFeatureEngine+HZOBProvider, AlphaSignalGenerator,
VioletAssetSelector) — so the only hand-replicated arithmetic left is ``_derive_mc_scale``
(begin_day's thresholds; pinned by test). Remaining REAL fidelity caveats:
  - OB faithfulness needs a PERSISTENT ``ob_engine`` + per-scan ``bar_idx`` (OBFeatureEngine
    accumulates a lookback window); a single-shot engine has no cross-scan history.
  - On stale exf (>12h) the ACB raises ValueError; BLUE keeps the prior boost/beta, VIOLET
    has no prior → neutral (1.0, 0.0).
Any change to BLUE's ENGINE_KWARGS or begin_day mc thresholds REQUIRES a matching change +
test here (see test_signal_gen_params_match_blue_engine_kwargs / the mc_scale formula test).
"""

from __future__ import annotations

import json
import logging
import os as _os
import sys
from collections import deque
from datetime import datetime, timezone
from collections.abc import Mapping
from dataclasses import dataclass
from dataclasses import field
from pathlib import Path
from typing import Any, Deque, Iterable, Optional

import hazelcast
import numpy as np

_PROJECT_ROOT = Path(__file__).resolve().parents[3]
for _p in (str(_PROJECT_ROOT), str(_PROJECT_ROOT / "nautilus_dolphin")):
    if _p not in sys.path:
        sys.path.insert(0, _p)

from nautilus_dolphin.nautilus.ob_features import OBFeatureEngine
from nautilus_dolphin.nautilus.hz_ob_provider import HZOBProvider
from nautilus_dolphin.nautilus.adaptive_circuit_breaker import AdaptiveCircuitBreaker
from nautilus_dolphin.nautilus.alpha_signal_generator import (
    AlphaSignalGenerator,
    VEL_DIV_THRESHOLD, VEL_DIV_EXTREME,
    LONG_VEL_DIV_THRESHOLD, LONG_VEL_DIV_EXTREME,
)

from .alpha_wrappers import VioletAssetSelector
from .decision_engine import SizingFactors
from .live_factor_source import esof_score_from_features, posture_from_engine_snapshot
from .live_factors import extract_live_sizing_factors

LOGGER = logging.getLogger("violet.live_blue_source")

# Hazelcast coordinates — MUST equal nautilus_event_trader.py:107-108 (BLUE's live
# cluster). HZOBProvider opens its own connection to these, exactly as BLUE's _wire_obf.
# TODO_HZBRIDGE: all VIOLET HZ access (the caller's client + HZOBProvider) must move to
# dolphinng5_predict/hzbridge when it ships — raw HazelcastClient is the silent-death/lockup
# surface (see prod/docs/VIOLET_OB_FEED_AND_AGENT_COORDINATION.md + hz_client_death memory).
HZ_CLUSTER = _os.environ.get("HZ_CLUSTER", "dolphin")
HZ_HOST = _os.environ.get("HZ_HOST", "127.0.0.1:5701")

# AlphaSignalGenerator construction — pinned to BLUE's live ENGINE_KWARGS
# (nautilus_event_trader.py:128-133). These equal AlphaSignalGenerator's own defaults
# TODAY, but BLUE constructs it EXPLICITLY from ENGINE_KWARGS, so a future champion
# retune (e.g. dc_lookback_bars→10) would silently diverge a bare AlphaSignalGenerator().
# We pin explicitly and assert the pin in test_signal_gen_params_match_blue_engine_kwargs.
# vel_div_* import the module constants directly so they auto-track the kernel.
BLUE_SIGNAL_GEN_KWARGS = dict(
    vel_div_threshold=VEL_DIV_THRESHOLD,        # -0.02
    vel_div_extreme=VEL_DIV_EXTREME,            # -0.05
    long_vel_div_threshold=LONG_VEL_DIV_THRESHOLD,  # 0.01
    long_vel_div_extreme=LONG_VEL_DIV_EXTREME,      # 0.04
    dc_lookback_bars=7,
    dc_min_magnitude_bps=0.75,
    dc_skip_contradicts=True,
    dc_leverage_boost=1.0,
    dc_leverage_reduce=0.5,
    use_direction_confirm=True,
)


def _jsonish(value: Any) -> Any:
    if isinstance(value, str):
        try:
            return json.loads(value)
        except Exception:
            return value
    return value


def _coerce_float(value: Any, default: Optional[float] = None) -> Optional[float]:
    try:
        if value is None:
            return default
        out = float(value)
        if not np.isfinite(out):
            return default
        return out
    except (TypeError, ValueError):
        return default


def _read_hz_map(client: hazelcast.HazelcastClient, map_name: str, key: str) -> Any:
    try:
        return client.get_map(map_name).blocking().get(key)
    except Exception:
        return None


def _derive_mc_scale(payload: Any) -> float:
    """Mirror BLUE ``begin_day``'s ``mc_scale`` EXACTLY (esf_alpha_orchestrator.py:956-962).

    BLUE does NOT use the MC service's published ``status`` label for sizing — that label
    (mc_forewarner_flow.py: GREEN<0.10 / ORANGE<0.30 / RED) is observability-only and uses
    DIFFERENT thresholds from the engine. The live trader re-derives the size haircut from
    the SAME published fields ``catastrophic_prob`` + ``envelope_score`` with the engine's
    own thresholds:

        mc_red    = catastrophic_prob > 0.25 or envelope_score < -1.0
        mc_orange = (not mc_red) and (envelope_score < 0 or catastrophic_prob > 0.10)
        mc_scale  = 0.5 if mc_orange else 1.0      # RED → 1.0 here; BLUE halts via regime_dd_halt

    Reading ``status`` instead diverges (e.g. cat=0.05/env=-0.5 → publisher GREEN→1.0 but
    BLUE orange→0.5).

    AMBIGUITY (flagged 2026-06-16): the MC service's ``status`` and the engine's begin_day
    thresholds genuinely disagree, and from OUTSIDE BLUE there is no way to know which is the
    "nominal" intent — they are two independent threshold sets over the same numbers. We go
    with the SOURCE FIELDS + begin_day formula because that is the path that actually drives
    BLUE's live sizing (the published ``status`` is consumed only by the TUI/observability).
    If BLUE's begin_day thresholds change, THIS must change with them. Operator confirmation
    of the canonical surface is still desirable.

    Missing/unparseable fields → neutral 1.0 (no haircut)."""
    data = _jsonish(payload)
    if not isinstance(data, Mapping):
        return 1.0
    cat = _coerce_float(data.get("catastrophic_prob"), None)
    env = _coerce_float(data.get("envelope_score"), None)
    if cat is None or env is None:
        return 1.0
    mc_red = cat > 0.25 or env < -1.0
    mc_orange = (not mc_red) and (env < 0.0 or cat > 0.10)
    return 0.5 if mc_orange else 1.0


# Inverse-ACB neutral identity — used ONLY at cold start (no prior yet AND no fresh exf).
# In continuous operation exf is warmed, so the first call seeds the prior and this is
# never the steady-state value.
_BOOST_BETA_NEUTRAL = (1.0, 0.0)


def _source_boost_beta(
    client: hazelcast.HazelcastClient,
    *,
    date_str: str,
    trade_direction: int,
    acb: Optional[AdaptiveCircuitBreaker] = None,
    prior: Optional[tuple[float, float]] = None,
) -> tuple[float, float]:
    """Recompute (boost, beta) EXACTLY as BLUE's live trader does — NOT from acb_boost.

    WHAT THE ACB IS DOING, IN EFFECT (AdaptiveCircuitBreaker v6, "inverse" mode):
      - ``boost`` = a DAILY position-size governor >= 1.0 driven by EXTERNAL FACTORS (ExF:
        funding_btc/dvol_btc/fng/taker from DOLPHIN_FEATURES.exf_latest). When stress
        signals fire, ``boost = 1 + 0.5*ln(1+signals)``; else 1.0. INVERSE = it leans size
        UP under stress, it does not cut (adaptive_circuit_breaker.py:591-593).
      - ``beta`` = regime sensitivity in {BETA_HIGH=0.8, BETA_LOW=0.2}, keyed on the w750
        eigenvalue-velocity (>= threshold → HIGH). It sets how strongly per-bar signal
        strength amplifies size: regime_size_mult = base_boost*(1 + beta*strength^3)*mc_scale.
      - Both become the engine's _day_base_boost / _day_beta via update_acb_boost
        (esf_alpha_orchestrator.py:771-772), which then feed BLUE's SIZING directly:
        _update_regime_size_mult (:898-909) = _day_base_boost * (1 + _day_beta*strength^3)
        * _day_mc_scale. So the ACB IS in the sizing layer — confirmed, not incidental.

    BLUE's LIVE path passes NO ob_engine (on_exf_update:4769 / rollover prewarm:2710), so the
    ACB's OB Sub-4 macro-regime beta modulation (x1.25 stress / x0.85 calm,
    adaptive_circuit_breaker.py:613-628) is DORMANT in live — only the NPZ backtest path
    get_dynamic_boost_for_date applies it. Per operator recollection (2026-06-16), OB Sub-4
    beta modulation was found NON-PERFORMANT and deliberately removed/bypassed — so passing
    no ob_engine is INTENTIONAL design, not an oversight. We pass no ob_engine to match BLUE
    bit-for-bit either way.
    # TODO_SOMEDAY: find the documentary confirmation of the OB Sub-4 non-performant bypass
    #   (operator: likely SYSTEM_BIBLE v7 lineage or a deep code comment / the in-progress
    #   INDEX_DOLPHINNG5_PREDICT doc). Behaviour is already bit-identical to live BLUE.

    The published DOLPHIN_FEATURES.acb_boost is acb_processor_service's SEPARATE daily value
    and is never read here.

    STALE / MISSING exf: BLUE keeps the prior _day_base_boost/_day_beta (update_acb_boost
    simply isn't called on the stale branch), so VIOLET KEEPS ``prior`` too. ``prior`` is
    seeded by the first successful compute — exf is warmed in continuous BLUE operation, so
    the steady state always has a prior; only a cold start with no prior AND no fresh exf
    falls back to the neutral identity."""
    fallback = prior if prior is not None else _BOOST_BETA_NEUTRAL
    exf = _jsonish(_read_hz_map(client, "DOLPHIN_FEATURES", "exf_latest"))
    if not isinstance(exf, Mapping):
        LOGGER.debug("ACB: no exf_latest — keeping prior boost/beta=%s", fallback)
        return fallback
    eigen = _jsonish(_read_hz_map(client, "DOLPHIN_FEATURES", "latest_eigen_scan"))
    w750 = _coerce_float(eigen.get("w750_velocity"), None) if isinstance(eigen, Mapping) else None
    acb = acb or AdaptiveCircuitBreaker()
    try:
        info = acb.get_dynamic_boost_from_hz(
            date_str=date_str,
            exf_snapshot=dict(exf),
            w750_velocity=float(w750) if w750 else None,   # 0.0 → None, matches BLUE
            direction=trade_direction,                     # NO ob_engine — matches BLUE live
        )
    except ValueError as exc:
        # BLUE logs "ACB Stale Data Fallback" and keeps the prior boost/beta.
        LOGGER.info("ACB Stale Data Fallback (%s) — keeping prior boost/beta=%s", exc, fallback)
        return fallback
    boost = max(0.0, _coerce_float(info.get("boost"), 1.0) or 1.0)
    beta = max(0.0, _coerce_float(info.get("beta"), 0.0) or 0.0)
    LOGGER.debug("ACB live boost=%.6f beta=%.6f (signals=%s w750=%s dir=%s)",
                 boost, beta, info.get("signals"), w750, trade_direction)
    return boost, beta


def _scan_view(payload: Any) -> Mapping[str, Any]:
    """Normalize NG7 nested or NG8 flat scan payloads into one mapping."""
    data = _jsonish(payload)
    if not isinstance(data, Mapping):
        return {}
    view: dict[str, Any] = dict(data)
    result = data.get("result")
    if isinstance(result, Mapping):
        view.update(result)
    return view


def _scan_assets(scan: Mapping[str, Any]) -> list[str]:
    assets = scan.get("assets")
    if isinstance(assets, list) and assets:
        return [str(a).upper() for a in assets if a is not None]
    target = scan.get("target_asset") or scan.get("asset")
    return [str(target).upper()] if target else []


def _scan_prices(scan: Mapping[str, Any]) -> list[float]:
    prices = scan.get("asset_prices") or scan.get("prices")
    if not isinstance(prices, list):
        return []
    out: list[float] = []
    for value in prices:
        px = _coerce_float(value, None)
        if px is None or px <= 0:
            continue
        out.append(px)
    return out


@dataclass
class LiveBlueScanHistory:
    """Stateful scan history mirror for DC confirmation.

    BLUE computes `dc_status` from the active asset price history. This helper keeps
    a read-only replay of the published scan stream so VIOLET can reproduce that
    state without touching BLUE or inventing a new schema.
    """

    maxlen: int = 32
    trade_direction: int = -1
    _prices: dict[str, Deque[float]] = field(default_factory=dict)
    last_scan_number: Optional[int] = None

    def ingest_scan(self, scan_payload: Any) -> Mapping[str, Any]:
        scan = _scan_view(scan_payload)
        assets = _scan_assets(scan)
        prices = _scan_prices(scan)
        for asset, price in zip(assets, prices):
            if price <= 0:
                continue
            hist = self._prices.setdefault(asset, deque(maxlen=self.maxlen))
            hist.append(float(price))
        sn = _coerce_float(scan.get("scan_number"), None)
        if sn is not None:
            self.last_scan_number = int(sn)
        return scan

    def price_history(self, asset: str) -> list[float]:
        return list(self._prices.get(str(asset).upper(), ()))

    def market_data(self, lookback: int) -> dict[str, list[float]]:
        need = max(1, int(lookback) + 1)
        return {asset: list(hist) for asset, hist in self._prices.items() if len(hist) >= need}

    def dc_status(self, scan_payload: Any, *, already_ingested: bool = False) -> str:
        scan = _scan_view(scan_payload) if already_ingested else self.ingest_scan(scan_payload)
        asset = _scan_assets(scan)
        if not asset:
            return "NONE"
        vel_div = _coerce_float(scan.get("vel_div"), None)
        if vel_div is None:
            return "NONE"
        history = self.price_history(asset[0])
        if not history:
            return "NONE"
        # BLUE constructs its signal_gen from ENGINE_KWARGS (the orchestrator threads them,
        # esf_alpha_orchestrator.py:180-191). Pin the SAME params — not bare defaults — so a
        # champion retune that changes dc_lookback_bars / dc_min_magnitude_bps / thresholds
        # diverges loudly (caught by test_signal_gen_params_match_blue_engine_kwargs), never
        # silently. dc_status is deterministic in the params + price history (counters unused).
        signal_gen = AlphaSignalGenerator(**BLUE_SIGNAL_GEN_KWARGS)
        sig = signal_gen.generate(
            vel_div=vel_div,
            vel_div_history=None,
            asset_price_history=history,
            trade_direction=self.trade_direction,
            asset=asset[0],
            current_timestamp=_coerce_float(scan.get("timestamp"), 0.0) or 0.0,
        )
        return sig.dc_status


def _source_ob_market(
    ob_assets: list[str],
    *,
    bar_idx: int,
    ob_engine: Optional[OBFeatureEngine] = None,
) -> tuple[Optional[float], Optional[float]]:
    """Derive (median_imbalance, agreement_pct) EXACTLY as BLUE does, via BLUE's HZOBProvider.

    BLUE wires OB once in _wire_obf (nautilus_event_trader.py:4967-4980):
        live_ob = HZOBProvider(hz_cluster=HZ_CLUSTER, hz_host=HZ_HOST, assets=assets)
        ob_eng = OBFeatureEngine(live_ob); eng.set_ob_engine(ob_eng)
    then per scan calls ``ob_eng.step_live(assets, bar_idx)`` and the orchestrator reads
    ``ob_eng.get_market(bar_idx, assets)`` (esf_alpha_orchestrator.py:590). We use BLUE's
    OWN HZOBProvider — NOT a reinvented reader — so OB parsing/shard semantics are BLUE's.

    OBFeatureEngine ACCUMULATES per-asset history across scans (lookback=10), so a faithful
    mirror requires a PERSISTENT ``ob_engine`` + per-scan-incrementing ``bar_idx`` (the
    caller/shadow loop owns it, exactly as BLUE keeps one ob_eng). When no engine is passed
    this builds a single-shot HZOBProvider-backed engine — correct wiring but no cross-scan
    history; use only for one-off reads / the live smoke."""
    if not ob_assets:
        return None, None
    if ob_engine is None:
        provider = HZOBProvider(hz_cluster=HZ_CLUSTER, hz_host=HZ_HOST, assets=list(ob_assets))
        ob_engine = OBFeatureEngine(provider)
    try:
        ob_engine.step_live(list(ob_assets), bar_idx)
        market = ob_engine.get_market(bar_idx, list(ob_assets))
        return float(market.median_imbalance), float(market.agreement_pct)
    except Exception:
        return None, None


@dataclass(frozen=True)
class LiveBlueSourceResult:
    factors: SizingFactors
    acb_boost: float
    acb_beta: float
    mc_scale: float
    posture: str
    dc_status: str
    selected_asset: str


def source_live_blue_sizing_factors(
    client: hazelcast.HazelcastClient,
    *,
    assets: Optional[Iterable[str]] = None,
    scan_history: Optional[LiveBlueScanHistory] = None,
    selector: Optional[VioletAssetSelector] = None,
    acb: Optional[AdaptiveCircuitBreaker] = None,
    ob_engine: Optional[OBFeatureEngine] = None,
    bar_idx: int = 0,
    date_str: Optional[str] = None,
    prior_boost_beta: Optional[tuple[float, float]] = None,
) -> LiveBlueSourceResult:
    """Read BLUE's live surfaces and RECONSTRUCT the factor plane the way BLUE computes it.

    boost/beta are recomputed via ``AdaptiveCircuitBreaker.get_dynamic_boost_from_hz`` (NOT
    the published acb_boost scalar); OB via BLUE's ``HZOBProvider`` + ``OBFeatureEngine``;
    dc_status via ``AlphaSignalGenerator`` pinned to BLUE's ENGINE_KWARGS. For full OB
    accumulation faithfulness the caller passes a PERSISTENT ``ob_engine`` + per-scan
    ``bar_idx`` (BLUE keeps one ob_eng). ``date_str`` defaults to today's UTC date (BLUE
    uses self.current_day)."""
    scan_history = scan_history or LiveBlueScanHistory()
    selector = selector or VioletAssetSelector()
    today = date_str or datetime.now(timezone.utc).strftime("%Y-%m-%d")

    engine_snapshot_raw = _read_hz_map(client, "DOLPHIN_STATE_BLUE", "latest_nautilus")
    if engine_snapshot_raw is None:
        engine_snapshot_raw = _read_hz_map(client, "DOLPHIN_STATE_BLUE", "engine_snapshot")
    engine_snapshot = _jsonish(engine_snapshot_raw)
    if isinstance(engine_snapshot, str):
        try:
            engine_snapshot = json.loads(engine_snapshot)
        except Exception:
            engine_snapshot = {}

    posture = posture_from_engine_snapshot(engine_snapshot if isinstance(engine_snapshot, Mapping) else None)

    esof_raw = _read_hz_map(client, "DOLPHIN_FEATURES", "esof_latest")
    if esof_raw is None:
        esof_raw = _read_hz_map(client, "DOLPHIN_FEATURES", "esof_advisor_latest")
    esof_score = esof_score_from_features(esof_raw)

    mc_raw = _read_hz_map(client, "DOLPHIN_FEATURES", "mc_forewarner_latest")
    mc_scale = _derive_mc_scale(mc_raw)

    scan_raw = _read_hz_map(client, "DOLPHIN_FEATURES", "latest_eigen_scan")
    scan = scan_history.ingest_scan(scan_raw)
    scan_assets = _scan_assets(scan)
    trade_direction = int(
        _coerce_float(
            engine_snapshot.get("trade_direction_runtime") if isinstance(engine_snapshot, Mapping) else None,
            None,
        )
        or _coerce_float(
            engine_snapshot.get("trade_direction_base") if isinstance(engine_snapshot, Mapping) else None,
            None,
        )
        or -1
    )

    # boost/beta — recompute via the ACB exactly as BLUE's trader does (NOT the published
    # acb_boost). Needs trade_direction, so computed after it.
    acb_boost, acb_beta = _source_boost_beta(
        client, date_str=today, trade_direction=trade_direction, acb=acb,
        prior=prior_boost_beta,
    )

    candidate_market = scan_history.market_data(selector.lookback)
    pick = selector.pick(candidate_market, regime_direction=trade_direction)
    selected_asset = pick.asset if pick is not None else (scan_assets[0] if scan_assets else "")
    dc_status = scan_history.dc_status(scan, already_ingested=True)

    # OB market consensus — BLUE's HZOBProvider + OBFeatureEngine (persistent engine if given).
    ob_assets = list(assets) if assets is not None else scan_assets
    ob_median_imbalance, ob_agreement_pct = _source_ob_market(
        ob_assets, bar_idx=bar_idx, ob_engine=ob_engine,
    )

    hz_snapshot = {
        "boost": acb_boost,
        "beta": acb_beta,
        "mc_scale": mc_scale,
        "esof_score": esof_score,
        "ob_median_imbalance": ob_median_imbalance,
        "ob_agreement_pct": ob_agreement_pct,
        "dc_status": dc_status,
        "posture": posture,
    }
    factors = extract_live_sizing_factors(hz_snapshot=hz_snapshot)
    LOGGER.debug(
        "live BLUE plane: asset=%s posture=%s boost=%.4f beta=%.4f mc_scale=%.2f "
        "esof=%s ob=(%s,%s) dc=%s dir=%s",
        selected_asset, posture, acb_boost, acb_beta, mc_scale, esof_score,
        ob_median_imbalance, ob_agreement_pct, dc_status, trade_direction,
    )
    return LiveBlueSourceResult(
        factors=factors,
        acb_boost=acb_boost,
        acb_beta=acb_beta,
        mc_scale=mc_scale,
        posture=posture,
        dc_status=dc_status,
        selected_asset=selected_asset,
    )