DOLPHIN/nautilus_dolphin/dvae/exp7_live_coupling.py

"""
Exp 7 — Live proxy_B Coupling: Scale, Hold-Limit, Rising-Exit

Context:
  proxy_B = instability_50 - v750_lambda_max_velocity
  AUC=0.715 for eigenspace stress, r=+0.42 (p=0.003) with intraday MAE.
  Orthogonal to vel_div (r=-0.03, ns).

  Exp6 confirmed: stop-tightening is DEAD (re-entry cascade, worse DD always).
  Exp4 retroactive modes A/B/C/D were 98% invalid (entry_bar alignment bug —
  entry_bar = global_bar_idx but day_proxy keyed by local ri).
  This experiment is the FIRST valid live test of those four coupling modes.

Modes tested:
  B. scale_boost:    size UP when proxy_B is LOW at entry (calm = better quality)
  A. scale_suppress: size DOWN when proxy_B is HIGH at entry (stress = worse quality)
  C. hold_limit:     exit early when proxy_B_max during hold exceeds threshold
  D. rising_exit:    exit early when current proxy_B rises above threshold during hold

Implementation notes:
  - Modes B and A: post-entry notional scaling. No timing change → no re-entry dynamics.
    ProxyScaleEngine._try_entry calls super() then scales self.position.notional.
  - Modes C and D: in process_day loop, per-bar condition check BEFORE step_bar.
    If triggered: _execute_exit(force reason) → position cleared → step_bar enters fresh.
    No re-entry on same bar (process_bar checks bar_idx > _last_exit_bar).
  - proxy_B history: rolling 500-bar window, minimum 20 samples before any gating activates.
  - threshold at entry: np.percentile(history, thr_pct * 100) computed from rolling window.

Configs:
  Baseline:          1
  scale_boost (B):   4  (thr in {0.25, 0.35} x alpha in {0.5, 1.0})
  scale_suppress (A): 4  (thr in {0.75, 0.85} x alpha in {0.5, 1.0}, s_min=0.25)
  hold_limit (C):    3  (frac=0.5, thr_pct in {0.65, 0.75, 0.85})
  rising_exit (D):   3  (frac=0.5, thr_pct in {0.65, 0.75, 0.85})
  Total: 15 configs x ~240s = ~60 min

Focus metric: DD < 15.05% AND ROI >= 84.1% (95% of gold=88.55%)
Results logged to exp7_live_coupling_results.json.
"""
import sys, time, json, math
sys.stdout.reconfigure(encoding='utf-8', errors='replace')
from pathlib import Path
import numpy as np

_HERE = Path(__file__).resolve().parent
sys.path.insert(0, str(_HERE.parent))

from exp_shared import (
    ensure_jit, ENGINE_KWARGS, GOLD, MC_BASE_CFG,
    load_data, load_forewarner, log_results, print_table,
)
from nautilus_dolphin.nautilus.esf_alpha_orchestrator import NDAlphaEngine
from nautilus_dolphin.nautilus.adaptive_circuit_breaker import AdaptiveCircuitBreaker


# ── Base: proxy_B per-bar tracking ───────────────────────────────────────────

class ProxyBaseEngine(NDAlphaEngine):
    """Tracks proxy_B = instability_50 - v750_lambda_max_velocity per bar."""

    def __init__(self, *args, **kwargs):
        super().__init__(*args, **kwargs)
        self._current_proxy_b: float = 0.0
        self._proxy_b_history: list = []   # rolling 500-bar window

    def _update_proxy(self, inst: float, v750: float) -> float:
        pb = inst - v750
        self._current_proxy_b = pb
        self._proxy_b_history.append(pb)
        if len(self._proxy_b_history) > 500:
            self._proxy_b_history = self._proxy_b_history[-500:]
        return pb

    def _proxy_prank(self) -> float:
        """Percentile rank of current proxy_B in history (0=low, 1=high)."""
        if not self._proxy_b_history:
            return 0.5
        n = len(self._proxy_b_history)
        return sum(v < self._current_proxy_b for v in self._proxy_b_history) / n

    def _proxy_threshold(self, thr_pct: float) -> float:
        """Absolute threshold: percentile thr_pct of rolling history."""
        if len(self._proxy_b_history) < 20:
            return float('inf')
        return float(np.percentile(self._proxy_b_history, thr_pct * 100.0))

    def process_day(self, date_str, df, asset_columns,
                    vol_regime_ok=None, direction=None, posture='APEX'):
        self.begin_day(date_str, posture=posture, direction=direction)
        bid = 0
        for ri in range(len(df)):
            row = df.iloc[ri]
            vd = row.get('vel_div')
            if vd is None or not np.isfinite(float(vd)):
                self._global_bar_idx += 1; bid += 1; continue

            def gf(col):
                v = row.get(col)
                if v is None: return 0.0
                try:   return float(v) if np.isfinite(float(v)) else 0.0
                except: return 0.0

            v50  = gf('v50_lambda_max_velocity')
            v750 = gf('v750_lambda_max_velocity')
            inst = gf('instability_50')
            self._update_proxy(inst, v750)

            prices = {}
            for ac in asset_columns:
                p = row.get(ac)
                if p is not None and p > 0 and np.isfinite(float(p)):
                    prices[ac] = float(p)

            if not prices:
                self._global_bar_idx += 1; bid += 1; continue

            vrok = bool(vol_regime_ok[ri]) if vol_regime_ok is not None else (bid >= 100)
            self._per_bar_hook(ri, float(vd), prices, v50, v750, vrok)
            self.step_bar(bar_idx=ri, vel_div=float(vd), prices=prices,
                          vol_regime_ok=vrok, v50_vel=v50, v750_vel=v750)
            bid += 1

        return self.end_day()

    def _per_bar_hook(self, ri, vd, prices, v50, v750, vrok):
        """Override in subclasses to inject pre-step_bar logic."""
        pass


# ── Mode B/A: position scaling ────────────────────────────────────────────────

class ProxyScaleEngine(ProxyBaseEngine):
    """
    Mode B (scale_boost):    size UP when proxy_B is LOW (calm entry conditions).
    Mode A (scale_suppress): size DOWN when proxy_B is HIGH (stress entry conditions).

    Scale factor computed from percentile rank of current proxy_B in rolling history.
      boost:    scale = 1.0 + alpha * max(0, threshold - prank)   [rk < thr → boost]
      suppress: scale = max(s_min, 1.0 - alpha * max(0, prank - threshold))  [rk > thr → reduce]

    Applied post-entry by multiplying self.position.notional directly.
    Does NOT change timing → no re-entry cascade dynamics.
    """
    def __init__(self, *args, mode: str = 'boost', threshold: float = 0.35,
                 alpha: float = 1.0, s_min: float = 0.25, **kwargs):
        super().__init__(*args, **kwargs)
        assert mode in ('boost', 'suppress')
        self.mode = mode
        self.threshold = threshold
        self.alpha = alpha
        self.s_min = s_min
        self._scale_history: list = []

    @property
    def sizing_scale_mean(self) -> float:
        return float(np.mean(self._scale_history)) if self._scale_history else 1.0

    def _try_entry(self, bar_idx, vel_div, prices, price_histories,
                   v50_vel=0.0, v750_vel=0.0):
        result = super()._try_entry(bar_idx, vel_div, prices, price_histories,
                                     v50_vel, v750_vel)
        if result and self.position:
            prank = self._proxy_prank()
            if self.mode == 'boost':
                scale = 1.0 + self.alpha * max(0.0, self.threshold - prank)
            else:  # suppress
                scale = max(self.s_min, 1.0 - self.alpha * max(0.0, prank - self.threshold))
            self.position.notional *= scale
            self._scale_history.append(scale)
        return result

    def reset(self):
        super().reset()
        self._scale_history = []


# ── Mode C: hold_limit ────────────────────────────────────────────────────────

class ProxyHoldLimitEngine(ProxyBaseEngine):
    """
    Exit early when:
      - proxy_B during hold has exceeded thr_pct-percentile threshold
      - AND bars_held >= frac * max_hold_bars

    Threshold computed at entry time from rolling proxy_B history.
    Tracks max proxy_B seen since entry; fires as soon as condition is met.
    """
    def __init__(self, *args, frac: float = 0.5, thr_pct: float = 0.75, **kwargs):
        super().__init__(*args, **kwargs)
        self.frac = frac
        self.thr_pct = thr_pct
        self._pb_hold_threshold: float = float('inf')  # set at entry
        self._pb_max_so_far: float = 0.0               # tracked during hold
        self.early_exits: int = 0

    def _try_entry(self, bar_idx, vel_div, prices, price_histories,
                   v50_vel=0.0, v750_vel=0.0):
        result = super()._try_entry(bar_idx, vel_div, prices, price_histories,
                                     v50_vel, v750_vel)
        if result:
            self._pb_hold_threshold = self._proxy_threshold(self.thr_pct)
            self._pb_max_so_far = self._current_proxy_b
        return result

    def _per_bar_hook(self, ri, vd, prices, v50, v750, vrok):
        if self.position is None:
            return

        # Track proxy_B max during hold
        self._pb_max_so_far = max(self._pb_max_so_far, self._current_proxy_b)

        bars_held = self._global_bar_idx - self.position.entry_bar
        min_bars = int(self.frac * self.exit_manager.max_hold_bars)

        if (bars_held >= min_bars
                and self._pb_max_so_far > self._pb_hold_threshold
                and self.position.asset in prices):
            self.position.current_price = prices[self.position.asset]
            self._execute_exit("PROXY_HOLD_LIMIT", self._global_bar_idx,
                               bars_held=bars_held)
            self._pb_max_so_far = 0.0
            self._pb_hold_threshold = float('inf')
            self.early_exits += 1

    def reset(self):
        super().reset()
        self._pb_hold_threshold = float('inf')
        self._pb_max_so_far = 0.0
        self.early_exits = 0


# ── Mode D: rising_exit ───────────────────────────────────────────────────────

class ProxyRisingExitEngine(ProxyBaseEngine):
    """
    Exit early when:
      - current proxy_B exceeds thr_pct-percentile of rolling history
      - AND bars_held >= frac * max_hold_bars

    Interpretation: proxy_B has risen to an elevated level during hold →
    eigenspace stress is actively high → exit before MAE deepens further.
    Threshold computed at entry time from rolling proxy_B history.
    """
    def __init__(self, *args, frac: float = 0.5, thr_pct: float = 0.80, **kwargs):
        super().__init__(*args, **kwargs)
        self.frac = frac
        self.thr_pct = thr_pct
        self._pb_entry_threshold: float = float('inf')  # set at entry
        self.early_exits: int = 0

    def _try_entry(self, bar_idx, vel_div, prices, price_histories,
                   v50_vel=0.0, v750_vel=0.0):
        result = super()._try_entry(bar_idx, vel_div, prices, price_histories,
                                     v50_vel, v750_vel)
        if result:
            self._pb_entry_threshold = self._proxy_threshold(self.thr_pct)
        return result

    def _per_bar_hook(self, ri, vd, prices, v50, v750, vrok):
        if self.position is None:
            return

        bars_held = self._global_bar_idx - self.position.entry_bar
        min_bars = int(self.frac * self.exit_manager.max_hold_bars)

        if (bars_held >= min_bars
                and self._current_proxy_b > self._pb_entry_threshold
                and self.position.asset in prices):
            self.position.current_price = prices[self.position.asset]
            self._execute_exit("PROXY_RISING_EXIT", self._global_bar_idx,
                               bars_held=bars_held)
            self._pb_entry_threshold = float('inf')
            self.early_exits += 1

    def reset(self):
        super().reset()
        self._pb_entry_threshold = float('inf')
        self.early_exits = 0


# ── Run harness ───────────────────────────────────────────────────────────────

def _run(engine_factory, name, d, fw):
    """Full 55-day backtest. Returns gold-comparable metrics + coupling-specific stats."""
    kw = ENGINE_KWARGS.copy()
    acb = AdaptiveCircuitBreaker()
    acb.preload_w750(d['date_strings'])

    eng = engine_factory(kw)
    eng.set_ob_engine(d['ob_eng'])
    eng.set_acb(acb)
    if fw is not None:
        eng.set_mc_forewarner(fw, MC_BASE_CFG)
    eng.set_esoteric_hazard_multiplier(0.0)

    daily_caps, daily_pnls = [], []
    for pf in d['parquet_files']:
        ds = pf.stem
        df, acols, dvol = d['pq_data'][ds]
        cap_before = eng.capital
        vol_ok = np.where(np.isfinite(dvol), dvol > d['vol_p60'], False)
        eng.process_day(ds, df, acols, vol_regime_ok=vol_ok)
        daily_caps.append(eng.capital)
        daily_pnls.append(eng.capital - cap_before)

    tr  = eng.trade_history
    n   = len(tr)
    roi = (eng.capital - 25000.0) / 25000.0 * 100.0

    if n == 0:
        return dict(name=name, roi=roi, pf=0.0, dd=0.0, wr=0.0, sharpe=0.0,
                    trades=0, early_exits=0, sizing_scale_mean=1.0, exit_reasons={})

    def _abs(t): return t.pnl_absolute if hasattr(t, 'pnl_absolute') else t.pnl_pct * 250.0
    wins   = [t for t in tr if _abs(t) > 0]
    losses = [t for t in tr if _abs(t) <= 0]
    wr     = len(wins) / n * 100.0
    pf     = sum(_abs(t) for t in wins) / max(abs(sum(_abs(t) for t in losses)), 1e-9)

    peak_cap, max_dd = 25000.0, 0.0
    for cap in daily_caps:
        peak_cap = max(peak_cap, cap)
        max_dd   = max(max_dd, (peak_cap - cap) / peak_cap * 100.0)

    dr     = np.array([p / 25000.0 * 100.0 for p in daily_pnls])
    sharpe = float(dr.mean() / (dr.std() + 1e-9) * math.sqrt(365)) if len(dr) > 1 else 0.0

    exit_reasons = {}
    for t in tr:
        r = t.exit_reason if hasattr(t, 'exit_reason') else 'UNKNOWN'
        exit_reasons[r] = exit_reasons.get(r, 0) + 1

    early_exits       = getattr(eng, 'early_exits', 0)
    sizing_scale_mean = getattr(eng, 'sizing_scale_mean', 1.0)

    return dict(
        name=name, roi=roi, pf=pf, dd=max_dd, wr=wr, sharpe=sharpe, trades=n,
        early_exits=early_exits,
        early_exit_rate=early_exits / n if n > 0 else 0.0,
        sizing_scale_mean=sizing_scale_mean,
        exit_reasons=exit_reasons,
    )


# ── Main ──────────────────────────────────────────────────────────────────────

def main():
    t_start = time.time()
    print("=" * 72)
    print("Exp 7 — Live proxy_B Coupling: Scale + Hold-Limit + Rising-Exit")
    print("Note: First valid live test — exp4 retroactive was 98% median-filled.")
    print("=" * 72)

    ensure_jit()
    d  = load_data()
    fw = load_forewarner()

    configs = []

    # Baseline
    configs.append(("A00_baseline", lambda kw: NDAlphaEngine(**kw)))

    # Mode B: scale_boost — size UP when proxy_B is LOW
    for thr in [0.25, 0.35]:
        for alpha in [0.5, 1.0]:
            tag = f"B_boost_thr={thr:.2f}_a={alpha:.1f}"
            def _make_boost(t, a):
                return lambda kw: ProxyScaleEngine(mode='boost', threshold=t, alpha=a, **kw)
            configs.append((tag, _make_boost(thr, alpha)))

    # Mode A: scale_suppress — size DOWN when proxy_B is HIGH
    for thr in [0.75, 0.85]:
        for alpha in [0.5, 1.0]:
            tag = f"A_suppress_thr={thr:.2f}_a={alpha:.1f}_smin=0.25"
            def _make_suppress(t, a):
                return lambda kw: ProxyScaleEngine(mode='suppress', threshold=t, alpha=a, s_min=0.25, **kw)
            configs.append((tag, _make_suppress(thr, alpha)))

    # Mode C: hold_limit — exit early when pb_max during hold exceeds threshold
    for thr_pct in [0.65, 0.75, 0.85]:
        tag = f"C_hold_limit_frac=0.5_thr={thr_pct:.2f}"
        def _make_hold(tp):
            return lambda kw: ProxyHoldLimitEngine(frac=0.5, thr_pct=tp, **kw)
        configs.append((tag, _make_hold(thr_pct)))

    # Mode D: rising_exit — exit early when current pb exceeds threshold during hold
    for thr_pct in [0.65, 0.75, 0.85]:
        tag = f"D_rising_exit_frac=0.5_thr={thr_pct:.2f}"
        def _make_rising(tp):
            return lambda kw: ProxyRisingExitEngine(frac=0.5, thr_pct=tp, **kw)
        configs.append((tag, _make_rising(thr_pct)))

    results = []
    for i, (name, factory) in enumerate(configs):
        t0 = time.time()
        print(f"\n[{i+1}/{len(configs)}] {name} ...")
        res = _run(factory, name, d, fw)
        elapsed = time.time() - t0
        extra = ""
        if res['early_exits'] > 0:
            extra = f" early_exits={res['early_exits']}"
        if abs(res['sizing_scale_mean'] - 1.0) > 0.001:
            extra += f" scale_mean={res['sizing_scale_mean']:.4f}"
        print(f"  ROI={res['roi']:.2f}% PF={res['pf']:.4f} DD={res['dd']:.2f}% "
              f"WR={res['wr']:.2f}% Sharpe={res['sharpe']:.3f} Trades={res['trades']}"
              f"{extra} ({elapsed:.0f}s)")
        results.append(res)

    # Verification
    baseline = results[0]
    gold_match = (
        abs(baseline['roi']    - GOLD['roi'])    < 0.5 and
        abs(baseline['pf']     - GOLD['pf'])     < 0.005 and
        abs(baseline['dd']     - GOLD['dd'])     < 0.5 and
        abs(baseline['trades'] - GOLD['trades']) < 10
    )
    print(f"\n{'='*72}")
    print(f"VERIFICATION: baseline ROI={baseline['roi']:.2f}% DD={baseline['dd']:.2f}%"
          f" Trades={baseline['trades']}  Match={'PASS ✓' if gold_match else 'FAIL ✗'}")

    base_roi = baseline['roi']
    base_dd  = baseline['dd']
    target_roi = GOLD['roi'] * 0.95   # 84.12%
    target_dd  = GOLD['dd']           # 15.05%

    print(f"\n{'='*72}")
    print(f"RESULTS TABLE (target: DD<{target_dd:.2f}% AND ROI>={target_roi:.1f}%)")
    hdr = f"{'Config':<46} {'ROI%':>7} {'PF':>6} {'DD%':>6} {'ΔDD':>6} {'ΔROI':>6} {'extra':>20} {'OK':>4}"
    print(hdr)
    print('-' * 100)
    for r in results:
        delta_roi = r['roi'] - base_roi
        delta_dd  = r['dd']  - base_dd
        ok = 'Y' if (r['dd'] < target_dd and r['roi'] >= target_roi) else 'N'
        extra = ''
        if r['early_exits'] > 0:
            extra = f"early={r['early_exits']}"
        elif abs(r['sizing_scale_mean'] - 1.0) > 0.001:
            extra = f"scale={r['sizing_scale_mean']:.4f}"
        print(f"{r['name']:<46} {r['roi']:>7.2f} {r['pf']:>6.4f} {r['dd']:>6.2f} "
              f"{delta_dd:>+6.2f} {delta_roi:>+6.2f} {extra:>20} {ok:>4}")

    # Summary by mode
    print(f"\n{'='*72}")
    print("BEST PER MODE:")
    def best_dd(group):
        return min(group, key=lambda r: r['dd'])
    groups = {'baseline': [], 'B_boost': [], 'A_suppress': [],
              'C_hold_limit': [], 'D_rising_exit': []}
    for r in results:
        for k in groups:
            if r['name'].startswith(k.split('_')[0]) or r['name'].startswith(k):
                groups[k].append(r); break
    for mode, rs in groups.items():
        if not rs: continue
        b = best_dd(rs)
        delta_roi = b['roi'] - base_roi
        delta_dd  = b['dd']  - base_dd
        ok = 'Y' if (b['dd'] < target_dd and b['roi'] >= target_roi) else 'N'
        print(f"  {mode:<16} best: {b['name']:<46} "
              f"DD={b['dd']:.2f}% ({delta_dd:+.2f}) ROI={b['roi']:.2f}% ({delta_roi:+.2f}) [{ok}]")

    # Exit breakdown
    print(f"\n{'='*72}")
    print("EXIT REASON BREAKDOWN:")
    for r in results:
        reasons = r.get('exit_reasons', {})
        parts = ', '.join(f"{k}={v}" for k, v in sorted(reasons.items()))
        print(f"  {r['name']:<46}: {parts}")

    # Save
    outfile = _HERE / "exp7_live_coupling_results.json"
    log_results(results, outfile, gold=GOLD, meta={
        "exp": "exp7",
        "note": "First valid live test of modes A/B/C/D — exp4 retroactive was invalid (entry_bar bug)",
        "total_elapsed_s": round(time.time() - t_start, 1),
        "gold_match": gold_match,
        "modes_tested": ["B_scale_boost", "A_scale_suppress", "C_hold_limit", "D_rising_exit"],
    })

    total = time.time() - t_start
    print(f"\nTotal elapsed: {total/60:.1f} min")
    print("Done.")


if __name__ == "__main__":
    main()