DOLPHIN/prod/diag_acb_inject.py

"""
ACB-inject diagnostic.
Pre-loads ACB factors via os.scandir (fast on remote mount, avoids materializing 15k files).
Injects _day_base_boost/_day_beta directly after begin_day().
Actor-style loop, static vol_ok, gidx fix → T=2155 base.
Measures ROI impact of ACB boost from ng6_data eigenvalues.
"""
import sys, math, os, pathlib
import numpy as np
import pandas as pd

sys.path.insert(0, '/mnt/dolphinng5_predict')
sys.path.insert(0, '/mnt/dolphinng5_predict/nautilus_dolphin')

print("Importing...", flush=True)
from nautilus_dolphin.nautilus.proxy_boost_engine import create_boost_engine
print("Import done.", flush=True)

PARQUET_DIR = pathlib.Path('/mnt/dolphinng5_predict/vbt_cache')
# Primary: dolphin_training (full Jan coverage); fallback: ng6_data (Jan21, Jan30-31)
EIGENVALUES_PATHS = [
    pathlib.Path('/mnt/ng6_data/eigenvalues'),          # extf backfill output (backfill_klines_exf.py)
    pathlib.Path('/mnt/dolphin_training/data/eigenvalues'),  # dolphin_training archive (Dec31-Jan12)
    pathlib.Path('/mnt/dolphinng6_data/eigenvalues'),   # ng3 live share (Jan2-8, Jan21, Jan30-31, Mar+)
]
VOL_P60_INWINDOW = 0.00009868

# ACB config constants (from adaptive_circuit_breaker.py)
BETA_HIGH = 0.8
BETA_LOW = 0.2
W750_THRESHOLD_PCT = 60
FUNDING_VERY_BEARISH = -0.0001
FUNDING_BEARISH = 0.0
DVOL_EXTREME = 80
DVOL_ELEVATED = 55
FNG_EXTREME_FEAR = 25
FNG_FEAR = 40
TAKER_SELLING = 0.8
TAKER_MILD_SELLING = 0.9

ENG_KWARGS = dict(
    max_hold_bars=120, min_irp_alignment=0.45, max_leverage=8.0,
    vel_div_threshold=-0.02, vel_div_extreme=-0.05,
    min_leverage=0.5, leverage_convexity=3.0,
    fraction=0.20, fixed_tp_pct=0.0095, stop_pct=1.0,
    use_direction_confirm=True, dc_lookback_bars=7, dc_min_magnitude_bps=0.75,
    dc_skip_contradicts=True, dc_leverage_boost=1.0, dc_leverage_reduce=0.5,
    use_asset_selection=True, use_sp_fees=True, use_sp_slippage=True,
    sp_maker_entry_rate=0.62, sp_maker_exit_rate=0.50,
    use_ob_edge=True, ob_edge_bps=5.0, ob_confirm_rate=0.40,
    lookback=100, use_alpha_layers=True, use_dynamic_leverage=True, seed=42,
)

def make_engine(cap=25000.0):
    eng = create_boost_engine(mode='d_liq', initial_capital=cap, **ENG_KWARGS)
    eng.set_esoteric_hazard_multiplier(0.0)
    return eng

def fast_get_npz_files(date_path, n=10):
    """Get first n NPZ indicator files using os.scandir (avoids full glob sort)."""
    files = []
    try:
        with os.scandir(date_path) as it:
            for entry in it:
                if entry.name.endswith('__Indicators.npz') and entry.name.startswith('scan_'):
                    files.append(pathlib.Path(entry.path))
                    if len(files) >= n:
                        break
    except Exception:
        pass
    return files

def load_acb_factors(date_str):
    """Load funding/dvol/fng/taker and w750 for a date from any available source."""
    # Try each eigenvalues path in priority order
    date_path = None
    for ep in EIGENVALUES_PATHS:
        candidate = ep / date_str
        if candidate.exists():
            files = fast_get_npz_files(candidate, n=1)
            if files:
                date_path = candidate
                break

    if date_path is None:
        return None

    files = fast_get_npz_files(date_path, n=10)
    if not files:
        return None

    indicators = {'funding_btc': [], 'dvol_btc': [], 'fng': [], 'taker': [], 'lambda_vel_w750': []}

    for f in files:
        try:
            data = np.load(f, allow_pickle=True)
            # External factors from api_indicators
            if 'api_names' in data and 'api_indicators' in data:
                names = list(data['api_names'])
                vals = data['api_indicators']
                succ = data['api_success'] if 'api_success' in data else np.ones(len(names), dtype=bool)
                for key in ['funding_btc', 'dvol_btc', 'fng', 'taker']:
                    if key in names:
                        idx = names.index(key)
                        if succ[idx] and np.isfinite(vals[idx]):
                            indicators[key].append(float(vals[idx]))
            # w750 from scan_global
            if 'scan_global_names' in data and 'scan_global' in data:
                gnames = list(data['scan_global_names'])
                gvals = data['scan_global']
                if 'lambda_vel_w750' in gnames:
                    idx = gnames.index('lambda_vel_w750')
                    if idx < len(gvals) and np.isfinite(gvals[idx]):
                        indicators['lambda_vel_w750'].append(float(gvals[idx]))
        except Exception:
            continue

    result = {
        'funding_btc': float(np.median(indicators['funding_btc'])) if indicators['funding_btc'] else 0.0,
        'dvol_btc': float(np.median(indicators['dvol_btc'])) if indicators['dvol_btc'] else 50.0,
        'fng': float(np.median(indicators['fng'])) if indicators['fng'] else 50.0,
        'taker': float(np.median(indicators['taker'])) if indicators['taker'] else 1.0,
        'w750_vel': float(np.median(indicators['lambda_vel_w750'])) if indicators['lambda_vel_w750'] else 0.0,
        'available': True,
    }
    return result

def compute_signals(f):
    """Replicate ACB get_cut_for_date signal count from factors dict."""
    signals = 0
    if f['funding_btc'] <= FUNDING_VERY_BEARISH:
        signals += 2
    elif f['funding_btc'] <= FUNDING_BEARISH:
        signals += 1
    dvol = f['dvol_btc']
    if dvol >= DVOL_EXTREME:
        signals += 2
    elif dvol >= DVOL_ELEVATED:
        signals += 1
    fng = f['fng']
    if fng <= FNG_EXTREME_FEAR:
        signals += 2
    elif fng <= FNG_FEAR:
        signals += 1
    taker = f['taker']
    if taker <= TAKER_SELLING:
        signals += 2
    elif taker <= TAKER_MILD_SELLING:
        signals += 1
    return signals

def compute_boost(signals):
    if signals >= 1.0:
        return 1.0 + 0.5 * math.log1p(signals)
    return 1.0

def preload_acb_all(all_dates):
    """Load ACB factors for all dates. Returns {date_str: {boost, beta, ...}}."""
    print("Pre-loading ACB factors (fast scandir)...", flush=True)
    factors_by_date = {}
    w750_vals = []

    for ds in all_dates:
        f = load_acb_factors(ds)
        if f:
            factors_by_date[ds] = f
            if f['w750_vel'] != 0.0:
                w750_vals.append(f['w750_vel'])

    n_loaded = len(factors_by_date)
    w750_thresh = float(np.percentile(w750_vals, W750_THRESHOLD_PCT)) if w750_vals else 0.0
    print(f"  Loaded: {n_loaded}/{len(all_dates)} dates, w750_thresh={w750_thresh:.6f}", flush=True)

    # Compute boost + beta per date
    acb_by_date = {}
    for ds in all_dates:
        if ds in factors_by_date:
            f = factors_by_date[ds]
            signals = compute_signals(f)
            boost = compute_boost(signals)
            w750 = f['w750_vel']
            beta = BETA_HIGH if (w750_thresh == 0.0 or w750 >= w750_thresh) else BETA_LOW
            acb_by_date[ds] = {'boost': boost, 'beta': beta, 'signals': signals, 'w750': w750, 'available': True}
            if boost > 1.0 or beta != BETA_LOW:
                print(f"    {ds}: signals={signals} boost={boost:.4f} beta={beta} w750={w750:.6f}", flush=True)
        else:
            # No data: boost=1.0, beta=midpoint (0.5) — unknown regime
            acb_by_date[ds] = {'boost': 1.0, 'beta': 0.5, 'signals': 0, 'w750': 0.0, 'available': False}

    return acb_by_date

def compute_vol_ok(df):
    btc_f = df['BTCUSDT'].values.astype('float64')
    n = len(btc_f)
    vol_ok = np.zeros(n, dtype=bool)
    for j in range(50, n):
        seg = btc_f[max(0, j-50):j]
        diffs = np.diff(seg)
        denom = seg[:-1]
        if np.any(denom == 0):
            continue
        v = float(np.std(diffs / denom))
        if math.isfinite(v) and v > 0:
            vol_ok[j] = v > VOL_P60_INWINDOW
    return vol_ok

def run_day(df, date_str, eng, acb_info=None, nan_fix=True):
    eng.begin_day(date_str)
    # Inject ACB boost/beta directly after begin_day
    if acb_info:
        eng._day_base_boost = acb_info['boost']
        eng._day_beta = acb_info['beta']

    data_arr = df.values
    cols = df.columns.tolist()
    vd_idx = cols.index('vel_div') if 'vel_div' in cols else -1
    v50_idx = cols.index('v50_lambda_max_velocity') if 'v50_lambda_max_velocity' in cols else -1
    v750_idx = cols.index('v750_lambda_max_velocity') if 'v750_lambda_max_velocity' in cols else -1
    i50_idx = cols.index('instability_50') if 'instability_50' in cols else -1
    usdt_idxs = [(c, cols.index(c)) for c in cols if c.endswith('USDT')]
    vol_ok = compute_vol_ok(df)
    trades = 0

    for i in range(len(df)):
        row_vals = data_arr[i]
        vd_raw = float(row_vals[vd_idx]) if vd_idx != -1 else float('nan')
        if not math.isfinite(vd_raw):
            if nan_fix:
                eng._global_bar_idx += 1
            continue

        v750 = float(row_vals[v750_idx]) if v750_idx != -1 and math.isfinite(float(row_vals[v750_idx])) else 0.0
        inst50 = float(row_vals[i50_idx]) if i50_idx != -1 and math.isfinite(float(row_vals[i50_idx])) else 0.0
        v50 = float(row_vals[v50_idx]) if v50_idx != -1 and math.isfinite(float(row_vals[v50_idx])) else 0.0
        prices = {sym: float(row_vals[ci]) for sym, ci in usdt_idxs
                  if math.isfinite(float(row_vals[ci])) and float(row_vals[ci]) > 0}

        prev_pos = eng.position
        if hasattr(eng, 'pre_bar_proxy_update'):
            eng.pre_bar_proxy_update(inst50, v750)

        eng.step_bar(
            bar_idx=i, vel_div=vd_raw, prices=prices,
            v50_vel=v50, v750_vel=v750,
            vol_regime_ok=bool(vol_ok[i]),
        )
        if prev_pos is not None and eng.position is None:
            trades += 1

    eng.end_day()
    return trades

def main():
    files = sorted(PARQUET_DIR.glob('*.parquet'))
    all_dates = [pf.stem for pf in files]
    print(f"Days: {len(files)}", flush=True)

    acb_by_date = preload_acb_all(all_dates)

    base_eng = make_engine()
    acb_eng = make_engine()
    base_T = acb_T = 0

    have_eigen = set(d for d in all_dates if acb_by_date[d].get('available', False))

    for pf in files:
        date_str = pf.stem
        df = pd.read_parquet(pf)

        tb = run_day(df, date_str, base_eng, acb_info=None, nan_fix=True)
        ta = run_day(df, date_str, acb_eng, acb_info=acb_by_date[date_str], nan_fix=True)
        base_T += tb
        acb_T += ta

        flag = '*' if date_str in have_eigen else ' '
        acb_d = acb_by_date[date_str]
        print(f"{date_str}{flag}[b={acb_d['boost']:.3f} β={acb_d['beta']:.1f} s={acb_d['signals']}]: "
              f"BASE+{tb:3d}(cum={base_T:4d} ${base_eng.capital:8.0f})  "
              f"ACB+{ta:3d}(cum={acb_T:4d} ${acb_eng.capital:8.0f})", flush=True)

    ic = 25000.0
    print(f"\nBASELINE: T={base_T}, cap=${base_eng.capital:.2f}, ROI={100*(base_eng.capital/ic-1):.2f}%", flush=True)
    print(f"ACB:      T={acb_T}, cap=${acb_eng.capital:.2f}, ROI={100*(acb_eng.capital/ic-1):.2f}%", flush=True)
    print(f"\nGold target: T=2155, ROI=+189.48%", flush=True)

if __name__ == '__main__':
    main()