DOLPHIN/prod/diag_vol_gold.py

"""
Verify gold vol_ok methodology: compare static-threshold FIX vs gold-vol FIX.
Confirms ROI improvement and T=2155 maintained.
"""
import sys, math, pathlib
import numpy as np
import pandas as pd

sys.path.insert(0, '/mnt/dolphinng5_predict')
sys.path.insert(0, '/mnt/dolphinng5_predict/nautilus_dolphin')

print("Importing...", flush=True)
from nautilus_dolphin.nautilus.proxy_boost_engine import create_boost_engine
print("Import done.", flush=True)

PARQUET_DIR = pathlib.Path('/mnt/dolphinng5_predict/vbt_cache')
VOL_P60_INWINDOW = 0.00009868

ENG_KWARGS = dict(
    max_hold_bars=120, min_irp_alignment=0.45, max_leverage=8.0,
    vel_div_threshold=-0.02, vel_div_extreme=-0.05,
    min_leverage=0.5, leverage_convexity=3.0,
    fraction=0.20, fixed_tp_pct=0.0095, stop_pct=1.0,
    use_direction_confirm=True, dc_lookback_bars=7, dc_min_magnitude_bps=0.75,
    dc_skip_contradicts=True, dc_leverage_boost=1.0, dc_leverage_reduce=0.5,
    use_asset_selection=True, use_sp_fees=True, use_sp_slippage=True,
    sp_maker_entry_rate=0.62, sp_maker_exit_rate=0.50,
    use_ob_edge=True, ob_edge_bps=5.0, ob_confirm_rate=0.40,
    lookback=100, use_alpha_layers=True, use_dynamic_leverage=True, seed=42,
)

def make_engine(cap=25000.0):
    return create_boost_engine(mode='d_liq', initial_capital=cap, **ENG_KWARGS)

def compute_static_vol_ok(df):
    """Static threshold, 49-ret window, stored at j (old actor method)."""
    btc_f = df['BTCUSDT'].values.astype('float64')
    n = len(btc_f)
    vol_ok = np.zeros(n, dtype=bool)
    for j in range(50, n):
        seg = btc_f[max(0, j-50):j]
        diffs = np.diff(seg)
        denom = seg[:-1]
        if np.any(denom == 0):
            continue
        v = float(np.std(diffs / denom))
        if math.isfinite(v) and v > 0:
            vol_ok[j] = v > VOL_P60_INWINDOW
    return vol_ok

def compute_gold_vol_ok_all_days(parquet_files):
    """Gold vol_ok: 50-ret window, dvol[j+1], accumulating vp60. Returns {ts_ns: bool}."""
    all_vols = []
    result = {}

    for pf in parquet_files:
        df = pd.read_parquet(pf)
        ts_ns_arr = df['timestamp'].values.astype('int64') if 'timestamp' in df.columns else None
        if ts_ns_arr is None:
            continue

        bp = df['BTCUSDT'].values if 'BTCUSDT' in df.columns else None
        n = len(df)
        dvol = np.zeros(n, dtype=np.float64)

        if bp is not None and len(bp) > 1:
            rets = np.diff(bp.astype('float64')) / (bp[:-1].astype('float64') + 1e-9)
            for j in range(50, len(rets)):
                v = float(np.std(rets[j - 50:j]))
                dvol[j + 1] = v
                if v > 0:
                    all_vols.append(v)

        vp60 = float(np.percentile(all_vols, 60)) if len(all_vols) > 1000 else VOL_P60_INWINDOW

        for i in range(n):
            result[int(ts_ns_arr[i])] = bool(dvol[i] > 0 and dvol[i] > vp60)

    return result

def run_day(df, date_str, eng, vol_ok_arr, nan_fix=True):
    eng.begin_day(date_str)
    data_arr = df.values
    cols = df.columns.tolist()
    vd_idx = cols.index('vel_div') if 'vel_div' in cols else -1
    v50_idx = cols.index('v50_lambda_max_velocity') if 'v50_lambda_max_velocity' in cols else -1
    v750_idx = cols.index('v750_lambda_max_velocity') if 'v750_lambda_max_velocity' in cols else -1
    i50_idx = cols.index('instability_50') if 'instability_50' in cols else -1
    usdt_idxs = [(c, cols.index(c)) for c in cols if c.endswith('USDT')]
    trades = 0

    for i in range(len(df)):
        row_vals = data_arr[i]
        vd_raw = float(row_vals[vd_idx]) if vd_idx != -1 else float('nan')
        if not math.isfinite(vd_raw):
            if nan_fix:
                eng._global_bar_idx += 1
            continue

        v750 = float(row_vals[v750_idx]) if v750_idx != -1 and math.isfinite(float(row_vals[v750_idx])) else 0.0
        inst50 = float(row_vals[i50_idx]) if i50_idx != -1 and math.isfinite(float(row_vals[i50_idx])) else 0.0
        v50 = float(row_vals[v50_idx]) if v50_idx != -1 and math.isfinite(float(row_vals[v50_idx])) else 0.0
        prices = {sym: float(row_vals[ci]) for sym, ci in usdt_idxs
                  if math.isfinite(float(row_vals[ci])) and float(row_vals[ci]) > 0}

        prev_pos = eng.position
        if hasattr(eng, 'pre_bar_proxy_update'):
            eng.pre_bar_proxy_update(inst50, v750)

        eng.step_bar(
            bar_idx=i, vel_div=vd_raw, prices=prices,
            v50_vel=v50, v750_vel=v750,
            vol_regime_ok=bool(vol_ok_arr[i]),
        )
        if prev_pos is not None and eng.position is None:
            trades += 1

    eng.end_day()
    return trades

def main():
    files = sorted(PARQUET_DIR.glob('*.parquet'))
    print(f"Days: {len(files)}", flush=True)

    # Precompute gold vol_ok
    print("Precomputing gold vol_ok...", flush=True)
    gold_vol_ok = compute_gold_vol_ok_all_days(files)
    n_true = sum(1 for v in gold_vol_ok.values() if v)
    print(f"Gold vol_ok: {n_true:,}/{len(gold_vol_ok):,} True ({100*n_true/len(gold_vol_ok):.1f}%)", flush=True)

    static_eng = make_engine()
    gold_eng = make_engine()
    static_T = gold_T = 0

    for pf in files:
        date_str = pf.stem
        df = pd.read_parquet(pf)

        static_vol = compute_static_vol_ok(df)

        ts_ns_arr = df['timestamp'].values.astype('int64')
        gold_vol = np.array([gold_vol_ok.get(int(ts), False) for ts in ts_ns_arr], dtype=bool)

        ts = static_eng.capital
        tg = static_eng.capital
        ta = run_day(df, date_str, static_eng, static_vol, nan_fix=True)
        tb = run_day(df, date_str, gold_eng, gold_vol, nan_fix=True)
        static_T += ta; gold_T += tb

        print(f"{date_str}: STATIC+{ta:3d}(cum={static_T:4d} ${static_eng.capital:8.0f})  "
              f"GOLD+{tb:3d}(cum={gold_T:4d} ${gold_eng.capital:8.0f})", flush=True)

    ic = 25000.0
    print(f"\nSTATIC: T={static_T}, cap=${static_eng.capital:.2f}, ROI={100*(static_eng.capital/ic-1):.2f}%", flush=True)
    print(f"GOLD:   T={gold_T}, cap=${gold_eng.capital:.2f}, ROI={100*(gold_eng.capital/ic-1):.2f}%", flush=True)
    print(f"\nGold target: T=2155, ROI=+189.48%", flush=True)

if __name__ == '__main__':
    main()