"""MacroPostureSwitcher Backtest — 5s Gold Standard (56 days) ============================================================= Same posture logic, applied to 5s data (vbt_cache/). At 5s resolution, 1 bar = 5s: MAX_HOLD = 240 bars = 20 min (matches 1m MAX_HOLD=20 bars) ExF from NPZ (same eigenvalues path, same dates). prev-day rvol computed from previous day's 5s prices. Outputs: ROI / WR / PF / Max-DD / Sharpe """ import sys, time, csv, gc, json sys.path.insert(0, str(__import__('pathlib').Path(__file__).parent.parent)) sys.stdout.reconfigure(encoding='utf-8', errors='replace') from pathlib import Path from datetime import datetime from collections import defaultdict import numpy as np import pandas as pd from nautilus_dolphin.nautilus.macro_posture_switcher import MacroPostureSwitcher, Posture VBT_DIR_5S = Path(r"C:\Users\Lenovo\Documents\- DOLPHIN NG HD HCM TSF Predict\vbt_cache") EIGEN_PATH = Path(r"C:\Users\Lenovo\Documents\- Dolphin NG HD (NG3)\correlation_arb512\eigenvalues") LOG_DIR = Path(r"C:\Users\Lenovo\Documents\- DOLPHIN NG HD HCM TSF Predict\nautilus_dolphin\run_logs") ENTRY_T = 0.020 MAX_HOLD = 240 # bars (240 × 5s = 20 min) EXF_KEYS = ['dvol_btc', 'fng', 'funding_btc', 'taker'] def load_exf(date_str: str) -> dict: defaults = {'dvol_btc': 50.0, 'fng': 50.0, 'funding_btc': 0.0, 'taker': 1.0} dp = EIGEN_PATH / date_str if not dp.exists(): return defaults files = sorted(dp.glob('scan_*__Indicators.npz'))[:5] if not files: return defaults buckets = defaultdict(list) for f in files: try: d = np.load(f, allow_pickle=True) if 'api_names' not in d: continue names = list(d['api_names']) vals = d['api_indicators'] for k in EXF_KEYS: if k in names: v = float(vals[names.index(k)]) if np.isfinite(v): buckets[k].append(v) except Exception: pass out = dict(defaults) for k, vs in buckets.items(): if vs: out[k] = float(np.median(vs)) return out parquet_files = sorted(VBT_DIR_5S.glob("*.parquet")) parquet_files = [p for p in parquet_files if 'catalog' not in str(p)] total = len(parquet_files) print(f"Files: {total} (5s gold standard) | Entry T: ±{ENTRY_T} MaxHold: {MAX_HOLD}b={MAX_HOLD*5}s") # Pass 1: prev-day rvol / btc_ret print("Pass 1: lag-1 rvol...") t0 = time.time() day_rvol = {} day_btcret = {} for pf in parquet_files: ds = pf.stem try: df = pd.read_parquet(pf, columns=['BTCUSDT']) except Exception: continue btc = df['BTCUSDT'].values.astype(np.float64) btc = btc[np.isfinite(btc) & (btc > 0)] if len(btc) < 2: continue log_r = np.diff(np.log(btc)) day_rvol[ds] = float(np.std(log_r)) day_btcret[ds] = float((btc[-1] - btc[0]) / btc[0]) dates_sorted = sorted(day_rvol.keys()) prev_rvol = {d: day_rvol.get(dates_sorted[i-1]) if i > 0 else None for i, d in enumerate(dates_sorted)} prev_btcret = {d: day_btcret.get(dates_sorted[i-1]) if i > 0 else None for i, d in enumerate(dates_sorted)} print(f" Pass 1 done: {time.time()-t0:.1f}s") # Pass 2: trade simulation print("Pass 2: posture + crossover trades...") switcher = MacroPostureSwitcher( enable_long_posture=True, # 5s-calibrated rvol thresholds (1m values don't transfer — 5s std is ~7x smaller): # 5y klines 1m Q1 = 0.000455; 5s equivalent p25 = 0.000203, p75 = 0.000337 rvol_pause_thresh=0.000203, # 5s p25 — was 0.000455 (1m Q1 = p90 of 5s → too restrictive) rvol_strong_thresh=0.000337, # 5s p75 # dvol Q1 gate: validated — removes 19 noisy days, PF 1.036→1.116, Sharpe 2.7→5.8 dvol_none_below=47.5, ) equity = 1.0 equity_curve = [equity] day_log = [] trade_log = [] stats_yr = defaultdict(lambda: {'wins':0,'losses':0,'gw':0.0,'gl':0.0,'n':0,'paused':0}) stats_pos = defaultdict(lambda: {'wins':0,'losses':0,'gw':0.0,'gl':0.0,'n':0}) for i, pf in enumerate(parquet_files): ds = pf.stem year = ds[:4] exf = load_exf(ds) pr = prev_rvol.get(ds) pb = prev_btcret.get(ds) decision = switcher.decide( dvol_btc=exf['dvol_btc'], fng=exf['fng'], funding_btc=exf['funding_btc'], realized_vol=pr, btc_day_return=pb, ) if decision.posture == Posture.NONE: stats_yr[year]['paused'] += 1 day_log.append({'date': ds, 'year': year, 'posture': 'NONE', 'fear': round(decision.fear_score, 3), 'n_trades': 0, 'day_ret': 0.0, 'equity': round(equity, 6)}) continue try: df = pd.read_parquet(pf) except Exception: continue if 'vel_div' not in df.columns or 'BTCUSDT' not in df.columns: continue vd = df['vel_div'].values.astype(np.float64) btc = df['BTCUSDT'].values.astype(np.float64) vd = np.where(np.isfinite(vd), vd, 0.0) btc = np.where(np.isfinite(btc) & (btc > 0), btc, np.nan) n = len(btc) del df if n < MAX_HOLD + 5: del vd, btc continue pos = decision.posture smult = decision.size_mult if pos == Posture.SHORT: entry_mask = (vd >= ENTRY_T) & np.isfinite(btc) cross_back = (vd <= -ENTRY_T) sign = -1 else: entry_mask = (vd <= -ENTRY_T) & np.isfinite(btc) cross_back = (vd >= ENTRY_T) sign = +1 day_rets_sized = [] for t in range(n - MAX_HOLD): if not entry_mask[t]: continue ep = btc[t] if not np.isfinite(ep) or ep <= 0: continue exit_bar = MAX_HOLD for k in range(1, MAX_HOLD + 1): tb = t + k if tb >= n: exit_bar = k; break if cross_back[tb]: exit_bar = k; break tb = t + exit_bar if tb >= n: continue xp = btc[tb] if not np.isfinite(xp) or xp <= 0: continue raw_ret = sign * (xp - ep) / ep sized_ret = raw_ret * smult day_rets_sized.append((raw_ret, sized_ret, exit_bar)) # Per-trade log (first 50K trades) if len(trade_log) < 50_000: trade_log.append({'date': ds, 'year': year, 'posture': pos.value, 't': t, 'hold': exit_bar, 'raw_ret': round(raw_ret, 6), 'sized_ret': round(sized_ret, 6), 'size_mult': round(smult, 3)}) del vd, btc, entry_mask, cross_back n_t = len(day_rets_sized) if n_t == 0: day_log.append({'date': ds, 'year': year, 'posture': pos.value, 'fear': round(decision.fear_score, 3), 'n_trades': 0, 'day_ret': 0.0, 'equity': round(equity, 6)}) continue wins = sum(1 for r, _, _ in day_rets_sized if r >= 0) losses = n_t - wins gw = sum(r for r, _, _ in day_rets_sized if r >= 0) gl = sum(abs(r) for r, _, _ in day_rets_sized if r < 0) pf_d = gw / gl if gl > 0 else 999.0 day_ret = sum(s for _, s, _ in day_rets_sized) day_ret_clamped = max(-0.5, min(day_ret, 2.0)) equity *= (1 + day_ret_clamped) equity_curve.append(equity) s = stats_yr[year] s['wins'] += wins; s['losses'] += losses s['gw'] += gw; s['gl'] += gl; s['n'] += n_t sp = stats_pos[pos.value] sp['wins'] += wins; sp['losses'] += losses sp['gw'] += gw; sp['gl'] += gl; sp['n'] += n_t day_log.append({ 'date': ds, 'year': year, 'posture': pos.value, 'fear': round(decision.fear_score, 3), 'dvol': round(exf['dvol_btc'], 1), 'fng': round(exf['fng'], 1), 'prev_rvol': round(pr, 7) if pr else None, 'n_trades': n_t, 'wins': wins, 'losses': losses, 'pf_day': round(pf_d, 4), 'day_ret': round(day_ret, 6), 'equity': round(equity, 6), }) elapsed = time.time() - t0 print(f"Pass 2 done: {elapsed:.1f}s") # ── Metrics ──────────────────────────────────────────────────────────────────── ec = np.array(equity_curve) roi = (ec[-1] - 1.0) * 100 running_max = np.maximum.accumulate(ec) dd_arr = (running_max - ec) / running_max max_dd = float(np.max(dd_arr)) * 100 daily_rets = np.array([d['day_ret'] for d in day_log if d['n_trades'] > 0]) sharpe = float(np.mean(daily_rets) / np.std(daily_rets) * np.sqrt(252)) if len(daily_rets) > 1 and np.std(daily_rets) > 0 else 0.0 tot_w = sum(s['wins'] for s in stats_yr.values()) tot_l = sum(s['losses'] for s in stats_yr.values()) tot_gw = sum(s['gw'] for s in stats_yr.values()) tot_gl = sum(s['gl'] for s in stats_yr.values()) tot_n = tot_w + tot_l pf = tot_gw / tot_gl if tot_gl > 0 else 999.0 wr = tot_w / tot_n * 100 if tot_n > 0 else 0.0 paused_days = sum(s['paused'] for s in stats_yr.values()) active_days = sum(1 for d in day_log if d['posture'] != 'NONE') # ── Console ──────────────────────────────────────────────────────────────────── print(f"\n{'='*80}") print(f" MacroPostureSwitcher — 5s Gold Standard Backtest") print(f" Entry: ±{ENTRY_T} MaxHold: {MAX_HOLD}b={MAX_HOLD*5}s Runtime: {elapsed:.0f}s") print(f"{'='*80}") print(f" ROI: {roi:>+8.2f}%") print(f" Max DD: {max_dd:>8.2f}%") print(f" Sharpe: {sharpe:>8.3f} (annualized, daily)") print(f" PF: {pf:>8.4f}") print(f" WR: {wr:>8.2f}%") print(f" N trades: {tot_n:>8,}") print(f" Active: {active_days} Paused: {paused_days}") print(f" Equity final: {ec[-1]:.4f}x") print(f"\n Per-year:") print(f" {'Year':<6} {'N':>7} {'WR%':>6} {'PF':>7} {'Paused':>6}") print(f" {'-'*40}") for yr in sorted(stats_yr.keys()): s = stats_yr[yr] n = s['wins'] + s['losses'] if n == 0: print(f" {yr:<6} {'—':>7} {'—':>6} {'—':>7} {s['paused']:>6}") continue print(f" {yr:<6} {n:>7,} {s['wins']/n*100:>6.2f}% {s['gw']/max(1e-9,s['gl']):>7.4f} {s['paused']:>6}") print(f"\n Per-posture:") for pos, s in sorted(stats_pos.items()): n = s['wins'] + s['losses'] if n == 0: continue print(f" {pos}: N={n:,} WR={s['wins']/n*100:.2f}% PF={s['gw']/max(1e-9,s['gl']):.4f}") # ── Save ─────────────────────────────────────────────────────────────────────── LOG_DIR.mkdir(exist_ok=True) ts = datetime.now().strftime("%Y%m%d_%H%M%S") day_csv = LOG_DIR / f"posture_5s_daily_{ts}.csv" if day_log: with open(day_csv, 'w', newline='') as f: w = csv.DictWriter(f, fieldnames=day_log[0].keys()) w.writeheader(); w.writerows(day_log) print(f"\n → {day_csv}") eq_csv = LOG_DIR / f"posture_5s_equity_{ts}.csv" with open(eq_csv, 'w', newline='') as f: w = csv.writer(f) w.writerow(['idx','equity']); w.writerows(enumerate(equity_curve)) print(f" → {eq_csv}") if trade_log: tr_csv = LOG_DIR / f"posture_5s_trades_{ts}.csv" with open(tr_csv, 'w', newline='') as f: w = csv.DictWriter(f, fieldnames=trade_log[0].keys()) w.writeheader(); w.writerows(trade_log) print(f" → {tr_csv}") summary = { 'mode': '5s_posture_backtest', 'ts': ts, 'runtime_s': round(elapsed, 1), 'entry_t': ENTRY_T, 'max_hold_bars': MAX_HOLD, 'max_hold_sec': MAX_HOLD * 5, 'roi_pct': round(roi, 4), 'max_dd_pct': round(max_dd, 4), 'sharpe': round(sharpe, 4), 'pf': round(pf, 4), 'wr_pct': round(wr, 3), 'n_trades': int(tot_n), 'active_days': active_days, 'paused_days': int(paused_days), 'equity_final': round(float(ec[-1]), 6), 'per_year': {yr: { 'n': int(stats_yr[yr]['n']), 'paused': int(stats_yr[yr]['paused']), 'wr': round(stats_yr[yr]['wins']/max(1,stats_yr[yr]['n'])*100, 3), 'pf': round(stats_yr[yr]['gw']/max(1e-9,stats_yr[yr]['gl']), 4), } for yr in sorted(stats_yr.keys())}, } sum_json = LOG_DIR / f"posture_5s_summary_{ts}.json" with open(sum_json, 'w') as f: json.dump(summary, f, indent=2) print(f" → {sum_json}") print(f"\n Runtime: {elapsed:.0f}s")