01c19662cb
Includes core prod + GREEN/BLUE subsystems: - prod/ (BLUE harness, configs, scripts, docs) - nautilus_dolphin/ (GREEN Nautilus-native impl + dvae/ preserved) - adaptive_exit/ (AEM engine + models/bucket_assignments.pkl) - Observability/ (EsoF advisor, TUI, dashboards) - external_factors/ (EsoF producer) - mc_forewarning_qlabs_fork/ (MC regime/envelope) Excludes runtime caches, logs, backups, and reproducible artifacts per .gitignore.
331 lines
12 KiB
Python
Executable File
331 lines
12 KiB
Python
Executable File
"""MacroPostureSwitcher Backtest — 5s Gold Standard (56 days)
|
||
=============================================================
|
||
Same posture logic, applied to 5s data (vbt_cache/).
|
||
|
||
At 5s resolution, 1 bar = 5s:
|
||
MAX_HOLD = 240 bars = 20 min (matches 1m MAX_HOLD=20 bars)
|
||
|
||
ExF from NPZ (same eigenvalues path, same dates).
|
||
prev-day rvol computed from previous day's 5s prices.
|
||
|
||
Outputs: ROI / WR / PF / Max-DD / Sharpe
|
||
"""
|
||
import sys, time, csv, gc, json
|
||
sys.path.insert(0, str(__import__('pathlib').Path(__file__).parent.parent))
|
||
sys.stdout.reconfigure(encoding='utf-8', errors='replace')
|
||
|
||
from pathlib import Path
|
||
from datetime import datetime
|
||
from collections import defaultdict
|
||
import numpy as np
|
||
import pandas as pd
|
||
|
||
from nautilus_dolphin.nautilus.macro_posture_switcher import MacroPostureSwitcher, Posture
|
||
|
||
VBT_DIR_5S = Path(r"C:\Users\Lenovo\Documents\- DOLPHIN NG HD HCM TSF Predict\vbt_cache")
|
||
EIGEN_PATH = Path(r"C:\Users\Lenovo\Documents\- Dolphin NG HD (NG3)\correlation_arb512\eigenvalues")
|
||
LOG_DIR = Path(r"C:\Users\Lenovo\Documents\- DOLPHIN NG HD HCM TSF Predict\nautilus_dolphin\run_logs")
|
||
|
||
ENTRY_T = 0.020
|
||
MAX_HOLD = 240 # bars (240 × 5s = 20 min)
|
||
|
||
EXF_KEYS = ['dvol_btc', 'fng', 'funding_btc', 'taker']
|
||
|
||
def load_exf(date_str: str) -> dict:
|
||
defaults = {'dvol_btc': 50.0, 'fng': 50.0, 'funding_btc': 0.0, 'taker': 1.0}
|
||
dp = EIGEN_PATH / date_str
|
||
if not dp.exists():
|
||
return defaults
|
||
files = sorted(dp.glob('scan_*__Indicators.npz'))[:5]
|
||
if not files:
|
||
return defaults
|
||
buckets = defaultdict(list)
|
||
for f in files:
|
||
try:
|
||
d = np.load(f, allow_pickle=True)
|
||
if 'api_names' not in d:
|
||
continue
|
||
names = list(d['api_names'])
|
||
vals = d['api_indicators']
|
||
for k in EXF_KEYS:
|
||
if k in names:
|
||
v = float(vals[names.index(k)])
|
||
if np.isfinite(v):
|
||
buckets[k].append(v)
|
||
except Exception:
|
||
pass
|
||
out = dict(defaults)
|
||
for k, vs in buckets.items():
|
||
if vs:
|
||
out[k] = float(np.median(vs))
|
||
return out
|
||
|
||
parquet_files = sorted(VBT_DIR_5S.glob("*.parquet"))
|
||
parquet_files = [p for p in parquet_files if 'catalog' not in str(p)]
|
||
total = len(parquet_files)
|
||
print(f"Files: {total} (5s gold standard) | Entry T: ±{ENTRY_T} MaxHold: {MAX_HOLD}b={MAX_HOLD*5}s")
|
||
|
||
# Pass 1: prev-day rvol / btc_ret
|
||
print("Pass 1: lag-1 rvol...")
|
||
t0 = time.time()
|
||
day_rvol = {}
|
||
day_btcret = {}
|
||
for pf in parquet_files:
|
||
ds = pf.stem
|
||
try:
|
||
df = pd.read_parquet(pf, columns=['BTCUSDT'])
|
||
except Exception:
|
||
continue
|
||
btc = df['BTCUSDT'].values.astype(np.float64)
|
||
btc = btc[np.isfinite(btc) & (btc > 0)]
|
||
if len(btc) < 2:
|
||
continue
|
||
log_r = np.diff(np.log(btc))
|
||
day_rvol[ds] = float(np.std(log_r))
|
||
day_btcret[ds] = float((btc[-1] - btc[0]) / btc[0])
|
||
|
||
dates_sorted = sorted(day_rvol.keys())
|
||
prev_rvol = {d: day_rvol.get(dates_sorted[i-1]) if i > 0 else None for i, d in enumerate(dates_sorted)}
|
||
prev_btcret = {d: day_btcret.get(dates_sorted[i-1]) if i > 0 else None for i, d in enumerate(dates_sorted)}
|
||
print(f" Pass 1 done: {time.time()-t0:.1f}s")
|
||
|
||
# Pass 2: trade simulation
|
||
print("Pass 2: posture + crossover trades...")
|
||
switcher = MacroPostureSwitcher(
|
||
enable_long_posture=True,
|
||
# 5s-calibrated rvol thresholds (1m values don't transfer — 5s std is ~7x smaller):
|
||
# 5y klines 1m Q1 = 0.000455; 5s equivalent p25 = 0.000203, p75 = 0.000337
|
||
rvol_pause_thresh=0.000203, # 5s p25 — was 0.000455 (1m Q1 = p90 of 5s → too restrictive)
|
||
rvol_strong_thresh=0.000337, # 5s p75
|
||
# dvol Q1 gate: validated — removes 19 noisy days, PF 1.036→1.116, Sharpe 2.7→5.8
|
||
dvol_none_below=47.5,
|
||
)
|
||
|
||
equity = 1.0
|
||
equity_curve = [equity]
|
||
day_log = []
|
||
trade_log = []
|
||
stats_yr = defaultdict(lambda: {'wins':0,'losses':0,'gw':0.0,'gl':0.0,'n':0,'paused':0})
|
||
stats_pos = defaultdict(lambda: {'wins':0,'losses':0,'gw':0.0,'gl':0.0,'n':0})
|
||
|
||
for i, pf in enumerate(parquet_files):
|
||
ds = pf.stem
|
||
year = ds[:4]
|
||
|
||
exf = load_exf(ds)
|
||
pr = prev_rvol.get(ds)
|
||
pb = prev_btcret.get(ds)
|
||
|
||
decision = switcher.decide(
|
||
dvol_btc=exf['dvol_btc'], fng=exf['fng'],
|
||
funding_btc=exf['funding_btc'],
|
||
realized_vol=pr, btc_day_return=pb,
|
||
)
|
||
|
||
if decision.posture == Posture.NONE:
|
||
stats_yr[year]['paused'] += 1
|
||
day_log.append({'date': ds, 'year': year, 'posture': 'NONE',
|
||
'fear': round(decision.fear_score, 3), 'n_trades': 0,
|
||
'day_ret': 0.0, 'equity': round(equity, 6)})
|
||
continue
|
||
|
||
try:
|
||
df = pd.read_parquet(pf)
|
||
except Exception:
|
||
continue
|
||
if 'vel_div' not in df.columns or 'BTCUSDT' not in df.columns:
|
||
continue
|
||
|
||
vd = df['vel_div'].values.astype(np.float64)
|
||
btc = df['BTCUSDT'].values.astype(np.float64)
|
||
vd = np.where(np.isfinite(vd), vd, 0.0)
|
||
btc = np.where(np.isfinite(btc) & (btc > 0), btc, np.nan)
|
||
n = len(btc)
|
||
del df
|
||
|
||
if n < MAX_HOLD + 5:
|
||
del vd, btc
|
||
continue
|
||
|
||
pos = decision.posture
|
||
smult = decision.size_mult
|
||
|
||
if pos == Posture.SHORT:
|
||
entry_mask = (vd >= ENTRY_T) & np.isfinite(btc)
|
||
cross_back = (vd <= -ENTRY_T)
|
||
sign = -1
|
||
else:
|
||
entry_mask = (vd <= -ENTRY_T) & np.isfinite(btc)
|
||
cross_back = (vd >= ENTRY_T)
|
||
sign = +1
|
||
|
||
day_rets_sized = []
|
||
for t in range(n - MAX_HOLD):
|
||
if not entry_mask[t]:
|
||
continue
|
||
ep = btc[t]
|
||
if not np.isfinite(ep) or ep <= 0:
|
||
continue
|
||
|
||
exit_bar = MAX_HOLD
|
||
for k in range(1, MAX_HOLD + 1):
|
||
tb = t + k
|
||
if tb >= n:
|
||
exit_bar = k; break
|
||
if cross_back[tb]:
|
||
exit_bar = k; break
|
||
|
||
tb = t + exit_bar
|
||
if tb >= n:
|
||
continue
|
||
xp = btc[tb]
|
||
if not np.isfinite(xp) or xp <= 0:
|
||
continue
|
||
|
||
raw_ret = sign * (xp - ep) / ep
|
||
sized_ret = raw_ret * smult
|
||
day_rets_sized.append((raw_ret, sized_ret, exit_bar))
|
||
|
||
# Per-trade log (first 50K trades)
|
||
if len(trade_log) < 50_000:
|
||
trade_log.append({'date': ds, 'year': year, 'posture': pos.value,
|
||
't': t, 'hold': exit_bar,
|
||
'raw_ret': round(raw_ret, 6),
|
||
'sized_ret': round(sized_ret, 6),
|
||
'size_mult': round(smult, 3)})
|
||
|
||
del vd, btc, entry_mask, cross_back
|
||
|
||
n_t = len(day_rets_sized)
|
||
if n_t == 0:
|
||
day_log.append({'date': ds, 'year': year, 'posture': pos.value,
|
||
'fear': round(decision.fear_score, 3), 'n_trades': 0,
|
||
'day_ret': 0.0, 'equity': round(equity, 6)})
|
||
continue
|
||
|
||
wins = sum(1 for r, _, _ in day_rets_sized if r >= 0)
|
||
losses = n_t - wins
|
||
gw = sum(r for r, _, _ in day_rets_sized if r >= 0)
|
||
gl = sum(abs(r) for r, _, _ in day_rets_sized if r < 0)
|
||
pf_d = gw / gl if gl > 0 else 999.0
|
||
day_ret = sum(s for _, s, _ in day_rets_sized)
|
||
|
||
day_ret_clamped = max(-0.5, min(day_ret, 2.0))
|
||
equity *= (1 + day_ret_clamped)
|
||
equity_curve.append(equity)
|
||
|
||
s = stats_yr[year]
|
||
s['wins'] += wins; s['losses'] += losses
|
||
s['gw'] += gw; s['gl'] += gl; s['n'] += n_t
|
||
sp = stats_pos[pos.value]
|
||
sp['wins'] += wins; sp['losses'] += losses
|
||
sp['gw'] += gw; sp['gl'] += gl; sp['n'] += n_t
|
||
|
||
day_log.append({
|
||
'date': ds, 'year': year, 'posture': pos.value,
|
||
'fear': round(decision.fear_score, 3),
|
||
'dvol': round(exf['dvol_btc'], 1), 'fng': round(exf['fng'], 1),
|
||
'prev_rvol': round(pr, 7) if pr else None,
|
||
'n_trades': n_t, 'wins': wins, 'losses': losses,
|
||
'pf_day': round(pf_d, 4), 'day_ret': round(day_ret, 6),
|
||
'equity': round(equity, 6),
|
||
})
|
||
|
||
elapsed = time.time() - t0
|
||
print(f"Pass 2 done: {elapsed:.1f}s")
|
||
|
||
# ── Metrics ────────────────────────────────────────────────────────────────────
|
||
ec = np.array(equity_curve)
|
||
roi = (ec[-1] - 1.0) * 100
|
||
running_max = np.maximum.accumulate(ec)
|
||
dd_arr = (running_max - ec) / running_max
|
||
max_dd = float(np.max(dd_arr)) * 100
|
||
|
||
daily_rets = np.array([d['day_ret'] for d in day_log if d['n_trades'] > 0])
|
||
sharpe = float(np.mean(daily_rets) / np.std(daily_rets) * np.sqrt(252)) if len(daily_rets) > 1 and np.std(daily_rets) > 0 else 0.0
|
||
|
||
tot_w = sum(s['wins'] for s in stats_yr.values())
|
||
tot_l = sum(s['losses'] for s in stats_yr.values())
|
||
tot_gw = sum(s['gw'] for s in stats_yr.values())
|
||
tot_gl = sum(s['gl'] for s in stats_yr.values())
|
||
tot_n = tot_w + tot_l
|
||
pf = tot_gw / tot_gl if tot_gl > 0 else 999.0
|
||
wr = tot_w / tot_n * 100 if tot_n > 0 else 0.0
|
||
paused_days = sum(s['paused'] for s in stats_yr.values())
|
||
active_days = sum(1 for d in day_log if d['posture'] != 'NONE')
|
||
|
||
# ── Console ────────────────────────────────────────────────────────────────────
|
||
print(f"\n{'='*80}")
|
||
print(f" MacroPostureSwitcher — 5s Gold Standard Backtest")
|
||
print(f" Entry: ±{ENTRY_T} MaxHold: {MAX_HOLD}b={MAX_HOLD*5}s Runtime: {elapsed:.0f}s")
|
||
print(f"{'='*80}")
|
||
print(f" ROI: {roi:>+8.2f}%")
|
||
print(f" Max DD: {max_dd:>8.2f}%")
|
||
print(f" Sharpe: {sharpe:>8.3f} (annualized, daily)")
|
||
print(f" PF: {pf:>8.4f}")
|
||
print(f" WR: {wr:>8.2f}%")
|
||
print(f" N trades: {tot_n:>8,}")
|
||
print(f" Active: {active_days} Paused: {paused_days}")
|
||
print(f" Equity final: {ec[-1]:.4f}x")
|
||
|
||
print(f"\n Per-year:")
|
||
print(f" {'Year':<6} {'N':>7} {'WR%':>6} {'PF':>7} {'Paused':>6}")
|
||
print(f" {'-'*40}")
|
||
for yr in sorted(stats_yr.keys()):
|
||
s = stats_yr[yr]
|
||
n = s['wins'] + s['losses']
|
||
if n == 0:
|
||
print(f" {yr:<6} {'—':>7} {'—':>6} {'—':>7} {s['paused']:>6}")
|
||
continue
|
||
print(f" {yr:<6} {n:>7,} {s['wins']/n*100:>6.2f}% {s['gw']/max(1e-9,s['gl']):>7.4f} {s['paused']:>6}")
|
||
|
||
print(f"\n Per-posture:")
|
||
for pos, s in sorted(stats_pos.items()):
|
||
n = s['wins'] + s['losses']
|
||
if n == 0: continue
|
||
print(f" {pos}: N={n:,} WR={s['wins']/n*100:.2f}% PF={s['gw']/max(1e-9,s['gl']):.4f}")
|
||
|
||
# ── Save ───────────────────────────────────────────────────────────────────────
|
||
LOG_DIR.mkdir(exist_ok=True)
|
||
ts = datetime.now().strftime("%Y%m%d_%H%M%S")
|
||
|
||
day_csv = LOG_DIR / f"posture_5s_daily_{ts}.csv"
|
||
if day_log:
|
||
with open(day_csv, 'w', newline='') as f:
|
||
w = csv.DictWriter(f, fieldnames=day_log[0].keys())
|
||
w.writeheader(); w.writerows(day_log)
|
||
print(f"\n → {day_csv}")
|
||
|
||
eq_csv = LOG_DIR / f"posture_5s_equity_{ts}.csv"
|
||
with open(eq_csv, 'w', newline='') as f:
|
||
w = csv.writer(f)
|
||
w.writerow(['idx','equity']); w.writerows(enumerate(equity_curve))
|
||
print(f" → {eq_csv}")
|
||
|
||
if trade_log:
|
||
tr_csv = LOG_DIR / f"posture_5s_trades_{ts}.csv"
|
||
with open(tr_csv, 'w', newline='') as f:
|
||
w = csv.DictWriter(f, fieldnames=trade_log[0].keys())
|
||
w.writeheader(); w.writerows(trade_log)
|
||
print(f" → {tr_csv}")
|
||
|
||
summary = {
|
||
'mode': '5s_posture_backtest', 'ts': ts, 'runtime_s': round(elapsed, 1),
|
||
'entry_t': ENTRY_T, 'max_hold_bars': MAX_HOLD, 'max_hold_sec': MAX_HOLD * 5,
|
||
'roi_pct': round(roi, 4), 'max_dd_pct': round(max_dd, 4),
|
||
'sharpe': round(sharpe, 4), 'pf': round(pf, 4),
|
||
'wr_pct': round(wr, 3), 'n_trades': int(tot_n),
|
||
'active_days': active_days, 'paused_days': int(paused_days),
|
||
'equity_final': round(float(ec[-1]), 6),
|
||
'per_year': {yr: {
|
||
'n': int(stats_yr[yr]['n']), 'paused': int(stats_yr[yr]['paused']),
|
||
'wr': round(stats_yr[yr]['wins']/max(1,stats_yr[yr]['n'])*100, 3),
|
||
'pf': round(stats_yr[yr]['gw']/max(1e-9,stats_yr[yr]['gl']), 4),
|
||
} for yr in sorted(stats_yr.keys())},
|
||
}
|
||
sum_json = LOG_DIR / f"posture_5s_summary_{ts}.json"
|
||
with open(sum_json, 'w') as f:
|
||
json.dump(summary, f, indent=2)
|
||
print(f" → {sum_json}")
|
||
print(f"\n Runtime: {elapsed:.0f}s")
|