Files
DOLPHIN/nautilus_dolphin/dvae/exp12_convnext_gate.py

348 lines
16 KiB
Python
Raw Normal View History

"""
exp12_convnext_gate.py ConvNeXt z-signal gate on D_LIQ_GOLD
==============================================================
exp10/11 used TitanSensor with broken LSTM weights z_recon ~10^14 noise.
This experiment uses the newly trained ConvNeXt-1D β-TCVAE (ep=17, val=19.26):
z[10] r=+0.973 with proxy_B (32-bar trajectory encoding)
z_post_std OOD indicator (>1 = uncertain/unusual regime)
The z[10] signal captures the 32-bar TRAJECTORY of proxy_B, whereas the current
D_LIQ_GOLD engine uses only the instantaneous proxy_B at entry. If trajectory
adds information, z[10] should produce a measurable Calmar improvement.
Configs (5 runs):
0. baseline D_LIQ_GOLD unmodified (control)
1. z10_analogue analogue_scale(z10) boost high proxy_B, cut low
2. z10_inv analogue_scale(-z10) inverse direction test
3. zstd_gate notional × max(0.4, 1 - z_post_std/4) OOD cut
4. z10_x_zstd z10_analogue × zstd_gate combined
analogue_scale (same as exp10/11):
z >= 0 : 1 + UP_STR * tanh(z / K) ceiling ~1.15
z < 0 : 1 + DOWN_STR * tanh(z / K) floor ~0.50
Threshold test: Calmar > baseline × 1.02 (same as exp10/11)
Gold baseline: ROI=181.81% DD=17.65% Calmar=10.30 (D_LIQ_GOLD memory)
"""
import sys, time, json, warnings
sys.stdout.reconfigure(encoding='utf-8', errors='replace')
warnings.filterwarnings('ignore')
from pathlib import Path
import numpy as np
import pandas as pd
ROOT = Path(__file__).parent.parent
sys.path.insert(0, str(ROOT))
from nautilus_dolphin.nautilus.alpha_asset_selector import compute_irp_nb, compute_ars_nb, rank_assets_irp_nb
from nautilus_dolphin.nautilus.alpha_bet_sizer import compute_sizing_nb
from nautilus_dolphin.nautilus.alpha_signal_generator import check_dc_nb
from nautilus_dolphin.nautilus.ob_features import (
OBFeatureEngine, compute_imbalance_nb, compute_depth_1pct_nb,
compute_depth_quality_nb, compute_fill_probability_nb, compute_spread_proxy_nb,
compute_depth_asymmetry_nb, compute_imbalance_persistence_nb,
compute_withdrawal_velocity_nb, compute_market_agreement_nb, compute_cascade_signal_nb,
)
from nautilus_dolphin.nautilus.ob_provider import MockOBProvider
from nautilus_dolphin.nautilus.adaptive_circuit_breaker import AdaptiveCircuitBreaker
from nautilus_dolphin.nautilus.proxy_boost_engine import LiquidationGuardEngine, create_d_liq_engine
from mc.mc_ml import DolphinForewarner
from dvae.convnext_sensor import ConvNextSensor, PROXY_B_DIM
# ── JIT warmup ────────────────────────────────────────────────────────────────
print("Warming up JIT...")
_p = np.array([1.,2.,3.], dtype=np.float64)
compute_irp_nb(_p,-1); compute_ars_nb(1.,.5,.01)
rank_assets_irp_nb(np.ones((10,2),dtype=np.float64),8,-1,5,500.,20,0.20)
compute_sizing_nb(-.03,-.02,-.05,3.,.5,5.,.20,True,True,0.,
np.zeros(4,dtype=np.int64),np.zeros(4,dtype=np.int64),
np.zeros(5,dtype=np.float64),0,-1,.01,.04)
check_dc_nb(_p,3,1,.75)
_b=np.array([100.,200.,300.,400.,500.],dtype=np.float64)
_a=np.array([110.,190.,310.,390.,510.],dtype=np.float64)
compute_imbalance_nb(_b,_a); compute_depth_1pct_nb(_b,_a)
compute_depth_quality_nb(210.,200.); compute_fill_probability_nb(1.)
compute_spread_proxy_nb(_b,_a); compute_depth_asymmetry_nb(_b,_a)
compute_imbalance_persistence_nb(np.array([.1,-.1],dtype=np.float64),2)
compute_withdrawal_velocity_nb(np.array([100.,110.],dtype=np.float64),1)
compute_market_agreement_nb(np.array([.1,-.05],dtype=np.float64),2)
compute_cascade_signal_nb(np.array([-.05,-.15],dtype=np.float64),2,-.10)
print(" JIT ready.")
# ── Paths ─────────────────────────────────────────────────────────────────────
VBT5s = Path(r"C:\Users\Lenovo\Documents\- DOLPHIN NG HD HCM TSF Predict\vbt_cache")
VBT1m = Path(r"C:\Users\Lenovo\Documents\- DOLPHIN NG HD HCM TSF Predict\vbt_cache_klines")
MODEL_PATH = Path(r"C:\Users\Lenovo\Documents\- DOLPHIN NG HD HCM TSF Predict\nautilus_dolphin\dvae\convnext_model.json")
MC_MODELS = str(ROOT / "mc_results" / "models")
OUT_FILE = Path(__file__).parent / "exp12_convnext_gate_results.json"
META_COLS = {'timestamp','scan_number','v50_lambda_max_velocity','v150_lambda_max_velocity',
'v300_lambda_max_velocity','v750_lambda_max_velocity','vel_div',
'instability_50','instability_150'}
BASE_ENGINE_KWARGS = dict(
initial_capital=25000., vel_div_threshold=-.02, vel_div_extreme=-.05,
min_leverage=.5, max_leverage=5., leverage_convexity=3.,
fraction=.20, fixed_tp_pct=.0095, stop_pct=1., max_hold_bars=120,
use_direction_confirm=True, dc_lookback_bars=7, dc_min_magnitude_bps=.75,
dc_skip_contradicts=True, dc_leverage_boost=1., dc_leverage_reduce=.5,
use_asset_selection=True, min_irp_alignment=.45,
use_sp_fees=True, use_sp_slippage=True,
sp_maker_entry_rate=.62, sp_maker_exit_rate=.50,
use_ob_edge=True, ob_edge_bps=5., ob_confirm_rate=.40,
lookback=100, use_alpha_layers=True, use_dynamic_leverage=True, seed=42,
)
D_LIQ_KWARGS = dict(
extended_soft_cap=8., extended_abs_cap=9., mc_leverage_ref=5.,
margin_buffer=.95, threshold=.35, alpha=1., adaptive_beta=True,
)
MC_BASE_CFG = {
'trial_id':0, 'vel_div_threshold':-.020, 'vel_div_extreme':-.050,
'use_direction_confirm':True, 'dc_lookback_bars':7, 'dc_min_magnitude_bps':.75,
'dc_skip_contradicts':True, 'dc_leverage_boost':1.00, 'dc_leverage_reduce':.50,
'vd_trend_lookback':10, 'min_leverage':.50, 'max_leverage':5.00,
'leverage_convexity':3.00, 'fraction':.20, 'use_alpha_layers':True,
'use_dynamic_leverage':True, 'fixed_tp_pct':.0095, 'stop_pct':1.00,
'max_hold_bars':120, 'use_sp_fees':True, 'use_sp_slippage':True,
'sp_maker_entry_rate':.62, 'sp_maker_exit_rate':.50, 'use_ob_edge':True,
'ob_edge_bps':5.00, 'ob_confirm_rate':.40, 'ob_imbalance_bias':-.09,
'ob_depth_scale':1.00, 'use_asset_selection':True, 'min_irp_alignment':.45,
'lookback':100, 'acb_beta_high':.80, 'acb_beta_low':.20, 'acb_w750_threshold_pct':60,
}
K_TANH = 1.5
UP_STR = 0.15
DOWN_STR = 0.50
def analogue_scale(z: float) -> float:
t = np.tanh(float(z) / K_TANH)
return 1.0 + (UP_STR if z >= 0. else DOWN_STR) * t
def zstd_scale(zstd: float) -> float:
"""OOD cut: normal zstd~0.94, high zstd = uncertain regime → cut notional."""
return float(max(0.4, 1.0 - (zstd - 0.94) / 4.0))
# ── Pre-compute signals per day ───────────────────────────────────────────────
def precompute_signals(parquet_files_5s, sensor: ConvNextSensor):
print("Pre-computing ConvNext z signals from 1m data...")
signals = {}
for pf5 in parquet_files_5s:
ds = pf5.stem
pf1 = VBT1m / f"{ds}.parquet"
if not pf1.exists():
signals[ds] = None
continue
df1 = pd.read_parquet(pf1).replace([np.inf, -np.inf], np.nan).fillna(0.)
n1 = len(df1)
n5 = len(pd.read_parquet(pf5, columns=['timestamp']))
z10_1m = np.zeros(n1, dtype=np.float64)
zstd_1m = np.zeros(n1, dtype=np.float64)
for j in range(n1):
z_mu, z_post_std = sensor.encode_window(df1, j)
z10_1m[j] = z_mu[PROXY_B_DIM]
zstd_1m[j] = z_post_std
# Map 1m → 5s by nearest index
z10_5s = np.array([z10_1m[min(int(i * n1 / n5), n1-1)] for i in range(n5)])
zstd_5s = np.array([zstd_1m[min(int(i * n1 / n5), n5-1)] for i in range(n5)])
signals[ds] = {'z10': z10_5s, 'zstd': zstd_5s}
print(f" {ds}: z10=[{z10_5s.min():.2f},{z10_5s.max():.2f}] "
f"zstd=[{zstd_5s.min():.3f},{zstd_5s.max():.3f}]")
n_ok = sum(1 for v in signals.values() if v is not None)
print(f" Signals ready: {n_ok}/{len(signals)} days\n")
return signals
# ── Engine subclass ───────────────────────────────────────────────────────────
class ConvNextGateEngine(LiquidationGuardEngine):
def __init__(self, scale_fn, **kwargs):
super().__init__(**kwargs)
self._scale_fn = scale_fn
self._bar_z10 = None
self._bar_zstd = None
self._scale_history = []
def set_signals(self, z10: np.ndarray, zstd: np.ndarray):
self._bar_z10 = z10
self._bar_zstd = zstd
def _try_entry(self, bar_idx, vel_div, prices, price_histories, v50_vel=0., v750_vel=0.):
result = super()._try_entry(bar_idx, vel_div, prices, price_histories, v50_vel, v750_vel)
if result and self.position is not None:
z10 = float(self._bar_z10[bar_idx]) if (self._bar_z10 is not None and bar_idx < len(self._bar_z10)) else 0.
zstd = float(self._bar_zstd[bar_idx]) if (self._bar_zstd is not None and bar_idx < len(self._bar_zstd)) else 0.94
s = float(self._scale_fn(z10, zstd))
s = max(0.2, min(2.0, s))
self.position.notional *= s
self._scale_history.append(s)
return result
def reset(self):
super().reset()
self._scale_history = []
# ── Scale functions ───────────────────────────────────────────────────────────
CONFIGS = {
"0_baseline": None,
"1_z10_analogue": lambda z10, zstd: analogue_scale(z10),
"2_z10_inv": lambda z10, zstd: analogue_scale(-z10),
"3_zstd_gate": lambda z10, zstd: zstd_scale(zstd),
"4_z10_x_zstd": lambda z10, zstd: analogue_scale(z10) * zstd_scale(zstd),
}
def run_one(config_name, scale_fn, parquet_files, pq_data, signals, vol_p60):
OB_ASSETS = sorted({a for ds, (df, ac, _) in pq_data.items() for a in ac})
_mock_ob = MockOBProvider(
imbalance_bias=-.09, depth_scale=1., assets=OB_ASSETS,
imbalance_biases={"BTCUSDT":-.086,"ETHUSDT":-.092,"BNBUSDT":+.05,"SOLUSDT":+.05},
)
ob_eng = OBFeatureEngine(_mock_ob)
ob_eng.preload_date("mock", OB_ASSETS)
forewarner = DolphinForewarner(models_dir=MC_MODELS)
acb = AdaptiveCircuitBreaker()
acb.preload_w750([pf.stem for pf in parquet_files])
if scale_fn is None:
engine = create_d_liq_engine(**BASE_ENGINE_KWARGS)
else:
engine = ConvNextGateEngine(scale_fn=scale_fn, **BASE_ENGINE_KWARGS, **D_LIQ_KWARGS)
engine.set_ob_engine(ob_eng)
engine.set_acb(acb)
engine.set_mc_forewarner(forewarner, MC_BASE_CFG)
engine.set_esoteric_hazard_multiplier(0.)
t0 = time.time()
for pf in parquet_files:
ds = pf.stem
df, acols, dvol = pq_data[ds]
vol_ok = np.where(np.isfinite(dvol), dvol > vol_p60, False)
if scale_fn is not None and isinstance(engine, ConvNextGateEngine):
sig = signals.get(ds)
if sig:
engine.set_signals(sig['z10'], sig['zstd'])
else:
engine.set_signals(np.zeros(len(df)), np.full(len(df), 0.94))
engine.process_day(ds, df, acols, vol_regime_ok=vol_ok)
elapsed = time.time() - t0
trades = engine.trade_history
roi = (engine.capital - 25000.) / 25000. * 100.
cap_curve = [25000.]
for t_ in sorted(trades, key=lambda x: getattr(x, 'exit_bar', 0)):
cap_curve.append(cap_curve[-1] + getattr(t_, 'pnl_absolute', 0.))
cap_arr = np.array(cap_curve)
peak = np.maximum.accumulate(cap_arr)
dd = float(np.max((peak - cap_arr) / (peak + 1e-10)) * 100.)
calmar = roi / max(dd, 1e-4)
sh = getattr(engine, '_scale_history', [])
return {
'config': config_name,
'T': len(trades),
'ROI': round(roi, 4),
'DD': round(dd, 4),
'Calmar': round(calmar, 4),
'elapsed_s': round(elapsed, 1),
'scale_mean': round(float(np.mean(sh)), 4) if sh else 1.0,
'scale_min': round(float(np.min(sh)), 4) if sh else 1.0,
'scale_max': round(float(np.max(sh)), 4) if sh else 1.0,
'n_scaled': len(sh),
}
def main():
parquet_files = sorted(VBT5s.glob("*.parquet"))
parquet_files = [p for p in parquet_files if 'catalog' not in str(p)]
print(f"Dataset: {len(parquet_files)} days (5s scans)\n")
print("Loading ConvNextSensor...")
sensor = ConvNextSensor(str(MODEL_PATH))
print(f" epoch={sensor.epoch} val_loss={sensor.val_loss:.4f} z_dim={sensor.z_dim}\n")
signals = precompute_signals(parquet_files, sensor)
print("Loading 5s parquet data...")
pq_data = {}
for pf in parquet_files:
df = pd.read_parquet(pf)
ac = [c for c in df.columns if c not in META_COLS]
bp = df['BTCUSDT'].values if 'BTCUSDT' in df.columns else None
dv = np.full(len(df), np.nan)
if bp is not None:
for i in range(50, len(bp)):
seg = bp[max(0, i-50):i]
if len(seg) >= 10:
dv[i] = float(np.std(np.diff(seg) / seg[:-1]))
pq_data[pf.stem] = (df, ac, dv)
all_vols = []
for pf in parquet_files[:2]:
df = pd.read_parquet(pf)
if 'BTCUSDT' not in df.columns: continue
pr = df['BTCUSDT'].values
for i in range(60, len(pr)):
seg = pr[max(0, i-50):i]
if len(seg) >= 10:
v = float(np.std(np.diff(seg) / seg[:-1]))
if v > 0: all_vols.append(v)
vol_p60 = float(np.percentile(all_vols, 60)) if all_vols else 0.
print(f"\nRunning {len(CONFIGS)} configs...\n")
results = []
for name, fn in CONFIGS.items():
print(f"[{name}]", flush=True)
r = run_one(name, fn, parquet_files, pq_data, signals, vol_p60)
results.append(r)
base = results[0]
dr = r['ROI'] - base['ROI']
dd = r['DD'] - base['DD']
print(f" T={r['T']} ROI={r['ROI']:+.2f}% DD={r['DD']:.2f}% "
f"Calmar={r['Calmar']:.2f} dROI={dr:+.2f}pp dDD={dd:+.2f}pp "
f"s_mean={r['scale_mean']:.3f} ({r['elapsed_s']:.0f}s)\n")
print("=" * 95)
print(f"{'Config':<22} {'T':>5} {'ROI%':>8} {'DD%':>7} {'Calmar':>7} "
f"{'dROI':>7} {'dDD':>6} {'s_mean':>7}")
print("-" * 95)
base = results[0]
for r in results:
dr = r['ROI'] - base['ROI']
dd = r['DD'] - base['DD']
print(f"{r['config']:<22} {r['T']:>5} {r['ROI']:>8.2f} {r['DD']:>7.2f} "
f"{r['Calmar']:>7.2f} {dr:>+7.2f} {dd:>+6.2f} {r['scale_mean']:>7.3f}")
with open(OUT_FILE, 'w') as f:
json.dump({'baseline': base, 'results': results,
'model_epoch': sensor.epoch, 'model_val_loss': sensor.val_loss}, f, indent=2)
print(f"\nResults -> {OUT_FILE}")
print("\n=== VERDICT ===")
best = max(results[1:], key=lambda x: x['Calmar'])
threshold = base['Calmar'] * 1.02
print(f"Best: [{best['config']}] Calmar={best['Calmar']:.2f} "
f"ROI={best['ROI']:.2f}% DD={best['DD']:.2f}%")
print(f"Threshold: Calmar > {threshold:.2f} (1.02x baseline {base['Calmar']:.2f})")
if best['Calmar'] > threshold:
print(" SIGNAL CONFIRMED — proceed to exp13 (productionization)")
else:
print(" MARGINAL / NO improvement over D_LIQ_GOLD")
if __name__ == '__main__':
main()