""" exp11_zrecon_inv.py — z_recon direction inversion test ======================================================= Exp10 showed z_recon now works (C: −0.78pp vs −8.65pp broken, D: −0.57pp vs −1.32pp) but the direction is WRONG: the current setup boosts on HIGH z_recon (OOD bars) and cuts on LOW z_recon (normal bars), but OOD bars at entry should be cut (not boosted). This experiment tests the INVERTED direction: use −ze instead of ze for the recon signal. D_inv: analogue_scale(−ze) → CUT on OOD (high z_recon), BOOST on quiet bars BD_inv: B × D_inv → combine z_roll analogue with inverted z_recon Configs (4 total): 0. Baseline — D_LIQ_GOLD unmodified (control) 1. B_analogue — z_roll analogue (positive control, same as exp10 config 2) 2. D_inv_analogue — z_recon analogue, INVERTED direction (−ze) 3. BD_inv — B_analogue × D_inv_analogue Signal from exp10 that was POSITIVE: B_analogue (z_roll i150, +0.54pp) Signal from exp10 that was NEGATIVE: D_analogue (z_recon, −0.57pp) Hypothesis: D_inv (−ze) should be positive since OOD = bad entry condition. No production code changes. Same model path, same data. """ import sys, time, json, warnings sys.stdout.reconfigure(encoding='utf-8', errors='replace') warnings.filterwarnings('ignore') from pathlib import Path import numpy as np import pandas as pd ROOT = Path(__file__).parent.parent sys.path.insert(0, str(ROOT)) from nautilus_dolphin.nautilus.alpha_asset_selector import compute_irp_nb, compute_ars_nb, rank_assets_irp_nb from nautilus_dolphin.nautilus.alpha_bet_sizer import compute_sizing_nb from nautilus_dolphin.nautilus.alpha_signal_generator import check_dc_nb from nautilus_dolphin.nautilus.ob_features import ( OBFeatureEngine, compute_imbalance_nb, compute_depth_1pct_nb, compute_depth_quality_nb, compute_fill_probability_nb, compute_spread_proxy_nb, compute_depth_asymmetry_nb, compute_imbalance_persistence_nb, compute_withdrawal_velocity_nb, compute_market_agreement_nb, compute_cascade_signal_nb, ) from nautilus_dolphin.nautilus.ob_provider import MockOBProvider from nautilus_dolphin.nautilus.adaptive_circuit_breaker import AdaptiveCircuitBreaker from nautilus_dolphin.nautilus.proxy_boost_engine import LiquidationGuardEngine, create_d_liq_engine from mc.mc_ml import DolphinForewarner from dvae.titan_sensor import TitanSensor, build_feature_vector # ── JIT warmup ──────────────────────────────────────────────────────────────── print("Warming up JIT...") _p = np.array([1.,2.,3.], dtype=np.float64) compute_irp_nb(_p,-1); compute_ars_nb(1.,.5,.01) rank_assets_irp_nb(np.ones((10,2),dtype=np.float64),8,-1,5,500.,20,0.20) compute_sizing_nb(-.03,-.02,-.05,3.,.5,5.,.20,True,True,0., np.zeros(4,dtype=np.int64),np.zeros(4,dtype=np.int64), np.zeros(5,dtype=np.float64),0,-1,.01,.04) check_dc_nb(_p,3,1,.75) _b=np.array([100.,200.,300.,400.,500.],dtype=np.float64) _a=np.array([110.,190.,310.,390.,510.],dtype=np.float64) compute_imbalance_nb(_b,_a); compute_depth_1pct_nb(_b,_a) compute_depth_quality_nb(210.,200.); compute_fill_probability_nb(1.) compute_spread_proxy_nb(_b,_a); compute_depth_asymmetry_nb(_b,_a) compute_imbalance_persistence_nb(np.array([.1,-.1],dtype=np.float64),2) compute_withdrawal_velocity_nb(np.array([100.,110.],dtype=np.float64),1) compute_market_agreement_nb(np.array([.1,-.05],dtype=np.float64),2) compute_cascade_signal_nb(np.array([-.05,-.15],dtype=np.float64),2,-.10) print(" JIT ready.") # ── Paths ───────────────────────────────────────────────────────────────────── VBT5s = Path(r"C:\Users\Lenovo\Documents\- DOLPHIN NG HD HCM TSF Predict\vbt_cache") VBT1m = Path(r"C:\Users\Lenovo\Documents\- DOLPHIN NG HD HCM TSF Predict\vbt_cache_klines") MODEL_PATH = Path(r"C:\Users\Lenovo\Documents\- DOLPHIN NG HD HCM TSF Predict\dvae_regime_model_TITAN_ULTRA_GD.json") MC_MODELS = str(ROOT / "mc_results" / "models") OUT_FILE = Path(__file__).parent / "exp11_zrecon_inv_results.json" META_COLS = {'timestamp','scan_number','v50_lambda_max_velocity','v150_lambda_max_velocity', 'v300_lambda_max_velocity','v750_lambda_max_velocity','vel_div', 'instability_50','instability_150'} BASE_ENGINE_KWARGS = dict( initial_capital=25000., vel_div_threshold=-.02, vel_div_extreme=-.05, min_leverage=.5, max_leverage=5., leverage_convexity=3., fraction=.20, fixed_tp_pct=.0095, stop_pct=1., max_hold_bars=120, use_direction_confirm=True, dc_lookback_bars=7, dc_min_magnitude_bps=.75, dc_skip_contradicts=True, dc_leverage_boost=1., dc_leverage_reduce=.5, use_asset_selection=True, min_irp_alignment=.45, use_sp_fees=True, use_sp_slippage=True, sp_maker_entry_rate=.62, sp_maker_exit_rate=.50, use_ob_edge=True, ob_edge_bps=5., ob_confirm_rate=.40, lookback=100, use_alpha_layers=True, use_dynamic_leverage=True, seed=42, ) D_LIQ_KWARGS = dict( extended_soft_cap=8., extended_abs_cap=9., mc_leverage_ref=5., margin_buffer=.95, threshold=.35, alpha=1., adaptive_beta=True, ) MC_BASE_CFG = { 'trial_id':0, 'vel_div_threshold':-.020, 'vel_div_extreme':-.050, 'use_direction_confirm':True, 'dc_lookback_bars':7, 'dc_min_magnitude_bps':.75, 'dc_skip_contradicts':True, 'dc_leverage_boost':1.00, 'dc_leverage_reduce':.50, 'vd_trend_lookback':10, 'min_leverage':.50, 'max_leverage':5.00, 'leverage_convexity':3.00, 'fraction':.20, 'use_alpha_layers':True, 'use_dynamic_leverage':True, 'fixed_tp_pct':.0095, 'stop_pct':1.00, 'max_hold_bars':120, 'use_sp_fees':True, 'use_sp_slippage':True, 'sp_maker_entry_rate':.62, 'sp_maker_exit_rate':.50, 'use_ob_edge':True, 'ob_edge_bps':5.00, 'ob_confirm_rate':.40, 'ob_imbalance_bias':-.09, 'ob_depth_scale':1.00, 'use_asset_selection':True, 'min_irp_alignment':.45, 'lookback':100, 'acb_beta_high':.80, 'acb_beta_low':.20, 'acb_w750_threshold_pct':60, } WINDOW_Z = 30 K_TANH = 1.5 UP_STR = 0.15 DOWN_STR = 0.50 def analogue_scale(z: float) -> float: t = np.tanh(float(z) / K_TANH) if z >= 0.: return 1.0 + UP_STR * t else: return 1.0 + DOWN_STR * t # ── Pre-compute 1m signals (same as exp10) ───────────────────────────────── def precompute_1m_signals(parquet_files_5s, sensor): print("Pre-computing 1m signals...") signals = {} for pf5 in parquet_files_5s: ds = pf5.stem pf1 = VBT1m / f"{ds}.parquet" if not pf1.exists(): signals[ds] = None continue df5 = pd.read_parquet(pf5) df1 = pd.read_parquet(pf1).replace([np.inf,-np.inf], np.nan).fillna(0.) n5, n1 = len(df5), len(df1) assets = [c for c in df1.columns if c not in META_COLS] i150 = df1['instability_150'].values.copy() if 'instability_150' in df1.columns else np.zeros(n1) z_roll_1m = np.zeros(n1) for j in range(WINDOW_Z, n1): seg = i150[max(0,j-WINDOW_Z):j] mu, sigma = np.mean(seg), np.std(seg) z_roll_1m[j] = (i150[j] - mu) / max(sigma, 1e-10) recon_1m = np.zeros(n1) for j in range(n1): feat = build_feature_vector(df1, j, assets) _, recon_err, _ = sensor.encode(feat) recon_1m[j] = recon_err z_recon_1m = np.zeros(n1) for j in range(WINDOW_Z, n1): seg = recon_1m[max(0,j-WINDOW_Z):j] mu, sigma = np.mean(seg), np.std(seg) z_recon_1m[j] = (recon_1m[j] - mu) / max(sigma, 1e-10) z_roll_5s = np.zeros(n5) z_recon_5s = np.zeros(n5) for i in range(n5): j = min(int(i * n1 / n5), n1-1) z_roll_5s[i] = z_roll_1m[j] z_recon_5s[i] = z_recon_1m[j] signals[ds] = {'z_roll': z_roll_5s, 'z_recon': z_recon_5s} print(f" {ds}: z_roll=[{z_roll_5s.min():.2f},{z_roll_5s.max():.2f}] " f"z_recon=[{z_recon_5s.min():.2f},{z_recon_5s.max():.2f}]") return signals # ── Engine subclass (same as exp10) ────────────────────────────────────────── class KeyframeGateEngine(LiquidationGuardEngine): def __init__(self, scale_fn, **kwargs): super().__init__(**kwargs) self._scale_fn = scale_fn self._bar_z_roll = None self._bar_z_recon = None self._1m_scale_history = [] def set_1m_signals(self, z_roll, z_recon): self._bar_z_roll = z_roll self._bar_z_recon = z_recon def _try_entry(self, bar_idx, vel_div, prices, price_histories, v50_vel=0., v750_vel=0.): result = super()._try_entry(bar_idx, vel_div, prices, price_histories, v50_vel, v750_vel) if result and self.position is not None: zr = float(self._bar_z_roll[bar_idx]) if (self._bar_z_roll is not None and bar_idx < len(self._bar_z_roll)) else 0. ze = float(self._bar_z_recon[bar_idx]) if (self._bar_z_recon is not None and bar_idx < len(self._bar_z_recon)) else 0. s = float(self._scale_fn(zr, ze)) s = max(0.2, min(2.0, s)) self.position.notional *= s self._1m_scale_history.append(s) return result def reset(self): super().reset() self._1m_scale_history = [] # ── Configs: focus on inverted z_recon direction ────────────────────────────── CONFIGS = { "0_baseline": None, "1_B_analogue": lambda zr, ze: analogue_scale(zr), # positive control "2_D_inv_analogue":lambda zr, ze: analogue_scale(-ze), # INVERTED z_recon "3_BD_inv": lambda zr, ze: analogue_scale(zr) * analogue_scale(-ze), # B × D_inv } def run_one(config_name, scale_fn, parquet_files, pq_data, signals, vol_p60): OB_ASSETS = sorted({a for ds,(df,ac,_) in pq_data.items() for a in ac}) _mock_ob = MockOBProvider( imbalance_bias=-.09, depth_scale=1., assets=OB_ASSETS, imbalance_biases={"BTCUSDT":-.086,"ETHUSDT":-.092,"BNBUSDT":+.05,"SOLUSDT":+.05}, ) ob_eng = OBFeatureEngine(_mock_ob) ob_eng.preload_date("mock", OB_ASSETS) forewarner = DolphinForewarner(models_dir=MC_MODELS) acb = AdaptiveCircuitBreaker() acb.preload_w750([pf.stem for pf in parquet_files]) if scale_fn is None: engine = create_d_liq_engine(**BASE_ENGINE_KWARGS) else: engine = KeyframeGateEngine(scale_fn=scale_fn, **BASE_ENGINE_KWARGS, **D_LIQ_KWARGS) engine.set_ob_engine(ob_eng) engine.set_acb(acb) engine.set_mc_forewarner(forewarner, MC_BASE_CFG) engine.set_esoteric_hazard_multiplier(0.) t0 = time.time() for pf in parquet_files: ds = pf.stem df, acols, dvol = pq_data[ds] vol_ok = np.where(np.isfinite(dvol), dvol > vol_p60, False) if scale_fn is not None and isinstance(engine, KeyframeGateEngine): sig = signals.get(ds) if sig: engine.set_1m_signals(sig['z_roll'], sig['z_recon']) else: engine.set_1m_signals(np.zeros(len(df)), np.zeros(len(df))) engine.process_day(ds, df, acols, vol_regime_ok=vol_ok) elapsed = time.time() - t0 trades = engine.trade_history roi = (engine.capital - 25000.) / 25000. * 100. cap_curve = [25000.] for t_ in sorted(trades, key=lambda x: getattr(x,'exit_bar',0)): cap_curve.append(cap_curve[-1] + getattr(t_,'pnl_absolute',0.)) cap_arr = np.array(cap_curve) peak = np.maximum.accumulate(cap_arr) dd = float(np.max((peak - cap_arr) / (peak + 1e-10)) * 100.) calmar = roi / max(dd, 1e-4) scale_hist = getattr(engine, '_1m_scale_history', []) return { 'config': config_name, 'T': len(trades), 'ROI': round(roi, 4), 'DD': round(dd, 4), 'Calmar': round(calmar, 4), 'elapsed_s': round(elapsed, 1), 'scale_mean': round(np.mean(scale_hist), 4) if scale_hist else 1.0, 'scale_min': round(np.min(scale_hist), 4) if scale_hist else 1.0, 'scale_max': round(np.max(scale_hist), 4) if scale_hist else 1.0, 'n_scale_applied': len(scale_hist), } def main(): parquet_files = sorted(VBT5s.glob("*.parquet")) parquet_files = [p for p in parquet_files if 'catalog' not in str(p)] print(f"Dataset: {len(parquet_files)} days") print("Loading parquet data...") pq_data = {} for pf in parquet_files: df = pd.read_parquet(pf) ac = [c for c in df.columns if c not in META_COLS] bp = df['BTCUSDT'].values if 'BTCUSDT' in df.columns else None dv = np.full(len(df), np.nan) if bp is not None: for i in range(50, len(bp)): seg = bp[max(0,i-50):i] if len(seg) >= 10: dv[i] = float(np.std(np.diff(seg)/seg[:-1])) pq_data[pf.stem] = (df, ac, dv) all_vols = [] for pf in parquet_files[:2]: df = pd.read_parquet(pf) if 'BTCUSDT' not in df.columns: continue pr = df['BTCUSDT'].values for i in range(60, len(pr)): seg = pr[max(0,i-50):i] if len(seg) >= 10: v = float(np.std(np.diff(seg)/seg[:-1])) if v > 0: all_vols.append(v) vol_p60 = float(np.percentile(all_vols, 60)) if all_vols else 0. print(f"\nLoading TitanSensor (GD-v2 with normalization)...") sensor = TitanSensor(str(MODEL_PATH)) print(f" lstm_weights_valid={sensor.lstm_weights_valid} " f"norm_mean is {'present' if sensor.norm_mean is not None else 'MISSING'}") signals = precompute_1m_signals(parquet_files, sensor) n_missing = sum(1 for v in signals.values() if v is None) print(f" 1m signals ready: {len(signals)-n_missing}/{len(signals)} days") print() results = [] for name, fn in CONFIGS.items(): print(f"Running [{name}]...", flush=True) r = run_one(name, fn, parquet_files, pq_data, signals, vol_p60) results.append(r) baseline_roi = results[0]['ROI'] delta_roi = r['ROI'] - baseline_roi delta_dd = r['DD'] - results[0]['DD'] if len(results) > 1 else 0. print(f" T={r['T']} ROI={r['ROI']:+.2f}% DD={r['DD']:.2f}% " f"Calmar={r['Calmar']:.2f} " f"(dROI={delta_roi:+.2f}pp dDD={delta_dd:+.2f}pp) " f"scale_mean={r['scale_mean']:.3f} {r['elapsed_s']:.0f}s") print() print("=" * 90) print(f"{'Config':<22} {'T':>5} {'ROI%':>8} {'DD%':>7} {'Calmar':>7} " f"{'dROI':>7} {'dDD':>6} {'s_mean':>7}") print("-" * 90) base = results[0] for r in results: dr = r['ROI'] - base['ROI'] dd = r['DD'] - base['DD'] print(f"{r['config']:<22} {r['T']:>5} {r['ROI']:>8.2f} {r['DD']:>7.2f} " f"{r['Calmar']:>7.2f} {dr:>+7.2f} {dd:>+6.2f} {r['scale_mean']:>7.3f}") with open(OUT_FILE, 'w') as f: json.dump({'baseline': base, 'results': results}, f, indent=2) print(f"\nResults: {OUT_FILE}") print("\n=== VERDICT ===") best = max(results[1:], key=lambda x: x['Calmar']) threshold = base['Calmar'] * 1.02 print(f"Best config: [{best['config']}] Calmar={best['Calmar']:.2f} " f"ROI={best['ROI']:+.2f}% DD={best['DD']:.2f}%") print(f"Threshold: Calmar > {threshold:.2f} (1.02× baseline {base['Calmar']:.2f})") if best['Calmar'] > threshold: print(" PROCEED: meaningful Calmar improvement vs baseline") else: print(" MARGINAL or NO improvement over D_LIQ_GOLD") # Direction analysis print("\n=== DIRECTION ANALYSIS ===") d_fwd = next((r for r in results if '4_D_analogue' in r['config']), None) d_inv = next((r for r in results if 'D_inv' in r['config']), None) if d_inv: print(f" D_inv_analogue (−ze): ROI={d_inv['ROI']:+.2f}% " f"Calmar={d_inv['Calmar']:.2f} dROI={d_inv['ROI']-base['ROI']:+.2f}pp") print(f" vs exp10 D_analogue (+ze): approx dROI=−0.57pp") if d_inv['ROI'] - base['ROI'] > 0: print(" CONFIRMED: inverted z_recon direction is POSITIVE — OOD = cut is right") else: print(" INCONCLUSIVE: inverted direction still negative") if __name__ == '__main__': main()