01c19662cb
Includes core prod + GREEN/BLUE subsystems: - prod/ (BLUE harness, configs, scripts, docs) - nautilus_dolphin/ (GREEN Nautilus-native impl + dvae/ preserved) - adaptive_exit/ (AEM engine + models/bucket_assignments.pkl) - Observability/ (EsoF advisor, TUI, dashboards) - external_factors/ (EsoF producer) - mc_forewarning_qlabs_fork/ (MC regime/envelope) Excludes runtime caches, logs, backups, and reproducible artifacts per .gitignore.
404 lines
18 KiB
Python
Executable File
404 lines
18 KiB
Python
Executable File
"""
|
||
exp10_1m_keyframe.py — 1m trajectory keyframe gate test
|
||
=========================================================
|
||
Tests whether instability_150 z-score at 1m timescale (and/or D-VAE recon_err
|
||
at 1m) provides actionable scale modulation ON TOP OF D_LIQ_GOLD (proxy_B boost
|
||
already baked in).
|
||
|
||
Signal forms tested (7 configs):
|
||
0. Baseline — D_LIQ_GOLD unmodified (control)
|
||
1. A_hard — hard threshold on rolling i150 z-score
|
||
2. B_analogue — continuous tanh on rolling i150 z-score
|
||
3. C_hard — hard threshold on VAE recon_err z-score at 1m
|
||
4. D_analogue — continuous tanh on VAE recon_err z-score at 1m
|
||
5. AC_hard — A × C (both hard, multiplicative)
|
||
6. BD_analogue — B × D (both analogue, multiplicative)
|
||
|
||
Analogue formula (split tanh, asymmetric):
|
||
z >= 0: scale = 1 + UP_STRENGTH * tanh(z / K) → [1.0, ~1.15)
|
||
z < 0: scale = 1 + DOWN_STRENGTH * tanh(z / K) → (~0.50, 1.0)
|
||
At z=-1.22: ~0.64; z=-3: ~0.50 floor; z=+1.11: ~1.12; z=+3: ~1.15 ceiling.
|
||
|
||
Zero changes to production code. D_LIQ_GOLD engine forked via subclass.
|
||
"""
|
||
|
||
import sys, time, json, warnings
|
||
sys.stdout.reconfigure(encoding='utf-8', errors='replace')
|
||
warnings.filterwarnings('ignore')
|
||
from pathlib import Path
|
||
import numpy as np
|
||
import pandas as pd
|
||
|
||
ROOT = Path(__file__).parent.parent
|
||
sys.path.insert(0, str(ROOT))
|
||
|
||
from nautilus_dolphin.nautilus.alpha_asset_selector import compute_irp_nb, compute_ars_nb, rank_assets_irp_nb
|
||
from nautilus_dolphin.nautilus.alpha_bet_sizer import compute_sizing_nb
|
||
from nautilus_dolphin.nautilus.alpha_signal_generator import check_dc_nb
|
||
from nautilus_dolphin.nautilus.ob_features import (
|
||
OBFeatureEngine, compute_imbalance_nb, compute_depth_1pct_nb,
|
||
compute_depth_quality_nb, compute_fill_probability_nb, compute_spread_proxy_nb,
|
||
compute_depth_asymmetry_nb, compute_imbalance_persistence_nb,
|
||
compute_withdrawal_velocity_nb, compute_market_agreement_nb, compute_cascade_signal_nb,
|
||
)
|
||
from nautilus_dolphin.nautilus.ob_provider import MockOBProvider
|
||
from nautilus_dolphin.nautilus.adaptive_circuit_breaker import AdaptiveCircuitBreaker
|
||
from nautilus_dolphin.nautilus.proxy_boost_engine import LiquidationGuardEngine, create_d_liq_engine
|
||
from mc.mc_ml import DolphinForewarner
|
||
from dvae.titan_sensor import TitanSensor, build_feature_vector
|
||
|
||
# ── JIT warmup ────────────────────────────────────────────────────────────────
|
||
print("Warming up JIT...")
|
||
_p = np.array([1.,2.,3.], dtype=np.float64)
|
||
compute_irp_nb(_p,-1); compute_ars_nb(1.,.5,.01)
|
||
rank_assets_irp_nb(np.ones((10,2),dtype=np.float64),8,-1,5,500.,20,0.20)
|
||
compute_sizing_nb(-.03,-.02,-.05,3.,.5,5.,.20,True,True,0.,
|
||
np.zeros(4,dtype=np.int64),np.zeros(4,dtype=np.int64),
|
||
np.zeros(5,dtype=np.float64),0,-1,.01,.04)
|
||
check_dc_nb(_p,3,1,.75)
|
||
_b=np.array([100.,200.,300.,400.,500.],dtype=np.float64)
|
||
_a=np.array([110.,190.,310.,390.,510.],dtype=np.float64)
|
||
compute_imbalance_nb(_b,_a); compute_depth_1pct_nb(_b,_a)
|
||
compute_depth_quality_nb(210.,200.); compute_fill_probability_nb(1.)
|
||
compute_spread_proxy_nb(_b,_a); compute_depth_asymmetry_nb(_b,_a)
|
||
compute_imbalance_persistence_nb(np.array([.1,-.1],dtype=np.float64),2)
|
||
compute_withdrawal_velocity_nb(np.array([100.,110.],dtype=np.float64),1)
|
||
compute_market_agreement_nb(np.array([.1,-.05],dtype=np.float64),2)
|
||
compute_cascade_signal_nb(np.array([-.05,-.15],dtype=np.float64),2,-.10)
|
||
print(" JIT ready.")
|
||
|
||
# ── Paths ─────────────────────────────────────────────────────────────────────
|
||
VBT5s = Path(r"C:\Users\Lenovo\Documents\- DOLPHIN NG HD HCM TSF Predict\vbt_cache")
|
||
VBT1m = Path(r"C:\Users\Lenovo\Documents\- DOLPHIN NG HD HCM TSF Predict\vbt_cache_klines")
|
||
MODEL_PATH = Path(r"C:\Users\Lenovo\Documents\- DOLPHIN NG HD HCM TSF Predict\dvae_regime_model_TITAN_ULTRA_GD.json")
|
||
MC_MODELS = str(ROOT / "mc_results" / "models")
|
||
OUT_FILE = Path(__file__).parent / "exp10_1m_keyframe_results.json"
|
||
|
||
META_COLS = {'timestamp','scan_number','v50_lambda_max_velocity','v150_lambda_max_velocity',
|
||
'v300_lambda_max_velocity','v750_lambda_max_velocity','vel_div',
|
||
'instability_50','instability_150'}
|
||
|
||
# Base kwargs accepted by NDAlphaEngine (no boost/leverage-guard params)
|
||
BASE_ENGINE_KWARGS = dict(
|
||
initial_capital=25000., vel_div_threshold=-.02, vel_div_extreme=-.05,
|
||
min_leverage=.5, max_leverage=5., leverage_convexity=3.,
|
||
fraction=.20, fixed_tp_pct=.0095, stop_pct=1., max_hold_bars=120,
|
||
use_direction_confirm=True, dc_lookback_bars=7, dc_min_magnitude_bps=.75,
|
||
dc_skip_contradicts=True, dc_leverage_boost=1., dc_leverage_reduce=.5,
|
||
use_asset_selection=True, min_irp_alignment=.45,
|
||
use_sp_fees=True, use_sp_slippage=True,
|
||
sp_maker_entry_rate=.62, sp_maker_exit_rate=.50,
|
||
use_ob_edge=True, ob_edge_bps=5., ob_confirm_rate=.40,
|
||
lookback=100, use_alpha_layers=True, use_dynamic_leverage=True, seed=42,
|
||
)
|
||
# D_LIQ_GOLD-specific params passed explicitly to KeyframeGateEngine / create_d_liq_engine
|
||
D_LIQ_KWARGS = dict(
|
||
extended_soft_cap=8., extended_abs_cap=9., mc_leverage_ref=5.,
|
||
margin_buffer=.95, threshold=.35, alpha=1., adaptive_beta=True,
|
||
)
|
||
|
||
MC_BASE_CFG = {
|
||
'trial_id':0, 'vel_div_threshold':-.020, 'vel_div_extreme':-.050,
|
||
'use_direction_confirm':True, 'dc_lookback_bars':7, 'dc_min_magnitude_bps':.75,
|
||
'dc_skip_contradicts':True, 'dc_leverage_boost':1.00, 'dc_leverage_reduce':.50,
|
||
'vd_trend_lookback':10, 'min_leverage':.50, 'max_leverage':5.00,
|
||
'leverage_convexity':3.00, 'fraction':.20, 'use_alpha_layers':True,
|
||
'use_dynamic_leverage':True, 'fixed_tp_pct':.0095, 'stop_pct':1.00,
|
||
'max_hold_bars':120, 'use_sp_fees':True, 'use_sp_slippage':True,
|
||
'sp_maker_entry_rate':.62, 'sp_maker_exit_rate':.50, 'use_ob_edge':True,
|
||
'ob_edge_bps':5.00, 'ob_confirm_rate':.40, 'ob_imbalance_bias':-.09,
|
||
'ob_depth_scale':1.00, 'use_asset_selection':True, 'min_irp_alignment':.45,
|
||
'lookback':100, 'acb_beta_high':.80, 'acb_beta_low':.20, 'acb_w750_threshold_pct':60,
|
||
}
|
||
|
||
WINDOW_Z = 30 # rolling bars for z-score (30-min at 1m)
|
||
K_TANH = 1.5 # tanh curvature
|
||
UP_STR = 0.15 # max upside boost
|
||
DOWN_STR = 0.50 # max downside cut
|
||
|
||
# ── Scale functions ───────────────────────────────────────────────────────────
|
||
def hard_scale(z: float) -> float:
|
||
if z < 0.: return 0.5
|
||
if z > 1.11: return 1.15
|
||
return 1.0
|
||
|
||
def analogue_scale(z: float) -> float:
|
||
t = np.tanh(float(z) / K_TANH)
|
||
if z >= 0.:
|
||
return 1.0 + UP_STR * t
|
||
else:
|
||
return 1.0 + DOWN_STR * t
|
||
|
||
# ── Pre-compute 1m signals ────────────────────────────────────────────────────
|
||
def precompute_1m_signals(parquet_files_5s, sensor):
|
||
"""
|
||
For each day, build per-5s-bar arrays of:
|
||
z_roll[bar] : rolling z-score of instability_150 at 1m
|
||
z_recon[bar] : rolling z-score of VAE recon_err at 1m
|
||
Returns dict[date_str] -> {'z_roll': np.ndarray, 'z_recon': np.ndarray}
|
||
"""
|
||
print("Pre-computing 1m signals...")
|
||
signals = {}
|
||
|
||
for pf5 in parquet_files_5s:
|
||
ds = pf5.stem
|
||
pf1 = VBT1m / f"{ds}.parquet"
|
||
if not pf1.exists():
|
||
signals[ds] = None
|
||
continue
|
||
|
||
df5 = pd.read_parquet(pf5)
|
||
df1 = pd.read_parquet(pf1).replace([np.inf,-np.inf], np.nan).fillna(0.)
|
||
n5, n1 = len(df5), len(df1)
|
||
assets = [c for c in df1.columns if c not in META_COLS]
|
||
|
||
# 1m instability_150 rolling z-score
|
||
i150 = df1['instability_150'].values.copy() if 'instability_150' in df1.columns else np.zeros(n1)
|
||
z_roll_1m = np.zeros(n1)
|
||
for j in range(WINDOW_Z, n1):
|
||
seg = i150[max(0,j-WINDOW_Z):j]
|
||
mu, sigma = np.mean(seg), np.std(seg)
|
||
z_roll_1m[j] = (i150[j] - mu) / max(sigma, 1e-10)
|
||
|
||
# 1m VAE recon_err rolling z-score
|
||
recon_1m = np.zeros(n1)
|
||
for j in range(n1):
|
||
feat = build_feature_vector(df1, j, assets)
|
||
_, recon_err, _ = sensor.encode(feat)
|
||
recon_1m[j] = recon_err
|
||
|
||
# z-score recon_err
|
||
z_recon_1m = np.zeros(n1)
|
||
for j in range(WINDOW_Z, n1):
|
||
seg = recon_1m[max(0,j-WINDOW_Z):j]
|
||
mu, sigma = np.mean(seg), np.std(seg)
|
||
z_recon_1m[j] = (recon_1m[j] - mu) / max(sigma, 1e-10)
|
||
|
||
# Map 1m arrays to 5s bar indices (proportional)
|
||
z_roll_5s = np.zeros(n5)
|
||
z_recon_5s = np.zeros(n5)
|
||
for i in range(n5):
|
||
j = int(i * n1 / n5)
|
||
j = min(j, n1-1)
|
||
z_roll_5s[i] = z_roll_1m[j]
|
||
z_recon_5s[i] = z_recon_1m[j]
|
||
|
||
signals[ds] = {'z_roll': z_roll_5s, 'z_recon': z_recon_5s}
|
||
print(f" {ds}: {n1} 1m bars -> {n5} 5s bars "
|
||
f"z_roll=[{z_roll_5s.min():.2f},{z_roll_5s.max():.2f}] "
|
||
f"z_recon=[{z_recon_5s.min():.2f},{z_recon_5s.max():.2f}]")
|
||
|
||
return signals
|
||
|
||
|
||
# ── Keyframe engine subclass ──────────────────────────────────────────────────
|
||
class KeyframeGateEngine(LiquidationGuardEngine):
|
||
"""
|
||
Adds a 1m keyframe scale multiplier on top of D_LIQ_GOLD.
|
||
_pending_1m_scale is set per bar by the backtest loop before _try_entry fires.
|
||
"""
|
||
|
||
def __init__(self, scale_fn, **kwargs):
|
||
super().__init__(**kwargs)
|
||
self._scale_fn = scale_fn # callable(z_roll, z_recon) -> float
|
||
self._bar_z_roll : np.ndarray | None = None
|
||
self._bar_z_recon : np.ndarray | None = None
|
||
self._1m_scale_history: list = []
|
||
|
||
def set_1m_signals(self, z_roll: np.ndarray, z_recon: np.ndarray):
|
||
self._bar_z_roll = z_roll
|
||
self._bar_z_recon = z_recon
|
||
|
||
def _try_entry(self, bar_idx, vel_div, prices, price_histories,
|
||
v50_vel=0., v750_vel=0.):
|
||
result = super()._try_entry(bar_idx, vel_div, prices, price_histories,
|
||
v50_vel, v750_vel)
|
||
if result and self.position is not None:
|
||
zr = float(self._bar_z_roll[bar_idx]) if (self._bar_z_roll is not None and bar_idx < len(self._bar_z_roll)) else 0.
|
||
ze = float(self._bar_z_recon[bar_idx]) if (self._bar_z_recon is not None and bar_idx < len(self._bar_z_recon)) else 0.
|
||
s = float(self._scale_fn(zr, ze))
|
||
s = max(0.2, min(2.0, s)) # hard clip, safety
|
||
self.position.notional *= s
|
||
self._1m_scale_history.append(s)
|
||
return result
|
||
|
||
def reset(self):
|
||
super().reset()
|
||
self._1m_scale_history = []
|
||
|
||
|
||
# ── Scale function definitions ─────────────────────────────────────────────────
|
||
CONFIGS = {
|
||
"0_baseline": None, # no gate
|
||
"1_A_hard": lambda zr, ze: hard_scale(zr),
|
||
"2_B_analogue": lambda zr, ze: analogue_scale(zr),
|
||
"3_C_hard": lambda zr, ze: hard_scale(ze),
|
||
"4_D_analogue": lambda zr, ze: analogue_scale(ze),
|
||
"5_AC_hard": lambda zr, ze: hard_scale(zr) * hard_scale(ze),
|
||
"6_BD_analogue":lambda zr, ze: analogue_scale(zr) * analogue_scale(ze),
|
||
}
|
||
|
||
|
||
# ── Backtest runner ───────────────────────────────────────────────────────────
|
||
def run_one(config_name, scale_fn, parquet_files, pq_data, signals, vol_p60):
|
||
"""Run one backtest config. Returns result dict."""
|
||
OB_ASSETS = sorted({a for ds,(df,ac,_) in pq_data.items() for a in ac})
|
||
_mock_ob = MockOBProvider(
|
||
imbalance_bias=-.09, depth_scale=1., assets=OB_ASSETS,
|
||
imbalance_biases={"BTCUSDT":-.086,"ETHUSDT":-.092,"BNBUSDT":+.05,"SOLUSDT":+.05},
|
||
)
|
||
ob_eng = OBFeatureEngine(_mock_ob)
|
||
ob_eng.preload_date("mock", OB_ASSETS)
|
||
|
||
forewarner = DolphinForewarner(models_dir=MC_MODELS)
|
||
acb = AdaptiveCircuitBreaker()
|
||
acb.preload_w750([pf.stem for pf in parquet_files])
|
||
|
||
if scale_fn is None:
|
||
engine = create_d_liq_engine(**BASE_ENGINE_KWARGS)
|
||
else:
|
||
engine = KeyframeGateEngine(scale_fn=scale_fn, **BASE_ENGINE_KWARGS, **D_LIQ_KWARGS)
|
||
|
||
engine.set_ob_engine(ob_eng)
|
||
engine.set_acb(acb)
|
||
engine.set_mc_forewarner(forewarner, MC_BASE_CFG)
|
||
engine.set_esoteric_hazard_multiplier(0.)
|
||
|
||
t0 = time.time()
|
||
bar_global = 0
|
||
for pf in parquet_files:
|
||
ds = pf.stem
|
||
df, acols, dvol = pq_data[ds]
|
||
vol_ok = np.where(np.isfinite(dvol), dvol > vol_p60, False)
|
||
|
||
if scale_fn is not None and isinstance(engine, KeyframeGateEngine):
|
||
sig = signals.get(ds)
|
||
if sig:
|
||
engine.set_1m_signals(sig['z_roll'], sig['z_recon'])
|
||
else:
|
||
engine.set_1m_signals(np.zeros(len(df)), np.zeros(len(df)))
|
||
|
||
engine.process_day(ds, df, acols, vol_regime_ok=vol_ok)
|
||
bar_global += len(df)
|
||
|
||
elapsed = time.time() - t0
|
||
trades = engine.trade_history
|
||
roi = (engine.capital - 25000.) / 25000. * 100.
|
||
|
||
# drawdown
|
||
cap_curve = [25000.]
|
||
for t_ in sorted(trades, key=lambda x: getattr(x,'exit_bar',0)):
|
||
cap_curve.append(cap_curve[-1] + getattr(t_,'pnl_absolute',0.))
|
||
cap_arr = np.array(cap_curve)
|
||
peak = np.maximum.accumulate(cap_arr)
|
||
dd = float(np.max((peak - cap_arr) / (peak + 1e-10)) * 100.)
|
||
calmar = roi / max(dd, 1e-4)
|
||
|
||
scale_hist = getattr(engine, '_1m_scale_history', [])
|
||
|
||
res = {
|
||
'config': config_name,
|
||
'T': len(trades),
|
||
'ROI': round(roi, 4),
|
||
'DD': round(dd, 4),
|
||
'Calmar': round(calmar, 4),
|
||
'elapsed_s': round(elapsed, 1),
|
||
'scale_mean': round(np.mean(scale_hist), 4) if scale_hist else 1.0,
|
||
'scale_min': round(np.min(scale_hist), 4) if scale_hist else 1.0,
|
||
'scale_max': round(np.max(scale_hist), 4) if scale_hist else 1.0,
|
||
'n_scale_applied': len(scale_hist),
|
||
}
|
||
return res
|
||
|
||
|
||
# ── Main ──────────────────────────────────────────────────────────────────────
|
||
def main():
|
||
parquet_files = sorted(VBT5s.glob("*.parquet"))
|
||
parquet_files = [p for p in parquet_files if 'catalog' not in str(p)]
|
||
print(f"Dataset: {len(parquet_files)} days ({parquet_files[0].stem} to {parquet_files[-1].stem})")
|
||
|
||
print("Loading parquet data...")
|
||
pq_data = {}
|
||
all_assets = set()
|
||
for pf in parquet_files:
|
||
df = pd.read_parquet(pf)
|
||
ac = [c for c in df.columns if c not in META_COLS]
|
||
all_assets.update(ac)
|
||
bp = df['BTCUSDT'].values if 'BTCUSDT' in df.columns else None
|
||
dv = np.full(len(df), np.nan)
|
||
if bp is not None:
|
||
for i in range(50, len(bp)):
|
||
seg = bp[max(0,i-50):i]
|
||
if len(seg)>=10:
|
||
dv[i] = float(np.std(np.diff(seg)/seg[:-1]))
|
||
pq_data[pf.stem] = (df, ac, dv)
|
||
|
||
# vol_p60
|
||
all_vols=[]
|
||
for pf in parquet_files[:2]:
|
||
df=pd.read_parquet(pf)
|
||
if 'BTCUSDT' not in df.columns: continue
|
||
pr=df['BTCUSDT'].values
|
||
for i in range(60,len(pr)):
|
||
seg=pr[max(0,i-50):i]
|
||
if len(seg)>=10:
|
||
v=float(np.std(np.diff(seg)/seg[:-1]))
|
||
if v>0: all_vols.append(v)
|
||
vol_p60 = float(np.percentile(all_vols,60)) if all_vols else 0.
|
||
|
||
# Load sensor
|
||
print(f"\nLoading TitanSensor...")
|
||
sensor = TitanSensor(str(MODEL_PATH))
|
||
|
||
# Pre-compute 1m signals
|
||
signals = precompute_1m_signals(parquet_files, sensor)
|
||
n_missing = sum(1 for v in signals.values() if v is None)
|
||
print(f" 1m signals ready: {len(signals)-n_missing}/{len(signals)} days")
|
||
|
||
# Run all configs
|
||
print()
|
||
results = []
|
||
for name, fn in CONFIGS.items():
|
||
print(f"Running [{name}]...", flush=True)
|
||
r = run_one(name, fn, parquet_files, pq_data, signals, vol_p60)
|
||
results.append(r)
|
||
baseline_roi = results[0]['ROI'] if results else 0.
|
||
delta_roi = r['ROI'] - baseline_roi
|
||
delta_dd = r['DD'] - results[0]['DD'] if len(results)>1 else 0.
|
||
print(f" T={r['T']} ROI={r['ROI']:+.2f}% DD={r['DD']:.2f}% "
|
||
f"Calmar={r['Calmar']:.2f} "
|
||
f"(dROI={delta_roi:+.2f}pp dDD={delta_dd:+.2f}pp) "
|
||
f"scale_mean={r['scale_mean']:.3f} {r['elapsed_s']:.0f}s")
|
||
|
||
# Summary table
|
||
print()
|
||
print("="*90)
|
||
print(f"{'Config':<20} {'T':>5} {'ROI%':>8} {'DD%':>7} {'Calmar':>7} "
|
||
f"{'dROI':>7} {'dDD':>6} {'s_mean':>7}")
|
||
print("-"*90)
|
||
base = results[0]
|
||
for r in results:
|
||
dr = r['ROI'] - base['ROI']
|
||
dd = r['DD'] - base['DD']
|
||
print(f"{r['config']:<20} {r['T']:>5} {r['ROI']:>8.2f} {r['DD']:>7.2f} "
|
||
f"{r['Calmar']:>7.2f} {dr:>+7.2f} {dd:>+6.2f} {r['scale_mean']:>7.3f}")
|
||
|
||
# Save
|
||
with open(OUT_FILE,'w') as f:
|
||
json.dump({'baseline': base, 'results': results}, f, indent=2)
|
||
print(f"\nResults: {OUT_FILE}")
|
||
|
||
# Verdict
|
||
print("\n=== VERDICT ===")
|
||
best = max(results[1:], key=lambda x: x['Calmar'])
|
||
print(f"Best config: [{best['config']}] Calmar={best['Calmar']:.2f} "
|
||
f"ROI={best['ROI']:+.2f}% DD={best['DD']:.2f}%")
|
||
if best['Calmar'] > base['Calmar'] * 1.02:
|
||
print(" PROCEED: meaningful Calmar improvement vs baseline")
|
||
else:
|
||
print(" MARGINAL or NO improvement over D_LIQ_GOLD")
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|