01c19662cb
Includes core prod + GREEN/BLUE subsystems: - prod/ (BLUE harness, configs, scripts, docs) - nautilus_dolphin/ (GREEN Nautilus-native impl + dvae/ preserved) - adaptive_exit/ (AEM engine + models/bucket_assignments.pkl) - Observability/ (EsoF advisor, TUI, dashboards) - external_factors/ (EsoF producer) - mc_forewarning_qlabs_fork/ (MC regime/envelope) Excludes runtime caches, logs, backups, and reproducible artifacts per .gitignore.
685 lines
31 KiB
Python
Executable File
685 lines
31 KiB
Python
Executable File
"""
|
||
exp14_sweep.py — z[13] / z_post_std / resonance-delta leverage gate sweep.
|
||
|
||
Tests three signal families against D_LIQ_GOLD baseline (56-day 5s scan data):
|
||
|
||
Family A — z[13] leverage gate:
|
||
When z[13] (proxy_B dim, always-positive in Dec-Jan 2026) exceeds a threshold,
|
||
cap the effective soft leverage below the D_LIQ 8x default.
|
||
High z[13] = high proxy_B context = expect adverse excursion → be smaller.
|
||
|
||
Family B — z_post_std OOD gate:
|
||
When z_post_std exceeds a threshold (market window is OOD / unusual),
|
||
cap leverage conservatively regardless of direction.
|
||
|
||
Family C — 2D combined gate (z[13] AND z_post_std):
|
||
Both signals gate simultaneously; take the min of their implied caps.
|
||
|
||
Family D — Resonance delta gate:
|
||
delta = live_proxy_B − implied_proxy_B(z[13]) [fitted linear map]
|
||
Three scenarios (see TODO.md exp14):
|
||
delta > thr → MORE turbulent than model expects → scale DOWN
|
||
delta < -thr → calmer than model expects → scale UP (carefully)
|
||
|delta| < resonance_thr → RESONANCE: two sensors agree → MAX CONFIDENCE
|
||
(disproportionately large weight: if both say danger → strong cap,
|
||
if both say calm → allow near-full leverage)
|
||
|
||
Family E — Combined best from A + B + D.
|
||
|
||
Baseline: D_LIQ_GOLD (soft=8x, hard=9x, mc_ref=5x, margin_buffer=0.95)
|
||
ROI=181.81% DD=17.65% Calmar=10.30 T=2155
|
||
|
||
Usage:
|
||
cd nautilus_dolphin/
|
||
python dvae/exp14_sweep.py --subset 14 --top_k 20 # Phase 1 (14-day screening)
|
||
python dvae/exp14_sweep.py --subset 0 --top_k 0 # Phase 2 (full 56 days)
|
||
"""
|
||
|
||
import sys, os, time, json, warnings, argparse
|
||
import io
|
||
sys.stdout = io.TextIOWrapper(sys.stdout.buffer, encoding='utf-8', errors='replace')
|
||
warnings.filterwarnings('ignore')
|
||
|
||
import numpy as np
|
||
import pandas as pd
|
||
from pathlib import Path
|
||
|
||
ROOT = Path(__file__).resolve().parent.parent.parent
|
||
ND_ROOT = ROOT / 'nautilus_dolphin'
|
||
sys.path.insert(0, str(ND_ROOT))
|
||
|
||
from dvae.convnext_sensor import ConvNextSensor
|
||
from nautilus_dolphin.nautilus.proxy_boost_engine import (
|
||
LiquidationGuardEngine,
|
||
D_LIQ_SOFT_CAP, D_LIQ_ABS_CAP, D_LIQ_MC_REF, D_LIQ_MARGIN_BUF,
|
||
create_d_liq_engine,
|
||
)
|
||
from nautilus_dolphin.nautilus.ob_features import (
|
||
OBFeatureEngine, compute_imbalance_nb, compute_depth_1pct_nb,
|
||
compute_depth_quality_nb, compute_fill_probability_nb, compute_spread_proxy_nb,
|
||
compute_depth_asymmetry_nb, compute_imbalance_persistence_nb,
|
||
compute_withdrawal_velocity_nb, compute_market_agreement_nb, compute_cascade_signal_nb,
|
||
)
|
||
from nautilus_dolphin.nautilus.ob_provider import MockOBProvider
|
||
from nautilus_dolphin.nautilus.adaptive_circuit_breaker import AdaptiveCircuitBreaker
|
||
from nautilus_dolphin.nautilus.alpha_asset_selector import compute_irp_nb, compute_ars_nb, rank_assets_irp_nb
|
||
from nautilus_dolphin.nautilus.alpha_bet_sizer import compute_sizing_nb
|
||
from nautilus_dolphin.nautilus.alpha_signal_generator import check_dc_nb
|
||
from mc.mc_ml import DolphinForewarner
|
||
|
||
# ── JIT warmup ────────────────────────────────────────────────────────────────
|
||
print("Warming up JIT...")
|
||
_p = np.array([1., 2., 3.], dtype=np.float64)
|
||
compute_irp_nb(_p, -1); compute_ars_nb(1., .5, .01)
|
||
rank_assets_irp_nb(np.ones((10, 2), dtype=np.float64), 8, -1, 5, 500., 20, 0.20)
|
||
compute_sizing_nb(-.03, -.02, -.05, 3., .5, 5., .20, True, True, 0.,
|
||
np.zeros(4, dtype=np.int64), np.zeros(4, dtype=np.int64),
|
||
np.zeros(5, dtype=np.float64), 0, -1, .01, .04)
|
||
check_dc_nb(_p, 3, 1, .75)
|
||
_b = np.array([100., 200., 300., 400., 500.], dtype=np.float64)
|
||
_a = np.array([110., 190., 310., 390., 510.], dtype=np.float64)
|
||
compute_imbalance_nb(_b, _a); compute_depth_1pct_nb(_b, _a)
|
||
compute_depth_quality_nb(210., 200.); compute_fill_probability_nb(1.)
|
||
compute_spread_proxy_nb(_b, _a); compute_depth_asymmetry_nb(_b, _a)
|
||
compute_imbalance_persistence_nb(np.array([.1, -.1], dtype=np.float64), 2)
|
||
compute_withdrawal_velocity_nb(np.array([100., 110.], dtype=np.float64), 1)
|
||
compute_market_agreement_nb(np.array([.1, -.05], dtype=np.float64), 2)
|
||
compute_cascade_signal_nb(np.array([-.05, -.15], dtype=np.float64), 2, -.10)
|
||
print(" JIT ready.")
|
||
|
||
MODEL_V2 = ND_ROOT / 'dvae' / 'convnext_model_v2.json'
|
||
SCANS_DIR = ROOT / 'vbt_cache'
|
||
KLINES_DIR = ROOT / 'vbt_cache_klines'
|
||
MC_MODELS = str(ROOT / 'nautilus_dolphin' / 'mc_results' / 'models')
|
||
OUT_FILE = ROOT / 'exp14_results.json'
|
||
|
||
META_COLS = {
|
||
'timestamp', 'scan_number',
|
||
'v50_lambda_max_velocity', 'v150_lambda_max_velocity',
|
||
'v300_lambda_max_velocity', 'v750_lambda_max_velocity',
|
||
'vel_div', 'instability_50', 'instability_150',
|
||
}
|
||
|
||
FEATURE_COLS = [
|
||
'v50_lambda_max_velocity','v150_lambda_max_velocity',
|
||
'v300_lambda_max_velocity','v750_lambda_max_velocity',
|
||
'vel_div','instability_50','instability_150',
|
||
]
|
||
|
||
BASE_ENGINE_KWARGS = dict(
|
||
initial_capital=25000., vel_div_threshold=-.02, vel_div_extreme=-.05,
|
||
min_leverage=.5, max_leverage=5., leverage_convexity=3.,
|
||
fraction=.20, fixed_tp_pct=.0095, stop_pct=1., max_hold_bars=120,
|
||
use_direction_confirm=True, dc_lookback_bars=7, dc_min_magnitude_bps=.75,
|
||
dc_skip_contradicts=True, dc_leverage_boost=1., dc_leverage_reduce=.5,
|
||
use_asset_selection=True, min_irp_alignment=.45,
|
||
use_sp_fees=True, use_sp_slippage=True,
|
||
sp_maker_entry_rate=.62, sp_maker_exit_rate=.50,
|
||
use_ob_edge=True, ob_edge_bps=5., ob_confirm_rate=.40,
|
||
lookback=100, use_alpha_layers=True, use_dynamic_leverage=True, seed=42,
|
||
)
|
||
D_LIQ_KWARGS = dict(
|
||
extended_soft_cap=D_LIQ_SOFT_CAP, extended_abs_cap=D_LIQ_ABS_CAP,
|
||
mc_leverage_ref=D_LIQ_MC_REF, margin_buffer=D_LIQ_MARGIN_BUF,
|
||
threshold=.35, alpha=1., adaptive_beta=True,
|
||
)
|
||
MC_BASE_CFG = {
|
||
'trial_id': 0, 'vel_div_threshold': -.020, 'vel_div_extreme': -.050,
|
||
'use_direction_confirm': True, 'dc_lookback_bars': 7, 'dc_min_magnitude_bps': .75,
|
||
'dc_skip_contradicts': True, 'dc_leverage_boost': 1.00, 'dc_leverage_reduce': .50,
|
||
'vd_trend_lookback': 10, 'min_leverage': .50, 'max_leverage': 5.00,
|
||
'leverage_convexity': 3.00, 'fraction': .20, 'use_alpha_layers': True,
|
||
'use_dynamic_leverage': True, 'fixed_tp_pct': .0095, 'stop_pct': 1.00,
|
||
'max_hold_bars': 120, 'use_sp_fees': True, 'use_sp_slippage': True,
|
||
'sp_maker_entry_rate': .62, 'sp_maker_exit_rate': .50, 'use_ob_edge': True,
|
||
'ob_edge_bps': 5.00, 'ob_confirm_rate': .40, 'ob_imbalance_bias': -.09,
|
||
'ob_depth_scale': 1.00, 'use_asset_selection': True, 'min_irp_alignment': .45,
|
||
'lookback': 100, 'acb_beta_high': .80, 'acb_beta_low': .20,
|
||
'acb_w750_threshold_pct': 60,
|
||
}
|
||
T_WIN = 32
|
||
PROXY_B_DIM = 13 # z[13] = proxy_B dim for v2 ep=13 (r=+0.933)
|
||
|
||
# ── ZLeverageGateEngine ──────────────────────────────────────────────────────
|
||
|
||
class ZLeverageGateEngine(LiquidationGuardEngine):
|
||
"""
|
||
LiquidationGuardEngine subclass that modulates the effective soft leverage
|
||
cap based on daily z[13], z_post_std, and resonance-delta signals.
|
||
|
||
Call set_day_signals(z13, z_post_std, delta) before each begin_day().
|
||
"""
|
||
|
||
def __init__(self, *args,
|
||
z13_thr: float = 1.0, # z[13] above → reduce
|
||
z13_scale: float = 0.75, # scale when z13 > thr
|
||
std_thr: float = 99.0, # z_post_std above → reduce
|
||
std_scale: float = 0.75, # scale when std > thr
|
||
delta_thr: float = 99.0, # |delta| threshold (std units)
|
||
delta_danger_scale: float = 0.80, # scale when delta > thr (danger)
|
||
delta_calm_scale: float = 1.00, # scale when delta < -thr (calm)
|
||
resonance_thr: float = 99.0, # |delta| < this → resonance
|
||
resonance_scale: float= 1.00, # scale at resonance
|
||
**kwargs):
|
||
super().__init__(*args, **kwargs)
|
||
self.z13_thr = z13_thr
|
||
self.z13_scale = z13_scale
|
||
self.std_thr = std_thr
|
||
self.std_scale = std_scale
|
||
self.delta_thr = delta_thr
|
||
self.delta_danger_scale = delta_danger_scale
|
||
self.delta_calm_scale = delta_calm_scale
|
||
self.resonance_thr = resonance_thr
|
||
self.resonance_scale = resonance_scale
|
||
# daily signals (set before each begin_day)
|
||
self._z13_today = 0.0
|
||
self._z_std_today = 1.0
|
||
self._delta_today = 0.0
|
||
self._active_scale = 1.0
|
||
|
||
def set_day_signals(self, z13: float, z_post_std: float, delta: float):
|
||
self._z13_today = z13
|
||
self._z_std_today = z_post_std
|
||
self._delta_today = delta
|
||
|
||
def begin_day(self, date_str: str, posture: str = 'APEX', direction=None) -> None:
|
||
super().begin_day(date_str, posture, direction)
|
||
# compute effective scale
|
||
scale = 1.0
|
||
z13 = self._z13_today
|
||
std = self._z_std_today
|
||
d = self._delta_today
|
||
|
||
# Family A: z[13] gate
|
||
if z13 > self.z13_thr:
|
||
scale = min(scale, self.z13_scale)
|
||
|
||
# Family B: OOD gate
|
||
if std > self.std_thr:
|
||
scale = min(scale, self.std_scale)
|
||
|
||
# Family D: resonance-delta gate (three scenarios)
|
||
if self.resonance_thr < 99.0 and abs(d) < self.resonance_thr:
|
||
# RESONANCE: both sensors agree → apply resonance_scale
|
||
# (if resonance_scale < 1 and both say danger, this amplifies caution;
|
||
# if resonance_scale >= 1 both say calm, this restores confidence)
|
||
if z13 > self.z13_thr: # resonance confirms danger
|
||
scale = min(scale, self.resonance_scale)
|
||
else: # resonance confirms calm
|
||
scale = max(scale, self.resonance_scale)
|
||
elif self.delta_thr < 99.0:
|
||
if d > self.delta_thr:
|
||
scale = min(scale, self.delta_danger_scale) # Scenario 1: danger
|
||
elif d < -self.delta_thr:
|
||
scale = max(scale, self.delta_calm_scale) # Scenario 2: calm
|
||
|
||
self._active_scale = scale
|
||
gated = max(self._extended_soft_cap * scale, 1.0)
|
||
self.bet_sizer.max_leverage = gated
|
||
self.base_max_leverage = gated
|
||
|
||
def reset(self):
|
||
super().reset()
|
||
self._z13_today = 0.0
|
||
self._z_std_today = 1.0
|
||
self._delta_today = 0.0
|
||
self._active_scale = 1.0
|
||
|
||
|
||
# ── Config generation ────────────────────────────────────────────────────────
|
||
|
||
def build_configs():
|
||
cfgs = []
|
||
|
||
# Family A: z[13] gate only
|
||
for z13_thr in [0.5, 0.8, 1.0, 1.2]:
|
||
for scale in [0.60, 0.70, 0.80, 0.90]:
|
||
cfgs.append(dict(
|
||
name=f'A_z13t{z13_thr}_s{scale}',
|
||
z13_thr=z13_thr, z13_scale=scale,
|
||
std_thr=99.0, std_scale=1.0,
|
||
delta_thr=99.0, resonance_thr=99.0,
|
||
))
|
||
|
||
# Family B: z_post_std OOD gate only
|
||
for std_thr in [0.90, 1.00, 1.10, 1.20, 1.50]:
|
||
for scale in [0.60, 0.70, 0.80, 0.90]:
|
||
cfgs.append(dict(
|
||
name=f'B_stdt{std_thr}_s{scale}',
|
||
z13_thr=99.0, z13_scale=1.0,
|
||
std_thr=std_thr, std_scale=scale,
|
||
delta_thr=99.0, resonance_thr=99.0,
|
||
))
|
||
|
||
# Family C: 2D combined (z[13] + z_post_std)
|
||
for z13_thr in [0.8, 1.0]:
|
||
for std_thr in [1.0, 1.2]:
|
||
for scale in [0.70, 0.80]:
|
||
cfgs.append(dict(
|
||
name=f'C_z13t{z13_thr}_stdt{std_thr}_s{scale}',
|
||
z13_thr=z13_thr, z13_scale=scale,
|
||
std_thr=std_thr, std_scale=scale,
|
||
delta_thr=99.0, resonance_thr=99.0,
|
||
))
|
||
|
||
# Family D: delta gate (3 scenarios)
|
||
for delta_thr in [0.25, 0.50, 1.00]:
|
||
for dscale in [0.70, 0.80, 0.90]:
|
||
# With resonance: |delta| < 0.2*delta_thr → resonance
|
||
res_thr = delta_thr * 0.25
|
||
cfgs.append(dict(
|
||
name=f'D_dt{delta_thr}_ds{dscale}_res{res_thr:.2f}',
|
||
z13_thr=99.0, z13_scale=1.0,
|
||
std_thr=99.0, std_scale=1.0,
|
||
delta_thr=delta_thr,
|
||
delta_danger_scale=dscale,
|
||
delta_calm_scale=1.0,
|
||
resonance_thr=res_thr,
|
||
resonance_scale=dscale, # resonance confirms danger → same scale
|
||
))
|
||
# Without resonance distinction
|
||
cfgs.append(dict(
|
||
name=f'D_dt{delta_thr}_ds{dscale}_nores',
|
||
z13_thr=99.0, z13_scale=1.0,
|
||
std_thr=99.0, std_scale=1.0,
|
||
delta_thr=delta_thr,
|
||
delta_danger_scale=dscale,
|
||
delta_calm_scale=1.0,
|
||
resonance_thr=99.0,
|
||
))
|
||
|
||
# Family E: combined A + B + D (best-guess params)
|
||
for z13_thr, std_thr, delta_thr, scale in [
|
||
(0.8, 1.0, 0.5, 0.75),
|
||
(1.0, 1.2, 0.5, 0.80),
|
||
(0.8, 1.0, 0.25, 0.70),
|
||
(1.0, 1.0, 0.5, 0.75),
|
||
]:
|
||
cfgs.append(dict(
|
||
name=f'E_z{z13_thr}_s{std_thr}_d{delta_thr}_sc{scale}',
|
||
z13_thr=z13_thr, z13_scale=scale,
|
||
std_thr=std_thr, std_scale=scale,
|
||
delta_thr=delta_thr,
|
||
delta_danger_scale=scale,
|
||
delta_calm_scale=1.0,
|
||
resonance_thr=delta_thr * 0.25,
|
||
resonance_scale=scale,
|
||
))
|
||
|
||
return cfgs
|
||
|
||
|
||
# ── Signal precomputation ────────────────────────────────────────────────────
|
||
|
||
def precompute_daily_signals(parquet_files_1m, sensor: ConvNextSensor):
|
||
"""
|
||
For each daily klines file, compute:
|
||
z13_mean — mean z[13] over all T_WIN windows in the day
|
||
z_std_mean — mean z_post_std
|
||
proxy_b_mean — mean raw proxy_B (instability_50 - v750_lambda_max_velocity)
|
||
Returns dict: date_str → {z13, z_post_std, proxy_b_raw}
|
||
"""
|
||
daily = {}
|
||
for f in parquet_files_1m:
|
||
date_str = Path(f).stem[:10]
|
||
try:
|
||
df = pd.read_parquet(f, columns=FEATURE_COLS).dropna()
|
||
if len(df) < T_WIN + 5:
|
||
continue
|
||
z13_vals, std_vals, pb_vals = [], [], []
|
||
for start in range(0, len(df) - T_WIN, T_WIN // 2):
|
||
window = df.iloc[start:start + T_WIN]
|
||
if len(window) < T_WIN:
|
||
continue
|
||
pb_val = float((window['instability_50'] - window['v750_lambda_max_velocity']).mean())
|
||
try:
|
||
z_mu, z_post_std = sensor.encode_window(df, start + T_WIN)
|
||
z13_vals.append(float(z_mu[PROXY_B_DIM]))
|
||
std_vals.append(float(z_post_std))
|
||
pb_vals.append(pb_val)
|
||
except Exception:
|
||
pass
|
||
if z13_vals:
|
||
daily[date_str] = {
|
||
'z13': float(np.mean(z13_vals)),
|
||
'z_post_std': float(np.mean(std_vals)),
|
||
'proxy_b_raw': float(np.mean(pb_vals)),
|
||
}
|
||
except Exception as e:
|
||
pass
|
||
return daily
|
||
|
||
|
||
def fit_delta_regression(daily_signals: dict):
|
||
"""
|
||
Fit linear map: proxy_b_raw = a * z13 + b
|
||
Returns (a, b, delta_std) — delta_std used to normalise delta to z-score units.
|
||
"""
|
||
dates = sorted(daily_signals.keys())
|
||
z13 = np.array([daily_signals[d]['z13'] for d in dates])
|
||
pb = np.array([daily_signals[d]['proxy_b_raw'] for d in dates])
|
||
# OLS
|
||
A = np.column_stack([z13, np.ones(len(z13))])
|
||
result = np.linalg.lstsq(A, pb, rcond=None)
|
||
a, b = result[0]
|
||
pb_hat = a * z13 + b
|
||
delta_raw = pb - pb_hat
|
||
delta_std = float(np.std(delta_raw)) if len(delta_raw) > 2 else 1.0
|
||
print(f" Delta regression: proxy_B = {a:.4f}*z[13] + {b:.4f} "
|
||
f"r={float(np.corrcoef(z13, pb)[0,1]):.4f} delta_std={delta_std:.4f}")
|
||
return float(a), float(b), delta_std
|
||
|
||
|
||
def add_delta(daily_signals: dict, a: float, b: float, delta_std: float):
|
||
"""Add normalised delta (z-score units) to each day's signals."""
|
||
for d, v in daily_signals.items():
|
||
raw_delta = v['proxy_b_raw'] - (a * v['z13'] + b)
|
||
v['delta'] = raw_delta / (delta_std + 1e-9) # normalised
|
||
|
||
|
||
# ── pq_data / vol helpers (same pattern as exp13) ────────────────────────────
|
||
|
||
def _load_pq_data(parquet_files):
|
||
"""Load all 5s parquet files into pq_data dict (date_str → (df, acols, dvol))."""
|
||
print("Loading 5s parquet data...")
|
||
pq_data = {}
|
||
for pf in parquet_files:
|
||
pf = Path(pf)
|
||
df = pd.read_parquet(pf)
|
||
ac = [c for c in df.columns if c not in META_COLS]
|
||
bp = df['BTCUSDT'].values if 'BTCUSDT' in df.columns else None
|
||
dv = np.full(len(df), np.nan)
|
||
if bp is not None:
|
||
for i in range(50, len(bp)):
|
||
seg = bp[max(0, i - 50):i]
|
||
if len(seg) >= 10:
|
||
dv[i] = float(np.std(np.diff(seg) / seg[:-1]))
|
||
pq_data[pf.stem] = (df, ac, dv)
|
||
print(f" Loaded {len(pq_data)} days")
|
||
return pq_data
|
||
|
||
|
||
def _compute_vol_p60(parquet_files):
|
||
pq = _load_pq_data(parquet_files[:2]) if parquet_files else {}
|
||
vols = []
|
||
for _, (_, _, dv) in pq.items():
|
||
vols.extend(dv[np.isfinite(dv)].tolist())
|
||
return float(np.percentile(vols, 60)) if vols else 0.0
|
||
|
||
|
||
def _make_ob_acb(parquet_files_paths, pq_data: dict):
|
||
"""Create fresh OBFeatureEngine + ACB + Forewarner combo for one run."""
|
||
pf_list = [Path(p) for p in parquet_files_paths]
|
||
OB_ASSETS = sorted({a for ds, (_, ac, _) in pq_data.items() for a in ac})
|
||
if not OB_ASSETS:
|
||
OB_ASSETS = ['BTCUSDT', 'ETHUSDT', 'BNBUSDT', 'SOLUSDT']
|
||
mock_ob = MockOBProvider(
|
||
imbalance_bias=-.09, depth_scale=1., assets=OB_ASSETS,
|
||
imbalance_biases={
|
||
"BTCUSDT": -.086, "ETHUSDT": -.092,
|
||
"BNBUSDT": +.05, "SOLUSDT": +.05,
|
||
},
|
||
)
|
||
ob_eng = OBFeatureEngine(mock_ob)
|
||
ob_eng.preload_date("mock", OB_ASSETS)
|
||
forewarner = DolphinForewarner(models_dir=MC_MODELS)
|
||
acb = AdaptiveCircuitBreaker()
|
||
acb.preload_w750([pf.stem for pf in pf_list])
|
||
return ob_eng, acb, forewarner
|
||
|
||
|
||
def _compute_metrics(engine, elapsed):
|
||
"""Extract ROI/DD/Calmar/T from a finished engine."""
|
||
trades = engine.trade_history
|
||
roi = (engine.capital - 25000.) / 25000. * 100.
|
||
cap_curve = [25000.]
|
||
for t_ in sorted(trades, key=lambda x: getattr(x, 'exit_bar', 0)):
|
||
cap_curve.append(cap_curve[-1] + getattr(t_, 'pnl_absolute', 0.))
|
||
cap_arr = np.array(cap_curve)
|
||
peak = np.maximum.accumulate(cap_arr)
|
||
dd = float(np.max((peak - cap_arr) / (peak + 1e-10)) * 100.)
|
||
calmar = roi / max(dd, 1e-4)
|
||
sh = getattr(engine, '_scale_history', [])
|
||
return {
|
||
'T': len(trades),
|
||
'roi': round(roi, 4),
|
||
'dd': round(dd, 4),
|
||
'calmar': round(calmar, 4),
|
||
'elapsed_s': round(elapsed, 1),
|
||
'scale_mean': round(float(np.mean(sh)), 4) if sh else 1.0,
|
||
}
|
||
|
||
|
||
# ── Single config runner ──────────────────────────────────────────────────────
|
||
|
||
def run_one(cfg: dict, daily_signals: dict, pq_data: dict,
|
||
parquet_files: list, vol_p60: float,
|
||
subset_days: int = 0) -> dict:
|
||
"""Run ZLeverageGateEngine for one config on pre-loaded pq_data."""
|
||
files = [Path(f) for f in parquet_files]
|
||
if subset_days > 0:
|
||
files = files[:subset_days]
|
||
|
||
ob_eng, acb, forewarner = _make_ob_acb([str(f) for f in files], pq_data)
|
||
|
||
engine = ZLeverageGateEngine(
|
||
**BASE_ENGINE_KWARGS,
|
||
**D_LIQ_KWARGS,
|
||
z13_thr = cfg.get('z13_thr', 99.0),
|
||
z13_scale = cfg.get('z13_scale', 1.0),
|
||
std_thr = cfg.get('std_thr', 99.0),
|
||
std_scale = cfg.get('std_scale', 1.0),
|
||
delta_thr = cfg.get('delta_thr', 99.0),
|
||
delta_danger_scale = cfg.get('delta_danger_scale', 1.0),
|
||
delta_calm_scale = cfg.get('delta_calm_scale', 1.0),
|
||
resonance_thr = cfg.get('resonance_thr', 99.0),
|
||
resonance_scale = cfg.get('resonance_scale', 1.0),
|
||
)
|
||
engine.set_ob_engine(ob_eng)
|
||
engine.set_acb(acb)
|
||
engine.set_mc_forewarner(forewarner, MC_BASE_CFG)
|
||
engine.set_esoteric_hazard_multiplier(0.)
|
||
|
||
t0 = time.time()
|
||
for pf in files:
|
||
ds = pf.stem
|
||
if ds not in pq_data:
|
||
continue
|
||
df, acols, dvol = pq_data[ds]
|
||
vol_ok = np.where(np.isfinite(dvol), dvol > vol_p60, False)
|
||
sig = daily_signals.get(ds, {})
|
||
engine.set_day_signals(
|
||
z13 = sig.get('z13', 0.0),
|
||
z_post_std = sig.get('z_post_std', 1.0),
|
||
delta = sig.get('delta', 0.0),
|
||
)
|
||
engine.process_day(ds, df, acols, vol_regime_ok=vol_ok)
|
||
|
||
return _compute_metrics(engine, time.time() - t0)
|
||
|
||
|
||
# ── Baseline runner ───────────────────────────────────────────────────────────
|
||
|
||
def run_baseline(pq_data: dict, parquet_files: list, vol_p60: float,
|
||
subset_days: int = 0) -> dict:
|
||
"""Run D_LIQ_GOLD baseline (no gate) on pre-loaded pq_data."""
|
||
files = [Path(f) for f in parquet_files]
|
||
if subset_days > 0:
|
||
files = files[:subset_days]
|
||
|
||
ob_eng, acb, forewarner = _make_ob_acb([str(f) for f in files], pq_data)
|
||
engine = create_d_liq_engine(**BASE_ENGINE_KWARGS)
|
||
engine.set_ob_engine(ob_eng)
|
||
engine.set_acb(acb)
|
||
engine.set_mc_forewarner(forewarner, MC_BASE_CFG)
|
||
engine.set_esoteric_hazard_multiplier(0.)
|
||
|
||
t0 = time.time()
|
||
for pf in files:
|
||
ds = pf.stem
|
||
if ds not in pq_data:
|
||
continue
|
||
df, acols, dvol = pq_data[ds]
|
||
vol_ok = np.where(np.isfinite(dvol), dvol > vol_p60, False)
|
||
engine.process_day(ds, df, acols, vol_regime_ok=vol_ok)
|
||
|
||
return _compute_metrics(engine, time.time() - t0)
|
||
|
||
|
||
# ── Main ─────────────────────────────────────────────────────────────────────
|
||
|
||
def main():
|
||
ap = argparse.ArgumentParser()
|
||
ap.add_argument('--subset', type=int, default=14,
|
||
help='Days for Phase 1 screening (0=full 56 days)')
|
||
ap.add_argument('--top_k', type=int, default=20,
|
||
help='Top-K configs to validate in Phase 2 (0=skip Phase 2)')
|
||
args = ap.parse_args()
|
||
|
||
print(f"exp14_sweep model=v2(ep=13) subset={args.subset} top_k={args.top_k}")
|
||
print(f" PROXY_B_DIM={PROXY_B_DIM} Families: A(z13) B(std) C(2D) D(delta) E(combined)")
|
||
|
||
# ── Load sensor ──────────────────────────────────────────────────────────
|
||
print(f"\nLoading v2 sensor from {MODEL_V2.name}...")
|
||
assert MODEL_V2.exists(), f"Model not found: {MODEL_V2}"
|
||
sensor = ConvNextSensor(str(MODEL_V2))
|
||
print(f" ep={sensor.epoch} val={sensor.val_loss:.4f} z_dim={sensor.z_dim}")
|
||
|
||
# ── Load data ─────────────────────────────────────────────────────────────
|
||
print("\nLoading data files...")
|
||
scans_5s = sorted(Path(SCANS_DIR).glob('*.parquet'))
|
||
klines_1m = sorted(Path(KLINES_DIR).glob('*.parquet'))
|
||
# align to same 56-day window (2025-12-31 to 2026-02-25)
|
||
scans_5s = [f for f in scans_5s if '2025-12-31' <= f.stem[:10] <= '2026-02-25']
|
||
klines_1m = [f for f in klines_1m if '2025-12-31' <= f.stem[:10] <= '2026-02-25']
|
||
print(f" 5s scans: {len(scans_5s)} 1m klines: {len(klines_1m)}")
|
||
|
||
# ── Pre-load pq_data (once, reused for every run) ─────────────────────────
|
||
print("\nPre-loading 5s parquet data (done once for all runs)...")
|
||
pq_data_full = _load_pq_data([str(f) for f in scans_5s])
|
||
# vol_p60 from full dataset
|
||
all_vols = []
|
||
for _, (_, _, dv) in pq_data_full.items():
|
||
all_vols.extend(dv[np.isfinite(dv)].tolist())
|
||
vol_p60 = float(np.percentile(all_vols, 60)) if all_vols else 0.0
|
||
print(f" vol_p60={vol_p60:.6f}")
|
||
|
||
# ── Precompute daily signals ──────────────────────────────────────────────
|
||
print("\nPrecomputing daily z[13] / z_post_std signals from 1m klines...")
|
||
t0 = time.time()
|
||
daily_sigs = precompute_daily_signals([str(f) for f in klines_1m], sensor)
|
||
print(f" {len(daily_sigs)} days with signals ({time.time()-t0:.0f}s)")
|
||
|
||
print("\nFitting delta regression (proxy_B = a*z[13] + b)...")
|
||
a, b, delta_std = fit_delta_regression(daily_sigs)
|
||
add_delta(daily_sigs, a, b, delta_std)
|
||
deltas = [v['delta'] for v in daily_sigs.values()]
|
||
print(f" delta stats: mean={np.mean(deltas):+.3f} std={np.std(deltas):.3f} "
|
||
f"min={np.min(deltas):+.3f} max={np.max(deltas):+.3f}")
|
||
|
||
# ── Build configs ─────────────────────────────────────────────────────────
|
||
configs = build_configs()
|
||
print(f"\n{len(configs)} configs across 5 families")
|
||
|
||
# ── Baseline ──────────────────────────────────────────────────────────────
|
||
print("\nRunning baseline (D_LIQ_GOLD)...")
|
||
t0 = time.time()
|
||
baseline = run_baseline(pq_data_full, [str(f) for f in scans_5s], vol_p60, args.subset)
|
||
bROI = baseline.get('roi', 0.0)
|
||
bDD = baseline.get('dd', 0.0)
|
||
bCal = baseline.get('calmar', 0.0)
|
||
bT = baseline.get('T', 0)
|
||
print(f" Baseline: T={bT} ROI={bROI:.2f}% DD={bDD:.2f}% Calmar={bCal:.2f} "
|
||
f"({time.time()-t0:.0f}s)")
|
||
|
||
# ── Phase 1: screening ───────────────────────────────────────────────────
|
||
print(f"\n{'='*65}")
|
||
print(f"Phase 1 — screening {len(configs)} configs on {args.subset or 56}-day window")
|
||
print(f"{'='*65}")
|
||
|
||
results = []
|
||
for i, cfg in enumerate(configs):
|
||
t0 = time.time()
|
||
res = run_one(cfg, daily_sigs, pq_data_full, [str(f) for f in scans_5s],
|
||
vol_p60, args.subset)
|
||
roi = res.get('roi', 0.0)
|
||
dd = res.get('dd', 0.0)
|
||
cal = res.get('calmar', 0.0)
|
||
T = res.get('T', 0)
|
||
dROI = roi - bROI
|
||
dDD = dd - bDD
|
||
dCal = cal - bCal
|
||
elapsed = time.time() - t0
|
||
print(f"[{i+1:3d}/{len(configs)}] {cfg['name']}")
|
||
print(f" T={T} ROI={roi:.2f}% DD={dd:.2f}% Calmar={cal:.2f} "
|
||
f"dROI={dROI:+.2f}pp dDD={dDD:+.2f}pp dCal={dCal:+.2f} "
|
||
f"({elapsed:.0f}s)")
|
||
results.append({**cfg, 'roi': roi, 'dd': dd, 'calmar': cal, 'trades': T,
|
||
'dROI': dROI, 'dDD': dDD, 'dCal': dCal})
|
||
|
||
results.sort(key=lambda x: x['dROI'], reverse=True)
|
||
print(f"\nPhase 1 Top 10:")
|
||
for r in results[:10]:
|
||
print(f" dROI={r['dROI']:+.2f}pp ROI={r['roi']:.2f}% "
|
||
f"Cal={r['calmar']:.2f} {r['name']}")
|
||
|
||
# ── Phase 2: full validation ─────────────────────────────────────────────
|
||
if args.top_k > 0 and args.subset > 0:
|
||
top_cfgs = results[:args.top_k]
|
||
print(f"\n{'='*65}")
|
||
print(f"Phase 2 — validating top {len(top_cfgs)} configs on FULL 56 days")
|
||
print(f"{'='*65}")
|
||
|
||
print("\nRunning baseline (full 56 days)...")
|
||
t0 = time.time()
|
||
base_full = run_baseline(pq_data_full, [str(f) for f in scans_5s], vol_p60, 0)
|
||
bROI_f = base_full.get('roi', 0.0)
|
||
bDD_f = base_full.get('dd', 0.0)
|
||
bCal_f = base_full.get('calmar', 0.0)
|
||
bT_f = base_full.get('T', 0)
|
||
print(f" Baseline full: T={bT_f} ROI={bROI_f:.2f}% DD={bDD_f:.2f}% "
|
||
f"Calmar={bCal_f:.2f} ({time.time()-t0:.0f}s)")
|
||
|
||
p2_results = []
|
||
for i, cfg in enumerate(top_cfgs):
|
||
t0 = time.time()
|
||
res = run_one(cfg, daily_sigs, pq_data_full, [str(f) for f in scans_5s],
|
||
vol_p60, 0)
|
||
roi = res.get('roi', 0.0)
|
||
dd = res.get('dd', 0.0)
|
||
cal = res.get('calmar', 0.0)
|
||
T = res.get('T', 0)
|
||
dROI = roi - bROI_f
|
||
dDD = dd - bDD_f
|
||
dCal = cal - bCal_f
|
||
elapsed = time.time() - t0
|
||
print(f"[P2 {i+1:2d}/{len(top_cfgs)}] {cfg['name']}")
|
||
print(f" T={T} ROI={roi:.2f}% DD={dd:.2f}% Calmar={cal:.2f} "
|
||
f"dROI={dROI:+.2f}pp dDD={dDD:+.2f}pp dCal={dCal:+.2f} "
|
||
f"({elapsed:.0f}s)")
|
||
p2_results.append({**cfg, 'roi': roi, 'dd': dd, 'calmar': cal,
|
||
'trades': T, 'dROI': dROI, 'dDD': dDD, 'dCal': dCal})
|
||
|
||
p2_results.sort(key=lambda x: x['dROI'], reverse=True)
|
||
print(f"\nPhase 2 Final Ranking:")
|
||
for r in p2_results[:10]:
|
||
beat = r['calmar'] > bCal_f * 1.02
|
||
print(f" dROI={r['dROI']:+.2f}pp dCal={r['dCal']:+.2f} "
|
||
f"{'✓ BEATS' if beat else '✗'} baseline {r['name']}")
|
||
|
||
# Save results
|
||
out = {
|
||
'baseline_full': {'roi': bROI_f, 'dd': bDD_f, 'calmar': bCal_f, 'trades': bT_f},
|
||
'phase2': p2_results,
|
||
'delta_regression': {'a': a, 'b': b, 'delta_std': delta_std},
|
||
}
|
||
out_path = ROOT / 'exp14_results.json'
|
||
json.dump(out, open(out_path, 'w'), indent=2)
|
||
print(f"\nResults saved to {out_path}")
|
||
|
||
print(f"\n[DONE]")
|
||
|
||
|
||
if __name__ == '__main__':
|
||
main()
|