Files
DOLPHIN/nautilus_dolphin/test_dynamic_tp_experiment.py

477 lines
22 KiB
Python
Raw Normal View History

"""Dynamic TP Experiment — per-trade TP variation strategies.
Background
----------
Noise experiment (2026-03-05) showed fixed TP=99bps is sub-optimal:
96% of seeds in ±3bp band beat baseline (+4.3% ΔROI, std=2.17%).
Mechanism: bar-close execution already gives +22bps average overshoot (mean TP exit = 1.21%
not 0.99%). Trades that hit TP at ~53 bars average still have 67 bars of slack momentum
continues past 99bps more often than it reverses.
Three TP strategies tested (all per-trade, not per-run):
S0 baseline fixed 99bps for all trades (deterministic reference)
S1a random_3bp Uniform[96.5, 102.5]bps per trade (anti-detection, ±3bp)
S1b random_7bp Uniform[92, 106]bps per trade (wider anti-detection radius)
S2a eigen_2bp TP += 2bp * signal_strength (stronger vel_div wider TP)
S2b eigen_4bp TP += 4bp * signal_strength
S2c eigen_6bp TP += 6bp * signal_strength
S3a regime_boost TP += 5bp * (acb_boost - 1.0) (high-boost day wider TP)
S3b regime_vol TP += 3bp * (regime_mult - 1.0) (high-regime wider TP)
S4 combined eigen_3bp + regime_2bp + random_jitter_1bp (meta-bias)
Logging
-------
Per-trade CSV: all engine state + entry features + tp_target_used + tp_strategy
Daily CSV: date, pnl, capital, boost, beta, mc_status
Summary JSON: all hyperparams + stats per strategy
Branch: experiment/dynamic-tp (no modifications to core engine files)
"""
import sys, time, math, json, csv
sys.stdout.reconfigure(encoding='utf-8', errors='replace')
from pathlib import Path
from datetime import datetime
from typing import Optional, Dict, List
import numpy as np
import pandas as pd
HCM = Path(r"C:\Users\Lenovo\Documents\- DOLPHIN NG HD HCM TSF Predict")
sys.path.insert(0, str(HCM / "nautilus_dolphin"))
VBT_DIR = HCM / "vbt_cache"
MC_MODELS_DIR = str(HCM / "nautilus_dolphin" / "mc_results" / "models")
LOG_DIR = HCM / "nautilus_dolphin" / "run_logs"
LOG_DIR.mkdir(exist_ok=True)
# ── champion engine config (exact) ──────────────────────────────────────────
META_COLS = {'timestamp', 'scan_number', 'v50_lambda_max_velocity',
'v150_lambda_max_velocity', 'v300_lambda_max_velocity',
'v750_lambda_max_velocity', 'vel_div', 'instability_50', 'instability_150'}
BASE_ENGINE_KWARGS = dict(
initial_capital=25000.0, vel_div_threshold=-0.02, vel_div_extreme=-0.05,
min_leverage=0.5, max_leverage=5.0, leverage_convexity=3.0,
fraction=0.20, fixed_tp_pct=0.0099, stop_pct=1.0, max_hold_bars=120,
use_direction_confirm=True, dc_lookback_bars=7, dc_min_magnitude_bps=0.75,
dc_skip_contradicts=True, dc_leverage_boost=1.0, dc_leverage_reduce=0.5,
use_asset_selection=True, min_irp_alignment=0.45,
use_sp_fees=True, use_sp_slippage=True,
sp_maker_entry_rate=0.62, sp_maker_exit_rate=0.50,
use_ob_edge=True, ob_edge_bps=5.0, ob_confirm_rate=0.40,
lookback=100, use_alpha_layers=True, use_dynamic_leverage=True, seed=42,
)
MC_BASE_CFG = {
'trial_id': 0, 'vel_div_threshold': -0.020, 'vel_div_extreme': -0.050,
'use_direction_confirm': True, 'dc_lookback_bars': 7,
'dc_min_magnitude_bps': 0.75, 'dc_skip_contradicts': True,
'dc_leverage_boost': 1.00, 'dc_leverage_reduce': 0.50,
'vd_trend_lookback': 10, 'min_leverage': 0.50, 'max_leverage': 5.00,
'leverage_convexity': 3.00, 'fraction': 0.20,
'use_alpha_layers': True, 'use_dynamic_leverage': True,
'fixed_tp_pct': 0.0099, 'stop_pct': 1.00, 'max_hold_bars': 120,
'use_sp_fees': True, 'use_sp_slippage': True,
'sp_maker_entry_rate': 0.62, 'sp_maker_exit_rate': 0.50,
'use_ob_edge': True, 'ob_edge_bps': 5.00, 'ob_confirm_rate': 0.40,
'ob_imbalance_bias': -0.09, 'ob_depth_scale': 1.00,
'use_asset_selection': True, 'min_irp_alignment': 0.45, 'lookback': 100,
'acb_beta_high': 0.80, 'acb_beta_low': 0.20, 'acb_w750_threshold_pct': 60,
}
# ── JIT warmup ───────────────────────────────────────────────────────────────
print("JIT warmup...", end='', flush=True)
t_jit = time.time()
from nautilus_dolphin.nautilus.alpha_asset_selector import compute_irp_nb, compute_ars_nb, rank_assets_irp_nb
from nautilus_dolphin.nautilus.alpha_bet_sizer import compute_sizing_nb
from nautilus_dolphin.nautilus.alpha_signal_generator import check_dc_nb
from nautilus_dolphin.nautilus.ob_features import (
OBFeatureEngine, compute_imbalance_nb, compute_depth_1pct_nb,
compute_depth_quality_nb, compute_fill_probability_nb, compute_spread_proxy_nb,
compute_depth_asymmetry_nb, compute_imbalance_persistence_nb,
compute_withdrawal_velocity_nb, compute_market_agreement_nb, compute_cascade_signal_nb,
)
from nautilus_dolphin.nautilus.ob_provider import MockOBProvider
_p = np.array([1.0, 2.0, 3.0], dtype=np.float64)
compute_irp_nb(_p, -1); compute_ars_nb(1.0, 0.5, 0.01)
rank_assets_irp_nb(np.ones((10, 2), dtype=np.float64), 8, -1, 5, 500.0, 20, 0.20)
compute_sizing_nb(-0.03, -0.02, -0.05, 3.0, 0.5, 5.0, 0.20, True, True, 0.0,
np.zeros(4, np.int64), np.zeros(4, np.int64), np.zeros(5, np.float64), 0, -1, 0.01, 0.04)
check_dc_nb(_p, 3, 1, 0.75)
_b = np.array([100., 200., 300., 400., 500.], dtype=np.float64)
_a = np.array([110., 190., 310., 390., 510.], dtype=np.float64)
compute_imbalance_nb(_b, _a); compute_depth_1pct_nb(_b, _a)
compute_depth_quality_nb(210., 200.); compute_fill_probability_nb(1.0)
compute_spread_proxy_nb(_b, _a); compute_depth_asymmetry_nb(_b, _a)
compute_imbalance_persistence_nb(np.array([0.1, -0.1], dtype=np.float64), 2)
compute_withdrawal_velocity_nb(np.array([100., 110.], dtype=np.float64), 1)
compute_market_agreement_nb(np.array([0.1, -0.05], dtype=np.float64), 2)
compute_cascade_signal_nb(np.array([-0.05, -0.15], dtype=np.float64), 2, -0.10)
print(f" {time.time()-t_jit:.1f}s")
from nautilus_dolphin.nautilus.esf_alpha_orchestrator import NDAlphaEngine
from nautilus_dolphin.nautilus.adaptive_circuit_breaker import AdaptiveCircuitBreaker
from mc.mc_ml import DolphinForewarner
# ── DynamicTPEngine — subclass, no core file modification ───────────────────
class DynamicTPEngine(NDAlphaEngine):
"""NDAlphaEngine subclass that applies per-trade dynamic TP strategies.
Injection point: overrides _try_entry() to set exit_manager.fixed_tp_pct
immediately before position open. Restores base TP afterward.
Records per-trade: tp_target, entry_vel_div, entry_boost, entry_beta,
entry_regime_mult, entry_leverage, entry_v50_vel, entry_v750_vel.
"""
def configure_tp_strategy(self, strategy: str, params: dict, seed: int = 0):
self._tp_strategy = strategy
self._tp_params = params
self._base_tp = self.exit_manager.fixed_tp_pct
self._tp_record: Dict[str, dict] = {} # trade_id → entry metadata
self._rng_tp = np.random.default_rng(seed)
self._vd_threshold = self.vel_div_threshold # -0.02
self._vd_extreme = self.vel_div_extreme # -0.05
def _compute_tp(self, vel_div: float) -> float:
base = self._base_tp
strat = self._tp_strategy
p = self._tp_params
if strat == 'baseline':
return base
elif strat == 'random_uniform':
# Anti-detection: uniform within ±radius
r = p['radius']
return float(self._rng_tp.uniform(base - r, base + r))
elif strat == 'eigen_biased':
# Signal strength in [0,1]: 0 = at threshold, 1 = at extreme
strength = min(1.0, max(0.0,
(self._vd_threshold - vel_div) /
(self._vd_threshold - self._vd_extreme)
))
return base + p['k'] * strength
elif strat == 'regime_biased':
# ACB boost contribution
boost_delta = max(0.0, getattr(self, '_day_base_boost', 1.0) - 1.0)
regime_delta = max(0.0, getattr(self, 'regime_size_mult', 1.0) - 1.0)
return base + p['alpha'] * boost_delta + p['gamma'] * regime_delta
elif strat == 'combined':
# Eigenvalue component
strength = min(1.0, max(0.0,
(self._vd_threshold - vel_div) /
(self._vd_threshold - self._vd_extreme)
))
eigen_part = p['k_eigen'] * strength
# Regime component
boost_delta = max(0.0, getattr(self, '_day_base_boost', 1.0) - 1.0)
regime_part = p['k_regime'] * boost_delta
# Random jitter (anti-detection noise on top of signal-derived TP)
jitter = float(self._rng_tp.normal(0, p['jitter_sigma']))
return base + eigen_part + regime_part + jitter
return base
def _try_entry(self, bar_idx: int, vel_div: float, prices: Dict[str, float],
price_histories, v50_vel: float = 0.0, v750_vel: float = 0.0):
# Compute and inject per-trade TP before position opens
dynamic_tp = float(np.clip(self._compute_tp(vel_div),
0.003, 0.030)) # hard bounds: 30bps300bps
self.exit_manager.fixed_tp_pct = dynamic_tp
# Record pre-entry state for logging
pre_n_trades = len(self.trade_history)
pre_pos = self.position
result = super()._try_entry(bar_idx, vel_div, prices, price_histories,
v50_vel, v750_vel)
# If a new position was opened, tag it
if self.position is not None and self.position is not pre_pos:
tid = self.position.trade_id
if tid not in self._tp_record:
self._tp_record[tid] = {
'tp_target_bps' : round(dynamic_tp * 10000, 3),
'entry_vel_div' : round(vel_div, 6),
'entry_v50_vel' : round(v50_vel, 8),
'entry_v750_vel' : round(v750_vel, 8),
'entry_boost' : round(getattr(self, '_day_base_boost', 1.0), 4),
'entry_beta' : round(getattr(self, '_day_beta', 0.0), 2),
'entry_regime' : round(getattr(self, 'regime_size_mult', 1.0), 4),
'signal_strength': round(min(1.0, max(0.0,
(self._vd_threshold - vel_div) /
(self._vd_threshold - self._vd_extreme))), 4),
}
return result
# ── shared infrastructure (loaded once) ─────────────────────────────────────
print("Loading MC-Forewarner...", end='', flush=True)
forewarner = DolphinForewarner(models_dir=MC_MODELS_DIR)
print(" OK")
parquet_files = sorted([p for p in VBT_DIR.glob("*.parquet") if 'catalog' not in str(p)])
date_strings = [pf.stem for pf in parquet_files]
print("Initializing ACB...", end='', flush=True)
acb = AdaptiveCircuitBreaker()
acb.preload_w750(date_strings)
print(f" OK (p60={acb._w750_threshold:.6f})")
OB_ASSETS = ["BTCUSDT", "ETHUSDT", "BNBUSDT", "SOLUSDT"]
_mock_ob = MockOBProvider(
imbalance_bias=-0.09, depth_scale=1.0, assets=OB_ASSETS,
imbalance_biases={"BTCUSDT": -0.086, "ETHUSDT": -0.092,
"BNBUSDT": +0.05, "SOLUSDT": +0.05},
)
ob_eng = OBFeatureEngine(_mock_ob)
ob_eng.preload_date("mock", OB_ASSETS)
print("Pre-loading parquet data...", end='', flush=True)
all_vols = []
for pf in parquet_files[:2]:
df = pd.read_parquet(pf)
if 'BTCUSDT' not in df.columns: continue
pr = df['BTCUSDT'].values
for i in range(60, len(pr)):
seg = pr[max(0, i-50):i]
if len(seg) >= 10:
v = float(np.std(np.diff(seg) / seg[:-1]))
if v > 0: all_vols.append(v)
vol_p60 = float(np.percentile(all_vols, 60))
pq_data = {}
for pf in parquet_files:
df = pd.read_parquet(pf)
ac = [c for c in df.columns if c not in META_COLS]
bp = df['BTCUSDT'].values if 'BTCUSDT' in df.columns else None
dv = np.full(len(df), np.nan)
if bp is not None:
for i in range(50, len(bp)):
seg = bp[max(0, i-50):i]
if len(seg) >= 10:
dv[i] = float(np.std(np.diff(seg) / seg[:-1]))
pq_data[pf.stem] = (df, ac, dv)
print(f" {len(pq_data)} days")
# ── run one engine configuration ─────────────────────────────────────────────
def run_one(strategy: str, params: dict, seed: int, label: str) -> dict:
eng = DynamicTPEngine(**BASE_ENGINE_KWARGS)
eng.set_ob_engine(ob_eng)
eng.set_acb(acb)
eng.set_mc_forewarner(forewarner, MC_BASE_CFG)
eng.set_esoteric_hazard_multiplier(0.0)
eng.configure_tp_strategy(strategy, params, seed=seed)
dstats = []
for pf in parquet_files:
ds = pf.stem
df, acols, dvol = pq_data[ds]
vol_ok = np.where(np.isfinite(dvol), dvol > vol_p60, False)
stats = eng.process_day(ds, df, acols, vol_regime_ok=vol_ok)
dstats.append({**stats, 'cap': eng.capital})
tr = eng.trade_history
wins = [t for t in tr if t.pnl_absolute > 0]
loss = [t for t in tr if t.pnl_absolute <= 0]
gw = sum(t.pnl_absolute for t in wins) if wins else 0.0
gl = abs(sum(t.pnl_absolute for t in loss)) if loss else 0.0
roi = (eng.capital - 25000.0) / 25000.0 * 100
pff = gw / gl if gl > 0 else 999.0
dr = np.array([s['pnl'] / 25000.0 * 100 for s in dstats])
sh = float(np.mean(dr) / np.std(dr) * np.sqrt(365)) if np.std(dr) > 0 else 0.0
pk = 25000.0; mdd = 0.0
for s in dstats:
pk = max(pk, s['cap']); mdd = max(mdd, (pk - s['cap']) / pk * 100)
wr = len(wins) / len(tr) * 100 if tr else 0.0
tp_exits = sum(1 for t in tr if t.exit_reason == 'FIXED_TP')
mh_exits = sum(1 for t in tr if t.exit_reason == 'MAX_HOLD')
# TP stats from record
tp_targets = [eng._tp_record.get(t.trade_id, {}).get('tp_target_bps', 99.0) for t in tr]
tp_arr = np.array(tp_targets)
return dict(
label=label, strategy=strategy, seed=seed,
roi=roi, pf=pff, dd=mdd, sharpe=sh, wr=wr, trades=len(tr), capital=eng.capital,
tp_exits=tp_exits, mh_exits=mh_exits,
tp_mean_bps=float(np.mean(tp_arr)), tp_std_bps=float(np.std(tp_arr)),
tp_min_bps=float(np.min(tp_arr)), tp_max_bps=float(np.max(tp_arr)),
_engine=eng, _dstats=dstats,
)
# ── experiment plan ───────────────────────────────────────────────────────────
N_RANDOM_SEEDS = 15
EXPERIMENTS = []
# S0 baseline
EXPERIMENTS.append(('baseline', 'baseline', {}, [0]))
# S1a random ±3bp
EXPERIMENTS.append(('random_3bp', 'random_uniform', {'radius': 0.0003}, list(range(N_RANDOM_SEEDS))))
# S1b random ±7bp
EXPERIMENTS.append(('random_7bp', 'random_uniform', {'radius': 0.0007}, list(range(N_RANDOM_SEEDS))))
# S2 eigenvalue-biased (deterministic — 1 seed each)
for k_bps, kname in [(0.0002, 'eigen_2bp'), (0.0004, 'eigen_4bp'), (0.0006, 'eigen_6bp')]:
EXPERIMENTS.append((kname, 'eigen_biased', {'k': k_bps}, [0]))
# S3 regime-biased (deterministic)
EXPERIMENTS.append(('regime_boost', 'regime_biased', {'alpha': 0.0005, 'gamma': 0.0000}, [0]))
EXPERIMENTS.append(('regime_vol', 'regime_biased', {'alpha': 0.0003, 'gamma': 0.0002}, [0]))
# S4 combined (random component → 10 seeds)
EXPERIMENTS.append(('combined', 'combined',
{'k_eigen': 0.0003, 'k_regime': 0.0002, 'jitter_sigma': 0.0001},
list(range(N_RANDOM_SEEDS))))
total_runs = sum(len(seeds) for _, _, _, seeds in EXPERIMENTS)
# ── output files ──────────────────────────────────────────────────────────────
run_ts = datetime.now().strftime("%Y%m%d_%H%M%S")
sum_path = LOG_DIR / f"dyntp_summary_{run_ts}.csv"
trade_path= LOG_DIR / f"dyntp_trades_{run_ts}.csv"
daily_path= LOG_DIR / f"dyntp_daily_{run_ts}.csv"
SUM_FIELDS = ['label','strategy','seed','roi','pf','dd','sharpe','wr','trades','capital',
'tp_exits','mh_exits','tp_mean_bps','tp_std_bps','tp_min_bps','tp_max_bps','elapsed_s']
TRADE_FIELDS = ['label','strategy','seed','trade_id','asset','direction',
'entry_price','exit_price','entry_bar','exit_bar','bars_held',
'leverage','notional','pnl_pct_pct','pnl_absolute','exit_reason','bucket_idx',
'tp_target_bps','entry_vel_div','entry_v50_vel','entry_v750_vel',
'entry_boost','entry_beta','entry_regime','signal_strength']
DAILY_FIELDS = ['label','strategy','seed','date','pnl','capital','dd_pct','boost','beta','mc_status','trades']
with open(sum_path, 'w', newline='') as f: csv.writer(f).writerow(SUM_FIELDS)
with open(trade_path, 'w', newline='') as f: csv.writer(f).writerow(TRADE_FIELDS)
with open(daily_path, 'w', newline='') as f: csv.writer(f).writerow(DAILY_FIELDS)
def append_summary(r, elapsed):
with open(sum_path, 'a', newline='') as f:
csv.writer(f).writerow([
r['label'], r['strategy'], r['seed'],
round(r['roi'],4), round(r['pf'],4), round(r['dd'],4),
round(r['sharpe'],4), round(r['wr'],4), r['trades'],
round(r['capital'],4), r['tp_exits'], r['mh_exits'],
round(r['tp_mean_bps'],3), round(r['tp_std_bps'],3),
round(r['tp_min_bps'],3), round(r['tp_max_bps'],3),
round(elapsed,1),
])
def append_trades(r):
eng = r['_engine']
with open(trade_path, 'a', newline='') as f:
cw = csv.writer(f)
for t in eng.trade_history:
meta = eng._tp_record.get(t.trade_id, {})
cw.writerow([
r['label'], r['strategy'], r['seed'],
t.trade_id, t.asset, t.direction,
f"{t.entry_price:.6f}", f"{t.exit_price:.6f}",
t.entry_bar, t.exit_bar, t.bars_held,
f"{t.leverage:.4f}", f"{t.notional:.4f}",
f"{t.pnl_pct*100:.6f}", f"{t.pnl_absolute:.4f}",
t.exit_reason, t.bucket_idx,
meta.get('tp_target_bps', 99.0),
meta.get('entry_vel_div', ''),
meta.get('entry_v50_vel', ''),
meta.get('entry_v750_vel', ''),
meta.get('entry_boost', ''),
meta.get('entry_beta', ''),
meta.get('entry_regime', ''),
meta.get('signal_strength', ''),
])
def append_daily(r):
eng = r['_engine']
with open(daily_path, 'a', newline='') as f:
cw = csv.writer(f)
pk = 25000.0
for s in r['_dstats']:
pk = max(pk, s['cap'])
cw.writerow([
r['label'], r['strategy'], r['seed'],
s['date'], f"{s['pnl']:.4f}", f"{s['cap']:.4f}",
f"{(pk-s['cap'])/pk*100:.4f}",
f"{s['boost']:.4f}", f"{s['beta']:.2f}",
s['mc_status'], s['trades'],
])
# ── main loop ────────────────────────────────────────────────────────────────
print(f"\n{'='*65}")
print(f" DYNAMIC TP EXPERIMENT — {total_runs} total runs")
print(f" Baseline TP: 99bps | Branch: experiment/dynamic-tp")
print(f"{'='*65}\n")
all_results = {}
completed = 0
t_exp = time.time()
for label, strategy, params, seeds in EXPERIMENTS:
n = len(seeds)
print(f" [{label}] strategy={strategy} params={params} n={n}")
all_results[label] = []
for seed in seeds:
t0 = time.time()
r = run_one(strategy, params, seed, label)
elapsed = time.time() - t0
all_results[label].append(r)
completed += 1
eta = (time.time() - t_exp) / completed * (total_runs - completed)
print(f" seed={seed:2d} ROI={r['roi']:+6.2f}% PF={r['pf']:.3f}"
f" DD={r['dd']:.2f}% T={r['trades']}"
f" TP_hits={r['tp_exits']} TP_mean={r['tp_mean_bps']:.1f}bps"
f" [{elapsed:.0f}s | ETA {eta/60:.0f}min]")
append_summary(r, elapsed)
append_trades(r)
append_daily(r)
# ── final analysis ────────────────────────────────────────────────────────────
print(f"\n{'='*65}")
print(f" RESULTS — Dynamic TP Strategies vs Baseline")
print(f"{'='*65}")
b_rois = [r['roi'] for r in all_results.get('baseline', [])]
b_roi = b_rois[0] if b_rois else 44.89
b_tp = all_results.get('baseline', [{}])[0].get('tp_exits', 301)
print(f"\n {'Label':<14} {'E[ROI]':>8} {'ΔROI':>7} {'E[PF]':>6} "
f"{'E[T]':>5} {'TP_exits':>8} {'TP_mean':>8} {'TP_std':>7} {'Beat%':>6}")
print(f" {'-'*85}")
for label, strategy, params, seeds in EXPERIMENTS:
results = all_results[label]
rois = [r['roi'] for r in results]
pfs = [r['pf'] for r in results]
ts = [r['trades'] for r in results]
tpe = [r['tp_exits'] for r in results]
tpmn = [r['tp_mean_bps'] for r in results]
tpsd = [r['tp_std_bps'] for r in results]
mean_roi = float(np.mean(rois))
delta = mean_roi - b_roi
mean_pf = float(np.mean(pfs))
mean_t = float(np.mean(ts))
mean_tpe = float(np.mean(tpe))
mean_tpmn = float(np.mean(tpmn))
mean_tpsd = float(np.mean(tpsd))
beat_pct = float(np.mean(np.array(rois) > b_roi)) * 100 if len(rois) > 1 else (100.0 if mean_roi > b_roi else 0.0)
print(f" {label:<14} {mean_roi:>+7.2f}% {delta:>+6.2f}% {mean_pf:>6.3f} "
f"{mean_t:>5.0f} {mean_tpe:>8.0f} {mean_tpmn:>7.1f}bps "
f"{mean_tpsd:>6.1f}bps {beat_pct:>5.1f}%")
print(f"\n{'='*65}")
print(f" Interpretation:")
print(f" random_*bp: Does randomisation itself help (anti-detection + luck)?")
print(f" eigen_*bp: Does signal-strength-proportional TP add edge?")
print(f" regime_*: Does ACB-boost-proportional TP add edge?")
print(f" combined: Does the meta-bias composite beat individual components?")
print(f"\n Key question: do TP exits increase (wider TP → fewer TP hits)?")
print(f" Or decrease (smarter TP → more precise captures)?")
print(f"\n Files: {sum_path.name} | {trade_path.name} | {daily_path.name}")
print(f" Total time: {(time.time()-t_exp)/60:.1f} min")