Files
DOLPHIN/prod/nautilus_native_backtest.py
hjnormey 01c19662cb initial: import DOLPHIN baseline 2026-04-21 from dolphinng5_predict working tree
Includes core prod + GREEN/BLUE subsystems:
- prod/ (BLUE harness, configs, scripts, docs)
- nautilus_dolphin/ (GREEN Nautilus-native impl + dvae/ preserved)
- adaptive_exit/ (AEM engine + models/bucket_assignments.pkl)
- Observability/ (EsoF advisor, TUI, dashboards)
- external_factors/ (EsoF producer)
- mc_forewarning_qlabs_fork/ (MC regime/envelope)

Excludes runtime caches, logs, backups, and reproducible artifacts per .gitignore.
2026-04-21 16:58:38 +02:00

505 lines
19 KiB
Python
Executable File

"""
nautilus_native_backtest.py -- Nautilus as the authoritative backtesting ledger
==================================================================================
Architecture (PROPER Nautilus certification):
- All VBT parquet asset columns registered as Nautilus instruments
- Each 5-second parquet row converted to Nautilus Bar objects (all assets)
- DolphinActor receives on_bar() for EVERY real 5s bar (not one synthetic)
- Orders filled by Nautilus at real parquet prices
- Nautilus account balance = authoritative capital ledger
- engine.capital = shadow book, reconciled vs Nautilus at end of each day
Day loop (per-day engine to limit RAM):
Per day:
1. Load parquet (~8000-17000 rows x N assets)
2. Convert rows -> Nautilus Bar objects for every asset
3. BacktestEngine.add_data(bars)
4. BacktestEngine.run() -> DolphinActor.on_bar() fires ~8000-17000 times
5. Orders fill at real bar-close prices (Nautilus settles)
6. End of day: Nautilus account balance carried to next day
Gold reference: ROI=+181.81%, Trades=2155, DD=17.65%
"""
import sys
import time
import json
from datetime import datetime, timezone, timedelta
from decimal import Decimal
from pathlib import Path
import numpy as np
import pandas as pd
_PROD_DIR = Path(__file__).resolve().parent
_HCM_DIR = _PROD_DIR.parent
_ND_DIR = _HCM_DIR / 'nautilus_dolphin'
sys.path.insert(0, str(_HCM_DIR))
sys.path.insert(0, str(_ND_DIR))
# ---------------------------------------------------------------------------
# Config
# ---------------------------------------------------------------------------
VBT_KLINES_DIR = _HCM_DIR / 'vbt_cache'
MC_MODELS_DIR = str(_ND_DIR / 'mc_results' / 'models')
RUN_LOGS_DIR = _ND_DIR / 'run_logs'
RUN_LOGS_DIR.mkdir(parents=True, exist_ok=True)
WINDOW_START = '2025-12-31'
WINDOW_END = '2026-02-25'
INITIAL_CAPITAL = 25000.0
BAR_INTERVAL_S = 5 # 5-second bars
RECON_TOLERANCE = 0.05 # 5% divergence threshold -> WARN
RECON_CRITICAL = 0.20 # 20% divergence -> HALT
TRADER_ID = 'DOLPHIN-NATIVE-BT-001'
CHAMPION_ENGINE_CFG = dict(
boost_mode='d_liq',
vel_div_threshold=-0.02, vel_div_extreme=-0.05,
min_leverage=0.5, max_leverage=5.0, leverage_convexity=3.0,
fraction=0.20, fixed_tp_pct=0.0095, stop_pct=1.0, max_hold_bars=120,
use_direction_confirm=True, dc_lookback_bars=7, dc_min_magnitude_bps=0.75,
dc_skip_contradicts=True, dc_leverage_boost=1.0, dc_leverage_reduce=0.5,
use_asset_selection=True, min_irp_alignment=0.45,
use_sp_fees=True, use_sp_slippage=True,
sp_maker_entry_rate=0.62, sp_maker_exit_rate=0.50,
use_ob_edge=True, ob_edge_bps=5.0, ob_confirm_rate=0.40,
lookback=100, use_alpha_layers=True, use_dynamic_leverage=True, seed=42,
)
MC_BASE_CFG = {
'trial_id': 0, 'vel_div_threshold': -0.020, 'vel_div_extreme': -0.050,
'use_direction_confirm': True, 'dc_lookback_bars': 7,
'dc_min_magnitude_bps': 0.75, 'dc_skip_contradicts': True,
'dc_leverage_boost': 1.00, 'dc_leverage_reduce': 0.50,
'vd_trend_lookback': 10, 'min_leverage': 0.50, 'max_leverage': 5.00,
'leverage_convexity': 3.00, 'fraction': 0.20, 'use_alpha_layers': True,
'use_dynamic_leverage': True, 'fixed_tp_pct': 0.0095, 'stop_pct': 1.00,
'max_hold_bars': 120, 'use_sp_fees': True, 'use_sp_slippage': True,
'sp_maker_entry_rate': 0.62, 'sp_maker_exit_rate': 0.50,
'use_ob_edge': True, 'ob_edge_bps': 5.00, 'ob_confirm_rate': 0.40,
'ob_imbalance_bias': -0.09, 'ob_depth_scale': 1.00,
'use_asset_selection': True, 'min_irp_alignment': 0.45, 'lookback': 100,
'acb_beta_high': 0.80, 'acb_beta_low': 0.20, 'acb_w750_threshold_pct': 60,
}
# ---------------------------------------------------------------------------
# Feature store (module-level singleton) - populated per day before engine.run()
# DolphinActor reads from this in native_mode
# ---------------------------------------------------------------------------
_FEATURE_STORE = {} # ts_event_ns -> {vel_div, v50, v750, inst50, vol_ok}
def get_parquet_files():
all_pq = sorted(VBT_KLINES_DIR.glob('*.parquet'))
return [p for p in all_pq
if 'catalog' not in str(p)
and WINDOW_START <= p.stem <= WINDOW_END]
# ---------------------------------------------------------------------------
# Instrument factory
# ---------------------------------------------------------------------------
def _make_instrument(symbol: str, venue, price_precision: int = 5):
"""Create a Nautilus CurrencyPair instrument for a crypto USDT pair."""
from nautilus_trader.model.instruments import CurrencyPair
from nautilus_trader.model.identifiers import InstrumentId, Symbol
from nautilus_trader.model.objects import Price, Quantity, Currency
from decimal import Decimal
if symbol.endswith("USDT"):
base = symbol[:-4]
quote = "USDT"
else:
base = symbol[:3]
quote = symbol[3:]
instr_id = InstrumentId(Symbol(symbol), venue)
return CurrencyPair(
instrument_id=instr_id,
raw_symbol=Symbol(symbol),
base_currency=Currency.from_str(base),
quote_currency=Currency.from_str(quote),
price_precision=price_precision,
size_precision=5,
price_increment=Price(10**(-price_precision), price_precision),
size_increment=Quantity(10**(-5), 5),
lot_size=None,
max_quantity=None,
min_quantity=None,
max_notional=None,
min_notional=None,
max_price=None,
min_price=None,
margin_init=Decimal("0.02"),
margin_maint=Decimal("0.01"),
maker_fee=Decimal("0.0002"),
taker_fee=Decimal("0.0002"),
ts_event=0,
ts_init=0,
)
# ---------------------------------------------------------------------------
# Bar factory
# ---------------------------------------------------------------------------
def _make_bar(bar_type, price_float: float, ts_event_ns: int,
price_precision: int = 5):
"""Create a Nautilus Bar with OHLC=close (flat bar from close-only data)."""
from nautilus_trader.model.data import Bar
from nautilus_trader.model.objects import Price, Quantity
if price_float <= 0 or not np.isfinite(price_float):
return None
px = Price(price_float, precision=price_precision)
qty = Quantity(1.0, precision=5)
return Bar(
bar_type=bar_type,
open=px, high=px, low=px, close=px,
volume=qty,
ts_event=ts_event_ns,
ts_init=ts_event_ns,
)
# ---------------------------------------------------------------------------
# Per-day runner
# ---------------------------------------------------------------------------
def run_one_day_native(
run_date: str,
initial_capital: float,
vol_p60: float,
preload_dates: list,
assets: list,
all_instruments: dict, # symbol -> Nautilus instrument
) -> dict:
"""
Run one day with Nautilus as authoritative ledger.
Steps:
1. Load parquet for run_date, build feature store + Bar objects
2. Populate _FEATURE_STORE (actor reads from it in native_mode)
3. Init BacktestEngine, register all instruments
4. Add all bars (all assets x all rows)
5. engine.run() -> on_bar() fires per BTCUSDT bar
6. Return Nautilus account balance + engine.capital (for reconciliation)
"""
global _FEATURE_STORE
from nautilus_trader.backtest.engine import BacktestEngine, BacktestEngineConfig
from nautilus_trader.model.identifiers import Venue
from nautilus_trader.model.data import BarType
from nautilus_trader.model.objects import Money, Currency
from nautilus_trader.model.enums import OmsType, AccountType
from nautilus_dolphin.nautilus.dolphin_actor import DolphinActor
# 1. Load parquet
pf = VBT_KLINES_DIR / f"{run_date}.parquet"
if not pf.exists():
return {'date': run_date, 'capital': initial_capital, 'pnl': 0.0,
'trades': 0, 'error': 'no_parquet', 'stale_state_events': 0}
df = pd.read_parquet(pf)
n_rows = len(df)
# Day start timestamp (midnight UTC) in nanoseconds
day_dt = datetime.strptime(run_date, '%Y-%m-%d').replace(tzinfo=timezone.utc)
day_start_ns = int(day_dt.timestamp() * 1e9)
# 2. Build vol_ok mask (same logic as _run_replay_day)
vol_ok_mask = np.zeros(n_rows, dtype=bool)
if 'BTCUSDT' in df.columns and vol_p60 > 0:
btc = df['BTCUSDT'].values
dvol = np.full(n_rows, np.nan)
for i in range(50, n_rows):
seg = btc[max(0, i-50):i]
if len(seg) >= 10 and (seg > 0).all():
dvol[i] = float(np.std(np.diff(seg) / seg[:-1]))
vol_ok_mask = np.where(np.isfinite(dvol), dvol > vol_p60, False)
# 3. Build feature store + Bar objects
_FEATURE_STORE.clear()
all_bars = []
bar_type_map = {}
for sym in assets:
bt_str = f"{sym}.BINANCE-5-SECOND-LAST-EXTERNAL"
bar_type_map[sym] = BarType.from_str(bt_str)
for row_i in range(n_rows):
row = df.iloc[row_i]
ts_ns = day_start_ns + row_i * BAR_INTERVAL_S * 1_000_000_000
vd = row.get('vel_div', 0.0)
v50 = row.get('v50_lambda_max_velocity', 0.0)
v750 = row.get('v750_lambda_max_velocity', 0.0)
inst50 = row.get('instability_50', 0.0)
_FEATURE_STORE[ts_ns] = {
'vel_div': float(vd) if (vd is not None and np.isfinite(float(vd))) else None,
'v50': float(v50) if (v50 is not None and np.isfinite(float(v50))) else 0.0,
'v750': float(v750) if (v750 is not None and np.isfinite(float(v750))) else 0.0,
'inst50': float(inst50) if (inst50 is not None and np.isfinite(float(inst50))) else 0.0,
'vol_ok': bool(vol_ok_mask[row_i]) and (row_i >= 100),
'row_i': row_i,
}
for sym in assets:
if sym not in df.columns:
continue
if sym not in all_instruments:
continue
price = row[sym]
if price is None:
continue
bar = _make_bar(bar_type_map[sym], float(price), ts_ns)
if bar is not None:
all_bars.append(bar)
if not all_bars:
return {'date': run_date, 'capital': initial_capital, 'pnl': 0.0,
'trades': 0, 'error': 'no_bars', 'stale_state_events': 0}
# 4. Build actor config
actor_cfg = {
'engine': dict(CHAMPION_ENGINE_CFG),
'paper_trade': {'initial_capital': initial_capital},
'posture_override': 'APEX',
'live_mode': False,
'native_mode': True, # new flag: actor reads from _FEATURE_STORE
'run_date': run_date,
'bar_type': 'BTCUSDT.BINANCE-5-SECOND-LAST-EXTERNAL',
'mc_models_dir': MC_MODELS_DIR,
'mc_base_cfg': MC_BASE_CFG,
'venue': 'BINANCE',
'vol_p60': vol_p60,
'acb_preload_dates': preload_dates,
'assets': assets,
'parquet_dir': 'vbt_cache',
'registered_assets': assets, # actor uses this to query cache
}
actor_cfg['engine']['initial_capital'] = initial_capital
# 5. Init Nautilus engine
be_cfg = BacktestEngineConfig(trader_id=TRADER_ID)
bt_engine = BacktestEngine(config=be_cfg)
venue = Venue('BINANCE')
usdt = Currency.from_str('USDT')
bt_engine.add_venue(
venue=venue,
oms_type=OmsType.HEDGING,
account_type=AccountType.MARGIN,
base_currency=usdt,
starting_balances=[Money(str(round(initial_capital, 8)), usdt)],
)
# Register ALL instruments
for sym, instr in all_instruments.items():
bt_engine.add_instrument(instr)
# 6. Register actor
actor = DolphinActor(config=actor_cfg)
bt_engine.add_strategy(actor)
# 7. Feed real bar data
bt_engine.add_data(all_bars)
# 8. Run
t0 = time.time()
try:
bt_engine.run()
except Exception as e:
print(f" [WARN] bt_engine.run() error on {run_date}: {e}")
elapsed = round(time.time() - t0, 1)
# 9. Read Nautilus account balance (authoritative)
try:
accounts = list(bt_engine.trader.portfolio.accounts())
nautilus_account = accounts[0] if accounts else None
if nautilus_account:
print(f"ACCOUNT BALANCES: {nautilus_account.balances()}")
naut_bal_obj = nautilus_account.balance_total(usdt)
naut_capital = float(naut_bal_obj.as_double()) if naut_bal_obj else initial_capital
else:
naut_capital = initial_capital
except Exception:
naut_capital = initial_capital
# 10. Read shadow book (engine.capital)
shadow_capital = float(getattr(actor.engine, 'capital', initial_capital))
trades = len(getattr(actor.engine, 'trade_history', []))
# 11. Reconciliation
if abs(naut_capital - initial_capital) < 1e-4 and trades > 0:
# Nautilus account not updating (orders not filling) -> use shadow
recon_status = 'SHADOW_USED'
final_capital = shadow_capital
else:
recon_status = 'NAUTILUS'
final_capital = naut_capital
divergence = abs(naut_capital - shadow_capital) / max(shadow_capital, 1.0)
if divergence > RECON_CRITICAL and trades > 0 and abs(naut_capital - initial_capital) > 1.0:
recon_status = f'DIVERGE_CRITICAL_{divergence:.1%}'
elif divergence > RECON_TOLERANCE and trades > 0 and abs(naut_capital - initial_capital) > 1.0:
recon_status = f'DIVERGE_WARN_{divergence:.1%}'
pnl = final_capital - initial_capital
bt_engine.dispose()
return {
'date': run_date,
'capital': final_capital,
'shadow_capital': shadow_capital,
'naut_capital': naut_capital,
'pnl': pnl,
'trades': trades,
'recon': recon_status,
'elapsed_s': elapsed,
'stale_state_events': getattr(actor, '_stale_state_events', 0),
}
# ---------------------------------------------------------------------------
# Main backtest loop
# ---------------------------------------------------------------------------
def run_native_backtest():
print('=' * 72)
print('NAUTILUS NATIVE BACKTEST - REAL BAR DATA + NAUTILUS LEDGER')
print(f'Window: {WINDOW_START} -> {WINDOW_END}')
print(f'Mode: VBT parquet -> Nautilus Bar stream -> DolphinActor.on_bar()')
print('=' * 72)
from nautilus_trader.model.identifiers import Venue as NVenue
NV = NVenue('BINANCE')
# Get parquet files
parquet_files = get_parquet_files()
if not parquet_files:
print(f'ERROR: No parquets in {VBT_KLINES_DIR}')
return None
print(f' Parquet files: {len(parquet_files)} ({parquet_files[0].stem} -> {parquet_files[-1].stem})')
# Get asset list
df0 = pd.read_parquet(parquet_files[0])
asset_cols = [c for c in df0.columns if c not in {
'timestamp', 'scan_number', 'v50_lambda_max_velocity',
'v150_lambda_max_velocity', 'v300_lambda_max_velocity',
'v750_lambda_max_velocity', 'vel_div', 'instability_50', 'instability_150'
}]
print(f' Assets: {len(asset_cols)} | {asset_cols[:5]}...')
# Build instruments (once, reused each day)
print(f' Building {len(asset_cols)} Nautilus instruments...')
all_instruments = {}
for sym in asset_cols:
try:
all_instruments[sym] = _make_instrument(sym, NV)
except Exception as e:
print(f' [WARN] Could not create instrument for {sym}: {e}')
print(f' Registered: {len(all_instruments)} instruments')
# Calibrate vol_p60 (full window)
print(' Calibrating vol_p60 (full window)...')
all_vols = []
for pf in parquet_files:
df = pd.read_parquet(pf)
if 'BTCUSDT' not in df.columns:
continue
btc = df['BTCUSDT'].values
for i in range(50, len(btc)):
seg = btc[max(0, i-50):i]
if len(seg) >= 10 and (seg > 0).all():
v = float(np.std(np.diff(seg) / seg[:-1]))
if v > 0 and np.isfinite(v):
all_vols.append(v)
vol_p60 = float(np.percentile(all_vols, 60)) if all_vols else 0.0002
print(f' vol_p60 = {vol_p60:.8f} (from {len(all_vols):,} samples)')
all_window_dates = [pf.stem for pf in parquet_files]
# Day loop
print()
print(f' {"Day":<5} {"Date":<12} {"Nautilus$":>11} {"Shadow$":>11} '
f'{"Trades":>6} {"DayPnL":>9} {"Recon":<25} {"Sec":>5}')
print(f' {"-"*5} {"-"*12} {"-"*11} {"-"*11} {"-"*6} {"-"*9} {"-"*25} {"-"*5}')
capital_dec = Decimal(str(INITIAL_CAPITAL))
daily_caps = []
daily_pnls = []
total_trades = 0
t_total = time.time()
for i, pf in enumerate(parquet_files):
ds = pf.stem
result = run_one_day_native(
ds, float(capital_dec), vol_p60,
all_window_dates, asset_cols, all_instruments
)
final_cap = result['capital']
pnl_f = result['pnl']
trades_d = result['trades']
recon = result.get('recon', '?')
elapsed = result.get('elapsed_s', 0)
capital_dec += Decimal(str(round(pnl_f, 8)))
total_trades += trades_d
daily_caps.append(float(capital_dec))
daily_pnls.append(pnl_f)
naut_str = f"${result.get('naut_capital', 0):,.2f}"
shad_str = f"${result.get('shadow_capital', 0):,.2f}"
print(f' {i+1:<5} {ds:<12} {naut_str:>11} {shad_str:>11} '
f'{trades_d:>6} {pnl_f:>+9.2f} {recon:<25} {elapsed:>5.0f}s')
# Final metrics
final_capital = float(capital_dec)
roi = (final_capital - INITIAL_CAPITAL) / INITIAL_CAPITAL * 100.0
arr = np.array(daily_caps)
peak = np.maximum.accumulate(arr)
dd_arr = (peak - arr) / np.maximum(peak, 1.0)
max_dd = float(np.max(dd_arr)) * 100.0
total_elapsed = time.time() - t_total
print()
print('=' * 72)
print(f' Final Capital : ${final_capital:,.2f}')
print(f' ROI : {roi:+.2f}% (gold ref: +181.81%)')
print(f' Max Drawdown : {max_dd:.2f}% (gold ref: 17.65%)')
print(f' Total Trades : {total_trades} (gold ref: 2155)')
print(f' Elapsed : {total_elapsed/60:.1f} min')
print('=' * 72)
result_payload = {
'roi': round(roi, 4),
'trades': total_trades,
'dd': round(max_dd, 4),
'capital': round(final_capital, 4),
'window': f'{WINDOW_START}:{WINDOW_END}',
'days': len(parquet_files),
'elapsed_s': round(total_elapsed, 1),
'engine': 'Nautilus Native (real bars + Nautilus ledger)',
'run_ts': datetime.now(timezone.utc).isoformat(),
}
out_path = RUN_LOGS_DIR / f"nautilus_native_{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}.json"
with open(out_path, 'w') as f:
json.dump(result_payload, f, indent=2)
print(f' Saved: {out_path.name}')
return result_payload
if __name__ == '__main__':
import sys
sys.modules['prod.nautilus_native_backtest'] = sys.modules['__main__']
run_native_backtest()