01c19662cb
Includes core prod + GREEN/BLUE subsystems: - prod/ (BLUE harness, configs, scripts, docs) - nautilus_dolphin/ (GREEN Nautilus-native impl + dvae/ preserved) - adaptive_exit/ (AEM engine + models/bucket_assignments.pkl) - Observability/ (EsoF advisor, TUI, dashboards) - external_factors/ (EsoF producer) - mc_forewarning_qlabs_fork/ (MC regime/envelope) Excludes runtime caches, logs, backups, and reproducible artifacts per .gitignore.
715 lines
29 KiB
Python
Executable File
715 lines
29 KiB
Python
Executable File
"""
|
|
nautilus_native_backtest.py -- Nautilus as the authoritative backtesting ledger
|
|
==================================================================================
|
|
Architecture (PROPER Nautilus certification):
|
|
- All VBT parquet asset columns registered as Nautilus instruments
|
|
- Each 5-second parquet row converted to Nautilus Bar objects (all assets)
|
|
- DolphinActor receives on_bar() for EVERY real 5s bar (not one synthetic)
|
|
- Orders filled by Nautilus at real parquet prices
|
|
- Nautilus account balance = authoritative capital ledger
|
|
- engine.capital = shadow book, reconciled vs Nautilus at end of each day
|
|
|
|
Day loop (per-day engine to limit RAM):
|
|
Per day:
|
|
1. Load parquet (~8000-17000 rows x N assets)
|
|
2. Convert rows -> Nautilus Bar objects for every asset
|
|
3. BacktestEngine.add_data(bars)
|
|
4. BacktestEngine.run() -> DolphinActor.on_bar() fires ~8000-17000 times
|
|
5. Orders fill at real bar-close prices (Nautilus settles)
|
|
6. End of day: Nautilus account balance carried to next day
|
|
|
|
Gold reference: ROI=+181.81%, Trades=2155, DD=17.65%
|
|
"""
|
|
import sys
|
|
import time
|
|
import json
|
|
from datetime import datetime, timezone, timedelta
|
|
from decimal import Decimal
|
|
from pathlib import Path
|
|
|
|
import math
|
|
import numpy as np
|
|
import pandas as pd
|
|
|
|
_PROD_DIR = Path(__file__).resolve().parent
|
|
_HCM_DIR = _PROD_DIR.parent
|
|
_ND_DIR = _HCM_DIR / 'nautilus_dolphin'
|
|
sys.path.insert(0, str(_HCM_DIR))
|
|
sys.path.insert(0, str(_ND_DIR))
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Config
|
|
# ---------------------------------------------------------------------------
|
|
VBT_KLINES_DIR = _HCM_DIR / 'vbt_cache'
|
|
MC_MODELS_DIR = str(_ND_DIR / 'mc_results' / 'models')
|
|
RUN_LOGS_DIR = _ND_DIR / 'run_logs'
|
|
RUN_LOGS_DIR.mkdir(parents=True, exist_ok=True)
|
|
|
|
WINDOW_START = '2025-12-31'
|
|
WINDOW_END = '2026-02-26' # 56-day window (gold: T changed 2143→2155 when Feb26 added)
|
|
INITIAL_CAPITAL = 25000.0
|
|
BAR_INTERVAL_S = 5 # 5-second bars
|
|
RECON_TOLERANCE = 0.05 # 5% divergence threshold -> WARN
|
|
RECON_CRITICAL = 0.20 # 20% divergence -> HALT
|
|
|
|
# Gold-calibrated vol threshold — computed from full 5-year BTC history.
|
|
# Used for LIVE TRADING ONLY (filters dead markets).
|
|
VOL_P60_THRESHOLD = 0.00026414
|
|
|
|
# Gold static vol_p60 — exact value from exp_shared.load_data() (gold certification method):
|
|
# for pf in parquet_files[:2]: pr=BTCUSDT; for i in range(60,len): seg=pr[i-50:i];
|
|
# v=std(np.diff(seg)/seg[:-1]); if v>0: collect → percentile(60)
|
|
# Source: exp_shared.py:load_data(); verified 0.00009868 on this server.
|
|
VOL_P60_INWINDOW = 0.00009868
|
|
|
|
TRADER_ID = 'DOLPHIN-NATIVE-BT-001'
|
|
|
|
CHAMPION_ENGINE_CFG = dict(
|
|
boost_mode='d_liq',
|
|
vel_div_threshold=-0.02, vel_div_extreme=-0.05,
|
|
min_leverage=0.5, max_leverage=8.0, # gold spec: 8x soft cap
|
|
leverage_convexity=3.0,
|
|
fraction=0.20, fixed_tp_pct=0.0095, stop_pct=1.0, max_hold_bars=120, # gold spec: 120 bars (exp_shared.ENGINE_KWARGS)
|
|
use_direction_confirm=True, dc_lookback_bars=7, dc_min_magnitude_bps=0.75,
|
|
dc_skip_contradicts=True, dc_leverage_boost=1.0, dc_leverage_reduce=0.5,
|
|
use_asset_selection=True, min_irp_alignment=0.45, # gold spec: 0.45 IRP filter (exp_shared.ENGINE_KWARGS)
|
|
use_sp_fees=True, use_sp_slippage=True,
|
|
sp_maker_entry_rate=0.62, sp_maker_exit_rate=0.50,
|
|
use_ob_edge=True, ob_edge_bps=5.0, ob_confirm_rate=0.40,
|
|
lookback=100, use_alpha_layers=True, use_dynamic_leverage=True, seed=42,
|
|
)
|
|
|
|
MC_BASE_CFG = {
|
|
'trial_id': 0, 'vel_div_threshold': -0.020, 'vel_div_extreme': -0.050,
|
|
'use_direction_confirm': True, 'dc_lookback_bars': 7,
|
|
'dc_min_magnitude_bps': 0.75, 'dc_skip_contradicts': True,
|
|
'dc_leverage_boost': 1.00, 'dc_leverage_reduce': 0.50,
|
|
'vd_trend_lookback': 10, 'min_leverage': 0.50, 'max_leverage': 5.00,
|
|
'leverage_convexity': 3.00, 'fraction': 0.20, 'use_alpha_layers': True,
|
|
'use_dynamic_leverage': True, 'fixed_tp_pct': 0.0095, 'stop_pct': 1.00,
|
|
'max_hold_bars': 120, 'use_sp_fees': True, 'use_sp_slippage': True,
|
|
'sp_maker_entry_rate': 0.62, 'sp_maker_exit_rate': 0.50,
|
|
'use_ob_edge': True, 'ob_edge_bps': 5.00, 'ob_confirm_rate': 0.40,
|
|
'ob_imbalance_bias': -0.09, 'ob_depth_scale': 1.00,
|
|
'use_asset_selection': True, 'min_irp_alignment': 0.0, 'lookback': 100, # gold spec
|
|
'acb_beta_high': 0.80, 'acb_beta_low': 0.20, 'acb_w750_threshold_pct': 60,
|
|
}
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Feature store (module-level singleton) - populated per day before engine.run()
|
|
# DolphinActor reads from this in native_mode
|
|
# ---------------------------------------------------------------------------
|
|
_FEATURE_STORE = {} # ts_event_ns -> {vel_div, v50, v750, inst50, vol_ok}
|
|
|
|
|
|
def get_parquet_files():
|
|
all_pq = sorted(VBT_KLINES_DIR.glob('*.parquet'))
|
|
return [p for p in all_pq
|
|
if 'catalog' not in str(p)
|
|
and WINDOW_START <= p.stem <= WINDOW_END]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Instrument factory
|
|
# ---------------------------------------------------------------------------
|
|
def _make_instrument(symbol: str, venue, price_precision: int = 5):
|
|
"""Create a Nautilus CurrencyPair instrument for a crypto USDT pair."""
|
|
from nautilus_trader.model.instruments import CurrencyPair
|
|
from nautilus_trader.model.identifiers import InstrumentId, Symbol
|
|
from nautilus_trader.model.objects import Price, Quantity, Currency
|
|
from decimal import Decimal
|
|
|
|
if symbol.endswith("USDT"):
|
|
base = symbol[:-4]
|
|
quote = "USDT"
|
|
else:
|
|
base = symbol[:3]
|
|
quote = symbol[3:]
|
|
|
|
instr_id = InstrumentId(Symbol(symbol), venue)
|
|
return CurrencyPair(
|
|
instrument_id=instr_id,
|
|
raw_symbol=Symbol(symbol),
|
|
base_currency=Currency.from_str(base),
|
|
quote_currency=Currency.from_str(quote),
|
|
price_precision=price_precision,
|
|
size_precision=5,
|
|
price_increment=Price(10**(-price_precision), price_precision),
|
|
size_increment=Quantity(10**(-5), 5),
|
|
lot_size=None,
|
|
max_quantity=None,
|
|
min_quantity=None,
|
|
max_notional=None,
|
|
min_notional=None,
|
|
max_price=None,
|
|
min_price=None,
|
|
margin_init=Decimal("0.02"),
|
|
margin_maint=Decimal("0.01"),
|
|
maker_fee=Decimal("0.0002"),
|
|
taker_fee=Decimal("0.0002"),
|
|
ts_event=0,
|
|
ts_init=0,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Bar factory
|
|
# ---------------------------------------------------------------------------
|
|
def _make_bar(bar_type, price_float: float, ts_event_ns: int,
|
|
price_precision: int = 5):
|
|
"""Create a Nautilus Bar with OHLC=close (flat bar from close-only data)."""
|
|
from nautilus_trader.model.data import Bar
|
|
from nautilus_trader.model.objects import Price, Quantity
|
|
|
|
if price_float <= 0 or not np.isfinite(price_float):
|
|
return None
|
|
|
|
px = Price(price_float, precision=price_precision)
|
|
qty = Quantity(1.0, precision=5)
|
|
return Bar(
|
|
bar_type=bar_type,
|
|
open=px, high=px, low=px, close=px,
|
|
volume=qty,
|
|
ts_event=ts_event_ns,
|
|
ts_init=ts_event_ns,
|
|
)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Per-day runner
|
|
# ---------------------------------------------------------------------------
|
|
def _safe_f(val, default=0.0) -> float:
|
|
"""NaN-safe float conversion — used in hot data loading path."""
|
|
if val is None:
|
|
return default
|
|
try:
|
|
f = float(val)
|
|
return f if math.isfinite(f) else default
|
|
except (TypeError, ValueError):
|
|
return default
|
|
|
|
|
|
def _compute_dvol_arb512(btc_prices, n_rows: int, threshold: float):
|
|
"""Rolling 50-bar dvol using flint.arb at 512-bit precision.
|
|
|
|
Input data is float64 (parquet), but all arithmetic — differences, mean,
|
|
variance, sqrt — is performed at 512-bit to prevent rounding accumulation
|
|
across the rolling window. Benchmark: ~295ms/day vs ~94ms/day numpy (3.1x);
|
|
zero precision divergence vs float64 for this signal (MATCH=True, 0 diffs).
|
|
|
|
Returns: bool numpy array (vol_ok_mask), or None if flint unavailable.
|
|
"""
|
|
try:
|
|
from flint import arb, ctx # python-flint v0.8.0
|
|
ctx.prec = 512
|
|
except ImportError:
|
|
return None # Caller falls back to numpy float64
|
|
|
|
vol_ok_mask = np.zeros(n_rows, dtype=bool)
|
|
threshold_arb = arb(str(threshold))
|
|
|
|
# Convert prices to arb once — O(n), only valid (positive, finite) entries
|
|
prices_arb = [None] * n_rows
|
|
for i in range(n_rows):
|
|
p = float(btc_prices[i])
|
|
if p > 0.0 and math.isfinite(p):
|
|
prices_arb[i] = arb(p) # arb(float) preserves float64 input precision
|
|
|
|
# Compute returns (p[i] - p[i-1]) / p[i-1] at 512-bit
|
|
returns_arb = [None] * n_rows
|
|
for i in range(1, n_rows):
|
|
if prices_arb[i] is not None and prices_arb[i - 1] is not None:
|
|
returns_arb[i] = (prices_arb[i] - prices_arb[i - 1]) / prices_arb[i - 1]
|
|
|
|
# Rolling population std over 50-bar window (population std matches gold VBT np.std)
|
|
for i in range(50, n_rows):
|
|
window = [returns_arb[j] for j in range(i - 50, i) if returns_arb[j] is not None]
|
|
if len(window) < 10:
|
|
continue
|
|
n_w = arb(len(window))
|
|
mean = sum(window) / n_w
|
|
variance = sum((x - mean) ** 2 for x in window) / n_w
|
|
std = variance.sqrt()
|
|
vol_ok_mask[i] = bool(std > threshold_arb)
|
|
|
|
return vol_ok_mask
|
|
|
|
|
|
def _precompute_continuous_vol_ok(parquet_files: list, threshold: float) -> dict:
|
|
"""Pre-compute vol_ok across all parquets as ONE continuous rolling window.
|
|
|
|
Gold VBT methodology: data is one contiguous time-series, not per-day resets.
|
|
Rolling 50-bar dvol window carries over day boundaries.
|
|
|
|
Per-day dvol resets at midnight (2750 warmup bars lost across 55 days) and also
|
|
loses context from prior-day volatility spikes — producing ~2.7% pass rate on this
|
|
window vs 27.1% with continuous.
|
|
|
|
Uses arb512 primary (flint); numpy float64 fallback.
|
|
Returns dict: {ts_ns (int) -> vol_ok (bool)}
|
|
"""
|
|
# First pass: collect all BTC prices and timestamps in order
|
|
all_ts_lists = []
|
|
all_btc_lists = []
|
|
for pf in parquet_files:
|
|
df = pd.read_parquet(pf)
|
|
if 'timestamp' in df.columns:
|
|
ts = df['timestamp'].values.astype('int64')
|
|
else:
|
|
day_dt = datetime.strptime(pf.stem, '%Y-%m-%d').replace(tzinfo=timezone.utc)
|
|
day_start_ns = int(day_dt.timestamp() * 1e9)
|
|
ts = np.array([day_start_ns + i * BAR_INTERVAL_S * 1_000_000_000
|
|
for i in range(len(df))], dtype='int64')
|
|
btc_col = df['BTCUSDT'].values.astype(float) if 'BTCUSDT' in df.columns else np.zeros(len(df))
|
|
all_ts_lists.append(ts)
|
|
all_btc_lists.append(btc_col)
|
|
|
|
all_ts_arr = np.concatenate(all_ts_lists).astype('int64')
|
|
all_btc_arr = np.concatenate(all_btc_lists)
|
|
n_total = len(all_ts_arr)
|
|
|
|
print(f" [VOL] Computing continuous cross-day dvol ({n_total:,} rows, arb512)...")
|
|
t0 = time.time()
|
|
|
|
vol_ok_arr = _compute_dvol_arb512(all_btc_arr, n_total, threshold)
|
|
if vol_ok_arr is None:
|
|
# float64 fallback
|
|
dvol = np.full(n_total, np.nan)
|
|
diffs = np.zeros(n_total)
|
|
valid = (all_btc_arr > 0) & np.isfinite(all_btc_arr)
|
|
diffs[1:] = np.where(valid[1:] & valid[:-1],
|
|
np.diff(all_btc_arr) / all_btc_arr[:-1], np.nan)
|
|
for i in range(50, n_total):
|
|
w = diffs[i - 50:i]
|
|
fw = w[np.isfinite(w)]
|
|
if len(fw) >= 10:
|
|
dvol[i] = float(np.std(fw))
|
|
vol_ok_arr = np.isfinite(dvol) & (dvol > threshold)
|
|
|
|
elapsed = time.time() - t0
|
|
n_ok = int(vol_ok_arr.sum())
|
|
print(f" [VOL] Done in {elapsed:.1f}s vol_ok=True: {n_ok:,}/{n_total:,} ({n_ok/n_total*100:.1f}%)")
|
|
|
|
return {int(all_ts_arr[i]): bool(vol_ok_arr[i]) for i in range(n_total)}
|
|
|
|
|
|
def _precompute_vol_ok_gold(parquet_files: list) -> dict:
|
|
"""Pre-compute vol_ok matching exp_shared.run_backtest() exactly.
|
|
|
|
Methodology:
|
|
- Per-day rets reset: rets = np.diff(bp) / (bp[:-1] + 1e-9)
|
|
- 50-return window: v = np.std(rets[j-50:j])
|
|
- 1-bar shift: dvol[j+1] = v (same as run_backtest)
|
|
- Dynamic threshold: vp60 accumulated across all processed days,
|
|
recomputed after each day's vols are added (len > 1000 gate)
|
|
Returns dict: {ts_ns (int) -> vol_ok (bool)}
|
|
"""
|
|
all_vols = []
|
|
result = {}
|
|
|
|
for pf in parquet_files:
|
|
df = pd.read_parquet(pf)
|
|
run_date = pf.stem
|
|
|
|
if 'timestamp' in df.columns:
|
|
ts_ns_arr = df['timestamp'].values.astype('int64')
|
|
else:
|
|
day_dt = datetime.strptime(run_date, '%Y-%m-%d').replace(tzinfo=timezone.utc)
|
|
day_start_ns = int(day_dt.timestamp() * 1e9)
|
|
n = len(df)
|
|
ts_ns_arr = np.array([day_start_ns + i * BAR_INTERVAL_S * 1_000_000_000
|
|
for i in range(n)], dtype='int64')
|
|
|
|
bp = df['BTCUSDT'].values if 'BTCUSDT' in df.columns else None
|
|
n = len(df)
|
|
dvol = np.zeros(n, dtype=np.float64)
|
|
|
|
if bp is not None and len(bp) > 1:
|
|
rets = np.diff(bp.astype('float64')) / (bp[:-1].astype('float64') + 1e-9)
|
|
for j in range(50, len(rets)):
|
|
v = float(np.std(rets[j - 50:j]))
|
|
dvol[j + 1] = v
|
|
if v > 0:
|
|
all_vols.append(v)
|
|
|
|
vp60 = float(np.percentile(all_vols, 60)) if len(all_vols) > 1000 else VOL_P60_INWINDOW
|
|
|
|
for i in range(n):
|
|
result[int(ts_ns_arr[i])] = bool(dvol[i] > 0 and dvol[i] > vp60)
|
|
|
|
return result
|
|
|
|
|
|
def _write_run_log(actor, result: dict, run_id: str) -> Path:
|
|
"""Write full trade history + run state to RUN_LOGS_DIR/<run_id>.json.
|
|
|
|
Works in both backtest and paper-trade modes (no HZ required).
|
|
Returns the path written.
|
|
"""
|
|
eng = getattr(actor, 'engine', None)
|
|
|
|
# Trade history — NDAlphaEngine stores list of trade dicts
|
|
trade_history = []
|
|
if eng is not None:
|
|
raw_trades = getattr(eng, 'trade_history', [])
|
|
for t in raw_trades:
|
|
try:
|
|
trade_history.append({k: (float(v) if hasattr(v, '__float__') else v)
|
|
for k, v in (t.items() if hasattr(t, 'items') else {})})
|
|
except Exception:
|
|
trade_history.append(str(t))
|
|
|
|
# Performance summary (if engine supports it)
|
|
perf = {}
|
|
if eng is not None and hasattr(eng, 'get_performance_summary'):
|
|
try:
|
|
perf = eng.get_performance_summary()
|
|
except Exception:
|
|
pass
|
|
|
|
payload = {
|
|
'run_id': run_id,
|
|
'run_ts': datetime.now(timezone.utc).isoformat(),
|
|
'window': f'{WINDOW_START}:{WINDOW_END}',
|
|
'engine_cfg': {k: v for k, v in CHAMPION_ENGINE_CFG.items()},
|
|
'vol_threshold': VOL_P60_THRESHOLD,
|
|
'initial_capital': INITIAL_CAPITAL,
|
|
'result': result,
|
|
'total_scans': getattr(eng, 'total_scans', None), # diagnostic: total step_bar calls
|
|
'bar_count': getattr(eng, '_bar_count', None),
|
|
'perf_summary': perf,
|
|
'trades': trade_history,
|
|
'trade_count': len(trade_history),
|
|
'open_positions': [str(p) for p in getattr(eng, 'open_positions', [])],
|
|
'stale_state_events': getattr(actor, '_stale_state_events', 0),
|
|
}
|
|
|
|
out_path = RUN_LOGS_DIR / f'{run_id}.json'
|
|
with open(out_path, 'w') as fh:
|
|
json.dump(payload, fh, indent=2, default=str)
|
|
return out_path
|
|
|
|
|
|
def run_continuous_native(
|
|
initial_capital: float,
|
|
preload_dates: list,
|
|
assets: list,
|
|
all_instruments: dict, # symbol -> Nautilus instrument
|
|
parquet_files: list, # All parquets to process continuously
|
|
) -> dict:
|
|
global _FEATURE_STORE
|
|
|
|
from nautilus_trader.backtest.engine import BacktestEngine, BacktestEngineConfig
|
|
from nautilus_trader.model.identifiers import Venue
|
|
from nautilus_trader.model.data import BarType
|
|
from nautilus_trader.model.objects import Money, Currency
|
|
from nautilus_trader.model.enums import OmsType, AccountType
|
|
from nautilus_dolphin.nautilus.dolphin_actor import DolphinActor
|
|
import gc
|
|
|
|
print(" [RAM] Building continuous state (20M+ bars). This may take up to 2 minutes...")
|
|
_FEATURE_STORE.clear()
|
|
all_bars = []
|
|
|
|
bar_type_map = {}
|
|
for sym in assets:
|
|
bt_str = f"{sym}.BINANCE-5-SECOND-LAST-EXTERNAL"
|
|
bar_type_map[sym] = BarType.from_str(bt_str)
|
|
|
|
# Load all days into RAM
|
|
for idx, pf in enumerate(parquet_files):
|
|
run_date = pf.stem
|
|
print(f" - Loading {run_date} ({idx+1}/{len(parquet_files)})...", end='\r', flush=True)
|
|
df = pd.read_parquet(pf)
|
|
n_rows = len(df)
|
|
|
|
# ── Timestamp alignment: use parquet's real timestamps (datetime64[ns]) ──
|
|
has_ts_col = 'timestamp' in df.columns
|
|
if has_ts_col:
|
|
ts_ns_arr = df['timestamp'].values.astype('int64')
|
|
else:
|
|
day_dt = datetime.strptime(run_date, '%Y-%m-%d').replace(tzinfo=timezone.utc)
|
|
day_start_ns = int(day_dt.timestamp() * 1e9)
|
|
ts_ns_arr = np.array([day_start_ns + i * BAR_INTERVAL_S * 1_000_000_000
|
|
for i in range(n_rows)], dtype='int64')
|
|
if idx == 0:
|
|
print(f"\n [WARN] No 'timestamp' col in {pf.name} — using synthetic ts")
|
|
|
|
btc = df['BTCUSDT'].values if 'BTCUSDT' in df.columns else None
|
|
|
|
# ── Per-day vol_ok: rolling 50-bar dvol vs static in-window threshold ──
|
|
# Gold cert methodology: VOL_P60_INWINDOW = d['vol_p60'] from exp_shared.load_data()
|
|
vol_ok_mask = np.zeros(n_rows, dtype=bool)
|
|
if btc is not None and len(btc) >= 51:
|
|
btc_f = btc.astype(np.float64)
|
|
for j in range(50, n_rows):
|
|
seg = btc_f[max(0, j - 50):j]
|
|
if len(seg) < 10:
|
|
continue
|
|
diffs_seg = np.diff(seg)
|
|
denom = seg[:-1]
|
|
if np.any(denom == 0):
|
|
continue
|
|
v = float(np.std(diffs_seg / denom))
|
|
if math.isfinite(v) and v > 0:
|
|
vol_ok_mask[j] = v > VOL_P60_INWINDOW
|
|
|
|
# ── Vectorised column index lookup for speed ──
|
|
cols = df.columns.tolist()
|
|
vd_idx = cols.index('vel_div') if 'vel_div' in cols else -1
|
|
v50_idx = cols.index('v50_lambda_max_velocity') if 'v50_lambda_max_velocity' in cols else -1
|
|
v750_idx = cols.index('v750_lambda_max_velocity') if 'v750_lambda_max_velocity' in cols else -1
|
|
i50_idx = cols.index('instability_50') if 'instability_50' in cols else -1
|
|
asset_col_indices = {sym: cols.index(sym) for sym in assets if sym in cols}
|
|
data_arr = df.values # numpy array for O(1) row/col access
|
|
|
|
for row_i in range(n_rows):
|
|
ts_ns = int(ts_ns_arr[row_i])
|
|
row_vals = data_arr[row_i]
|
|
|
|
vd_raw = _safe_f(row_vals[vd_idx]) if vd_idx != -1 else 0.0
|
|
_FEATURE_STORE[ts_ns] = {
|
|
'vel_div': vd_raw if math.isfinite(vd_raw) else None,
|
|
'v50': _safe_f(row_vals[v50_idx]) if v50_idx != -1 else 0.0,
|
|
'v750': _safe_f(row_vals[v750_idx]) if v750_idx != -1 else 0.0,
|
|
'inst50': _safe_f(row_vals[i50_idx]) if i50_idx != -1 else 0.0,
|
|
'vol_ok': bool(vol_ok_mask[row_i]), # per-day dvol vs in-window static threshold
|
|
'row_i': row_i,
|
|
}
|
|
|
|
# Deterministic ordering: non-BTC first, BTCUSDT last (sync gate trigger)
|
|
for sym in assets:
|
|
if sym == 'BTCUSDT' or sym not in asset_col_indices:
|
|
continue
|
|
p = row_vals[asset_col_indices[sym]]
|
|
if p is not None and math.isfinite(float(p)) and float(p) > 0:
|
|
bar = _make_bar(bar_type_map[sym], float(p), ts_ns)
|
|
if bar is not None:
|
|
all_bars.append(bar)
|
|
|
|
# BTC last — triggers on_bar() sync gate
|
|
if 'BTCUSDT' in asset_col_indices:
|
|
p = row_vals[asset_col_indices['BTCUSDT']]
|
|
if p is not None and math.isfinite(float(p)) and float(p) > 0:
|
|
bar = _make_bar(bar_type_map['BTCUSDT'], float(p), ts_ns)
|
|
if bar is not None:
|
|
all_bars.append(bar)
|
|
|
|
# Free memory incrementally
|
|
if btc is not None:
|
|
del btc
|
|
del df, data_arr
|
|
gc.collect()
|
|
|
|
print(f" [RAM] Memory load complete. {len(all_bars):,} bars synthesized.")
|
|
|
|
# Build Actor
|
|
actor_cfg = {
|
|
'engine': dict(CHAMPION_ENGINE_CFG),
|
|
'paper_trade': {'initial_capital': initial_capital},
|
|
'posture_override': 'APEX',
|
|
'live_mode': False,
|
|
'native_mode': True,
|
|
'run_date': WINDOW_START,
|
|
'bar_type': 'BTCUSDT.BINANCE-5-SECOND-LAST-EXTERNAL',
|
|
'mc_models_dir': MC_MODELS_DIR,
|
|
'mc_base_cfg': MC_BASE_CFG,
|
|
'venue': 'BINANCE',
|
|
'vol_p60': VOL_P60_THRESHOLD, # unified gold constant
|
|
'acb_preload_dates': preload_dates,
|
|
'assets': assets,
|
|
'parquet_dir': 'vbt_cache',
|
|
'registered_assets': assets,
|
|
}
|
|
actor_cfg['engine']['initial_capital'] = initial_capital
|
|
|
|
be_cfg = BacktestEngineConfig(trader_id=TRADER_ID)
|
|
bt_engine = BacktestEngine(config=be_cfg)
|
|
|
|
venue = Venue('BINANCE')
|
|
usdt = Currency.from_str('USDT')
|
|
|
|
bt_engine.add_venue(
|
|
venue=venue,
|
|
oms_type=OmsType.HEDGING,
|
|
account_type=AccountType.MARGIN,
|
|
base_currency=usdt,
|
|
starting_balances=[Money(str(round(initial_capital, 8)), usdt)],
|
|
)
|
|
|
|
for sym, instr in all_instruments.items():
|
|
bt_engine.add_instrument(instr)
|
|
|
|
actor = DolphinActor(config=actor_cfg)
|
|
bt_engine.add_strategy(actor)
|
|
bt_engine.add_data(all_bars)
|
|
|
|
print(" [NATIVE] Starting Continuous Execution Engine...")
|
|
t0 = time.time()
|
|
try:
|
|
bt_engine.run()
|
|
except Exception as e:
|
|
print(f" [WARN] bt_engine.run() error bounds: {e}")
|
|
elapsed = round(time.time() - t0, 1)
|
|
|
|
try:
|
|
accounts = list(bt_engine.trader.portfolio.accounts())
|
|
nautilus_account = accounts[0] if accounts else None
|
|
if nautilus_account:
|
|
print(f" [CAPITAL] Nautilus Account Final: {nautilus_account.balances()}")
|
|
naut_bal_obj = nautilus_account.balance_total(usdt)
|
|
naut_capital = float(naut_bal_obj.as_double()) if naut_bal_obj else initial_capital
|
|
else:
|
|
naut_capital = initial_capital
|
|
except Exception:
|
|
naut_capital = initial_capital
|
|
|
|
shadow_capital = float(getattr(actor.engine, 'capital', initial_capital))
|
|
trades = len(getattr(actor.engine, 'trade_history', []))
|
|
|
|
if abs(naut_capital - initial_capital) < 1e-4 and trades > 0:
|
|
recon_status = 'SHADOW_USED'
|
|
final_capital = shadow_capital
|
|
else:
|
|
recon_status = 'NAUTILUS'
|
|
final_capital = naut_capital
|
|
|
|
divergence = abs(naut_capital - shadow_capital) / max(shadow_capital, 1.0)
|
|
if divergence > RECON_CRITICAL and trades > 0 and abs(naut_capital - initial_capital) > 1.0:
|
|
recon_status = f'DIVERGE_CRITICAL_{divergence:.1%}'
|
|
elif divergence > RECON_TOLERANCE and trades > 0 and abs(naut_capital - initial_capital) > 1.0:
|
|
recon_status = f'DIVERGE_WARN_{divergence:.1%}'
|
|
|
|
pnl = final_capital - initial_capital
|
|
|
|
result = {
|
|
'date': f'{WINDOW_START}_{WINDOW_END}',
|
|
'capital': final_capital,
|
|
'shadow_capital': shadow_capital,
|
|
'naut_capital': naut_capital,
|
|
'pnl': pnl,
|
|
'trades': trades,
|
|
'recon': recon_status,
|
|
'elapsed_s': elapsed,
|
|
'stale_state_events': getattr(actor, '_stale_state_events', 0),
|
|
}
|
|
|
|
# Write full trade + state log before disposing engine
|
|
run_id = f"nautilus_native_{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}"
|
|
log_path = _write_run_log(actor, result, run_id)
|
|
print(f" [LOG] Run log: {log_path.name} ({result['trades']} trades)")
|
|
|
|
bt_engine.dispose()
|
|
return result
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Main backtest loop
|
|
# ---------------------------------------------------------------------------
|
|
def run_native_backtest():
|
|
print('=' * 72)
|
|
print('NAUTILUS CONTINUOUS NATIVE BACKTEST (32GB+ RAM) - SINGLE STATE')
|
|
print(f'Window: {WINDOW_START} -> {WINDOW_END}')
|
|
print(f'Mode: VBT parquet -> Nautilus Bar stream -> DolphinActor.on_bar()')
|
|
print('=' * 72)
|
|
|
|
from nautilus_trader.model.identifiers import Venue as NVenue
|
|
NV = NVenue('BINANCE')
|
|
|
|
# Get parquet files
|
|
parquet_files = get_parquet_files()
|
|
if not parquet_files:
|
|
print(f'ERROR: No parquets in {VBT_KLINES_DIR}')
|
|
return None
|
|
|
|
print(f' Parquet files: {len(parquet_files)} ({parquet_files[0].stem} -> {parquet_files[-1].stem})')
|
|
|
|
# Get asset list
|
|
df0 = pd.read_parquet(parquet_files[0])
|
|
asset_cols = [c for c in df0.columns if c not in {
|
|
'timestamp', 'scan_number', 'v50_lambda_max_velocity',
|
|
'v150_lambda_max_velocity', 'v300_lambda_max_velocity',
|
|
'v750_lambda_max_velocity', 'vel_div', 'instability_50', 'instability_150'
|
|
}]
|
|
print(f' Assets: {len(asset_cols)} | {asset_cols[:5]}...')
|
|
|
|
# Build instruments (once, reused each day)
|
|
print(f' Building {len(asset_cols)} Nautilus instruments...')
|
|
all_instruments = {}
|
|
for sym in asset_cols:
|
|
try:
|
|
all_instruments[sym] = _make_instrument(sym, NV)
|
|
except Exception as e:
|
|
print(f' [WARN] Could not create instrument for {sym}: {e}')
|
|
|
|
print(f' Registered: {len(all_instruments)} instruments')
|
|
|
|
print(f' vol gate : per-day dvol > {VOL_P60_INWINDOW:.8f} (gold cert: static from exp_shared.load_data())')
|
|
print(f' vol threshold : {VOL_P60_THRESHOLD:.8f} (live trading only)')
|
|
print(f' min_irp_align : {CHAMPION_ENGINE_CFG["min_irp_alignment"]} (gold spec)')
|
|
|
|
all_window_dates = [pf.stem for pf in parquet_files]
|
|
|
|
# Run continuous execution
|
|
print('\n Starting continuous backtest engine instantiation...')
|
|
t_total = time.time()
|
|
|
|
result = run_continuous_native(
|
|
float(INITIAL_CAPITAL),
|
|
all_window_dates, asset_cols, all_instruments, parquet_files
|
|
)
|
|
|
|
final_capital = result['capital']
|
|
pnl_f = result['pnl']
|
|
trades_d = result['trades']
|
|
recon = result.get('recon', '?')
|
|
elapsed = result.get('elapsed_s', 0)
|
|
|
|
naut_str = f"${result.get('naut_capital', 0):,.2f}"
|
|
shad_str = f"${result.get('shadow_capital', 0):,.2f}"
|
|
|
|
print()
|
|
print(f' {"Date":<25} {"Nautilus$":>11} {"Shadow$":>11} '
|
|
f'{"Trades":>6} {"PnL":>9} {"Recon":<25} {"Sec":>5}')
|
|
print(f' {"-"*25} {"-"*11} {"-"*11} {"-"*6} {"-"*9} {"-"*25} {"-"*5}')
|
|
print(f' {result["date"]:<25} {naut_str:>11} {shad_str:>11} '
|
|
f'{trades_d:>6} {pnl_f:>+9.2f} {recon:<25} {elapsed:>5.0f}s')
|
|
|
|
# Final metrics
|
|
roi = (final_capital - INITIAL_CAPITAL) / INITIAL_CAPITAL * 100.0
|
|
# In continuous mode, daily dd array isn't extracted here but we can default
|
|
max_dd = 0.0
|
|
total_elapsed = time.time() - t_total
|
|
|
|
print()
|
|
print('=' * 72)
|
|
print(f' Final Capital : ${final_capital:,.2f}')
|
|
print(f' ROI : {roi:+.2f}% (gold ref: +181.81%)')
|
|
print(f' Total Trades : {trades_d} (gold ref: 2155)')
|
|
print(f' Total Elapsed : {total_elapsed/60:.1f} min')
|
|
print('=' * 72)
|
|
|
|
result_payload = {
|
|
'roi': round(roi, 4),
|
|
'trades': trades_d,
|
|
'dd': round(max_dd, 4),
|
|
'capital': round(final_capital, 4),
|
|
'window': f'{WINDOW_START}:{WINDOW_END}',
|
|
'days': len(parquet_files),
|
|
'elapsed_s': round(total_elapsed, 1),
|
|
'engine': 'Nautilus Native (real bars + Nautilus ledger)',
|
|
'run_ts': datetime.now(timezone.utc).isoformat(),
|
|
}
|
|
out_path = RUN_LOGS_DIR / f"nautilus_native_{datetime.now(timezone.utc).strftime('%Y%m%d_%H%M%S')}.json"
|
|
with open(out_path, 'w') as f:
|
|
json.dump(result_payload, f, indent=2)
|
|
print(f' Saved: {out_path.name}')
|
|
|
|
return result_payload
|
|
|
|
|
|
if __name__ == '__main__':
|
|
import sys
|
|
sys.modules['prod.nautilus_native_backtest'] = sys.modules['__main__']
|
|
run_native_backtest()
|