661 lines
32 KiB
Python
661 lines
32 KiB
Python
|
"""DOLPHIN Paper Trading — Prefect Flow.
|
|||
|
|
|
|||
|
|
Runs daily at 00:05 UTC. Processes yesterday's live scan data through
|
|||
|
|
the NDAlphaEngine champion stack. Logs virtual P&L to disk + Hazelcast.
|
|||
|
|
|
|||
|
|
Blue deployment: champion SHORT (configs/blue.yml)
|
|||
|
|
Green deployment: bidirectional SHORT+LONG (configs/green.yml) [pending LONG validation]
|
|||
|
|
|
|||
|
|
Usage:
|
|||
|
|
# Register flows (run once, after Prefect server is up):
|
|||
|
|
PREFECT_API_URL=http://localhost:4200/api python paper_trade_flow.py --register
|
|||
|
|
|
|||
|
|
# Run manually for a specific date:
|
|||
|
|
PREFECT_API_URL=http://localhost:4200/api python paper_trade_flow.py \\
|
|||
|
|
--date 2026-02-25 --config configs/blue.yml
|
|||
|
|
"""
|
|||
|
|
import sys, json, yaml, logging, argparse, csv, urllib.request
|
|||
|
|
from pathlib import Path
|
|||
|
|
from datetime import datetime, timedelta, date, timezone
|
|||
|
|
import numpy as np
|
|||
|
|
import pandas as pd
|
|||
|
|
|
|||
|
|
HCM_DIR = Path(__file__).parent.parent
|
|||
|
|
sys.path.insert(0, str(HCM_DIR / 'nautilus_dolphin'))
|
|||
|
|
|
|||
|
|
from prefect import flow, task, get_run_logger
|
|||
|
|
from prefect.schedules import Cron
|
|||
|
|
|
|||
|
|
import hazelcast
|
|||
|
|
|
|||
|
|
logging.basicConfig(level=logging.WARNING) # suppress Prefect noise below WARNING
|
|||
|
|
|
|||
|
|
# ── Paths ───────────────────────────────────────────────────────────────────────
|
|||
|
|
from dolphin_paths import get_eigenvalues_path, get_klines_dir
|
|||
|
|
SCANS_DIR = get_eigenvalues_path() # platform-aware: Win → NG3 dir, Linux → /mnt/ng6_data/eigenvalues
|
|||
|
|
KLINES_DIR = get_klines_dir() # vbt_cache_klines/ — NG5 parquet source (preferred)
|
|||
|
|
MC_MODELS_DIR = str(HCM_DIR / 'nautilus_dolphin' / 'mc_results' / 'models')
|
|||
|
|
|
|||
|
|
# Columns that are eigenvalue metadata, not asset prices
|
|||
|
|
META_COLS = {
|
|||
|
|
'timestamp', 'scan_number',
|
|||
|
|
'v50_lambda_max_velocity', 'v150_lambda_max_velocity',
|
|||
|
|
'v300_lambda_max_velocity', 'v750_lambda_max_velocity',
|
|||
|
|
'vel_div', 'instability_50', 'instability_150',
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
HZ_HOST = "localhost:5701"
|
|||
|
|
HZ_CLUSTER = "dolphin"
|
|||
|
|
|
|||
|
|
# Number of historical eigenvalue dates to use for ACB w750 threshold calibration
|
|||
|
|
ACB_HISTORY_DAYS = 60
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ── Helpers ──────────────────────────────────────────────────────────────────────
|
|||
|
|
|
|||
|
|
def _get_recent_scan_dates(n: int) -> list:
|
|||
|
|
"""Return sorted list of up to n most-recent eigenvalue date dirs."""
|
|||
|
|
try:
|
|||
|
|
dirs = sorted(
|
|||
|
|
d.name for d in SCANS_DIR.iterdir()
|
|||
|
|
if d.is_dir() and len(d.name) == 10 and d.name.startswith('20')
|
|||
|
|
)
|
|||
|
|
return dirs[-n:]
|
|||
|
|
except Exception:
|
|||
|
|
return []
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _fetch_btcusdt_klines_fallback(date_str: str) -> "dict[str, float]":
|
|||
|
|
"""Fetch BTCUSDT 1m klines from Binance futures for date_str.
|
|||
|
|
|
|||
|
|
Returns a dict mapping ISO timestamp strings (minute precision) → close price.
|
|||
|
|
Falls back to empty dict on any error (caller handles missing prices gracefully).
|
|||
|
|
"""
|
|||
|
|
try:
|
|||
|
|
from datetime import timezone as tz
|
|||
|
|
day_start = datetime.strptime(date_str, "%Y-%m-%d").replace(tzinfo=tz.utc)
|
|||
|
|
day_end = day_start + timedelta(days=1)
|
|||
|
|
start_ms = int(day_start.timestamp() * 1000)
|
|||
|
|
end_ms = int(day_end.timestamp() * 1000)
|
|||
|
|
prices: dict[str, float] = {}
|
|||
|
|
# Binance returns max 1500 bars per request; 1440 bars/day fits in one call
|
|||
|
|
url = (
|
|||
|
|
f"https://fapi.binance.com/fapi/v1/klines"
|
|||
|
|
f"?symbol=BTCUSDT&interval=1m"
|
|||
|
|
f"&startTime={start_ms}&endTime={end_ms}&limit=1500"
|
|||
|
|
)
|
|||
|
|
with urllib.request.urlopen(url, timeout=10) as resp:
|
|||
|
|
bars = json.loads(resp.read())
|
|||
|
|
for bar in bars:
|
|||
|
|
ts_ms = int(bar[0])
|
|||
|
|
close = float(bar[4])
|
|||
|
|
ts_iso = datetime.fromtimestamp(ts_ms / 1000, tz=tz.utc).strftime("%Y-%m-%dT%H:%M")
|
|||
|
|
prices[ts_iso] = close
|
|||
|
|
return prices
|
|||
|
|
except Exception:
|
|||
|
|
return {}
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _load_scan_df_from_json(date_str: str) -> pd.DataFrame:
|
|||
|
|
"""Load daily scan JSON files → DataFrame with vel_div + asset prices.
|
|||
|
|
|
|||
|
|
Each scan JSON has:
|
|||
|
|
- windows: {50, 150, 300, 750} → tracking_data.lambda_max_velocity
|
|||
|
|
- pricing_data.current_prices → per-asset USD prices
|
|||
|
|
Returns one row per scan, sorted by scan_number.
|
|||
|
|
When current_prices is empty (scanner bug), falls back to Binance 1m klines.
|
|||
|
|
"""
|
|||
|
|
scan_dir = SCANS_DIR / date_str
|
|||
|
|
if not scan_dir.exists():
|
|||
|
|
return pd.DataFrame()
|
|||
|
|
|
|||
|
|
json_files = sorted(scan_dir.glob("scan_*.json"), key=lambda f: f.name)
|
|||
|
|
# Exclude the _Indicators.json companion files if any
|
|||
|
|
json_files = [f for f in json_files if '__Indicators' not in f.name]
|
|||
|
|
if not json_files:
|
|||
|
|
return pd.DataFrame()
|
|||
|
|
|
|||
|
|
total = len(json_files)
|
|||
|
|
rows = []
|
|||
|
|
for i, jf in enumerate(json_files):
|
|||
|
|
if i % 500 == 0 and i > 0:
|
|||
|
|
print(f" [scan load] {i}/{total} files...", flush=True)
|
|||
|
|
try:
|
|||
|
|
with open(jf, 'r', encoding='utf-8') as fh:
|
|||
|
|
data = json.load(fh)
|
|||
|
|
|
|||
|
|
row = {
|
|||
|
|
'scan_number': data.get('scan_number', 0),
|
|||
|
|
'timestamp': data.get('timestamp', ''),
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
# Eigenvalue velocity per window
|
|||
|
|
windows = data.get('windows', {})
|
|||
|
|
for w_key, vel_col, inst_col in [
|
|||
|
|
('50', 'v50_lambda_max_velocity', 'instability_50'),
|
|||
|
|
('150', 'v150_lambda_max_velocity', 'instability_150'),
|
|||
|
|
('300', 'v300_lambda_max_velocity', None),
|
|||
|
|
('750', 'v750_lambda_max_velocity', None),
|
|||
|
|
]:
|
|||
|
|
w_data = windows.get(str(w_key), {})
|
|||
|
|
td = w_data.get('tracking_data', {})
|
|||
|
|
vel = td.get('lambda_max_velocity')
|
|||
|
|
row[vel_col] = float(vel) if vel is not None else np.nan
|
|||
|
|
if inst_col is not None:
|
|||
|
|
rs = w_data.get('regime_signals', {})
|
|||
|
|
row[inst_col] = float(rs.get('instability_score', 0.0) or 0.0)
|
|||
|
|
|
|||
|
|
# vel_div = w50_vel − w150_vel (the primary alpha signal)
|
|||
|
|
v50 = row.get('v50_lambda_max_velocity', np.nan)
|
|||
|
|
v150 = row.get('v150_lambda_max_velocity', np.nan)
|
|||
|
|
row['vel_div'] = (v50 - v150) if (np.isfinite(v50) and np.isfinite(v150)) else np.nan
|
|||
|
|
|
|||
|
|
# Asset prices
|
|||
|
|
pricing = data.get('pricing_data', {})
|
|||
|
|
prices = pricing.get('current_prices', {})
|
|||
|
|
row.update({sym: float(px) for sym, px in prices.items() if px is not None})
|
|||
|
|
|
|||
|
|
rows.append(row)
|
|||
|
|
except Exception:
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
if not rows:
|
|||
|
|
return pd.DataFrame()
|
|||
|
|
|
|||
|
|
df = pd.DataFrame(rows).sort_values('scan_number').reset_index(drop=True)
|
|||
|
|
|
|||
|
|
# Fallback: if BTCUSDT prices are missing (scanner current_prices bug),
|
|||
|
|
# fetch 1m klines from Binance and fill by timestamp.
|
|||
|
|
if 'BTCUSDT' not in df.columns or df['BTCUSDT'].isna().all():
|
|||
|
|
btc_klines = _fetch_btcusdt_klines_fallback(date_str)
|
|||
|
|
if btc_klines:
|
|||
|
|
def _lookup_btc(ts_str: str) -> float:
|
|||
|
|
# ts_str is ISO format from scan JSON; match to minute precision
|
|||
|
|
ts_min = str(ts_str)[:16] # "YYYY-MM-DDTHH:MM"
|
|||
|
|
return btc_klines.get(ts_min, np.nan)
|
|||
|
|
df['BTCUSDT'] = df['timestamp'].apply(_lookup_btc)
|
|||
|
|
# Forward-fill any gaps (scan timestamps between 1m bars)
|
|||
|
|
df['BTCUSDT'] = df['BTCUSDT'].ffill().bfill()
|
|||
|
|
|
|||
|
|
return df
|
|||
|
|
|
|||
|
|
|
|||
|
|
def _load_scan_df_from_parquet(date_str: str) -> pd.DataFrame:
|
|||
|
|
"""Load daily scan data from vbt_cache_klines/ parquet (NG5+ preferred path).
|
|||
|
|
|
|||
|
|
Returns DataFrame with vel_div + asset prices, or empty DataFrame if not available.
|
|||
|
|
"""
|
|||
|
|
parq_path = KLINES_DIR / f"{date_str}.parquet"
|
|||
|
|
if not parq_path.exists():
|
|||
|
|
return pd.DataFrame()
|
|||
|
|
try:
|
|||
|
|
df = pd.read_parquet(parq_path)
|
|||
|
|
if df.empty or 'vel_div' not in df.columns:
|
|||
|
|
return pd.DataFrame()
|
|||
|
|
return df.reset_index(drop=True)
|
|||
|
|
except Exception:
|
|||
|
|
return pd.DataFrame()
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ── Tasks ───────────────────────────────────────────────────────────────────────
|
|||
|
|
|
|||
|
|
@task(name="load_config", retries=0)
|
|||
|
|
def load_config(config_path: str) -> dict:
|
|||
|
|
with open(config_path) as f:
|
|||
|
|
return yaml.safe_load(f)
|
|||
|
|
|
|||
|
|
|
|||
|
|
@task(name="load_day_scans", retries=2, retry_delay_seconds=10)
|
|||
|
|
def load_day_scans(date_str: str) -> pd.DataFrame:
|
|||
|
|
"""Load scan data for one date → VBT-compatible DataFrame.
|
|||
|
|
|
|||
|
|
Prefers vbt_cache_klines/ parquet (NG5 native, single file, fast).
|
|||
|
|
Falls back to eigenvalues/ JSON (NG3 legacy) if parquet unavailable.
|
|||
|
|
Extracts vel_div, eigenvalue features, and asset prices.
|
|||
|
|
"""
|
|||
|
|
log = get_run_logger()
|
|||
|
|
|
|||
|
|
# ── Parquet path (NG5 native — preferred) ──────────────────────────────────
|
|||
|
|
df = _load_scan_df_from_parquet(date_str)
|
|||
|
|
if not df.empty:
|
|||
|
|
log.info(f" [parquet] Loaded {len(df)} scans for {date_str} | cols={len(df.columns)}")
|
|||
|
|
else:
|
|||
|
|
# ── JSON fallback (NG3 legacy) ─────────────────────────────────────────
|
|||
|
|
df = _load_scan_df_from_json(date_str)
|
|||
|
|
if df.empty:
|
|||
|
|
log.warning(f"No usable scan data for {date_str} in {SCANS_DIR}")
|
|||
|
|
return pd.DataFrame()
|
|||
|
|
log.info(f" [json] Loaded {len(df)} scans for {date_str} | cols={len(df.columns)}")
|
|||
|
|
|
|||
|
|
if df.empty:
|
|||
|
|
log.warning(f"No usable scan data for {date_str} in {SCANS_DIR}")
|
|||
|
|
return pd.DataFrame()
|
|||
|
|
|
|||
|
|
# Drop rows with NaN vel_div (warmup period at start of day)
|
|||
|
|
valid = df['vel_div'].notna()
|
|||
|
|
n_dropped = (~valid).sum()
|
|||
|
|
df = df[valid].reset_index(drop=True)
|
|||
|
|
|
|||
|
|
# Verify BTCUSDT prices present (required for vol gate and DC)
|
|||
|
|
if 'BTCUSDT' not in df.columns:
|
|||
|
|
log.error(f"BTCUSDT prices missing from scan data on {date_str} — cannot run engine")
|
|||
|
|
return pd.DataFrame()
|
|||
|
|
|
|||
|
|
log.info(f" Loaded {len(df)} scans for {date_str} | cols={len(df.columns)} | "
|
|||
|
|
f"vel_div range=[{df['vel_div'].min():.4f}, {df['vel_div'].max():.4f}] "
|
|||
|
|
f"| {n_dropped} warmup rows dropped")
|
|||
|
|
return df
|
|||
|
|
|
|||
|
|
|
|||
|
|
@task(name="run_engine_day", retries=0, persist_result=False)
|
|||
|
|
def run_engine_day(date_str: str, df: pd.DataFrame, engine, vol_p60: float, posture: str = 'APEX', direction: int = -1) -> dict:
|
|||
|
|
"""Run one day through NDAlphaEngine. Returns daily stats dict."""
|
|||
|
|
log = get_run_logger()
|
|||
|
|
if df.empty or posture == 'HIBERNATE':
|
|||
|
|
log.warning(f"Empty DataFrame or HIBERNATE for {date_str} — skipping.")
|
|||
|
|
return {'date': date_str, 'pnl': 0.0, 'capital': engine.capital,
|
|||
|
|
'trades': 0, 'boost': 1.0, 'beta': 0.0, 'mc_status': 'NO_DATA', 'posture': posture}
|
|||
|
|
|
|||
|
|
asset_cols = [c for c in df.columns if c not in META_COLS]
|
|||
|
|
|
|||
|
|
# Vol gate: rolling 50-bar std of BTC returns
|
|||
|
|
bp = df['BTCUSDT'].values
|
|||
|
|
dvol = np.full(len(df), np.nan)
|
|||
|
|
for i in range(50, len(bp)):
|
|||
|
|
seg = bp[max(0, i - 50):i]
|
|||
|
|
if len(seg) >= 10 and seg[0] > 0:
|
|||
|
|
dvol[i] = float(np.std(np.diff(seg) / seg[:-1]))
|
|||
|
|
vol_ok = np.where(np.isfinite(dvol), dvol > vol_p60, False)
|
|||
|
|
|
|||
|
|
# 1. Setup day
|
|||
|
|
engine.begin_day(date_str, posture=posture, direction=direction)
|
|||
|
|
|
|||
|
|
# 2. Bar stream (replaces batch process_day)
|
|||
|
|
for ri in range(len(df)):
|
|||
|
|
row = df.iloc[ri]
|
|||
|
|
vd = row.get('vel_div')
|
|||
|
|
if vd is None or not np.isfinite(float(vd)):
|
|||
|
|
engine._global_bar_idx += 1
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
v50_raw = row.get('v50_lambda_max_velocity')
|
|||
|
|
v750_raw = row.get('v750_lambda_max_velocity')
|
|||
|
|
v50_val = float(v50_raw) if (v50_raw is not None and np.isfinite(float(v50_raw))) else 0.0
|
|||
|
|
v750_val = float(v750_raw) if (v750_raw is not None and np.isfinite(float(v750_raw))) else 0.0
|
|||
|
|
|
|||
|
|
prices = {}
|
|||
|
|
for ac in asset_cols:
|
|||
|
|
p = row.get(ac)
|
|||
|
|
if p is not None and p > 0 and np.isfinite(p):
|
|||
|
|
prices[ac] = float(p)
|
|||
|
|
|
|||
|
|
if not prices:
|
|||
|
|
engine._global_bar_idx += 1
|
|||
|
|
continue
|
|||
|
|
|
|||
|
|
# OB live step: fetch HZ snapshots and compute features BEFORE step_bar().
|
|||
|
|
# This populates the live caches so get_placement/get_signal/get_market()
|
|||
|
|
# return real OBF values instead of NEUTRAL defaults.
|
|||
|
|
if engine.ob_engine is not None:
|
|||
|
|
try:
|
|||
|
|
engine.ob_engine.step_live(list(prices.keys()), ri)
|
|||
|
|
except Exception:
|
|||
|
|
pass # OBF degraded → NEUTRAL values, trading continues
|
|||
|
|
|
|||
|
|
engine.step_bar(
|
|||
|
|
bar_idx=ri,
|
|||
|
|
vel_div=float(vd),
|
|||
|
|
prices=prices,
|
|||
|
|
vol_regime_ok=bool(vol_ok[ri]),
|
|||
|
|
v50_vel=v50_val,
|
|||
|
|
v750_vel=v750_val,
|
|||
|
|
)
|
|||
|
|
|
|||
|
|
# 3. Finalize day
|
|||
|
|
result = engine.end_day()
|
|||
|
|
result['posture'] = posture
|
|||
|
|
log.info(f" {date_str}: PnL={result.get('pnl', 0):+.2f} "
|
|||
|
|
f"T={result.get('trades', 0)} boost={result.get('boost', 1.0):.2f}x "
|
|||
|
|
f"MC={result.get('mc_status', '?')} Posture={posture}")
|
|||
|
|
return result
|
|||
|
|
|
|||
|
|
|
|||
|
|
@task(name="write_hz_state", retries=3, retry_delay_seconds=5, persist_result=False)
|
|||
|
|
def write_hz_state(hz_host: str, hz_cluster: str, imap_name: str, key: str, value: dict):
|
|||
|
|
"""Write state dict to Hazelcast IMap. Creates own client per call (serialization-safe)."""
|
|||
|
|
client = hazelcast.HazelcastClient(cluster_name=hz_cluster, cluster_members=[hz_host])
|
|||
|
|
try:
|
|||
|
|
client.get_map(imap_name).blocking().put(key, json.dumps(value))
|
|||
|
|
finally:
|
|||
|
|
client.shutdown()
|
|||
|
|
|
|||
|
|
|
|||
|
|
@task(name="log_pnl", retries=0, persist_result=False)
|
|||
|
|
def log_pnl(log_dir: Path, date_str: str, result: dict, capital: float):
|
|||
|
|
log_dir.mkdir(parents=True, exist_ok=True)
|
|||
|
|
row = {**result, 'date': date_str, 'capital': capital,
|
|||
|
|
'logged_at': datetime.now(timezone.utc).isoformat()}
|
|||
|
|
log_file = log_dir / f"paper_pnl_{date_str[:7]}.jsonl"
|
|||
|
|
with open(log_file, 'a') as f:
|
|||
|
|
f.write(json.dumps(row) + '\n')
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ── Flow ────────────────────────────────────────────────────────────────────────
|
|||
|
|
|
|||
|
|
@flow(name="dolphin-paper-trade", log_prints=True)
|
|||
|
|
def dolphin_paper_trade_flow(config_path: str = "configs/blue.yml",
|
|||
|
|
run_date: str = None,
|
|||
|
|
instrument: bool = False):
|
|||
|
|
"""Daily paper trading flow. Processes one day of live eigenvalue scans.
|
|||
|
|
|
|||
|
|
Scheduled at 00:05 UTC — processes yesterday's data.
|
|||
|
|
Run manually with run_date='YYYY-MM-DD' for backtesting or debugging.
|
|||
|
|
"""
|
|||
|
|
log = get_run_logger()
|
|||
|
|
|
|||
|
|
cfg = load_config(config_path)
|
|||
|
|
strategy_name = cfg['strategy_name']
|
|||
|
|
eng_cfg = cfg['engine']
|
|||
|
|
pt_cfg = cfg['paper_trade']
|
|||
|
|
hz_cfg = cfg['hazelcast']
|
|||
|
|
|
|||
|
|
target_date = run_date or (date.today() - timedelta(days=1)).isoformat()
|
|||
|
|
log.info(f"=== {strategy_name.upper()} paper trade: {target_date} ===")
|
|||
|
|
|
|||
|
|
# ── Lazy imports (numba JIT happens here) ──────────────────────────────────
|
|||
|
|
from nautilus_dolphin.nautilus.proxy_boost_engine import create_d_liq_engine
|
|||
|
|
from nautilus_dolphin.nautilus.adaptive_circuit_breaker import AdaptiveCircuitBreaker
|
|||
|
|
from mc.mc_ml import DolphinForewarner
|
|||
|
|
from nautilus_dolphin.nautilus.ob_features import OBFeatureEngine
|
|||
|
|
from nautilus_dolphin.nautilus.hz_ob_provider import HZOBProvider
|
|||
|
|
|
|||
|
|
client = hazelcast.HazelcastClient(cluster_name=HZ_CLUSTER, cluster_members=[HZ_HOST])
|
|||
|
|
imap_state = client.get_map(hz_cfg['imap_state']).blocking()
|
|||
|
|
|
|||
|
|
# ---- Restore capital ----
|
|||
|
|
STATE_KEY = f"state_{strategy_name}_{target_date}"
|
|||
|
|
restored_capital = pt_cfg['initial_capital']
|
|||
|
|
peak_capital = pt_cfg['initial_capital']
|
|||
|
|
stored_state = {}
|
|||
|
|
engine_state = None
|
|||
|
|
try:
|
|||
|
|
raw = imap_state.get(STATE_KEY) or imap_state.get('latest') or '{}'
|
|||
|
|
stored_state = json.loads(raw)
|
|||
|
|
if stored_state.get('strategy') == strategy_name and stored_state.get('capital', 0) > 0:
|
|||
|
|
restored_capital = float(stored_state['capital'])
|
|||
|
|
peak_capital = float(stored_state.get('peak_capital', restored_capital))
|
|||
|
|
engine_state = stored_state.get('engine_state')
|
|||
|
|
log.info(f"[STATE] Restored capital={restored_capital:.2f} from HZ")
|
|||
|
|
except Exception as e:
|
|||
|
|
log.warning(f"[STATE] HZ restore failed: {e} — using config capital")
|
|||
|
|
|
|||
|
|
# ── Engine — D_LIQ_GOLD config (8x/9x LiquidationGuardEngine) ───────────
|
|||
|
|
# create_d_liq_engine() overrides max_leverage→8.0 / abs_max_leverage→9.0
|
|||
|
|
# internally (D_LIQ_SOFT_CAP / D_LIQ_ABS_CAP constants), regardless of what
|
|||
|
|
# eng_cfg says. This is the certified gold leverage stack.
|
|||
|
|
engine = create_d_liq_engine(
|
|||
|
|
initial_capital = restored_capital,
|
|||
|
|
vel_div_threshold = eng_cfg['vel_div_threshold'],
|
|||
|
|
vel_div_extreme = eng_cfg['vel_div_extreme'],
|
|||
|
|
min_leverage = eng_cfg['min_leverage'],
|
|||
|
|
max_leverage = eng_cfg.get('max_leverage', 8.0),
|
|||
|
|
abs_max_leverage = eng_cfg.get('abs_max_leverage', 9.0),
|
|||
|
|
leverage_convexity = eng_cfg['leverage_convexity'],
|
|||
|
|
fraction = eng_cfg['fraction'],
|
|||
|
|
fixed_tp_pct = eng_cfg['fixed_tp_pct'],
|
|||
|
|
stop_pct = eng_cfg['stop_pct'],
|
|||
|
|
max_hold_bars = eng_cfg['max_hold_bars'],
|
|||
|
|
use_direction_confirm= eng_cfg['use_direction_confirm'],
|
|||
|
|
dc_lookback_bars = eng_cfg['dc_lookback_bars'],
|
|||
|
|
dc_min_magnitude_bps = eng_cfg['dc_min_magnitude_bps'],
|
|||
|
|
dc_skip_contradicts = eng_cfg['dc_skip_contradicts'],
|
|||
|
|
dc_leverage_boost = eng_cfg['dc_leverage_boost'],
|
|||
|
|
dc_leverage_reduce = eng_cfg['dc_leverage_reduce'],
|
|||
|
|
use_asset_selection = eng_cfg['use_asset_selection'],
|
|||
|
|
min_irp_alignment = eng_cfg['min_irp_alignment'],
|
|||
|
|
use_sp_fees = eng_cfg['use_sp_fees'],
|
|||
|
|
use_sp_slippage = eng_cfg['use_sp_slippage'],
|
|||
|
|
sp_maker_entry_rate = eng_cfg['sp_maker_entry_rate'],
|
|||
|
|
sp_maker_exit_rate = eng_cfg['sp_maker_exit_rate'],
|
|||
|
|
use_ob_edge = eng_cfg['use_ob_edge'],
|
|||
|
|
ob_edge_bps = eng_cfg['ob_edge_bps'],
|
|||
|
|
ob_confirm_rate = eng_cfg['ob_confirm_rate'],
|
|||
|
|
lookback = eng_cfg['lookback'],
|
|||
|
|
use_alpha_layers = eng_cfg['use_alpha_layers'],
|
|||
|
|
use_dynamic_leverage = eng_cfg['use_dynamic_leverage'],
|
|||
|
|
seed = eng_cfg.get('seed', 42),
|
|||
|
|
)
|
|||
|
|
engine.set_esoteric_hazard_multiplier(0.0) # gold spec: hazard=0 → base_max_leverage=8.0
|
|||
|
|
|
|||
|
|
if engine_state:
|
|||
|
|
try:
|
|||
|
|
engine.restore_state(engine_state)
|
|||
|
|
log.info("[STATE] Restored full engine state (including open positions)")
|
|||
|
|
except Exception as e:
|
|||
|
|
log.error(f"[STATE] Failed to restore engine state: {e}")
|
|||
|
|
|
|||
|
|
# ── ACB — preload w750 from recent history for valid p60 threshold ─────────
|
|||
|
|
# w750 calibration always uses NPZ history (threshold is a population statistic).
|
|||
|
|
# Daily ExF factors are sourced from HZ exf_latest (pre-lagged) when available;
|
|||
|
|
# fall back to NPZ disk scan if HZ data is absent or stale (>12 h).
|
|||
|
|
acb = AdaptiveCircuitBreaker()
|
|||
|
|
acb.config.EIGENVALUES_PATH = SCANS_DIR # CRITICAL: override Windows default for Linux
|
|||
|
|
recent_dates = _get_recent_scan_dates(ACB_HISTORY_DAYS)
|
|||
|
|
if target_date not in recent_dates:
|
|||
|
|
recent_dates = (recent_dates + [target_date])[-ACB_HISTORY_DAYS:]
|
|||
|
|
acb.preload_w750(recent_dates)
|
|||
|
|
log.info(f" ACB preloaded {len(recent_dates)} dates | w750_threshold={acb._w750_threshold:.6f}")
|
|||
|
|
|
|||
|
|
# ── ACB HZ warm-up: pre-load today's boost from live exf_latest ────────────
|
|||
|
|
# The ExF service applies per-indicator lag BEFORE pushing to HZ, so the
|
|||
|
|
# values in exf_latest are already delay-adjusted. Do NOT re-lag them.
|
|||
|
|
_acb_hz_ok = False
|
|||
|
|
try:
|
|||
|
|
features_map = client.get_map('DOLPHIN_FEATURES').blocking()
|
|||
|
|
exf_raw = features_map.get('exf_latest')
|
|||
|
|
if exf_raw:
|
|||
|
|
exf_snapshot = json.loads(exf_raw)
|
|||
|
|
# Live w750 from latest scan (may be absent during warmup)
|
|||
|
|
scan_raw = features_map.get('latest_eigen_scan')
|
|||
|
|
w750_live: float | None = None
|
|||
|
|
if scan_raw:
|
|||
|
|
scan_data = json.loads(scan_raw)
|
|||
|
|
w750_live = scan_data.get('w750_velocity')
|
|||
|
|
|
|||
|
|
boost_info = acb.get_dynamic_boost_from_hz(
|
|||
|
|
target_date, exf_snapshot, w750_velocity=w750_live
|
|||
|
|
)
|
|||
|
|
stale_s = boost_info.get('max_staleness_s', 0)
|
|||
|
|
log.info(
|
|||
|
|
f" ACB HZ: boost={boost_info['boost']:.4f} beta={boost_info['beta']:.2f} "
|
|||
|
|
f"signals={boost_info['signals']:.1f} staleness={stale_s:.0f}s"
|
|||
|
|
)
|
|||
|
|
_acb_hz_ok = True
|
|||
|
|
else:
|
|||
|
|
log.warning(" ACB HZ: exf_latest not found — falling back to NPZ disk scan")
|
|||
|
|
except ValueError as _ve:
|
|||
|
|
log.warning(f" ACB HZ: snapshot stale ({_ve}) — falling back to NPZ disk scan")
|
|||
|
|
except Exception as _e:
|
|||
|
|
log.warning(f" ACB HZ: read failed ({_e}) — falling back to NPZ disk scan")
|
|||
|
|
|
|||
|
|
if not _acb_hz_ok:
|
|||
|
|
# NPZ fallback: get_dynamic_boost_for_date() will read eigenvalues/ on demand
|
|||
|
|
log.info(f" ACB: using NPZ disk path for {target_date}")
|
|||
|
|
|
|||
|
|
# ── Data Loading & Live OB Integration ────────────────────────────────────
|
|||
|
|
df = load_day_scans(target_date)
|
|||
|
|
OB_ASSETS = [c for c in df.columns if c not in META_COLS] if not df.empty else ["BTCUSDT", "ETHUSDT", "BNBUSDT", "SOLUSDT"]
|
|||
|
|
|
|||
|
|
live_ob = HZOBProvider(hz_cluster=HZ_CLUSTER, hz_host=HZ_HOST)
|
|||
|
|
ob_eng = OBFeatureEngine(live_ob)
|
|||
|
|
# NOTE: HZOBProvider has no historical snapshots, so preload_date() is a no-op.
|
|||
|
|
# OB features are fetched live via ob_eng.step_live() before each step_bar() call.
|
|||
|
|
|
|||
|
|
# ── MC-Forewarner ──────────────────────────────────────────────────────────
|
|||
|
|
forewarner = DolphinForewarner(models_dir=MC_MODELS_DIR)
|
|||
|
|
mc_base_cfg = {
|
|||
|
|
'trial_id': 0, 'vel_div_threshold': -0.020, 'vel_div_extreme': -0.050,
|
|||
|
|
'use_direction_confirm': True, 'dc_lookback_bars': 7,
|
|||
|
|
'dc_min_magnitude_bps': 0.75, 'dc_skip_contradicts': True,
|
|||
|
|
'dc_leverage_boost': 1.00, 'dc_leverage_reduce': 0.50,
|
|||
|
|
'vd_trend_lookback': 10, 'min_leverage': 0.50, 'max_leverage': 5.00,
|
|||
|
|
'leverage_convexity': 3.00, 'fraction': 0.20,
|
|||
|
|
'use_alpha_layers': True, 'use_dynamic_leverage': True,
|
|||
|
|
'fixed_tp_pct': 0.0095, 'stop_pct': 1.00, 'max_hold_bars': 120,
|
|||
|
|
'use_sp_fees': True, 'use_sp_slippage': True,
|
|||
|
|
'sp_maker_entry_rate': 0.62, 'sp_maker_exit_rate': 0.50,
|
|||
|
|
'use_ob_edge': True, 'ob_edge_bps': 5.00, 'ob_confirm_rate': 0.40,
|
|||
|
|
'ob_imbalance_bias': -0.09, 'ob_depth_scale': 1.00,
|
|||
|
|
'use_asset_selection': True, 'min_irp_alignment': 0.45, 'lookback': 100,
|
|||
|
|
'acb_beta_high': 0.80, 'acb_beta_low': 0.20, 'acb_w750_threshold_pct': 60,
|
|||
|
|
}
|
|||
|
|
|
|||
|
|
engine.set_ob_engine(ob_eng)
|
|||
|
|
engine.set_acb(acb)
|
|||
|
|
engine.set_mc_forewarner(forewarner, mc_base_cfg)
|
|||
|
|
engine.set_esoteric_hazard_multiplier(0.0)
|
|||
|
|
if instrument:
|
|||
|
|
engine._bar_log_enabled = True
|
|||
|
|
|
|||
|
|
# vol_p60: 60th percentile of rolling 50-bar BTC return std
|
|||
|
|
# Calibrated from 55-day NG3 champion window (Dec31–Feb25).
|
|||
|
|
# TODO: compute adaptively from rolling scan history (Phase MIG3)
|
|||
|
|
vol_p60 = pt_cfg.get('vol_p60', 0.000099)
|
|||
|
|
|
|||
|
|
# ── Run ────────────────────────────────────────────────────────────────────
|
|||
|
|
# df was already loaded above to define OB_ASSETS
|
|||
|
|
|
|||
|
|
# ── DOLPHIN_SAFETY (MIG3) ──────────────────────────────────────────────────
|
|||
|
|
try:
|
|||
|
|
safety_ref = client.get_cp_subsystem().get_atomic_reference('DOLPHIN_SAFETY').blocking()
|
|||
|
|
safety_raw = safety_ref.get()
|
|||
|
|
except Exception:
|
|||
|
|
safety_ref = client.get_map('DOLPHIN_SAFETY').blocking()
|
|||
|
|
safety_raw = safety_ref.get('latest')
|
|||
|
|
|
|||
|
|
safety_state = json.loads(safety_raw) if safety_raw else {}
|
|||
|
|
posture = safety_state.get('posture', 'APEX')
|
|||
|
|
Rm = safety_state.get('Rm', 1.0)
|
|||
|
|
log.info(f"[SURVIVAL STACK] Posture={posture} | Rm={Rm:.3f}")
|
|||
|
|
|
|||
|
|
# Apply Rm to absolute max leverage
|
|||
|
|
effective_max_lev = engine.abs_max_leverage * Rm
|
|||
|
|
engine.abs_max_leverage = max(1.0, effective_max_lev)
|
|||
|
|
|
|||
|
|
if posture == 'STALKER':
|
|||
|
|
engine.abs_max_leverage = min(engine.abs_max_leverage, 2.0)
|
|||
|
|
|
|||
|
|
dir_str = cfg.get('direction', 'short_only')
|
|||
|
|
direction_val = 1 if dir_str in ['long', 'long_only'] else -1
|
|||
|
|
|
|||
|
|
result = run_engine_day(target_date, df, engine, vol_p60, posture=posture, direction=direction_val)
|
|||
|
|
result['strategy'] = strategy_name
|
|||
|
|
result['capital'] = engine.capital
|
|||
|
|
|
|||
|
|
# ── Hazelcast & State Persistence ──────────────────────────────────────────
|
|||
|
|
try:
|
|||
|
|
# PnL write
|
|||
|
|
imap_pnl = client.get_map(hz_cfg['imap_pnl']).blocking()
|
|||
|
|
imap_pnl.put(target_date, json.dumps(result))
|
|||
|
|
|
|||
|
|
# State persist write
|
|||
|
|
new_peak = max(engine.capital, peak_capital)
|
|||
|
|
new_drawdown = 1.0 - (engine.capital / new_peak) if new_peak > 0 else 0.0
|
|||
|
|
|
|||
|
|
new_state = {
|
|||
|
|
'strategy': strategy_name,
|
|||
|
|
'capital': engine.capital,
|
|||
|
|
'date': target_date,
|
|||
|
|
'pnl': result.get('pnl', 0.0),
|
|||
|
|
'trades': result.get('trades', 0),
|
|||
|
|
'peak_capital': new_peak,
|
|||
|
|
'drawdown': new_drawdown,
|
|||
|
|
'last_date': target_date,
|
|||
|
|
'updated_at': datetime.now(timezone.utc).isoformat(),
|
|||
|
|
'engine_state': engine.get_state(),
|
|||
|
|
}
|
|||
|
|
imap_state.put('latest', json.dumps(new_state))
|
|||
|
|
imap_state.put(STATE_KEY, json.dumps(new_state))
|
|||
|
|
log.info(f" HZ write OK → state & {hz_cfg['imap_pnl']}[{target_date}]")
|
|||
|
|
except Exception as e:
|
|||
|
|
log.error(f"[STATE] HZ persist failed: {e}")
|
|||
|
|
# Fallback: write to local JSON ledger
|
|||
|
|
ledger_dir = Path(__file__).parent / pt_cfg.get('log_dir', 'paper_logs')
|
|||
|
|
ledger_dir.mkdir(parents=True, exist_ok=True)
|
|||
|
|
ledger_path = ledger_dir / f"state_ledger_{strategy_name}.jsonl"
|
|||
|
|
with open(ledger_path, 'a') as f:
|
|||
|
|
f.write(json.dumps({
|
|||
|
|
'strategy': strategy_name, 'capital': engine.capital,
|
|||
|
|
'date': target_date, 'pnl': result.get('pnl', 0.0),
|
|||
|
|
'trades': result.get('trades', 0), 'peak_capital': peak_capital,
|
|||
|
|
'drawdown': 1.0 - engine.capital / max(engine.capital, peak_capital) if max(engine.capital, peak_capital) > 0 else 0.0
|
|||
|
|
}) + '\n')
|
|||
|
|
finally:
|
|||
|
|
live_ob.close()
|
|||
|
|
client.shutdown()
|
|||
|
|
|
|||
|
|
# ── Instrumentation (--instrument flag) ────────────────────────────────────
|
|||
|
|
if instrument:
|
|||
|
|
instr_dir = Path(__file__).parent / pt_cfg['log_dir']
|
|||
|
|
instr_dir.mkdir(parents=True, exist_ok=True)
|
|||
|
|
trades_instr_path = instr_dir / f"E2E_trades_{target_date}.csv"
|
|||
|
|
with open(trades_instr_path, 'w', newline='') as f:
|
|||
|
|
cw = csv.writer(f)
|
|||
|
|
cw.writerow(['trade_id', 'asset', 'direction', 'entry_price', 'exit_price',
|
|||
|
|
'entry_bar', 'exit_bar', 'bars_held', 'leverage', 'notional',
|
|||
|
|
'pnl_pct', 'pnl_absolute', 'exit_reason', 'bucket_idx'])
|
|||
|
|
for t in engine.trade_history:
|
|||
|
|
cw.writerow([t.trade_id, t.asset, t.direction,
|
|||
|
|
f"{t.entry_price:.6f}", f"{t.exit_price:.6f}",
|
|||
|
|
t.entry_bar, t.exit_bar, t.bars_held,
|
|||
|
|
f"{t.leverage:.4f}", f"{t.notional:.4f}",
|
|||
|
|
f"{t.pnl_pct:.8f}", f"{t.pnl_absolute:.4f}",
|
|||
|
|
t.exit_reason, t.bucket_idx])
|
|||
|
|
bars_instr_path = instr_dir / f"E2E_bars_{target_date}.csv"
|
|||
|
|
with open(bars_instr_path, 'w', newline='') as f:
|
|||
|
|
cw = csv.writer(f)
|
|||
|
|
cw.writerow(['date', 'bar_idx', 'vel_div', 'vol_ok', 'posture',
|
|||
|
|
'regime_size_mult', 'position_open', 'boost', 'beta'])
|
|||
|
|
for b in engine._bar_log:
|
|||
|
|
cw.writerow([target_date, b['bar_idx'], f"{b['vel_div']:.8f}",
|
|||
|
|
b['vol_ok'], b['posture'], f"{b['regime_size_mult']:.6f}",
|
|||
|
|
b['position_open'], f"{b['boost']:.4f}", f"{b['beta']:.2f}"])
|
|||
|
|
log.info(f" Instrumentation → {trades_instr_path.name} ({len(engine.trade_history)} trades), "
|
|||
|
|
f"{bars_instr_path.name} ({len(engine._bar_log)} bars)")
|
|||
|
|
|
|||
|
|
# ── Disk log ───────────────────────────────────────────────────────────────
|
|||
|
|
log_pnl(Path(__file__).parent / pt_cfg['log_dir'], target_date, result, engine.capital)
|
|||
|
|
|
|||
|
|
log.info(f"=== DONE: {strategy_name} {target_date} | "
|
|||
|
|
f"PnL={result.get('pnl', 0):+.2f} | Capital={engine.capital:,.2f} ===")
|
|||
|
|
return result
|
|||
|
|
|
|||
|
|
|
|||
|
|
# ── CLI entry point ──────────────────────────────────────────────────────────────
|
|||
|
|
if __name__ == '__main__':
|
|||
|
|
parser = argparse.ArgumentParser(description='DOLPHIN paper trading flow')
|
|||
|
|
parser.add_argument('--config', default='configs/blue.yml', help='Strategy config YAML')
|
|||
|
|
parser.add_argument('--date', default=None, help='YYYY-MM-DD (default: yesterday)')
|
|||
|
|
parser.add_argument('--register', action='store_true', help='Register Prefect deployments')
|
|||
|
|
parser.add_argument('--instrument', action='store_true', help='Write per-trade + bar CSVs to log_dir')
|
|||
|
|
args = parser.parse_args()
|
|||
|
|
|
|||
|
|
if args.register:
|
|||
|
|
from prefect.client.schemas.schedules import CronSchedule as CS
|
|||
|
|
for color, cfg_path in [('blue', 'configs/blue.yml'), ('green', 'configs/green.yml')]:
|
|||
|
|
abs_cfg = str(Path(__file__).parent / cfg_path)
|
|||
|
|
deployment = dolphin_paper_trade_flow.to_deployment(
|
|||
|
|
name=f"dolphin-paper-{color}",
|
|||
|
|
parameters={"config_path": abs_cfg},
|
|||
|
|
schedule=CS(cron="5 0 * * *", timezone="UTC"),
|
|||
|
|
work_pool_name="dolphin",
|
|||
|
|
tags=[color, "paper-trade", "dolphin"],
|
|||
|
|
)
|
|||
|
|
deployment.apply()
|
|||
|
|
print(f"Registered: dolphin-paper-{color}")
|
|||
|
|
else:
|
|||
|
|
import os
|
|||
|
|
os.environ.setdefault('PREFECT_API_URL', 'http://localhost:4200/api')
|
|||
|
|
dolphin_paper_trade_flow(config_path=args.config, run_date=args.date,
|
|||
|
|
instrument=args.instrument)
|