Wire long-capable prod alpha path
This commit is contained in:
Codex
200
prod/acb_processor_service.py
Normal file
200
prod/acb_processor_service.py
Normal file
@@ -0,0 +1,200 @@
|
|||||||
|
"""
|
||||||
|
MIG6.1 & MIG6.2: ACB Processor Service
|
||||||
|
Watches for new scan arrivals and atomically computes/writes ACB boost
|
||||||
|
to the Hazelcast DOLPHIN_FEATURES map using CP Subsystem lock for atomicity.
|
||||||
|
"""
|
||||||
|
|
||||||
|
import sys
|
||||||
|
import time
|
||||||
|
import json
|
||||||
|
import logging
|
||||||
|
from pathlib import Path
|
||||||
|
from datetime import datetime
|
||||||
|
import hazelcast
|
||||||
|
|
||||||
|
HCM_DIR = Path(__file__).parent.parent
|
||||||
|
|
||||||
|
# Use platform-independent paths from dolphin_paths
|
||||||
|
sys.path.insert(0, str(HCM_DIR))
|
||||||
|
sys.path.insert(0, str(HCM_DIR / 'prod'))
|
||||||
|
from dolphin_paths import get_eigenvalues_path
|
||||||
|
|
||||||
|
SCANS_DIR = get_eigenvalues_path()
|
||||||
|
|
||||||
|
sys.path.insert(0, str(HCM_DIR / 'nautilus_dolphin'))
|
||||||
|
from nautilus_dolphin.nautilus.adaptive_circuit_breaker import AdaptiveCircuitBreaker
|
||||||
|
|
||||||
|
logging.basicConfig(level=logging.INFO, format='%(asctime)s %(levelname)s:%(message)s')
|
||||||
|
|
||||||
|
class ACBProcessorService:
|
||||||
|
def __init__(self, hz_cluster="dolphin", hz_host="localhost:5701"):
|
||||||
|
try:
|
||||||
|
self.hz_client = hazelcast.HazelcastClient(
|
||||||
|
cluster_name=hz_cluster,
|
||||||
|
cluster_members=[hz_host]
|
||||||
|
)
|
||||||
|
self.imap = self.hz_client.get_map("DOLPHIN_FEATURES").blocking()
|
||||||
|
|
||||||
|
# Using CP Subsystem lock as per MIG6.1
|
||||||
|
self.lock = self.hz_client.cp_subsystem.get_lock("acb_update_lock").blocking()
|
||||||
|
except Exception as e:
|
||||||
|
logging.error(f"Failed to connect to Hazelcast: {e}")
|
||||||
|
raise
|
||||||
|
|
||||||
|
self.acb = AdaptiveCircuitBreaker()
|
||||||
|
self.acb.config.EIGENVALUES_PATH = SCANS_DIR # CRITICAL: override Windows default for Linux
|
||||||
|
self.acb.preload_w750(self._get_recent_dates(60))
|
||||||
|
self.last_scan_count = 0
|
||||||
|
self.last_date = None
|
||||||
|
|
||||||
|
def _get_recent_dates(self, n=60):
|
||||||
|
try:
|
||||||
|
dirs = sorted([d.name for d in SCANS_DIR.iterdir() if d.is_dir() and len(d.name)==10])
|
||||||
|
return dirs[-n:]
|
||||||
|
except Exception:
|
||||||
|
return []
|
||||||
|
|
||||||
|
def get_today_str(self):
|
||||||
|
return datetime.utcnow().strftime('%Y-%m-%d')
|
||||||
|
|
||||||
|
def check_new_scans(self, date_str):
|
||||||
|
today_dir = SCANS_DIR / date_str
|
||||||
|
if not today_dir.exists():
|
||||||
|
return False
|
||||||
|
|
||||||
|
json_files = list(today_dir.glob("scan_*.json"))
|
||||||
|
count = len(json_files)
|
||||||
|
|
||||||
|
if self.last_date != date_str:
|
||||||
|
self.last_date = date_str
|
||||||
|
self.last_scan_count = 0
|
||||||
|
# Preload updated dates when day rolls over
|
||||||
|
self.acb.preload_w750(self._get_recent_dates(60))
|
||||||
|
|
||||||
|
if count > self.last_scan_count:
|
||||||
|
self.last_scan_count = count
|
||||||
|
return True
|
||||||
|
|
||||||
|
return False
|
||||||
|
|
||||||
|
def process_and_write(self, date_str):
|
||||||
|
"""Compute ACB boost and write to HZ acb_boost.
|
||||||
|
|
||||||
|
Preference order:
|
||||||
|
1. HZ exf_latest — live, pre-lagged values (preferred, ~0.5 s latency)
|
||||||
|
2. NPZ disk scan — fallback when HZ data absent or stale (>12 h)
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
boost_info = None
|
||||||
|
long_boost_info = None
|
||||||
|
|
||||||
|
# ── HZ path (preferred) ────────────────────────────────────────────
|
||||||
|
try:
|
||||||
|
exf_raw = self.imap.get('exf_latest')
|
||||||
|
if exf_raw:
|
||||||
|
exf_snapshot = json.loads(exf_raw)
|
||||||
|
scan_raw = self.imap.get('latest_eigen_scan')
|
||||||
|
w750_live = None
|
||||||
|
if scan_raw:
|
||||||
|
scan_data = json.loads(scan_raw)
|
||||||
|
w750_live = scan_data.get('w750_velocity')
|
||||||
|
boost_info = self.acb.get_dynamic_boost_from_hz(
|
||||||
|
date_str, exf_snapshot, w750_velocity=w750_live, direction=-1
|
||||||
|
)
|
||||||
|
long_boost_info = self.acb.get_dynamic_boost_from_hz(
|
||||||
|
date_str, exf_snapshot, w750_velocity=w750_live, direction=1
|
||||||
|
)
|
||||||
|
logging.debug(
|
||||||
|
f"ACB computed from HZ: short={boost_info['boost']:.4f} "
|
||||||
|
f"long={long_boost_info['boost']:.4f}"
|
||||||
|
)
|
||||||
|
except ValueError as ve:
|
||||||
|
logging.warning(f"ACB HZ snapshot stale: {ve} — falling back to NPZ")
|
||||||
|
boost_info = None
|
||||||
|
except Exception as e:
|
||||||
|
logging.warning(f"ACB HZ read failed: {e} — falling back to NPZ")
|
||||||
|
boost_info = None
|
||||||
|
|
||||||
|
# ── NPZ fallback ───────────────────────────────────────────────────
|
||||||
|
if boost_info is None:
|
||||||
|
boost_info = self.acb.get_dynamic_boost_for_date(date_str, direction=-1)
|
||||||
|
long_boost_info = self.acb.get_dynamic_boost_for_date(date_str, direction=1)
|
||||||
|
logging.debug(
|
||||||
|
f"ACB computed from NPZ: short={boost_info['boost']:.4f} "
|
||||||
|
f"long={long_boost_info['boost']:.4f}"
|
||||||
|
)
|
||||||
|
|
||||||
|
payload = json.dumps(boost_info)
|
||||||
|
long_payload = json.dumps(long_boost_info or boost_info)
|
||||||
|
|
||||||
|
# Atomic Write via CP Subsystem Lock
|
||||||
|
self.lock.lock()
|
||||||
|
try:
|
||||||
|
# Legacy key remains SHORT for BLUE/PRODGREEN compatibility.
|
||||||
|
self.imap.put("acb_boost", payload)
|
||||||
|
self.imap.put("acb_boost_short", payload)
|
||||||
|
self.imap.put("acb_boost_long", long_payload)
|
||||||
|
logging.info(
|
||||||
|
f"acb_boost updated (src={boost_info.get('source','npz')}): "
|
||||||
|
f"short={boost_info['boost']:.4f}/{boost_info['signals']:.1f}sig "
|
||||||
|
f"long={(long_boost_info or {}).get('boost', 0.0):.4f}/"
|
||||||
|
f"{(long_boost_info or {}).get('signals', 0.0):.1f}sig"
|
||||||
|
)
|
||||||
|
try:
|
||||||
|
from ch_writer import ch_put, ts_us as _ts
|
||||||
|
ch_put("acb_state", {
|
||||||
|
"ts": _ts(),
|
||||||
|
"boost": float(boost_info.get("boost", 0)),
|
||||||
|
"beta": float(boost_info.get("beta", 0)),
|
||||||
|
"signals": float(boost_info.get("signals", 0)),
|
||||||
|
})
|
||||||
|
except Exception:
|
||||||
|
pass
|
||||||
|
finally:
|
||||||
|
self.lock.unlock()
|
||||||
|
|
||||||
|
except Exception as e:
|
||||||
|
logging.error(f"Error processing ACB: {e}")
|
||||||
|
|
||||||
|
def run(self, poll_interval=1.0, hz_refresh_interval=30.0):
|
||||||
|
"""Main service loop.
|
||||||
|
|
||||||
|
Two update triggers:
|
||||||
|
1. New scan files arrive for today → compute from HZ (preferred) or NPZ.
|
||||||
|
2. hz_refresh_interval elapsed → re-push acb_boost from live exf_latest
|
||||||
|
even when no new scans exist (covers live-only operation days when
|
||||||
|
scan files land in a different directory or not at all).
|
||||||
|
"""
|
||||||
|
logging.info("Starting ACB Processor Service (Python CP Subsystem)...")
|
||||||
|
today = self.get_today_str()
|
||||||
|
# Write immediately on startup so acb_boost is populated from the first second
|
||||||
|
logging.info(f"Startup write for {today}")
|
||||||
|
self.process_and_write(today)
|
||||||
|
last_hz_refresh = time.monotonic()
|
||||||
|
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
today = self.get_today_str()
|
||||||
|
now = time.monotonic()
|
||||||
|
|
||||||
|
# Trigger 1: new scan files
|
||||||
|
if self.check_new_scans(today):
|
||||||
|
self.process_and_write(today)
|
||||||
|
last_hz_refresh = now
|
||||||
|
|
||||||
|
# Trigger 2: periodic HZ refresh (ensures acb_boost stays current
|
||||||
|
# even on days with no new NPZ scan files)
|
||||||
|
elif (now - last_hz_refresh) >= hz_refresh_interval:
|
||||||
|
self.process_and_write(today)
|
||||||
|
last_hz_refresh = now
|
||||||
|
|
||||||
|
time.sleep(poll_interval)
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
break
|
||||||
|
except Exception as e:
|
||||||
|
logging.error(f"Loop error: {e}")
|
||||||
|
time.sleep(5.0)
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
service = ACBProcessorService()
|
||||||
|
service.run()
|
||||||
@@ -3386,4 +3386,9 @@ The canonical vel_div is `v50_lambda_max_velocity − v750_lambda_max_velocity`
|
|||||||
> 2026-05-08 lowered post-win-threshold addendum: the post-win overlay is stronger than the original narrow sample implied, but only when conditioned on realized exhaustion. Dollar-only thresholds below about `$300` are harmful. With a prior-return filter (`pnl_pct >= +0.75%`), lower thresholds become useful: e.g. `$100-$150` prior wins produced `63-67` immediate next-trade cases with about `+$2.4k` marginal delta and positive flipped-LONG PnL. High-leverage `$300-$500` wins support a next-`2`-trade rebound/cooldown signal (`+$2.7k` to `+$3.7k` marginal delta). The edge is payoff-asymmetry / loss-tail avoidance, not WR improvement, and should be researched as a guarded next-1/next-2 overlay or abstain gate.
|
> 2026-05-08 lowered post-win-threshold addendum: the post-win overlay is stronger than the original narrow sample implied, but only when conditioned on realized exhaustion. Dollar-only thresholds below about `$300` are harmful. With a prior-return filter (`pnl_pct >= +0.75%`), lower thresholds become useful: e.g. `$100-$150` prior wins produced `63-67` immediate next-trade cases with about `+$2.4k` marginal delta and positive flipped-LONG PnL. High-leverage `$300-$500` wins support a next-`2`-trade rebound/cooldown signal (`+$2.7k` to `+$3.7k` marginal delta). The edge is payoff-asymmetry / loss-tail avoidance, not WR improvement, and should be researched as a guarded next-1/next-2 overlay or abstain gate.
|
||||||
> 2026-05-08 post-win EFSM implementation addendum: the candidate BLUE overlay is now the post-win **EFSM** (**Execution FSM**) at `adaptive_exit/post_win_long_overlay.py` with tests in `prod/tests/test_post_win_long_overlay.py`. Canonical class names are `PostWinExecutionFSM` and `PostWinExecutionFSMConfig` (`PostWinLongOverlay` names remain compatibility aliases). Codified rule: `pnl_abs > $397` arms next `1` FLIP_LONG slot; `pnl_abs > $397 and lev > 8.6` arms next `2`; `0 < pnl_abs < $250 and pnl_pct >= +0.75%` arms next `1`; consumed slots reset to SHORT. Active slots cannot re-arm and overlay-flipped LONG outcomes cannot re-arm. This reset invariant is mandatory: unsafe recursive re-arm replay turned `+$1.51k` marginal delta into `-$5.43k`. V7 is side-aware but SHORT-calibrated; validate LONG overlay exits in shadow or with conservative LONG-specific settings before live use.
|
> 2026-05-08 post-win EFSM implementation addendum: the candidate BLUE overlay is now the post-win **EFSM** (**Execution FSM**) at `adaptive_exit/post_win_long_overlay.py` with tests in `prod/tests/test_post_win_long_overlay.py`. Canonical class names are `PostWinExecutionFSM` and `PostWinExecutionFSMConfig` (`PostWinLongOverlay` names remain compatibility aliases). Codified rule: `pnl_abs > $397` arms next `1` FLIP_LONG slot; `pnl_abs > $397 and lev > 8.6` arms next `2`; `0 < pnl_abs < $250 and pnl_pct >= +0.75%` arms next `1`; consumed slots reset to SHORT. Active slots cannot re-arm and overlay-flipped LONG outcomes cannot re-arm. This reset invariant is mandatory: unsafe recursive re-arm replay turned `+$1.51k` marginal delta into `-$5.43k`. V7 is side-aware but SHORT-calibrated; validate LONG overlay exits in shadow or with conservative LONG-specific settings before live use.
|
||||||
> 2026-05-08 AlphaExitEngineV7 LONG calibration addendum: V7 threshold/gate constants are now surfaced as `AlphaExitV7Config` in `nautilus_dolphin/nautilus_dolphin/nautilus/alpha_exit_v7_engine.py`. Default `AlphaExitEngineV7()` remains the deployed SHORT-calibrated surface: `exit_pressure_threshold=2.69`, `retract_pressure_threshold=1.0`, `extend_pressure_threshold=-0.5`, vol-normalized MAE tiers `max(floor, k * rv_comp)` with `k=(3.5,7.0,12.0)` and floors `(0.005,0.012,0.025)`, and bounce soft weights `(0.15,0.35)`. A separate LONG engine can now be initialized with a different `AlphaExitV7Config` without mutating BLUE SHORT defaults. Synthetic LONG replay over BLUE V7 journal paths (`97` paths, `6,812` rows, bounce disabled because the current bounce model is SHORT-trained) found natural LONG PnL `-$328.84`; deployed default V7 improved this to `+$1.43` (`+$330.26` delta); best tested LONG proxy was reducing MFE-risk contributions by half while keeping pressure threshold `2.69`, yielding `+$205.32` (`+$534.15` delta), `36/97` exits, and `1.69%` max DD. Do not deploy this live from proxy alone; first shadow it on actual EFSM-flipped LONG contexts. Detailed method/results: `prod/docs/LONG_DETERMINISTIC_RULE_RESEARCH.md`.
|
> 2026-05-08 AlphaExitEngineV7 LONG calibration addendum: V7 threshold/gate constants are now surfaced as `AlphaExitV7Config` in `nautilus_dolphin/nautilus_dolphin/nautilus/alpha_exit_v7_engine.py`. Default `AlphaExitEngineV7()` remains the deployed SHORT-calibrated surface: `exit_pressure_threshold=2.69`, `retract_pressure_threshold=1.0`, `extend_pressure_threshold=-0.5`, vol-normalized MAE tiers `max(floor, k * rv_comp)` with `k=(3.5,7.0,12.0)` and floors `(0.005,0.012,0.025)`, and bounce soft weights `(0.15,0.35)`. A separate LONG engine can now be initialized with a different `AlphaExitV7Config` without mutating BLUE SHORT defaults. Synthetic LONG replay over BLUE V7 journal paths (`97` paths, `6,812` rows, bounce disabled because the current bounce model is SHORT-trained) found natural LONG PnL `-$328.84`; deployed default V7 improved this to `+$1.43` (`+$330.26` delta); best tested LONG proxy was reducing MFE-risk contributions by half while keeping pressure threshold `2.69`, yielding `+$205.32` (`+$534.15` delta), `36/97` exits, and `1.69%` max DD. Do not deploy this live from proxy alone; first shadow it on actual EFSM-flipped LONG contexts. Detailed method/results: `prod/docs/LONG_DETERMINISTIC_RULE_RESEARCH.md`.
|
||||||
|
> 2026-05-08 LONG-capability addendum: BLUE and PRODGREEN Alpha Engine code paths are now LONG-capable without changing the default deployed side. `short_only` remains the default everywhere. LONG is activated only by explicit config/env direction (`long`, `long_only`, `buy`, `1`, `+1`). The signal generator exposes configurable LONG thresholds (`long_vel_div_threshold=+0.01`, `long_vel_div_extreme=+0.04`) and keeps the canonical SHORT thresholds (`-0.02`, `-0.05`). `NDAlphaEngine.begin_day(..., direction=+1)` now propagates LONG semantics into signal gating, DC interpretation, IRP expected action, sizing, PnL, exit price slippage, and ACB meta-strength. The sizing trend multiplier is side-aware: negative `vel_div` trend remains favorable for SHORT; positive `vel_div` trend is favorable for LONG.
|
||||||
|
> 2026-05-08 ACBv6 side-awareness addendum: ACBv6 remains SHORT/risk-off by default and preserves the legacy cache key for default calls. LONG/risk-on ACB is opt-in via `direction=+1` and uses separate cache entries (`date|long`) so HZ prewarm cannot pollute SHORT BLUE state. SHORT signals are unchanged: bearish funding, high DVOL, fear, and taker selling. LONG signals are explicit and separate: positive funding, calm DVOL, greed/risk appetite, and taker buying. OB beta modulation is also side-aware: stress/cascade raises beta for SHORT and reduces it for LONG; calm/liquidity-building raises beta for LONG and reduces it for SHORT.
|
||||||
|
> 2026-05-08 ACB HZ keying addendum: `prod/acb_processor_service.py` now publishes `acb_boost` and `acb_boost_short` as the legacy SHORT payload and `acb_boost_long` as the LONG/risk-on payload. BLUE continues to use `acb_boost` unless explicitly run with `DOLPHIN_DIRECTION=long_only`, in which case its local prewarm calls ACB with `direction=+1`. PRODGREEN's Nautilus actor subscribes to `acb_boost_long` when its config direction is LONG, otherwise to legacy `acb_boost`.
|
||||||
|
> 2026-05-08 LONG exit-layer addendum: base TP/SL/max-hold exits were already direction-aware through signed PnL. Optional `AlphaExitManager` vel_div invalidation/exhaustion and TF-spread recovery exits are now side-aware too. They remain disabled by default unless explicitly enabled, but if enabled for a LONG invocation they now treat falling/negative `vel_div` and adverse TF-spread recovery as LONG invalidation rather than applying hidden SHORT semantics.
|
||||||
|
> 2026-05-08 validation addendum: targeted regression after LONG-capability wiring passed `prod/tests/test_long_capability_layers.py` (`9 passed`), existing ACB HZ + V7 + EFSM suites (`57 passed`), and ACB signal-threshold integrity (`11 passed`). Compile checks passed for modified Alpha/ACB/live runner files. These tests verify SHORT default preservation, explicit LONG entries, side-separated ACB caches, side-aware OB beta modulation, side-aware optional VD exits, and case-insensitive PRODGREEN direction parsing.
|
||||||
> The retrieval spec is documented in `prod/docs/ASSET_FINGERPRINT_CANDIDATE_SYSTEM.md`.
|
> The retrieval spec is documented in `prod/docs/ASSET_FINGERPRINT_CANDIDATE_SYSTEM.md`.
|
||||||
|
|||||||
296
prod/launch_dolphin_live.py
Normal file
296
prod/launch_dolphin_live.py
Normal file
@@ -0,0 +1,296 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
"""
|
||||||
|
Dolphin Live Node — DolphinActor inside NT TradingNode
|
||||||
|
=======================================================
|
||||||
|
Phase 1: paper_trading=True (live Binance Futures data, paper fills).
|
||||||
|
Validates signal parity with nautilus_event_trader.py before live exec.
|
||||||
|
|
||||||
|
To go live (Phase 2): set paper_trading=False in build_node().
|
||||||
|
"""
|
||||||
|
import os
|
||||||
|
import sys
|
||||||
|
import asyncio
|
||||||
|
from copy import deepcopy
|
||||||
|
from pathlib import Path
|
||||||
|
|
||||||
|
PROJECT_ROOT = Path(__file__).parent.parent
|
||||||
|
sys.path.insert(0, str(PROJECT_ROOT / 'nautilus_dolphin'))
|
||||||
|
sys.path.insert(0, str(PROJECT_ROOT / 'prod'))
|
||||||
|
sys.path.insert(0, str(PROJECT_ROOT))
|
||||||
|
|
||||||
|
from dotenv import load_dotenv
|
||||||
|
load_dotenv(PROJECT_ROOT / '.env')
|
||||||
|
|
||||||
|
from nautilus_trader.live.node import TradingNode
|
||||||
|
from nautilus_trader.config import TradingNodeConfig, LiveDataEngineConfig, CacheConfig
|
||||||
|
from nautilus_trader.adapters.binance.config import BinanceDataClientConfig
|
||||||
|
from nautilus_trader.adapters.binance.common.enums import BinanceAccountType
|
||||||
|
from nautilus_trader.adapters.binance.factories import BinanceLiveDataClientFactory
|
||||||
|
from nautilus_trader.model.identifiers import TraderId
|
||||||
|
|
||||||
|
from prod.bingx.config import BingxExecClientConfig
|
||||||
|
from prod.bingx.data_config import BingxDataClientConfig
|
||||||
|
from prod.bingx.enums import BingxEnvironment
|
||||||
|
from prod.bingx.data_factories import BingxLiveDataClientFactory
|
||||||
|
from prod.bingx.factories import BingxLiveExecClientFactory
|
||||||
|
|
||||||
|
# Nautilus changed this enum name across versions.
|
||||||
|
_BINANCE_USDT_FUTURES_ACCOUNT_TYPE = getattr(
|
||||||
|
BinanceAccountType,
|
||||||
|
"USDT_FUTURES",
|
||||||
|
getattr(BinanceAccountType, "USDT_FUTURE", None),
|
||||||
|
)
|
||||||
|
if _BINANCE_USDT_FUTURES_ACCOUNT_TYPE is None:
|
||||||
|
raise AttributeError("BinanceAccountType is missing both USDT_FUTURES and USDT_FUTURE")
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# Universe — 50 OBF assets. Subscribed for live quote cache (order sizing).
|
||||||
|
# Must cover the full eigen universe so _exec_submit_entry finds live quotes.
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
LIVE_ASSETS = [
|
||||||
|
"BTCUSDT", "ETHUSDT", "BNBUSDT", "SOLUSDT", "XRPUSDT",
|
||||||
|
"ADAUSDT", "DOGEUSDT", "TRXUSDT", "DOTUSDT", "MATICUSDT",
|
||||||
|
"LTCUSDT", "AVAXUSDT", "LINKUSDT", "UNIUSDT", "ATOMUSDT",
|
||||||
|
"ETCUSDT", "XLMUSDT", "BCHUSDT", "NEARUSDT", "ALGOUSDT",
|
||||||
|
"VETUSDT", "FILUSDT", "APTUSDT", "OPUSDT", "ARBUSDT",
|
||||||
|
"INJUSDT", "SUIUSDT", "SEIUSDT", "TIAUSDT", "ORDIUSDT",
|
||||||
|
"WLDUSDT", "FETUSDT", "AGIXUSDT", "RENDERUSDT", "IOTAUSDT",
|
||||||
|
"AAVEUSDT", "SNXUSDT", "CRVUSDT", "COMPUSDT", "MKRUSDT",
|
||||||
|
"ENJUSDT", "MANAUSDT", "SANDUSDT", "AXSUSDT", "GALAUSDT",
|
||||||
|
"ZECUSDT", "DASHUSDT", "XMRUSDT", "NEOUSDT", "QTUMUSDT",
|
||||||
|
]
|
||||||
|
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
# DolphinActor config — gold-standard params, must match nautilus_event_trader
|
||||||
|
# ---------------------------------------------------------------------------
|
||||||
|
DOLPHIN_CONFIG = {
|
||||||
|
'live_mode': True,
|
||||||
|
'venue': 'BINANCE',
|
||||||
|
'data_venue': 'BINANCE',
|
||||||
|
'exec_venue': 'BINANCE',
|
||||||
|
'direction': 'short_only',
|
||||||
|
'hazelcast': {
|
||||||
|
'host': '127.0.0.1:5701',
|
||||||
|
'cluster': 'dolphin',
|
||||||
|
'state_map': 'DOLPHIN_STATE_PRODGREEN',
|
||||||
|
'imap_pnl': 'DOLPHIN_PNL_PRODGREEN',
|
||||||
|
},
|
||||||
|
'paper_trade': {'initial_capital': 25000.0},
|
||||||
|
'assets': LIVE_ASSETS,
|
||||||
|
'engine': {
|
||||||
|
'boost_mode': 'd_liq',
|
||||||
|
# Signal
|
||||||
|
'vel_div_threshold': -0.020,
|
||||||
|
'vel_div_extreme': -0.050,
|
||||||
|
# Leverage — gold spec: 8x soft / 9x hard
|
||||||
|
'min_leverage': 0.5,
|
||||||
|
'max_leverage': 8.0,
|
||||||
|
'abs_max_leverage': 9.0,
|
||||||
|
'leverage_convexity': 3.0,
|
||||||
|
'fraction': 0.20,
|
||||||
|
'max_account_leverage': 3.0,
|
||||||
|
# Exits — gold spec: 250 bars max hold
|
||||||
|
'fixed_tp_pct': 0.0095,
|
||||||
|
'stop_pct': 1.0,
|
||||||
|
'max_hold_bars': 250,
|
||||||
|
# Direction confirm
|
||||||
|
'use_direction_confirm': True,
|
||||||
|
'dc_lookback_bars': 7,
|
||||||
|
'dc_min_magnitude_bps': 0.75,
|
||||||
|
'dc_skip_contradicts': True,
|
||||||
|
'dc_leverage_boost': 1.0,
|
||||||
|
'dc_leverage_reduce': 0.5,
|
||||||
|
# Asset selection — gold spec: IRP filter disabled in live
|
||||||
|
'use_asset_selection': True,
|
||||||
|
'min_irp_alignment': 0.0,
|
||||||
|
'asset_selector_lookback': 10,
|
||||||
|
# Fees / slippage
|
||||||
|
'use_sp_fees': True,
|
||||||
|
'use_sp_slippage': True,
|
||||||
|
'sp_maker_entry_rate': 0.62,
|
||||||
|
'sp_maker_exit_rate': 0.50,
|
||||||
|
# OB edge
|
||||||
|
'use_ob_edge': True,
|
||||||
|
'ob_edge_bps': 5.0,
|
||||||
|
'ob_confirm_rate': 0.40,
|
||||||
|
'ob_imbalance_bias': -0.09,
|
||||||
|
'ob_depth_scale': 1.0,
|
||||||
|
# Alpha layers
|
||||||
|
'lookback': 100,
|
||||||
|
'use_alpha_layers': True,
|
||||||
|
'use_dynamic_leverage': True,
|
||||||
|
'seed': 42,
|
||||||
|
# V7 RT exit engine (GREEN only)
|
||||||
|
'use_exit_v7': True,
|
||||||
|
'use_exit_v6': False,
|
||||||
|
'v6_bar_duration_sec': 5.0,
|
||||||
|
'bounce_model_path': str(PROJECT_ROOT / 'prod' / 'models' / 'bounce_detector_v3.pkl'),
|
||||||
|
},
|
||||||
|
'strategy_name': 'prodgreen',
|
||||||
|
'vol_p60': 0.00009868,
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
def _env_upper(name: str, default: str = "") -> str:
|
||||||
|
return str(os.environ.get(name, default)).strip().upper()
|
||||||
|
|
||||||
|
|
||||||
|
def _env_bool(name: str, default: bool = False) -> bool:
|
||||||
|
raw = str(os.environ.get(name, str(default))).strip().lower()
|
||||||
|
return raw in ("1", "true", "yes", "on")
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_bingx_environment(value: str | None = None) -> BingxEnvironment:
|
||||||
|
name = str(value or os.environ.get("DOLPHIN_BINGX_ENV", "VST")).strip().upper()
|
||||||
|
return BingxEnvironment.LIVE if name == "LIVE" else BingxEnvironment.VST
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_bingx_allow_mainnet(value: str | None = None) -> bool:
|
||||||
|
if isinstance(value, bool):
|
||||||
|
return value
|
||||||
|
raw = str(value or os.environ.get("DOLPHIN_BINGX_ALLOW_MAINNET", "0")).strip().lower()
|
||||||
|
return raw in ("1", "true", "yes", "on")
|
||||||
|
|
||||||
|
|
||||||
|
def _resolve_bingx_recv_window_ms(value: str | None = None) -> int:
|
||||||
|
raw = str(value or os.environ.get("DOLPHIN_BINGX_RECV_WINDOW_MS", "")).strip()
|
||||||
|
try:
|
||||||
|
parsed = int(raw)
|
||||||
|
return parsed if parsed > 0 else 5_000
|
||||||
|
except (TypeError, ValueError):
|
||||||
|
return 5_000
|
||||||
|
|
||||||
|
|
||||||
|
def build_actor_config(
|
||||||
|
*,
|
||||||
|
data_venue: str | None = None,
|
||||||
|
exec_venue: str | None = None,
|
||||||
|
) -> dict:
|
||||||
|
actor_cfg = deepcopy(DOLPHIN_CONFIG)
|
||||||
|
resolved_data_venue = (data_venue or _env_upper("DOLPHIN_DATA_VENUE", actor_cfg["data_venue"])).upper()
|
||||||
|
resolved_exec_venue = (exec_venue or _env_upper("DOLPHIN_EXEC_VENUE", actor_cfg["exec_venue"])).upper()
|
||||||
|
actor_cfg["data_venue"] = resolved_data_venue
|
||||||
|
actor_cfg["exec_venue"] = resolved_exec_venue
|
||||||
|
actor_cfg["venue"] = resolved_exec_venue
|
||||||
|
actor_cfg["direction"] = os.environ.get("DOLPHIN_DIRECTION", actor_cfg.get("direction", "short_only"))
|
||||||
|
return actor_cfg
|
||||||
|
|
||||||
|
|
||||||
|
def build_bingx_exec_client_config(
|
||||||
|
*,
|
||||||
|
resolved_bingx_env: BingxEnvironment,
|
||||||
|
resolved_bingx_allow_mainnet: bool,
|
||||||
|
resolved_bingx_recv_window_ms: int | None,
|
||||||
|
assets: list[str] | None = None,
|
||||||
|
) -> BingxExecClientConfig:
|
||||||
|
from prod.bingx.config import BingxInstrumentProviderConfig
|
||||||
|
default_leverage = int(os.environ.get("DOLPHIN_BINGX_DEFAULT_LEVERAGE", "1"))
|
||||||
|
symbol_filters = tuple(assets) if assets else None
|
||||||
|
return BingxExecClientConfig(
|
||||||
|
api_key=os.environ.get("BINGX_API_KEY"),
|
||||||
|
secret_key=os.environ.get("BINGX_SECRET_KEY"),
|
||||||
|
environment=resolved_bingx_env,
|
||||||
|
allow_mainnet=resolved_bingx_allow_mainnet,
|
||||||
|
recv_window_ms=resolved_bingx_recv_window_ms if resolved_bingx_recv_window_ms is not None else 5_000,
|
||||||
|
default_leverage=default_leverage,
|
||||||
|
leverage_by_symbol={symbol: default_leverage for symbol in (assets or [])} if assets else None,
|
||||||
|
prefer_websocket=_env_bool("DOLPHIN_BINGX_PREFER_WEBSOCKET", True),
|
||||||
|
instrument_provider=BingxInstrumentProviderConfig(
|
||||||
|
load_all=True,
|
||||||
|
symbol_filters=symbol_filters,
|
||||||
|
),
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def build_node(
|
||||||
|
*,
|
||||||
|
data_venue: str | None = None,
|
||||||
|
exec_venue: str | None = None,
|
||||||
|
trader_id: str | None = None,
|
||||||
|
bingx_environment: BingxEnvironment | None = None,
|
||||||
|
bingx_allow_mainnet: bool | None = None,
|
||||||
|
bingx_recv_window_ms: int | None = None,
|
||||||
|
) -> TradingNode:
|
||||||
|
resolved_bingx_env = bingx_environment or _resolve_bingx_environment()
|
||||||
|
resolved_bingx_allow_mainnet = (
|
||||||
|
bingx_allow_mainnet if bingx_allow_mainnet is not None else _resolve_bingx_allow_mainnet()
|
||||||
|
)
|
||||||
|
resolved_bingx_recv_window_ms = (
|
||||||
|
bingx_recv_window_ms if bingx_recv_window_ms is not None else _resolve_bingx_recv_window_ms()
|
||||||
|
)
|
||||||
|
if resolved_bingx_env is BingxEnvironment.LIVE and not resolved_bingx_allow_mainnet:
|
||||||
|
raise RuntimeError(
|
||||||
|
"BingX LIVE requested but DOLPHIN_BINGX_ALLOW_MAINNET is not enabled"
|
||||||
|
)
|
||||||
|
|
||||||
|
actor_cfg = build_actor_config(data_venue=data_venue, exec_venue=exec_venue)
|
||||||
|
actor_cfg["bingx_environment"] = str(resolved_bingx_env.value)
|
||||||
|
resolved_data_venue = actor_cfg["data_venue"]
|
||||||
|
resolved_exec_venue = actor_cfg["exec_venue"]
|
||||||
|
|
||||||
|
data_clients = {}
|
||||||
|
exec_clients = {}
|
||||||
|
|
||||||
|
if resolved_data_venue == "BINANCE":
|
||||||
|
api_key = os.environ["BINANCE_API_KEY"]
|
||||||
|
api_secret = os.environ["BINANCE_API_SECRET"]
|
||||||
|
data_clients["BINANCE"] = BinanceDataClientConfig(
|
||||||
|
account_type=_BINANCE_USDT_FUTURES_ACCOUNT_TYPE,
|
||||||
|
api_key=api_key,
|
||||||
|
api_secret=api_secret,
|
||||||
|
testnet=False,
|
||||||
|
)
|
||||||
|
elif resolved_data_venue == "BINGX":
|
||||||
|
from prod.bingx.config import BingxInstrumentProviderConfig
|
||||||
|
data_clients["BINGX"] = BingxDataClientConfig(
|
||||||
|
environment=resolved_bingx_env,
|
||||||
|
allow_mainnet=resolved_bingx_allow_mainnet,
|
||||||
|
instrument_provider=BingxInstrumentProviderConfig(
|
||||||
|
load_all=True,
|
||||||
|
symbol_filters=tuple(actor_cfg.get("assets", [])),
|
||||||
|
),
|
||||||
|
)
|
||||||
|
else:
|
||||||
|
raise ValueError(f"Unsupported data venue: {resolved_data_venue}")
|
||||||
|
|
||||||
|
if resolved_exec_venue == "BINGX":
|
||||||
|
exec_clients["BINGX"] = build_bingx_exec_client_config(
|
||||||
|
resolved_bingx_env=resolved_bingx_env,
|
||||||
|
resolved_bingx_allow_mainnet=resolved_bingx_allow_mainnet,
|
||||||
|
resolved_bingx_recv_window_ms=resolved_bingx_recv_window_ms,
|
||||||
|
assets=actor_cfg.get("assets"),
|
||||||
|
)
|
||||||
|
|
||||||
|
trader_id_value = trader_id or os.environ.get("DOLPHIN_TRADER_ID", "DOLPHIN-LIVE-001")
|
||||||
|
|
||||||
|
from nautilus_dolphin.nautilus.dolphin_actor import DolphinActor
|
||||||
|
actor = DolphinActor(config=actor_cfg)
|
||||||
|
|
||||||
|
node_config = TradingNodeConfig(
|
||||||
|
trader_id=TraderId(trader_id_value),
|
||||||
|
data_clients=data_clients,
|
||||||
|
exec_clients=exec_clients if exec_clients else None,
|
||||||
|
data_engine=LiveDataEngineConfig(time_bars_build_with_no_updates=False),
|
||||||
|
cache=CacheConfig(database=None),
|
||||||
|
)
|
||||||
|
node = TradingNode(config=node_config)
|
||||||
|
|
||||||
|
if "BINANCE" in data_clients:
|
||||||
|
node.add_data_client_factory("BINANCE", BinanceLiveDataClientFactory)
|
||||||
|
if "BINGX" in data_clients:
|
||||||
|
node.add_data_client_factory("BINGX", BingxLiveDataClientFactory)
|
||||||
|
if "BINGX" in exec_clients:
|
||||||
|
node.add_exec_client_factory("BINGX", BingxLiveExecClientFactory)
|
||||||
|
|
||||||
|
node.trader.add_strategy(actor)
|
||||||
|
node.build()
|
||||||
|
return node
|
||||||
|
|
||||||
|
|
||||||
|
async def run() -> None:
|
||||||
|
node = build_node()
|
||||||
|
await node.run_async()
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
asyncio.run(run())
|
||||||
2010
prod/nautilus_event_trader.py
Normal file
2010
prod/nautilus_event_trader.py
Normal file
File diff suppressed because it is too large
Load Diff
658
prod/paper_trade_flow.py
Normal file
658
prod/paper_trade_flow.py
Normal file
@@ -0,0 +1,658 @@
|
|||||||
|
"""DOLPHIN Paper Trading — Prefect Flow.
|
||||||
|
|
||||||
|
Runs daily at 00:05 UTC. Processes yesterday's live scan data through
|
||||||
|
the NDAlphaEngine champion stack. Logs virtual P&L to disk + Hazelcast.
|
||||||
|
|
||||||
|
Blue deployment: champion SHORT (configs/blue.yml)
|
||||||
|
Green deployment: bidirectional SHORT+LONG (configs/green.yml) [pending LONG validation]
|
||||||
|
|
||||||
|
Usage:
|
||||||
|
# Register flows (run once, after Prefect server is up):
|
||||||
|
PREFECT_API_URL=http://localhost:4200/api python paper_trade_flow.py --register
|
||||||
|
|
||||||
|
# Run manually for a specific date:
|
||||||
|
PREFECT_API_URL=http://localhost:4200/api python paper_trade_flow.py \\
|
||||||
|
--date 2026-02-25 --config configs/blue.yml
|
||||||
|
"""
|
||||||
|
import sys, json, yaml, logging, argparse, csv, urllib.request, os
|
||||||
|
from pathlib import Path
|
||||||
|
from datetime import datetime, timedelta, date, timezone
|
||||||
|
import numpy as np
|
||||||
|
import pandas as pd
|
||||||
|
|
||||||
|
HCM_DIR = Path(__file__).parent.parent
|
||||||
|
sys.path.insert(0, str(HCM_DIR / 'nautilus_dolphin'))
|
||||||
|
|
||||||
|
from prefect import flow, task, get_run_logger
|
||||||
|
from prefect.schedules import Cron
|
||||||
|
|
||||||
|
import hazelcast
|
||||||
|
|
||||||
|
logging.basicConfig(level=logging.WARNING) # suppress Prefect noise below WARNING
|
||||||
|
|
||||||
|
# ── Paths ───────────────────────────────────────────────────────────────────────
|
||||||
|
from dolphin_paths import get_eigenvalues_path, get_klines_dir
|
||||||
|
SCANS_DIR = get_eigenvalues_path() # platform-aware: Win → NG3 dir, Linux → /mnt/ng6_data/eigenvalues
|
||||||
|
KLINES_DIR = get_klines_dir() # vbt_cache_klines/ — NG5 parquet source (preferred)
|
||||||
|
MC_MODELS_DIR = str(HCM_DIR / 'nautilus_dolphin' / 'mc_results' / 'models')
|
||||||
|
|
||||||
|
# Columns that are eigenvalue metadata, not asset prices
|
||||||
|
META_COLS = {
|
||||||
|
'timestamp', 'scan_number',
|
||||||
|
'v50_lambda_max_velocity', 'v150_lambda_max_velocity',
|
||||||
|
'v300_lambda_max_velocity', 'v750_lambda_max_velocity',
|
||||||
|
'vel_div', 'instability_50', 'instability_150',
|
||||||
|
}
|
||||||
|
|
||||||
|
HZ_HOST = "localhost:5701"
|
||||||
|
HZ_CLUSTER = "dolphin"
|
||||||
|
|
||||||
|
# Number of historical eigenvalue dates to use for ACB w750 threshold calibration
|
||||||
|
ACB_HISTORY_DAYS = 60
|
||||||
|
|
||||||
|
|
||||||
|
# ── Helpers ──────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
def _get_recent_scan_dates(n: int) -> list:
|
||||||
|
"""Return sorted list of up to n most-recent eigenvalue date dirs."""
|
||||||
|
try:
|
||||||
|
dirs = sorted(
|
||||||
|
d.name for d in SCANS_DIR.iterdir()
|
||||||
|
if d.is_dir() and len(d.name) == 10 and d.name.startswith('20')
|
||||||
|
)
|
||||||
|
return dirs[-n:]
|
||||||
|
except Exception:
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def _fetch_btcusdt_klines_fallback(date_str: str) -> "dict[str, float]":
|
||||||
|
"""Fetch BTCUSDT 1m klines from Binance futures for date_str.
|
||||||
|
|
||||||
|
Returns a dict mapping ISO timestamp strings (minute precision) → close price.
|
||||||
|
Falls back to empty dict on any error (caller handles missing prices gracefully).
|
||||||
|
"""
|
||||||
|
try:
|
||||||
|
from datetime import timezone as tz
|
||||||
|
day_start = datetime.strptime(date_str, "%Y-%m-%d").replace(tzinfo=tz.utc)
|
||||||
|
day_end = day_start + timedelta(days=1)
|
||||||
|
start_ms = int(day_start.timestamp() * 1000)
|
||||||
|
end_ms = int(day_end.timestamp() * 1000)
|
||||||
|
prices: dict[str, float] = {}
|
||||||
|
# Binance returns max 1500 bars per request; 1440 bars/day fits in one call
|
||||||
|
url = (
|
||||||
|
f"https://fapi.binance.com/fapi/v1/klines"
|
||||||
|
f"?symbol=BTCUSDT&interval=1m"
|
||||||
|
f"&startTime={start_ms}&endTime={end_ms}&limit=1500"
|
||||||
|
)
|
||||||
|
with urllib.request.urlopen(url, timeout=10) as resp:
|
||||||
|
bars = json.loads(resp.read())
|
||||||
|
for bar in bars:
|
||||||
|
ts_ms = int(bar[0])
|
||||||
|
close = float(bar[4])
|
||||||
|
ts_iso = datetime.fromtimestamp(ts_ms / 1000, tz=tz.utc).strftime("%Y-%m-%dT%H:%M")
|
||||||
|
prices[ts_iso] = close
|
||||||
|
return prices
|
||||||
|
except Exception:
|
||||||
|
return {}
|
||||||
|
|
||||||
|
|
||||||
|
def _load_scan_df_from_json(date_str: str) -> pd.DataFrame:
|
||||||
|
"""Load daily scan JSON files → DataFrame with vel_div + asset prices.
|
||||||
|
|
||||||
|
Each scan JSON has:
|
||||||
|
- windows: {50, 150, 300, 750} → tracking_data.lambda_max_velocity
|
||||||
|
- pricing_data.current_prices → per-asset USD prices
|
||||||
|
Returns one row per scan, sorted by scan_number.
|
||||||
|
When current_prices is empty (scanner bug), falls back to Binance 1m klines.
|
||||||
|
"""
|
||||||
|
scan_dir = SCANS_DIR / date_str
|
||||||
|
if not scan_dir.exists():
|
||||||
|
return pd.DataFrame()
|
||||||
|
|
||||||
|
json_files = sorted(scan_dir.glob("scan_*.json"), key=lambda f: f.name)
|
||||||
|
# Exclude the _Indicators.json companion files if any
|
||||||
|
json_files = [f for f in json_files if '__Indicators' not in f.name]
|
||||||
|
if not json_files:
|
||||||
|
return pd.DataFrame()
|
||||||
|
|
||||||
|
total = len(json_files)
|
||||||
|
rows = []
|
||||||
|
for i, jf in enumerate(json_files):
|
||||||
|
if i % 500 == 0 and i > 0:
|
||||||
|
print(f" [scan load] {i}/{total} files...", flush=True)
|
||||||
|
try:
|
||||||
|
with open(jf, 'r', encoding='utf-8') as fh:
|
||||||
|
data = json.load(fh)
|
||||||
|
|
||||||
|
row = {
|
||||||
|
'scan_number': data.get('scan_number', 0),
|
||||||
|
'timestamp': data.get('timestamp', ''),
|
||||||
|
}
|
||||||
|
|
||||||
|
# Eigenvalue velocity per window
|
||||||
|
windows = data.get('windows', {})
|
||||||
|
for w_key, vel_col, inst_col in [
|
||||||
|
('50', 'v50_lambda_max_velocity', 'instability_50'),
|
||||||
|
('150', 'v150_lambda_max_velocity', 'instability_150'),
|
||||||
|
('300', 'v300_lambda_max_velocity', None),
|
||||||
|
('750', 'v750_lambda_max_velocity', None),
|
||||||
|
]:
|
||||||
|
w_data = windows.get(str(w_key), {})
|
||||||
|
td = w_data.get('tracking_data', {})
|
||||||
|
vel = td.get('lambda_max_velocity')
|
||||||
|
row[vel_col] = float(vel) if vel is not None else np.nan
|
||||||
|
if inst_col is not None:
|
||||||
|
rs = w_data.get('regime_signals', {})
|
||||||
|
row[inst_col] = float(rs.get('instability_score', 0.0) or 0.0)
|
||||||
|
|
||||||
|
# vel_div = w50_vel - w750_vel (canonical v2_gold_fix_v50-v750 formula)
|
||||||
|
v50 = row.get('v50_lambda_max_velocity', np.nan)
|
||||||
|
v750 = row.get('v750_lambda_max_velocity', np.nan)
|
||||||
|
row['vel_div'] = (v50 - v750) if (np.isfinite(v50) and np.isfinite(v750)) else np.nan
|
||||||
|
|
||||||
|
# Asset prices
|
||||||
|
pricing = data.get('pricing_data', {})
|
||||||
|
prices = pricing.get('current_prices', {})
|
||||||
|
row.update({sym: float(px) for sym, px in prices.items() if px is not None})
|
||||||
|
|
||||||
|
rows.append(row)
|
||||||
|
except Exception:
|
||||||
|
continue
|
||||||
|
|
||||||
|
if not rows:
|
||||||
|
return pd.DataFrame()
|
||||||
|
|
||||||
|
df = pd.DataFrame(rows).sort_values('scan_number').reset_index(drop=True)
|
||||||
|
|
||||||
|
# Fallback: if BTCUSDT prices are missing (scanner current_prices bug),
|
||||||
|
# fetch 1m klines from Binance and fill by timestamp.
|
||||||
|
if 'BTCUSDT' not in df.columns or df['BTCUSDT'].isna().all():
|
||||||
|
btc_klines = _fetch_btcusdt_klines_fallback(date_str)
|
||||||
|
if btc_klines:
|
||||||
|
def _lookup_btc(ts_str: str) -> float:
|
||||||
|
# ts_str is ISO format from scan JSON; match to minute precision
|
||||||
|
ts_min = str(ts_str)[:16] # "YYYY-MM-DDTHH:MM"
|
||||||
|
return btc_klines.get(ts_min, np.nan)
|
||||||
|
df['BTCUSDT'] = df['timestamp'].apply(_lookup_btc)
|
||||||
|
# Forward-fill any gaps (scan timestamps between 1m bars)
|
||||||
|
df['BTCUSDT'] = df['BTCUSDT'].ffill().bfill()
|
||||||
|
|
||||||
|
return df
|
||||||
|
|
||||||
|
|
||||||
|
def _load_scan_df_from_parquet(date_str: str) -> pd.DataFrame:
|
||||||
|
"""Load daily scan data from vbt_cache_klines/ parquet (NG5+ preferred path).
|
||||||
|
|
||||||
|
Returns DataFrame with vel_div + asset prices, or empty DataFrame if not available.
|
||||||
|
"""
|
||||||
|
parq_path = KLINES_DIR / f"{date_str}.parquet"
|
||||||
|
if not parq_path.exists():
|
||||||
|
return pd.DataFrame()
|
||||||
|
try:
|
||||||
|
df = pd.read_parquet(parq_path)
|
||||||
|
if df.empty or 'vel_div' not in df.columns:
|
||||||
|
return pd.DataFrame()
|
||||||
|
return df.reset_index(drop=True)
|
||||||
|
except Exception:
|
||||||
|
return pd.DataFrame()
|
||||||
|
|
||||||
|
|
||||||
|
# ── Tasks ───────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
@task(name="load_config", retries=0)
|
||||||
|
def load_config(config_path: str) -> dict:
|
||||||
|
with open(config_path) as f:
|
||||||
|
return yaml.safe_load(f)
|
||||||
|
|
||||||
|
|
||||||
|
@task(name="load_day_scans", retries=2, retry_delay_seconds=10)
|
||||||
|
def load_day_scans(date_str: str) -> pd.DataFrame:
|
||||||
|
"""Load scan data for one date → VBT-compatible DataFrame.
|
||||||
|
|
||||||
|
Prefers vbt_cache_klines/ parquet (NG5 native, single file, fast).
|
||||||
|
Falls back to eigenvalues/ JSON (NG3 legacy) if parquet unavailable.
|
||||||
|
Extracts vel_div, eigenvalue features, and asset prices.
|
||||||
|
"""
|
||||||
|
log = get_run_logger()
|
||||||
|
|
||||||
|
# ── Parquet path (NG5 native — preferred) ──────────────────────────────────
|
||||||
|
df = _load_scan_df_from_parquet(date_str)
|
||||||
|
if not df.empty:
|
||||||
|
log.info(f" [parquet] Loaded {len(df)} scans for {date_str} | cols={len(df.columns)}")
|
||||||
|
else:
|
||||||
|
# ── JSON fallback (NG3 legacy) ─────────────────────────────────────────
|
||||||
|
df = _load_scan_df_from_json(date_str)
|
||||||
|
if df.empty:
|
||||||
|
log.warning(f"No usable scan data for {date_str} in {SCANS_DIR}")
|
||||||
|
return pd.DataFrame()
|
||||||
|
log.info(f" [json] Loaded {len(df)} scans for {date_str} | cols={len(df.columns)}")
|
||||||
|
|
||||||
|
if df.empty:
|
||||||
|
log.warning(f"No usable scan data for {date_str} in {SCANS_DIR}")
|
||||||
|
return pd.DataFrame()
|
||||||
|
|
||||||
|
# Drop rows with NaN vel_div (warmup period at start of day)
|
||||||
|
valid = df['vel_div'].notna()
|
||||||
|
n_dropped = (~valid).sum()
|
||||||
|
df = df[valid].reset_index(drop=True)
|
||||||
|
|
||||||
|
# Verify BTCUSDT prices present (required for vol gate and DC)
|
||||||
|
if 'BTCUSDT' not in df.columns:
|
||||||
|
log.error(f"BTCUSDT prices missing from scan data on {date_str} — cannot run engine")
|
||||||
|
return pd.DataFrame()
|
||||||
|
|
||||||
|
log.info(f" Loaded {len(df)} scans for {date_str} | cols={len(df.columns)} | "
|
||||||
|
f"vel_div range=[{df['vel_div'].min():.4f}, {df['vel_div'].max():.4f}] "
|
||||||
|
f"| {n_dropped} warmup rows dropped")
|
||||||
|
return df
|
||||||
|
|
||||||
|
|
||||||
|
@task(name="run_engine_day", retries=0, persist_result=False)
|
||||||
|
def run_engine_day(date_str: str, df: pd.DataFrame, engine, vol_p60: float, posture: str = 'APEX', direction: int = -1) -> dict:
|
||||||
|
"""Run one day through NDAlphaEngine. Returns daily stats dict."""
|
||||||
|
log = get_run_logger()
|
||||||
|
if df.empty or posture == 'HIBERNATE':
|
||||||
|
log.warning(f"Empty DataFrame or HIBERNATE for {date_str} — skipping.")
|
||||||
|
return {'date': date_str, 'pnl': 0.0, 'capital': engine.capital,
|
||||||
|
'trades': 0, 'boost': 1.0, 'beta': 0.0, 'mc_status': 'NO_DATA', 'posture': posture}
|
||||||
|
|
||||||
|
asset_cols = [c for c in df.columns if c not in META_COLS]
|
||||||
|
|
||||||
|
# Vol gate: rolling 50-bar std of BTC returns
|
||||||
|
bp = df['BTCUSDT'].values
|
||||||
|
dvol = np.full(len(df), np.nan)
|
||||||
|
for i in range(50, len(bp)):
|
||||||
|
seg = bp[max(0, i - 50):i]
|
||||||
|
if len(seg) >= 10 and seg[0] > 0:
|
||||||
|
dvol[i] = float(np.std(np.diff(seg) / seg[:-1]))
|
||||||
|
vol_ok = np.where(np.isfinite(dvol), dvol > vol_p60, False)
|
||||||
|
|
||||||
|
# 1. Setup day
|
||||||
|
engine.begin_day(date_str, posture=posture, direction=direction)
|
||||||
|
|
||||||
|
# 2. Bar stream (replaces batch process_day)
|
||||||
|
for ri in range(len(df)):
|
||||||
|
row = df.iloc[ri]
|
||||||
|
vd = row.get('vel_div')
|
||||||
|
if vd is None or not np.isfinite(float(vd)):
|
||||||
|
engine._global_bar_idx += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
v50_raw = row.get('v50_lambda_max_velocity')
|
||||||
|
v750_raw = row.get('v750_lambda_max_velocity')
|
||||||
|
v50_val = float(v50_raw) if (v50_raw is not None and np.isfinite(float(v50_raw))) else 0.0
|
||||||
|
v750_val = float(v750_raw) if (v750_raw is not None and np.isfinite(float(v750_raw))) else 0.0
|
||||||
|
|
||||||
|
prices = {}
|
||||||
|
for ac in asset_cols:
|
||||||
|
p = row.get(ac)
|
||||||
|
if p is not None and p > 0 and np.isfinite(p):
|
||||||
|
prices[ac] = float(p)
|
||||||
|
|
||||||
|
if not prices:
|
||||||
|
engine._global_bar_idx += 1
|
||||||
|
continue
|
||||||
|
|
||||||
|
# OB live step: fetch HZ snapshots and compute features BEFORE step_bar().
|
||||||
|
# This populates the live caches so get_placement/get_signal/get_market()
|
||||||
|
# return real OBF values instead of NEUTRAL defaults.
|
||||||
|
if engine.ob_engine is not None:
|
||||||
|
try:
|
||||||
|
engine.ob_engine.step_live(list(prices.keys()), ri)
|
||||||
|
except Exception:
|
||||||
|
pass # OBF degraded → NEUTRAL values, trading continues
|
||||||
|
|
||||||
|
engine.step_bar(
|
||||||
|
bar_idx=ri,
|
||||||
|
vel_div=float(vd),
|
||||||
|
prices=prices,
|
||||||
|
vol_regime_ok=bool(vol_ok[ri]),
|
||||||
|
v50_vel=v50_val,
|
||||||
|
v750_vel=v750_val,
|
||||||
|
)
|
||||||
|
|
||||||
|
# 3. Finalize day
|
||||||
|
result = engine.end_day()
|
||||||
|
result['posture'] = posture
|
||||||
|
log.info(f" {date_str}: PnL={result.get('pnl', 0):+.2f} "
|
||||||
|
f"T={result.get('trades', 0)} boost={result.get('boost', 1.0):.2f}x "
|
||||||
|
f"MC={result.get('mc_status', '?')} Posture={posture}")
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
@task(name="write_hz_state", retries=3, retry_delay_seconds=5, persist_result=False)
|
||||||
|
def write_hz_state(hz_host: str, hz_cluster: str, imap_name: str, key: str, value: dict):
|
||||||
|
"""Write state dict to Hazelcast IMap. Creates own client per call (serialization-safe)."""
|
||||||
|
client = hazelcast.HazelcastClient(cluster_name=hz_cluster, cluster_members=[hz_host])
|
||||||
|
try:
|
||||||
|
client.get_map(imap_name).blocking().put(key, json.dumps(value))
|
||||||
|
finally:
|
||||||
|
client.shutdown()
|
||||||
|
|
||||||
|
|
||||||
|
@task(name="log_pnl", retries=0, persist_result=False)
|
||||||
|
def log_pnl(log_dir: Path, date_str: str, result: dict, capital: float):
|
||||||
|
log_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
row = {**result, 'date': date_str, 'capital': capital,
|
||||||
|
'logged_at': datetime.now(timezone.utc).isoformat()}
|
||||||
|
log_file = log_dir / f"paper_pnl_{date_str[:7]}.jsonl"
|
||||||
|
with open(log_file, 'a') as f:
|
||||||
|
f.write(json.dumps(row) + '\n')
|
||||||
|
|
||||||
|
|
||||||
|
# ── Flow ────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
@flow(name="dolphin-paper-trade", log_prints=True)
|
||||||
|
def dolphin_paper_trade_flow(config_path: str = "configs/blue.yml",
|
||||||
|
run_date: str = None,
|
||||||
|
instrument: bool = False):
|
||||||
|
"""Daily paper trading flow. Processes one day of live eigenvalue scans.
|
||||||
|
|
||||||
|
Scheduled at 00:05 UTC — processes yesterday's data.
|
||||||
|
Run manually with run_date='YYYY-MM-DD' for backtesting or debugging.
|
||||||
|
"""
|
||||||
|
log = get_run_logger()
|
||||||
|
|
||||||
|
cfg = load_config(config_path)
|
||||||
|
strategy_name = cfg['strategy_name']
|
||||||
|
eng_cfg = cfg['engine']
|
||||||
|
pt_cfg = cfg['paper_trade']
|
||||||
|
hz_cfg = cfg['hazelcast']
|
||||||
|
dir_str = os.environ.get('DOLPHIN_DIRECTION', cfg.get('direction', 'short_only'))
|
||||||
|
direction_val = 1 if str(dir_str).strip().lower() in ['long', 'long_only', 'buy', '+1', '1'] else -1
|
||||||
|
|
||||||
|
target_date = run_date or (date.today() - timedelta(days=1)).isoformat()
|
||||||
|
log.info(f"=== {strategy_name.upper()} paper trade: {target_date} ===")
|
||||||
|
|
||||||
|
# ── Lazy imports (numba JIT happens here) ──────────────────────────────────
|
||||||
|
from nautilus_dolphin.nautilus.proxy_boost_engine import create_d_liq_engine
|
||||||
|
from nautilus_dolphin.nautilus.adaptive_circuit_breaker import AdaptiveCircuitBreaker
|
||||||
|
from mc.mc_ml import DolphinForewarner
|
||||||
|
from nautilus_dolphin.nautilus.ob_features import OBFeatureEngine
|
||||||
|
from nautilus_dolphin.nautilus.hz_ob_provider import HZOBProvider
|
||||||
|
|
||||||
|
client = hazelcast.HazelcastClient(cluster_name=HZ_CLUSTER, cluster_members=[HZ_HOST])
|
||||||
|
imap_state = client.get_map(hz_cfg['imap_state']).blocking()
|
||||||
|
|
||||||
|
# ---- Restore capital ----
|
||||||
|
STATE_KEY = f"state_{strategy_name}_{target_date}"
|
||||||
|
restored_capital = pt_cfg['initial_capital']
|
||||||
|
peak_capital = pt_cfg['initial_capital']
|
||||||
|
stored_state = {}
|
||||||
|
engine_state = None
|
||||||
|
try:
|
||||||
|
raw = imap_state.get(STATE_KEY) or imap_state.get('latest') or '{}'
|
||||||
|
stored_state = json.loads(raw)
|
||||||
|
if stored_state.get('strategy') == strategy_name and stored_state.get('capital', 0) > 0:
|
||||||
|
restored_capital = float(stored_state['capital'])
|
||||||
|
peak_capital = float(stored_state.get('peak_capital', restored_capital))
|
||||||
|
engine_state = stored_state.get('engine_state')
|
||||||
|
log.info(f"[STATE] Restored capital={restored_capital:.2f} from HZ")
|
||||||
|
except Exception as e:
|
||||||
|
log.warning(f"[STATE] HZ restore failed: {e} — using config capital")
|
||||||
|
|
||||||
|
# ── Engine — D_LIQ_GOLD config (8x/9x LiquidationGuardEngine) ───────────
|
||||||
|
# create_d_liq_engine() overrides max_leverage→8.0 / abs_max_leverage→9.0
|
||||||
|
# internally (D_LIQ_SOFT_CAP / D_LIQ_ABS_CAP constants), regardless of what
|
||||||
|
# eng_cfg says. This is the certified gold leverage stack.
|
||||||
|
engine = create_d_liq_engine(
|
||||||
|
initial_capital = restored_capital,
|
||||||
|
vel_div_threshold = eng_cfg['vel_div_threshold'],
|
||||||
|
vel_div_extreme = eng_cfg['vel_div_extreme'],
|
||||||
|
min_leverage = eng_cfg['min_leverage'],
|
||||||
|
max_leverage = eng_cfg.get('max_leverage', 8.0),
|
||||||
|
abs_max_leverage = eng_cfg.get('abs_max_leverage', 9.0),
|
||||||
|
leverage_convexity = eng_cfg['leverage_convexity'],
|
||||||
|
fraction = eng_cfg['fraction'],
|
||||||
|
fixed_tp_pct = eng_cfg['fixed_tp_pct'],
|
||||||
|
stop_pct = eng_cfg['stop_pct'],
|
||||||
|
max_hold_bars = eng_cfg['max_hold_bars'],
|
||||||
|
use_direction_confirm= eng_cfg['use_direction_confirm'],
|
||||||
|
dc_lookback_bars = eng_cfg['dc_lookback_bars'],
|
||||||
|
dc_min_magnitude_bps = eng_cfg['dc_min_magnitude_bps'],
|
||||||
|
dc_skip_contradicts = eng_cfg['dc_skip_contradicts'],
|
||||||
|
dc_leverage_boost = eng_cfg['dc_leverage_boost'],
|
||||||
|
dc_leverage_reduce = eng_cfg['dc_leverage_reduce'],
|
||||||
|
use_asset_selection = eng_cfg['use_asset_selection'],
|
||||||
|
min_irp_alignment = eng_cfg['min_irp_alignment'],
|
||||||
|
use_sp_fees = eng_cfg['use_sp_fees'],
|
||||||
|
use_sp_slippage = eng_cfg['use_sp_slippage'],
|
||||||
|
sp_maker_entry_rate = eng_cfg['sp_maker_entry_rate'],
|
||||||
|
sp_maker_exit_rate = eng_cfg['sp_maker_exit_rate'],
|
||||||
|
use_ob_edge = eng_cfg['use_ob_edge'],
|
||||||
|
ob_edge_bps = eng_cfg['ob_edge_bps'],
|
||||||
|
ob_confirm_rate = eng_cfg['ob_confirm_rate'],
|
||||||
|
lookback = eng_cfg['lookback'],
|
||||||
|
use_alpha_layers = eng_cfg['use_alpha_layers'],
|
||||||
|
use_dynamic_leverage = eng_cfg['use_dynamic_leverage'],
|
||||||
|
seed = eng_cfg.get('seed', 42),
|
||||||
|
)
|
||||||
|
engine.set_esoteric_hazard_multiplier(0.0) # gold spec: hazard=0 → base_max_leverage=8.0
|
||||||
|
|
||||||
|
if engine_state:
|
||||||
|
try:
|
||||||
|
engine.restore_state(engine_state)
|
||||||
|
log.info("[STATE] Restored full engine state (including open positions)")
|
||||||
|
except Exception as e:
|
||||||
|
log.error(f"[STATE] Failed to restore engine state: {e}")
|
||||||
|
|
||||||
|
# ── ACB — preload w750 from recent history for valid p60 threshold ─────────
|
||||||
|
# w750 calibration always uses NPZ history (threshold is a population statistic).
|
||||||
|
# Daily ExF factors are sourced from HZ exf_latest (pre-lagged) when available;
|
||||||
|
# fall back to NPZ disk scan if HZ data is absent or stale (>12 h).
|
||||||
|
acb = AdaptiveCircuitBreaker()
|
||||||
|
acb.config.EIGENVALUES_PATH = SCANS_DIR # CRITICAL: override Windows default for Linux
|
||||||
|
recent_dates = _get_recent_scan_dates(ACB_HISTORY_DAYS)
|
||||||
|
if target_date not in recent_dates:
|
||||||
|
recent_dates = (recent_dates + [target_date])[-ACB_HISTORY_DAYS:]
|
||||||
|
acb.preload_w750(recent_dates)
|
||||||
|
log.info(f" ACB preloaded {len(recent_dates)} dates | w750_threshold={acb._w750_threshold:.6f}")
|
||||||
|
|
||||||
|
# ── ACB HZ warm-up: pre-load today's boost from live exf_latest ────────────
|
||||||
|
# The ExF service applies per-indicator lag BEFORE pushing to HZ, so the
|
||||||
|
# values in exf_latest are already delay-adjusted. Do NOT re-lag them.
|
||||||
|
_acb_hz_ok = False
|
||||||
|
try:
|
||||||
|
features_map = client.get_map('DOLPHIN_FEATURES').blocking()
|
||||||
|
exf_raw = features_map.get('exf_latest')
|
||||||
|
if exf_raw:
|
||||||
|
exf_snapshot = json.loads(exf_raw)
|
||||||
|
# Live w750 from latest scan (may be absent during warmup)
|
||||||
|
scan_raw = features_map.get('latest_eigen_scan')
|
||||||
|
w750_live: float | None = None
|
||||||
|
if scan_raw:
|
||||||
|
scan_data = json.loads(scan_raw)
|
||||||
|
w750_live = scan_data.get('w750_velocity')
|
||||||
|
|
||||||
|
boost_info = acb.get_dynamic_boost_from_hz(
|
||||||
|
target_date, exf_snapshot, w750_velocity=w750_live, direction=direction_val
|
||||||
|
)
|
||||||
|
stale_s = boost_info.get('max_staleness_s', 0)
|
||||||
|
log.info(
|
||||||
|
f" ACB HZ: boost={boost_info['boost']:.4f} beta={boost_info['beta']:.2f} "
|
||||||
|
f"signals={boost_info['signals']:.1f} staleness={stale_s:.0f}s"
|
||||||
|
)
|
||||||
|
_acb_hz_ok = True
|
||||||
|
else:
|
||||||
|
log.warning(" ACB HZ: exf_latest not found — falling back to NPZ disk scan")
|
||||||
|
except ValueError as _ve:
|
||||||
|
log.warning(f" ACB HZ: snapshot stale ({_ve}) — falling back to NPZ disk scan")
|
||||||
|
except Exception as _e:
|
||||||
|
log.warning(f" ACB HZ: read failed ({_e}) — falling back to NPZ disk scan")
|
||||||
|
|
||||||
|
if not _acb_hz_ok:
|
||||||
|
# NPZ fallback: get_dynamic_boost_for_date() will read eigenvalues/ on demand
|
||||||
|
log.info(f" ACB: using NPZ disk path for {target_date}")
|
||||||
|
|
||||||
|
# ── Data Loading & Live OB Integration ────────────────────────────────────
|
||||||
|
df = load_day_scans(target_date)
|
||||||
|
OB_ASSETS = [c for c in df.columns if c not in META_COLS] if not df.empty else ["BTCUSDT", "ETHUSDT", "BNBUSDT", "SOLUSDT"]
|
||||||
|
|
||||||
|
live_ob = HZOBProvider(hz_cluster=HZ_CLUSTER, hz_host=HZ_HOST)
|
||||||
|
ob_eng = OBFeatureEngine(live_ob)
|
||||||
|
# NOTE: HZOBProvider has no historical snapshots, so preload_date() is a no-op.
|
||||||
|
# OB features are fetched live via ob_eng.step_live() before each step_bar() call.
|
||||||
|
|
||||||
|
# ── MC-Forewarner ──────────────────────────────────────────────────────────
|
||||||
|
forewarner = DolphinForewarner(models_dir=MC_MODELS_DIR)
|
||||||
|
mc_base_cfg = {
|
||||||
|
'trial_id': 0, 'vel_div_threshold': -0.020, 'vel_div_extreme': -0.050,
|
||||||
|
'use_direction_confirm': True, 'dc_lookback_bars': 7,
|
||||||
|
'dc_min_magnitude_bps': 0.75, 'dc_skip_contradicts': True,
|
||||||
|
'dc_leverage_boost': 1.00, 'dc_leverage_reduce': 0.50,
|
||||||
|
'vd_trend_lookback': 10, 'min_leverage': 0.50, 'max_leverage': 5.00,
|
||||||
|
'leverage_convexity': 3.00, 'fraction': 0.20,
|
||||||
|
'use_alpha_layers': True, 'use_dynamic_leverage': True,
|
||||||
|
'fixed_tp_pct': 0.0095, 'stop_pct': 1.00, 'max_hold_bars': 120,
|
||||||
|
'use_sp_fees': True, 'use_sp_slippage': True,
|
||||||
|
'sp_maker_entry_rate': 0.62, 'sp_maker_exit_rate': 0.50,
|
||||||
|
'use_ob_edge': True, 'ob_edge_bps': 5.00, 'ob_confirm_rate': 0.40,
|
||||||
|
'ob_imbalance_bias': -0.09, 'ob_depth_scale': 1.00,
|
||||||
|
'use_asset_selection': True, 'min_irp_alignment': 0.45, 'lookback': 100,
|
||||||
|
'acb_beta_high': 0.80, 'acb_beta_low': 0.20, 'acb_w750_threshold_pct': 60,
|
||||||
|
}
|
||||||
|
|
||||||
|
engine.set_ob_engine(ob_eng)
|
||||||
|
engine.set_acb(acb)
|
||||||
|
engine.set_mc_forewarner(forewarner, mc_base_cfg)
|
||||||
|
engine.set_esoteric_hazard_multiplier(0.0)
|
||||||
|
if instrument:
|
||||||
|
engine._bar_log_enabled = True
|
||||||
|
|
||||||
|
# vol_p60: 60th percentile of rolling 50-bar BTC return std
|
||||||
|
# Calibrated from 55-day NG3 champion window (Dec31–Feb25).
|
||||||
|
# TODO: compute adaptively from rolling scan history (Phase MIG3)
|
||||||
|
vol_p60 = pt_cfg.get('vol_p60', 0.000099)
|
||||||
|
|
||||||
|
# ── Run ────────────────────────────────────────────────────────────────────
|
||||||
|
# df was already loaded above to define OB_ASSETS
|
||||||
|
|
||||||
|
# ── DOLPHIN_SAFETY (MIG3) ──────────────────────────────────────────────────
|
||||||
|
try:
|
||||||
|
safety_ref = client.get_cp_subsystem().get_atomic_reference('DOLPHIN_SAFETY').blocking()
|
||||||
|
safety_raw = safety_ref.get()
|
||||||
|
except Exception:
|
||||||
|
safety_ref = client.get_map('DOLPHIN_SAFETY').blocking()
|
||||||
|
safety_raw = safety_ref.get('latest')
|
||||||
|
|
||||||
|
safety_state = json.loads(safety_raw) if safety_raw else {}
|
||||||
|
posture = safety_state.get('posture', 'APEX')
|
||||||
|
Rm = safety_state.get('Rm', 1.0)
|
||||||
|
log.info(f"[SURVIVAL STACK] Posture={posture} | Rm={Rm:.3f}")
|
||||||
|
|
||||||
|
# Apply Rm to absolute max leverage
|
||||||
|
effective_max_lev = engine.abs_max_leverage * Rm
|
||||||
|
engine.abs_max_leverage = max(1.0, effective_max_lev)
|
||||||
|
|
||||||
|
if posture == 'STALKER':
|
||||||
|
engine.abs_max_leverage = min(engine.abs_max_leverage, 2.0)
|
||||||
|
|
||||||
|
result = run_engine_day(target_date, df, engine, vol_p60, posture=posture, direction=direction_val)
|
||||||
|
result['strategy'] = strategy_name
|
||||||
|
result['capital'] = engine.capital
|
||||||
|
|
||||||
|
# ── Hazelcast & State Persistence ──────────────────────────────────────────
|
||||||
|
try:
|
||||||
|
# PnL write
|
||||||
|
imap_pnl = client.get_map(hz_cfg['imap_pnl']).blocking()
|
||||||
|
imap_pnl.put(target_date, json.dumps(result))
|
||||||
|
|
||||||
|
# State persist write
|
||||||
|
new_peak = max(engine.capital, peak_capital)
|
||||||
|
new_drawdown = 1.0 - (engine.capital / new_peak) if new_peak > 0 else 0.0
|
||||||
|
|
||||||
|
new_state = {
|
||||||
|
'strategy': strategy_name,
|
||||||
|
'capital': engine.capital,
|
||||||
|
'date': target_date,
|
||||||
|
'pnl': result.get('pnl', 0.0),
|
||||||
|
'trades': result.get('trades', 0),
|
||||||
|
'peak_capital': new_peak,
|
||||||
|
'drawdown': new_drawdown,
|
||||||
|
'last_date': target_date,
|
||||||
|
'updated_at': datetime.now(timezone.utc).isoformat(),
|
||||||
|
'engine_state': engine.get_state(),
|
||||||
|
}
|
||||||
|
imap_state.put('latest', json.dumps(new_state))
|
||||||
|
imap_state.put(STATE_KEY, json.dumps(new_state))
|
||||||
|
log.info(f" HZ write OK → state & {hz_cfg['imap_pnl']}[{target_date}]")
|
||||||
|
except Exception as e:
|
||||||
|
log.error(f"[STATE] HZ persist failed: {e}")
|
||||||
|
# Fallback: write to local JSON ledger
|
||||||
|
ledger_dir = Path(__file__).parent / pt_cfg.get('log_dir', 'paper_logs')
|
||||||
|
ledger_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
ledger_path = ledger_dir / f"state_ledger_{strategy_name}.jsonl"
|
||||||
|
with open(ledger_path, 'a') as f:
|
||||||
|
f.write(json.dumps({
|
||||||
|
'strategy': strategy_name, 'capital': engine.capital,
|
||||||
|
'date': target_date, 'pnl': result.get('pnl', 0.0),
|
||||||
|
'trades': result.get('trades', 0), 'peak_capital': peak_capital,
|
||||||
|
'drawdown': 1.0 - engine.capital / max(engine.capital, peak_capital) if max(engine.capital, peak_capital) > 0 else 0.0
|
||||||
|
}) + '\n')
|
||||||
|
finally:
|
||||||
|
live_ob.close()
|
||||||
|
client.shutdown()
|
||||||
|
|
||||||
|
# ── Instrumentation (--instrument flag) ────────────────────────────────────
|
||||||
|
if instrument:
|
||||||
|
instr_dir = Path(__file__).parent / pt_cfg['log_dir']
|
||||||
|
instr_dir.mkdir(parents=True, exist_ok=True)
|
||||||
|
trades_instr_path = instr_dir / f"E2E_trades_{target_date}.csv"
|
||||||
|
with open(trades_instr_path, 'w', newline='') as f:
|
||||||
|
cw = csv.writer(f)
|
||||||
|
cw.writerow(['trade_id', 'asset', 'direction', 'entry_price', 'exit_price',
|
||||||
|
'entry_bar', 'exit_bar', 'bars_held', 'leverage', 'notional',
|
||||||
|
'pnl_pct', 'pnl_absolute', 'exit_reason', 'bucket_idx'])
|
||||||
|
for t in engine.trade_history:
|
||||||
|
cw.writerow([t.trade_id, t.asset, t.direction,
|
||||||
|
f"{t.entry_price:.6f}", f"{t.exit_price:.6f}",
|
||||||
|
t.entry_bar, t.exit_bar, t.bars_held,
|
||||||
|
f"{t.leverage:.4f}", f"{t.notional:.4f}",
|
||||||
|
f"{t.pnl_pct:.8f}", f"{t.pnl_absolute:.4f}",
|
||||||
|
t.exit_reason, t.bucket_idx])
|
||||||
|
bars_instr_path = instr_dir / f"E2E_bars_{target_date}.csv"
|
||||||
|
with open(bars_instr_path, 'w', newline='') as f:
|
||||||
|
cw = csv.writer(f)
|
||||||
|
cw.writerow(['date', 'bar_idx', 'vel_div', 'vol_ok', 'posture',
|
||||||
|
'regime_size_mult', 'position_open', 'boost', 'beta'])
|
||||||
|
for b in engine._bar_log:
|
||||||
|
cw.writerow([target_date, b['bar_idx'], f"{b['vel_div']:.8f}",
|
||||||
|
b['vol_ok'], b['posture'], f"{b['regime_size_mult']:.6f}",
|
||||||
|
b['position_open'], f"{b['boost']:.4f}", f"{b['beta']:.2f}"])
|
||||||
|
log.info(f" Instrumentation → {trades_instr_path.name} ({len(engine.trade_history)} trades), "
|
||||||
|
f"{bars_instr_path.name} ({len(engine._bar_log)} bars)")
|
||||||
|
|
||||||
|
# ── Disk log ───────────────────────────────────────────────────────────────
|
||||||
|
log_pnl(Path(__file__).parent / pt_cfg['log_dir'], target_date, result, engine.capital)
|
||||||
|
|
||||||
|
log.info(f"=== DONE: {strategy_name} {target_date} | "
|
||||||
|
f"PnL={result.get('pnl', 0):+.2f} | Capital={engine.capital:,.2f} ===")
|
||||||
|
return result
|
||||||
|
|
||||||
|
|
||||||
|
# ── CLI entry point ──────────────────────────────────────────────────────────────
|
||||||
|
if __name__ == '__main__':
|
||||||
|
parser = argparse.ArgumentParser(description='DOLPHIN paper trading flow')
|
||||||
|
parser.add_argument('--config', default='configs/blue.yml', help='Strategy config YAML')
|
||||||
|
parser.add_argument('--date', default=None, help='YYYY-MM-DD (default: yesterday)')
|
||||||
|
parser.add_argument('--register', action='store_true', help='Register Prefect deployments')
|
||||||
|
parser.add_argument('--instrument', action='store_true', help='Write per-trade + bar CSVs to log_dir')
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
if args.register:
|
||||||
|
from prefect.client.schemas.schedules import CronSchedule as CS
|
||||||
|
for color, cfg_path in [('blue', 'configs/blue.yml'), ('green', 'configs/green.yml')]:
|
||||||
|
abs_cfg = str(Path(__file__).parent / cfg_path)
|
||||||
|
deployment = dolphin_paper_trade_flow.to_deployment(
|
||||||
|
name=f"dolphin-paper-{color}",
|
||||||
|
parameters={"config_path": abs_cfg},
|
||||||
|
schedule=CS(cron="5 0 * * *", timezone="UTC"),
|
||||||
|
work_pool_name="dolphin",
|
||||||
|
tags=[color, "paper-trade", "dolphin"],
|
||||||
|
)
|
||||||
|
deployment.apply()
|
||||||
|
print(f"Registered: dolphin-paper-{color}")
|
||||||
|
else:
|
||||||
|
os.environ.setdefault('PREFECT_API_URL', 'http://localhost:4200/api')
|
||||||
|
dolphin_paper_trade_flow(config_path=args.config, run_date=args.date,
|
||||||
|
instrument=args.instrument)
|
||||||
194
prod/tests/test_long_capability_layers.py
Normal file
194
prod/tests/test_long_capability_layers.py
Normal file
@@ -0,0 +1,194 @@
|
|||||||
|
import math
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
ROOT = Path("/mnt/dolphinng5_predict")
|
||||||
|
sys.path.insert(0, str(ROOT / "nautilus_dolphin"))
|
||||||
|
sys.path.insert(1, str(ROOT))
|
||||||
|
if "nautilus_dolphin" in sys.modules:
|
||||||
|
pkg = sys.modules["nautilus_dolphin"]
|
||||||
|
pkg_file = str(getattr(pkg, "__file__", "") or "")
|
||||||
|
if not pkg_file.endswith("nautilus_dolphin/nautilus_dolphin/__init__.py"):
|
||||||
|
del sys.modules["nautilus_dolphin"]
|
||||||
|
|
||||||
|
from nautilus_dolphin.nautilus.adaptive_circuit_breaker import AdaptiveCircuitBreaker, ACBConfig
|
||||||
|
from nautilus_dolphin.nautilus.alpha_bet_sizer import AlphaBetSizer
|
||||||
|
from nautilus_dolphin.nautilus.alpha_exit_manager import AlphaExitManager
|
||||||
|
from nautilus_dolphin.nautilus.alpha_signal_generator import AlphaSignalGenerator
|
||||||
|
from nautilus_dolphin.nautilus.esf_alpha_orchestrator import NDAlphaEngine
|
||||||
|
from nautilus_dolphin.nautilus.dolphin_actor import _trade_direction_from_config
|
||||||
|
|
||||||
|
|
||||||
|
def test_signal_generator_long_gate_and_dc_are_side_aware():
|
||||||
|
gen = AlphaSignalGenerator(use_direction_confirm=True)
|
||||||
|
rising_prices = [100.0, 100.1, 100.2, 100.3, 100.4, 100.5, 100.6, 101.0]
|
||||||
|
falling_prices = [101.0, 100.8, 100.6, 100.4, 100.2, 100.0, 99.8, 99.5]
|
||||||
|
|
||||||
|
long_sig = gen.generate(
|
||||||
|
vel_div=0.025,
|
||||||
|
vel_div_history=[0.012] * 10,
|
||||||
|
asset_price_history=rising_prices,
|
||||||
|
trade_direction=1,
|
||||||
|
)
|
||||||
|
assert long_sig.is_valid
|
||||||
|
assert long_sig.direction == 1
|
||||||
|
assert long_sig.dc_status == "CONFIRM"
|
||||||
|
|
||||||
|
contradicted = gen.generate(
|
||||||
|
vel_div=0.025,
|
||||||
|
vel_div_history=[0.012] * 10,
|
||||||
|
asset_price_history=falling_prices,
|
||||||
|
trade_direction=1,
|
||||||
|
)
|
||||||
|
assert not contradicted.is_valid
|
||||||
|
assert contradicted.dc_status == "SKIP_CONTRADICT"
|
||||||
|
|
||||||
|
|
||||||
|
def test_bet_sizer_trend_multiplier_is_direction_aware_for_long():
|
||||||
|
sizer = AlphaBetSizer(
|
||||||
|
base_fraction=0.20,
|
||||||
|
min_leverage=0.5,
|
||||||
|
max_leverage=8.0,
|
||||||
|
use_alpha_layers=True,
|
||||||
|
use_dynamic_leverage=True,
|
||||||
|
)
|
||||||
|
favorable_long = sizer.calculate_size(25000, 0.025, vel_div_trend=0.02, trade_direction=1)
|
||||||
|
adverse_long = sizer.calculate_size(25000, 0.025, vel_div_trend=-0.02, trade_direction=1)
|
||||||
|
favorable_short = sizer.calculate_size(25000, -0.035, vel_div_trend=-0.02, trade_direction=-1)
|
||||||
|
adverse_short = sizer.calculate_size(25000, -0.035, vel_div_trend=0.02, trade_direction=-1)
|
||||||
|
|
||||||
|
assert favorable_long["fraction"] > adverse_long["fraction"]
|
||||||
|
assert favorable_short["fraction"] > adverse_short["fraction"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_ndalphaengine_enters_long_when_begin_day_direction_is_long():
|
||||||
|
engine = NDAlphaEngine(
|
||||||
|
initial_capital=1000.0,
|
||||||
|
use_asset_selection=False,
|
||||||
|
use_direction_confirm=False,
|
||||||
|
use_sp_fees=False,
|
||||||
|
use_sp_slippage=False,
|
||||||
|
use_ob_edge=False,
|
||||||
|
use_alpha_layers=False,
|
||||||
|
use_dynamic_leverage=False,
|
||||||
|
lookback=1,
|
||||||
|
)
|
||||||
|
engine.begin_day("2026-05-08", posture="APEX", direction=1)
|
||||||
|
|
||||||
|
res = engine.step_bar(0, vel_div=0.025, prices={"BTCUSDT": 100.0}, vol_regime_ok=True)
|
||||||
|
|
||||||
|
assert res["entry"] is not None
|
||||||
|
assert res["entry"]["direction"] == 1
|
||||||
|
assert engine.position is not None
|
||||||
|
assert engine.position.direction == 1
|
||||||
|
|
||||||
|
|
||||||
|
def test_ndalphaengine_short_default_preserved():
|
||||||
|
engine = NDAlphaEngine(
|
||||||
|
initial_capital=1000.0,
|
||||||
|
use_asset_selection=False,
|
||||||
|
use_direction_confirm=False,
|
||||||
|
use_sp_fees=False,
|
||||||
|
use_sp_slippage=False,
|
||||||
|
use_ob_edge=False,
|
||||||
|
use_alpha_layers=False,
|
||||||
|
use_dynamic_leverage=False,
|
||||||
|
lookback=1,
|
||||||
|
)
|
||||||
|
engine.begin_day("2026-05-08", posture="APEX")
|
||||||
|
|
||||||
|
res = engine.step_bar(0, vel_div=-0.035, prices={"BTCUSDT": 100.0}, vol_regime_ok=True)
|
||||||
|
|
||||||
|
assert res["entry"] is not None
|
||||||
|
assert res["entry"]["direction"] == -1
|
||||||
|
|
||||||
|
|
||||||
|
def test_acb_short_default_and_long_cache_are_side_separated():
|
||||||
|
acb = AdaptiveCircuitBreaker()
|
||||||
|
acb._w750_threshold = 0.001
|
||||||
|
bullish = {
|
||||||
|
"funding_btc": 0.0002,
|
||||||
|
"dvol_btc": 30.0,
|
||||||
|
"fng": 80.0,
|
||||||
|
"taker": 1.25,
|
||||||
|
"available": True,
|
||||||
|
}
|
||||||
|
short = acb._calculate_signals(bullish)
|
||||||
|
long = acb._calculate_signals(bullish, direction=1)
|
||||||
|
|
||||||
|
assert short["signals"] == pytest.approx(0.0)
|
||||||
|
assert long["signals"] == pytest.approx(4.0)
|
||||||
|
|
||||||
|
snap = dict(bullish, _acb_ready=True, _staleness_s={})
|
||||||
|
short_hz = acb.get_dynamic_boost_from_hz("2026-05-08", snap, w750_velocity=0.002, direction=-1)
|
||||||
|
long_hz = acb.get_dynamic_boost_from_hz("2026-05-08", snap, w750_velocity=0.002, direction=1)
|
||||||
|
|
||||||
|
assert short_hz["side"] == "SHORT"
|
||||||
|
assert long_hz["side"] == "LONG"
|
||||||
|
assert short_hz["boost"] == pytest.approx(1.0)
|
||||||
|
assert long_hz["boost"] == pytest.approx(1.0 + 0.5 * math.log1p(4.0))
|
||||||
|
assert acb.get_dynamic_boost_for_date("2026-05-08")["side"] == "SHORT"
|
||||||
|
assert acb.get_dynamic_boost_for_date("2026-05-08", direction=1)["side"] == "LONG"
|
||||||
|
|
||||||
|
|
||||||
|
def test_acb_short_threshold_regression_values_still_match_v6():
|
||||||
|
acb = AdaptiveCircuitBreaker()
|
||||||
|
factors = {
|
||||||
|
"funding_btc": -0.0002,
|
||||||
|
"dvol_btc": 85.0,
|
||||||
|
"fng": 20.0,
|
||||||
|
"taker": 0.75,
|
||||||
|
"available": True,
|
||||||
|
}
|
||||||
|
|
||||||
|
result = acb._calculate_signals(factors)
|
||||||
|
|
||||||
|
assert result["signals"] == pytest.approx(4.0)
|
||||||
|
assert result["severity"] == 7
|
||||||
|
|
||||||
|
|
||||||
|
def test_acb_ob_beta_modulation_is_side_aware():
|
||||||
|
acb = AdaptiveCircuitBreaker()
|
||||||
|
acb._w750_threshold = 0.001
|
||||||
|
calm_ob = SimpleNamespace(
|
||||||
|
get_macro=lambda: SimpleNamespace(regime_signal=-1, depth_velocity=0.1, cascade_count=0)
|
||||||
|
)
|
||||||
|
stress_ob = SimpleNamespace(
|
||||||
|
get_macro=lambda: SimpleNamespace(regime_signal=1, depth_velocity=-0.3, cascade_count=2)
|
||||||
|
)
|
||||||
|
|
||||||
|
long_calm = acb.get_dynamic_boost_from_hz(
|
||||||
|
"2026-05-08", {"_acb_ready": True, "_staleness_s": {}}, w750_velocity=0.002, ob_engine=calm_ob, direction=1
|
||||||
|
)
|
||||||
|
short_calm = acb.get_dynamic_boost_from_hz(
|
||||||
|
"2026-05-09", {"_acb_ready": True, "_staleness_s": {}}, w750_velocity=0.002, ob_engine=calm_ob, direction=-1
|
||||||
|
)
|
||||||
|
short_stress = acb.get_dynamic_boost_from_hz(
|
||||||
|
"2026-05-10", {"_acb_ready": True, "_staleness_s": {}}, w750_velocity=0.002, ob_engine=stress_ob, direction=-1
|
||||||
|
)
|
||||||
|
|
||||||
|
assert long_calm["beta"] == pytest.approx(1.0)
|
||||||
|
assert short_calm["beta"] == pytest.approx(0.68)
|
||||||
|
assert short_stress["beta"] == pytest.approx(1.0)
|
||||||
|
|
||||||
|
|
||||||
|
def test_exit_manager_optional_vd_exit_is_long_aware():
|
||||||
|
manager = AlphaExitManager(vd_enabled=True, vd_consec_bars=2)
|
||||||
|
manager.setup_position("long-1", entry_price=100.0, direction=1, entry_bar=0)
|
||||||
|
|
||||||
|
first = manager.evaluate("long-1", current_price=100.1, current_bar=1, vel_div=-0.02)
|
||||||
|
second = manager.evaluate("long-1", current_price=100.1, current_bar=2, vel_div=-0.02)
|
||||||
|
|
||||||
|
assert first["action"] == "HOLD"
|
||||||
|
assert second["action"] == "EXIT"
|
||||||
|
assert second["reason"] == "VD_INVALIDATION"
|
||||||
|
|
||||||
|
|
||||||
|
def test_prodgreen_direction_parser_is_explicit_and_case_insensitive():
|
||||||
|
assert _trade_direction_from_config("LONG_ONLY") == 1
|
||||||
|
assert _trade_direction_from_config("short_only") == -1
|
||||||
|
with pytest.raises(ValueError):
|
||||||
|
_trade_direction_from_config("bidirectional")
|
||||||
Reference in New Issue
Block a user