diff --git a/adaptive_exit/calibrate_v7_long_from_journal.py b/adaptive_exit/calibrate_v7_long_from_journal.py new file mode 100644 index 0000000..4902423 --- /dev/null +++ b/adaptive_exit/calibrate_v7_long_from_journal.py @@ -0,0 +1,315 @@ +"""Calibrate AlphaExitEngineV7 thresholds for synthetic LONG EFSM paths. + +This script replays BLUE V7 decision journal price paths with side inverted to +LONG. It follows the original V7 SHORT calibration pattern: + +1. Reconstruct per-trade path from V7 journal rows. +2. Compute the natural end-of-path LONG outcome. +3. Replay AlphaExitEngineV7 on the same path using side=LONG. +4. Sweep configurable threshold surfaces. +5. Compare the first V7 EXIT against the natural end outcome. + +The output is a calibration proxy for EFSM FLIP_LONG trades, not proof from +actual exchange-filled LONG trades. +""" + +from __future__ import annotations + +import argparse +import base64 +import csv +import json +import logging +import math +import sys +import urllib.request +from collections import defaultdict +from dataclasses import asdict +from pathlib import Path +from statistics import fmean +from typing import Any + +ROOT = Path("/mnt/dolphinng5_predict") +sys.path.insert(0, str(ROOT / "nautilus_dolphin")) +sys.path.insert(1, str(ROOT)) + +from nautilus_dolphin.nautilus.alpha_exit_v7_engine import AlphaExitEngineV7, AlphaExitV7Config # noqa: E402 + + +CH_URL = "http://localhost:8123/?database=dolphin" +AUTH = "Basic " + base64.b64encode(b"dolphin:dolphin_ch_2026").decode() +FEE_PCT = 0.0004 + +logging.getLogger("nautilus_dolphin.nautilus.alpha_exit_v7_engine").setLevel(logging.ERROR) + + +def _query(sql: str) -> str: + req = urllib.request.Request(CH_URL, data=sql.encode(), headers={"Authorization": AUTH}) + return urllib.request.urlopen(req, timeout=60).read().decode() + + +def load_v7_rows(limit_trades: int = 0) -> list[dict[str, Any]]: + trade_filter = "" + if limit_trades > 0: + trade_filter = ( + "AND trade_id IN (" + "SELECT trade_id FROM (" + "SELECT trade_id, max(ts) AS mx FROM v7_decision_events " + "WHERE strategy='blue' AND side='SHORT' GROUP BY trade_id ORDER BY mx DESC " + f"LIMIT {int(limit_trades)}" + "))" + ) + sql = f""" + SELECT + ts, trade_id, asset, entry_price, current_price, quantity, leverage, + bar_idx, decision_seq, bars_held, ob_imbalance, + exf_funding, exf_dvol, exf_fear_greed, exf_taker + FROM v7_decision_events + WHERE strategy='blue' AND side='SHORT' {trade_filter} + ORDER BY trade_id ASC, decision_seq ASC, ts ASC + FORMAT CSVWithNames + """ + text = _query(sql) + rows: list[dict[str, Any]] = [] + for r in csv.DictReader(text.splitlines()): + rows.append({ + "ts": r["ts"], + "trade_id": r["trade_id"], + "asset": r["asset"], + "entry_price": float(r["entry_price"] or 0.0), + "current_price": float(r["current_price"] or 0.0), + "quantity": float(r["quantity"] or 0.0), + "leverage": float(r["leverage"] or 0.0), + "bar_idx": int(float(r["bar_idx"] or 0)), + "decision_seq": int(float(r["decision_seq"] or 0)), + "bars_held": int(float(r["bars_held"] or 0)), + "ob_imbalance": float(r["ob_imbalance"] or 0.0), + "exf_funding": float(r["exf_funding"] or 0.0), + "exf_dvol": float(r["exf_dvol"] or 0.0), + "exf_fear_greed": float(r["exf_fear_greed"] or 0.0), + "exf_taker": float(r["exf_taker"] or 0.0), + }) + return rows + + +def group_paths(rows: list[dict[str, Any]]) -> list[list[dict[str, Any]]]: + grouped: dict[str, list[dict[str, Any]]] = defaultdict(list) + for row in rows: + grouped[row["trade_id"]].append(row) + paths = [] + for path in grouped.values(): + path.sort(key=lambda r: (r["decision_seq"], r["bar_idx"], r["ts"])) + clean = [r for r in path if r["entry_price"] > 0 and r["current_price"] > 0] + if len(clean) >= 2: + paths.append(clean) + paths.sort(key=lambda p: p[-1]["ts"]) + return paths + + +def natural_long_return(path: list[dict[str, Any]]) -> float: + entry = path[0]["entry_price"] + last = path[-1]["current_price"] + return (last - entry) / entry - FEE_PCT if entry > 0 else 0.0 + + +def pnl_dollars(path: list[dict[str, Any]], ret: float) -> float: + notional = abs(path[0]["entry_price"] * path[0]["quantity"]) + return notional * ret + + +def replay_path(path: list[dict[str, Any]], cfg: AlphaExitV7Config) -> dict[str, Any]: + engine = AlphaExitEngineV7( + bar_duration_sec=11.0, + bounce_model_path="/tmp/nonexistent-bounce-model.pkl", + config=cfg, + ) + ctx = engine.make_context(entry_price=path[0]["entry_price"], entry_bar=path[0]["bar_idx"], side=0) + first_exit = None + decisions = [] + for row in path: + if hasattr(ctx, "set_exf"): + ctx.set_exf( + funding=row["exf_funding"], + dvol=row["exf_dvol"], + fear_greed=row["exf_fear_greed"], + taker=row["exf_taker"], + ) + dec = engine.evaluate( + ctx, + current_price=row["current_price"], + current_bar=row["bar_idx"], + ob_imbalance=row["ob_imbalance"], + asset=row["asset"], + ) + decisions.append(dec) + if first_exit is None and dec["action"] == "EXIT": + first_exit = (row, dec) + break + nat_ret = natural_long_return(path) + if first_exit is None: + exit_ret = nat_ret + exit_row = path[-1] + exit_dec = decisions[-1] + exited = False + else: + exit_row, exit_dec = first_exit + exit_ret = (exit_row["current_price"] - path[0]["entry_price"]) / path[0]["entry_price"] - FEE_PCT + exited = True + return { + "trade_id": path[0]["trade_id"], + "asset": path[0]["asset"], + "n_rows": len(path), + "natural_ret": nat_ret, + "natural_pnl": pnl_dollars(path, nat_ret), + "exit_ret": exit_ret, + "exit_pnl": pnl_dollars(path, exit_ret), + "delta_pnl": pnl_dollars(path, exit_ret) - pnl_dollars(path, nat_ret), + "exited": exited, + "exit_action": exit_dec.get("action"), + "exit_reason": exit_dec.get("reason") or "", + "exit_pressure": float(exit_dec.get("exit_pressure", 0.0) or 0.0), + "exit_bars_held": int(exit_dec.get("bars_held", 0) or 0), + "exit_mae": float(exit_dec.get("mae", 0.0) or 0.0), + "exit_mfe": float(exit_dec.get("mfe", 0.0) or 0.0), + "exit_mae_risk": float(exit_dec.get("mae_risk", 0.0) or 0.0), + "exit_mfe_risk": float(exit_dec.get("mfe_risk", 0.0) or 0.0), + } + + +def equity_stats(vals: list[float]) -> dict[str, float]: + eq = 1.0 + peak = 1.0 + dd = 0.0 + for r in vals: + eq *= max(0.0, 1.0 + r) + peak = max(peak, eq) + dd = max(dd, (peak - eq) / peak if peak else 0.0) + return { + "n": len(vals), + "wr": sum(1 for r in vals if r > 0) / len(vals) if vals else 0.0, + "mean": fmean(vals) if vals else 0.0, + "compound": eq - 1.0, + "max_dd": dd, + } + + +def summarize(results: list[dict[str, Any]], cfg: AlphaExitV7Config, name: str) -> dict[str, Any]: + natural_rets = [r["natural_ret"] for r in results] + exit_rets = [r["exit_ret"] for r in results] + deltas = [r["delta_pnl"] for r in results] + return { + "name": name, + "config": asdict(cfg), + "n": len(results), + "exits": sum(1 for r in results if r["exited"]), + "exit_rate": sum(1 for r in results if r["exited"]) / len(results) if results else 0.0, + "natural": { + **equity_stats(natural_rets), + "pnl": sum(r["natural_pnl"] for r in results), + }, + "v7": { + **equity_stats(exit_rets), + "pnl": sum(r["exit_pnl"] for r in results), + }, + "delta_pnl": sum(deltas), + "positive_delta_trades": sum(1 for d in deltas if d > 0), + "negative_delta_trades": sum(1 for d in deltas if d < 0), + "avg_exit_pressure": fmean([r["exit_pressure"] for r in results if r["exited"]]) if any(r["exited"] for r in results) else 0.0, + "reasons": dict(sorted({ + reason: sum(1 for r in results if r["exit_reason"] == reason) + for reason in {r["exit_reason"] for r in results} + }.items())), + } + + +def candidate_configs() -> list[tuple[str, AlphaExitV7Config]]: + out = [("short_default", AlphaExitV7Config())] + for threshold in [1.4, 1.7, 2.0, 2.35, 2.69, 3.0]: + out.append((f"exit_p{threshold}", AlphaExitV7Config(exit_pressure_threshold=threshold))) + for tier_scale in [0.5, 0.75, 1.0, 1.25, 1.5]: + out.append(( + f"mae_scale_{tier_scale}", + AlphaExitV7Config( + mae_tier1_k=3.5 * tier_scale, + mae_tier2_k=7.0 * tier_scale, + mae_tier3_k=12.0 * tier_scale, + mae_tier1_floor=0.005 * tier_scale, + mae_tier2_floor=0.012 * tier_scale, + mae_tier3_floor=0.025 * tier_scale, + ), + )) + for mfe_scale in [0.5, 0.75, 1.25, 1.5]: + out.append(( + f"mfe_risk_scale_{mfe_scale}", + AlphaExitV7Config( + mfe_convexity_exit_risk=1.5 * mfe_scale, + mfe_convexity_soft_risk=0.3 * mfe_scale, + mfe_accel_risk=0.2 * mfe_scale, + ), + )) + for late_start in [0.3, 0.45, 0.6, 0.75]: + out.append((f"late_start_{late_start}", AlphaExitV7Config(mae_late_start_frac=late_start))) + for threshold in [1.7, 2.0, 2.35]: + for mae_scale in [0.5, 0.75, 1.25]: + out.append(( + f"combo_p{threshold}_mae{mae_scale}", + AlphaExitV7Config( + exit_pressure_threshold=threshold, + mae_tier1_k=3.5 * mae_scale, + mae_tier2_k=7.0 * mae_scale, + mae_tier3_k=12.0 * mae_scale, + mae_tier1_floor=0.005 * mae_scale, + mae_tier2_floor=0.012 * mae_scale, + mae_tier3_floor=0.025 * mae_scale, + ), + )) + return out + + +def main() -> None: + parser = argparse.ArgumentParser() + parser.add_argument("--limit-trades", type=int, default=0) + parser.add_argument("--out", default="/tmp/v7_long_calibration.json") + args = parser.parse_args() + + rows = load_v7_rows(limit_trades=args.limit_trades) + paths = group_paths(rows) + summaries = [] + for name, cfg in candidate_configs(): + results = [replay_path(path, cfg) for path in paths] + summaries.append(summarize(results, cfg, name)) + summaries.sort(key=lambda r: (r["delta_pnl"], r["v7"]["pnl"], -r["exits"]), reverse=True) + payload = { + "method": "Synthetic LONG replay of BLUE SHORT V7 decision journal paths; bounce model disabled.", + "input": { + "rows": len(rows), + "paths": len(paths), + "limit_trades": args.limit_trades, + }, + "top_by_delta": summaries[:20], + "all": summaries, + } + Path(args.out).write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8") + print(json.dumps({ + "input": payload["input"], + "top_by_delta": [ + { + "name": s["name"], + "n": s["n"], + "exits": s["exits"], + "exit_rate": s["exit_rate"], + "natural_pnl": s["natural"]["pnl"], + "v7_pnl": s["v7"]["pnl"], + "delta_pnl": s["delta_pnl"], + "natural_compound": s["natural"]["compound"], + "v7_compound": s["v7"]["compound"], + "v7_dd": s["v7"]["max_dd"], + "reasons": s["reasons"], + } + for s in summaries[:12] + ], + }, indent=2, sort_keys=True)) + + +if __name__ == "__main__": + main() diff --git a/adaptive_exit/post_win_long_overlay.py b/adaptive_exit/post_win_long_overlay.py new file mode 100644 index 0000000..d0741fc --- /dev/null +++ b/adaptive_exit/post_win_long_overlay.py @@ -0,0 +1,351 @@ +"""Deterministic post-win LONG overlay EFSM. + +This module does not place orders. It tags future entries after realized BLUE +SHORT exhaustion wins so the live/shadow caller can decide whether to flip the +next one or more SHORT-engine opportunities to LONG. + +EFSM means Execution FSM. The EFSM is deliberately slot-based: + +- a trigger arms N future slots +- each future entry consumes exactly one slot +- when slots reach zero, state resets to SHORT +- flipped LONG trades do not re-arm the overlay +- triggers observed while an arm is active are ignored unless explicitly + enabled by config + +That prevents the bug class where a one/two-trade rebound probe becomes a +self-extending regime switch. +""" + +from __future__ import annotations + +from dataclasses import dataclass, field +from datetime import datetime, timezone +from typing import Any, Mapping, Optional, Sequence + + +def _to_float(value: Any, default: float = 0.0) -> float: + try: + out = float(value) + except (TypeError, ValueError): + return default + return out if out == out else default + + +def _to_utc(ts: datetime | None) -> datetime | None: + if ts is None: + return None + if ts.tzinfo is None: + return ts.replace(tzinfo=timezone.utc) + return ts.astimezone(timezone.utc) + + +@dataclass(frozen=True) +class PostWinFlipTrigger: + """A configurable trigger that arms future LONG flip slots.""" + + name: str + slots: int + min_pnl_abs: float = 0.0 + max_pnl_abs: Optional[float] = None + min_pnl_pct: Optional[float] = None + min_leverage: Optional[float] = None + strict_min_pnl_abs: bool = True + strict_max_pnl_abs: bool = True + strict_min_leverage: bool = True + + def matches(self, *, pnl: float, pnl_pct: float, leverage: float) -> bool: + if self.slots <= 0: + return False + if self.strict_min_pnl_abs: + if not pnl > self.min_pnl_abs: + return False + elif pnl < self.min_pnl_abs: + return False + if self.max_pnl_abs is not None: + if self.strict_max_pnl_abs: + if not pnl < self.max_pnl_abs: + return False + elif pnl > self.max_pnl_abs: + return False + if self.min_pnl_pct is not None and pnl_pct < self.min_pnl_pct: + return False + if self.min_leverage is not None: + if self.strict_min_leverage: + if not leverage > self.min_leverage: + return False + elif leverage < self.min_leverage: + return False + return True + + +@dataclass(frozen=True) +class PostWinExecutionFSMConfig: + """Configuration for the BLUE post-win Execution FSM.""" + + enabled: bool = True + rules: Sequence[PostWinFlipTrigger] = field( + default_factory=lambda: ( + # Order matters: the high-leverage big-win rule must win before the + # generic big-win rule, otherwise it would be capped at one slot. + PostWinFlipTrigger( + name="big_win_high_lev", + slots=2, + min_pnl_abs=397.0, + min_leverage=8.6, + strict_min_pnl_abs=True, + strict_min_leverage=True, + ), + PostWinFlipTrigger( + name="big_win", + slots=1, + min_pnl_abs=397.0, + strict_min_pnl_abs=True, + ), + PostWinFlipTrigger( + name="small_dollar_high_return", + slots=1, + min_pnl_abs=0.0, + max_pnl_abs=250.0, + min_pnl_pct=0.0075, + strict_min_pnl_abs=True, + strict_max_pnl_abs=True, + ), + ) + ) + max_arm_age_sec: Optional[float] = None + allow_rearm_while_armed: bool = False + allow_triggers_from_overlay_flips: bool = False + + +@dataclass(frozen=True) +class ActiveFlipArm: + """Currently armed future LONG flip slots.""" + + arm_id: int + trigger_name: str + slots_total: int + slots_remaining: int + trigger_trade_id: str = "" + trigger_asset: str = "" + trigger_ts: datetime | None = None + trigger_pnl: float = 0.0 + trigger_pnl_pct: float = 0.0 + trigger_leverage: float = 0.0 + + def with_remaining(self, slots_remaining: int) -> "ActiveFlipArm": + return ActiveFlipArm( + arm_id=self.arm_id, + trigger_name=self.trigger_name, + slots_total=self.slots_total, + slots_remaining=max(0, int(slots_remaining)), + trigger_trade_id=self.trigger_trade_id, + trigger_asset=self.trigger_asset, + trigger_ts=self.trigger_ts, + trigger_pnl=self.trigger_pnl, + trigger_pnl_pct=self.trigger_pnl_pct, + trigger_leverage=self.trigger_leverage, + ) + + def to_dict(self) -> dict[str, Any]: + return { + "arm_id": self.arm_id, + "trigger_name": self.trigger_name, + "slots_total": self.slots_total, + "slots_remaining": self.slots_remaining, + "trigger_trade_id": self.trigger_trade_id, + "trigger_asset": self.trigger_asset, + "trigger_ts": self.trigger_ts.isoformat() if self.trigger_ts else None, + "trigger_pnl": self.trigger_pnl, + "trigger_pnl_pct": self.trigger_pnl_pct, + "trigger_leverage": self.trigger_leverage, + } + + +@dataclass(frozen=True) +class OverlayDecision: + """Result returned by observe/entry tagging calls.""" + + action: str + side: str = "SHORT" + reason: str = "" + arm: ActiveFlipArm | None = None + consumed_slot: int = 0 + reset: bool = False + + def to_dict(self) -> dict[str, Any]: + return { + "action": self.action, + "side": self.side, + "reason": self.reason, + "arm": self.arm.to_dict() if self.arm else None, + "consumed_slot": self.consumed_slot, + "reset": self.reset, + } + + +class PostWinExecutionFSM: + """Multi-slot post-win LONG tag Execution FSM.""" + + def __init__(self, config: PostWinExecutionFSMConfig | None = None) -> None: + self.config = config or PostWinExecutionFSMConfig() + self._arm: ActiveFlipArm | None = None + self._next_arm_id = 1 + self.ignored_rearm_attempts = 0 + self.ignored_overlay_flip_triggers = 0 + self.expired_arms = 0 + self.consumed_arms = 0 + + @property + def active_arm(self) -> ActiveFlipArm | None: + return self._arm + + @property + def pending_slots(self) -> int: + return int(self._arm.slots_remaining) if self._arm else 0 + + def reset(self, reason: str = "manual") -> OverlayDecision: + old = self._arm + self._arm = None + return OverlayDecision(action="RESET", reason=reason, arm=old, reset=True) + + def observe_closed_trade( + self, + *, + trade_id: str = "", + asset: str = "", + side: str = "SHORT", + pnl: float = 0.0, + pnl_pct: float = 0.0, + leverage: float = 0.0, + closed_ts: datetime | None = None, + was_overlay_flip: bool = False, + metadata: Mapping[str, Any] | None = None, + ) -> OverlayDecision: + """Observe a completed trade and possibly arm future LONG slots. + + Parameters are intentionally primitive so this can be called from live + code, replay code, or ClickHouse/log readers. + """ + + del metadata # reserved for future feature logging without API churn + self._expire_if_needed(_to_utc(closed_ts)) + + if not self.config.enabled: + return OverlayDecision(action="NOOP", reason="disabled", arm=self._arm) + + side_u = str(side or "SHORT").upper() + if was_overlay_flip or side_u == "LONG": + self.ignored_overlay_flip_triggers += 1 + return OverlayDecision(action="IGNORED", reason="overlay_flip_outcome", arm=self._arm) + + pnl_f = _to_float(pnl) + pnl_pct_f = _to_float(pnl_pct) + lev_f = _to_float(leverage) + rule = self._match_rule(pnl=pnl_f, pnl_pct=pnl_pct_f, leverage=lev_f) + if rule is None: + return OverlayDecision(action="NO_TRIGGER", reason="no_rule_match", arm=self._arm) + + if self._arm is not None and not self.config.allow_rearm_while_armed: + self.ignored_rearm_attempts += 1 + return OverlayDecision(action="IGNORED", reason="active_arm_no_rearm", arm=self._arm) + + arm = ActiveFlipArm( + arm_id=self._next_arm_id, + trigger_name=rule.name, + slots_total=int(rule.slots), + slots_remaining=int(rule.slots), + trigger_trade_id=str(trade_id or ""), + trigger_asset=str(asset or ""), + trigger_ts=_to_utc(closed_ts), + trigger_pnl=pnl_f, + trigger_pnl_pct=pnl_pct_f, + trigger_leverage=lev_f, + ) + self._next_arm_id += 1 + self._arm = arm + return OverlayDecision(action="ARMED", reason=rule.name, arm=arm) + + def tag_next_entry( + self, + *, + asset: str = "", + entry_ts: datetime | None = None, + metadata: Mapping[str, Any] | None = None, + ) -> OverlayDecision: + """Return the side tag for the next engine entry and consume one slot.""" + + del asset, metadata # reserved for future asset-specific slot routing + self._expire_if_needed(_to_utc(entry_ts)) + if self._arm is None or self._arm.slots_remaining <= 0: + return OverlayDecision(action="PASS", side="SHORT", reason="no_active_arm") + + arm_before = self._arm + consumed_slot = arm_before.slots_total - arm_before.slots_remaining + 1 + remaining = arm_before.slots_remaining - 1 + if remaining <= 0: + self._arm = None + self.consumed_arms += 1 + return OverlayDecision( + action="TAG", + side="LONG", + reason=arm_before.trigger_name, + arm=arm_before.with_remaining(0), + consumed_slot=consumed_slot, + reset=True, + ) + + self._arm = arm_before.with_remaining(remaining) + return OverlayDecision( + action="TAG", + side="LONG", + reason=arm_before.trigger_name, + arm=self._arm, + consumed_slot=consumed_slot, + reset=False, + ) + + def snapshot(self) -> dict[str, Any]: + return { + "enabled": self.config.enabled, + "active_arm": self._arm.to_dict() if self._arm else None, + "pending_slots": self.pending_slots, + "ignored_rearm_attempts": self.ignored_rearm_attempts, + "ignored_overlay_flip_triggers": self.ignored_overlay_flip_triggers, + "expired_arms": self.expired_arms, + "consumed_arms": self.consumed_arms, + } + + def _match_rule(self, *, pnl: float, pnl_pct: float, leverage: float) -> PostWinFlipTrigger | None: + for rule in self.config.rules: + if rule.matches(pnl=pnl, pnl_pct=pnl_pct, leverage=leverage): + return rule + return None + + def _expire_if_needed(self, now: datetime | None) -> None: + if self._arm is None: + return + if self.config.max_arm_age_sec is None: + return + if now is None or self._arm.trigger_ts is None: + return + age = (now - self._arm.trigger_ts).total_seconds() + if age > float(self.config.max_arm_age_sec): + self._arm = None + self.expired_arms += 1 + + +# Compatibility aliases for earlier research scripts and tests. +PostWinLongOverlayConfig = PostWinExecutionFSMConfig +PostWinLongOverlay = PostWinExecutionFSM + + +__all__ = [ + "ActiveFlipArm", + "OverlayDecision", + "PostWinExecutionFSM", + "PostWinExecutionFSMConfig", + "PostWinFlipTrigger", + "PostWinLongOverlay", + "PostWinLongOverlayConfig", +] diff --git a/prod/docs/LONG_DETERMINISTIC_RULE_RESEARCH.md b/prod/docs/LONG_DETERMINISTIC_RULE_RESEARCH.md new file mode 100644 index 0000000..c4208fd --- /dev/null +++ b/prod/docs/LONG_DETERMINISTIC_RULE_RESEARCH.md @@ -0,0 +1,1305 @@ +# LONG Deterministic Rule Research + +Date: 2026-05-07 + +## Goal + +Find the simplest deterministic long-side market rule, using primarily Dolphin +NG eigendata, that behaves like the original short Alpha Engine rule in spirit: + +- few moving parts +- market-structural +- explainable in one breath +- reliable enough to serve as a basal gate before asset selection and later + overlays + +This note is explicitly **not** about a fitted long model. + +## Data source + +The analysis uses the raw daily scan cache summarized by: + +- `adaptive_exit/characterize_long_signals.py` +- `/mnt/dolphin_training/long_signal_research/long_signal_scan_summary_h24.parquet` +- `/mnt/dolphin_training/long_signal_research/long_signal_characterization_report.json` + +Only eigendata and scan-price-derived outcomes are used here: + +- `instability_50` +- `v50/v150/v300/v750_lambda_max_velocity` +- `vel_div` +- `vel_div` lag / delta terms + +No ExF, EsoF, or OBF are required for the core finding. + +## What does **not** work as the basal long rule + +The obvious mirror thesis, + +- `vel_div > 0.01` + +is too weak to be the basal long edge. + +Recent HQ slice (`2025-12-31` onward): + +- support: `39.65%` +- `strong_long` lift: `1.15x` +- `broad_long` lift: `1.22x` + +That is not useless, but it is not elegant enough nor selective enough to be +the long analogue of `vel_div < -0.02`. + +## Strongest deterministic shape + +The long side shows up most clearly as a **stressed unwind / squeeze** regime, +not as a generic bullish breakout regime. + +### Candidate primary deterministic rule + +```text +LONG_REGIME if + instability_50 >= 20.5 + and v300_lambda_max_velocity < 0 + and v750_lambda_max_velocity < 0 +``` + +Interpretation: + +- `instability_50 >= 20.5`: the market is structurally stressed +- `v300 < 0` and `v750 < 0`: the slower eigenspace is still negative / damaged +- together: this is a high-stress unwind state where long opportunities tend to + appear as reversals / squeezes on the same manifold that produces short + dislocations + +### Why `20.5` + +`20.5` is the rounded recent-HQ `instability_50` 90th-percentile threshold +(`20.546996...`). It is the most practical fixed threshold found in the +recent-era characterization. + +## Empirical support + +### Recent HQ (`2025-12-31` onward) + +Base rates: + +- `strong_long`: `0.1648` +- `broad_long`: `0.1367` + +Rule: + +- support: `6356` rows (`3.58%`) +- `strong_long`: `0.3409` (`2.07x` lift) +- `broad_long`: `0.3538` (`2.59x` lift) + +### Full history + +Base rates: + +- `strong_long`: `0.2603` +- `broad_long`: `0.2472` + +Rule: + +- support: `300,728` rows (`12.59%`) +- `strong_long`: `0.3330` (`1.28x` lift) +- `broad_long`: `0.3375` (`1.37x` lift) + +## Simpler fallback + +If maximum elegance is preferred over extra selectivity, the one-factor +fallback is: + +```text +LONG_REGIME_SIMPLE if instability_50 >= 20.5 +``` + +Recent HQ: + +- support: `10.10%` +- `strong_long`: `0.3297` (`2.00x` lift) +- `broad_long`: `0.3420` (`2.50x` lift) + +This is surprisingly strong for a one-variable rule. It is the closest thing +found to a pure long-side analogue of the short `vel_div < -0.02` gate. + +Tradeoff: + +- simpler +- broader +- slightly less selective than adding `v300 < 0` and `v750 < 0` + +## Optional stricter confirmation + +If later tuning wants more explicit “healing after stress” confirmation, the +strict variant is: + +```text +LONG_REGIME_STRICT if + instability_50 >= 20.5 + and vel_div_lag6 < -0.03 + and vel_div_delta6 > 0.02 +``` + +This is directionally sensible, but it is not materially better than the +`instability_50 + v300 + v750` rule, so it should be treated as an optional +refinement, not the basal rule. + +## Monthly sanity check + +For the candidate primary rule (`instability_50 >= 20.5 && v300 < 0 && v750 < 0`) +in the recent HQ window: + +- `2026-01`: `strong_long = 0.348` +- `2026-02`: `strong_long = 0.344` +- `2026-03`: `strong_long = 0.312` + +The monthly base rates for the same period were: + +- `2026-01`: `0.289` +- `2026-02`: `0.271` +- `2026-03`: `0.068` + +So even into the weak March tape, the rule remains elevated relative to base. + +## Practical interpretation + +This should be viewed as a **market-state gate**, not a complete trade engine. + +It says: + +- “the market is in the sort of stressed, damaged regime where long squeeze / + unwind opportunities become meaningfully more likely” + +It does **not** by itself say: + +- which asset is the best expression +- how to size +- how to exit + +That is where the next layers belong: + +- deterministic or learned asset selection +- OBF / ARS / bounce overlays +- TP / MAX_HOLD policy + +## Recommendation + +If a single deterministic long gate must be named now, use: + +```text +LONG_REGIME if instability_50 >= 20.5 and v300 < 0 and v750 < 0 +``` + +If maximum simplicity is the priority, use: + +```text +LONG_REGIME_SIMPLE if instability_50 >= 20.5 +``` + +And explicitly do **not** promote `vel_div > 0.01` as the basal long rule. + +## Deferred analysis idea: dual-shadow regime sampler + +This is a **later analysis / control-layer research note**, not a live-rule +recommendation. + +One plausible way to sample the market in real time without committing the full +system immediately is a very lightweight **dual-shadow engine**: + +- Shadow A: the basal SHORT engine (`vel_div < -0.02` Alpha Engine posture) +- Shadow B: the basal LONG engine (currently the older negative-`vel_div` + mean-reversion LONG posture is the best simple candidate) + +The intent is not merely paper PnL logging. It is to use live, recent +sample-trade outcomes as a **micro-regime probe**: + +- if SHORT shadow performance degrades while LONG shadow performance improves, + the tape may have rotated into a LONG-favorable regime +- if LONG degrades while SHORT improves, the inverse may be true +- if both are performing acceptably, the tape may be permissive / broad enough + that either side can express edge +- if both are failing, the tape is likely choppy / non-coherent and abstention + becomes a first-class candidate + +This should be implemented, if ever pursued, as: + +- very fast +- very lightweight +- explicitly shadow-only at first +- based on small, recent sample trades rather than a heavy fitted model + +Longer-term, the entire shadow stream can itself become training data: + +- market fingerprints at shadow-entry time +- concurrent SHORT-shadow and LONG-shadow outcomes +- relative WR / ROI-per-trade / drawdown / time-to-win asymmetries + +That would allow a later learner to predict or simplify the regime switcher. +But even before ML, the dual-shadow process may already serve as a useful +real-time market-sampling / regime-detection mechanism. + +## Dual-shadow persistence characterization + +This section records the first persistence pass over extant trades. The goal +was not to prove a full regime-switch system, but to test whether the observed +short-loss streaks are durable enough to justify a regime-favorableness probe. + +Important caveat: + +- the live SHORT series and the replay LONG series are on different date spans +- this is therefore a side-specific persistence study, not a same-bar paired + dominance study +- the numbers below are still useful for run-length and hysteresis design + +### Live SHORT stream + +From the current BLUE trader log: + +- trades: `234` +- win rate: `44.44%` +- mean `pnl_pct`: `+0.000506` +- median `pnl_pct`: `-0.000234` +- average win streak: `1.65 trades` +- average loss streak: `2.03 trades` +- `P(win -> win) = 0.394` +- `P(loss -> loss) = 0.512` +- average positive-day run: `1.5 days` +- average negative-day run: `1.5 days` + +Interpretation: + +- short failures do cluster +- the cluster is real enough to notice +- but it is only mildly persistent +- by itself, it is not strong enough to justify a raw ping-pong switch + +### Basal LONG shadow, old mirror posture + +Using the recent bullish-month replay and the single comparable `10-bar / +worst_10bar` configuration: + +- trades: `2,243` +- win rate: `48.33%` +- mean `pnl_pct`: `+0.000320` +- median `pnl_pct`: `-0.000400` +- average win streak: `1.93 trades` +- average loss streak: `2.07 trades` +- `P(win -> win) = 0.483` +- `P(loss -> loss) = 0.517` +- average positive-day run: `3.0 days` +- average negative-day run: `1.86 days` + +Interpretation: + +- this is the clearest durable long-favorable candidate seen so far +- the multi-day positive run length is materially better than the live short + stream +- this supports a long-favorable regime probe, but not an unconditional flip + +### Basal LONG shadow, new stressed-unwind posture + +Same replay setup: + +- trades: `569` +- win rate: `50.44%` +- mean `pnl_pct`: `-0.000078` +- median `pnl_pct`: `+0.000068` +- average win streak: `2.24 trades` +- average loss streak: `2.20 trades` +- `P(win -> win) = 0.556` +- `P(loss -> loss) = 0.546` +- average positive-day run: `1.36 days` +- average negative-day run: `1.18 days` + +Interpretation: + +- the new long posture has decent local persistence +- but it is more fragile than the mirror-long posture as a regime switch +- it does not yet justify itself as the primary flip trigger + +### Conclusion for regime switching + +The data support a **smoothed regime-favorableness detector**, not a raw +flip-on-first-loss system. + +Practical reading: + +- short-loss streak persistence is real but modest +- long-favorable states exist and can persist +- persistence is on the order of a few trades, not a dramatic regime lock +- the correct implementation is a shadow score with hysteresis and abstain + logic, not a hard immediate SHORT/LONG switch + +Suggested rule shape for later analysis: + +- compute rolling shadow scores for SHORT and LONG +- use persistence thresholds before flipping +- require stronger evidence to reverse than to stay put +- abstain when both shadows are weak or both are losing + +This is enough to justify the next engineering step: + +- live dual-shadow logging on the same bars +- market-fingerprint tagging of each shadow entry +- later ML over shadow outcomes if the deterministic layer proves stable + +## Rolling flip-worthiness test + +To make the side-switch question stricter, the recent live short slice was +retested with a `5-trade` rolling shadow-delta proxy: + +- short shadow return = actual live short `pnl_pct` +- long shadow return = counterfactual `-pnl_pct - fee` +- rolling delta = rolling mean of `(long_shadow - short_shadow)` + +Recent 3-day slice (`2026-05-04` to `2026-05-06`): + +- trades: `168` +- short actual WR: `39.88%` +- short actual compounded return: `+10.02%` +- long counterfactual WR: `47.62%` +- long counterfactual compounded return: `-16.92%` +- flip-to-long signals from the `5-trade` rolling delta: `68` +- flip-to-short signals from the `5-trade` rolling delta: `79` + +Interpretation: + +- the rolling delta does detect alternating regime pockets +- but it does so often enough that a raw flip would be too twitchy +- on the most recent 30 live trades, the regime buckets were: + - `13` long-favorable + - `7` short-favorable + - `10` neutral +- the long-favorable bucket had positive expected PnL, but the short-favorable + bucket was also positive and slightly stronger + +The important point is that the signal is not “switch now on first loss.” +It is: + +- keep a smoothed side-dominance score +- require persistence before flipping +- use hysteresis +- abstain when the shadow spread is weak or oscillatory + +So the stricter test reinforces the earlier conclusion: + +- there is enough structure to justify a regime-favorableness detector +- there is not yet enough stability to justify a raw mechanical flip +- the right next step is live dual-shadow logging on the same bars, then + threshold and persistence calibration on that shared stream + +## Flip-after-loss counterfactual + +The actual live short ledger was also replayed under a simple finite-state +side-switch rule: + +- start `SHORT` +- if the current side loses `N` trades in a row, flip to the other side +- keep applying the same rule across the whole trade sequence + +This is the cleanest way to test the idea “short losses are the long cue.” + +On the current `234`-trade live ledger: + +- always short: WR `44.44%`, compounded return `+11.35%`, max DD `5.71%` +- always long: WR `44.87%`, compounded return `-20.13%`, max DD `23.09%` + +Threshold sweep: + +- `N=1`: WR `40.60%`, compounded return `+5.33%`, max DD `11.11%`, flips `139` +- `N=2`: WR `44.44%`, compounded return `-17.72%`, max DD `17.77%`, flips `43` +- `N=3`: WR `48.29%`, compounded return `+5.48%`, max DD `6.35%`, flips `13` +- `N=4`: WR `47.86%`, compounded return `+6.21%`, max DD `6.55%`, flips `7` +- `N=5`: WR `43.59%`, compounded return `+10.52%`, max DD `5.59%`, flips `5` +- `N=6`: WR `45.73%`, compounded return `+15.17%`, max DD `4.84%`, flips `3` + +Interpretation: + +- side switching can help +- it helps best when the flip threshold is fairly high +- the best observed threshold in this small grid was `N=6` +- low thresholds are too twitchy and can destroy the edge + +So the practical conclusion is: + +- a raw flip-on-first-loss rule is not justified +- a slower loss-cluster regime switcher is plausible +- the switcher must be hysteretic and persistence-gated + +This is consistent with the earlier shadow-score recommendation and explains +why the observed “8 or 9 losses, then a couple wins” pattern can be useful +without being directly automatable at a low threshold. + +## Condition-gated flip replay + +I then reran the side-switch counterfactual with an additional gate: + +- the current side must first hit `N` consecutive losses +- the opposite side must also satisfy its own deterministic long/short entry condition +- the replay uses the same 10-bar tape skeleton and the worst-10-bar asset expression + +Two long theories were tested separately: + +- **Old mirror-long**: `vel_div < -0.02` and cross-sectional 10-bar momentum `< 0` +- **New stressed-unwind long**: `instability_50 >= 20.5` and `v300 < 0` and `v750 < 0` + +Results on the long research windows: + +- old mirror-long becomes marginally usable only at high thresholds: + - `N=5`: WR `47.00%`, compounded return `+6.34%`, DD `46.23%`, flips `11` + - `N=6`: WR `46.52%`, compounded return `+28.34%`, DD `43.78%`, flips `5` +- the new stressed-unwind long does **not** survive this gate cleanly: + - `N=1..6`: compounded return stays negative, with severe drawdown + +Interpretation: + +- the condition gate does not rescue the new long theory +- it does preserve the old mirror-long as a late, low-frequency fallback +- the market still looks too unstable for a low-threshold flip rule +- if we keep this path, it should be a smoothed regime sampler, not an immediate switcher + +Report: + +- [`flip_on_loss_condition_gate_report.md`]() + +## Full-history condition-gated replay + +I then ran the same condition-gated flip simulator across the entire +available price tape: + +- root: `/mnt/dolphin_training/share_offload/vbt_cache_klines` +- rows: `2,553,401` +- span: `2021-06-15 00:01:00+00:00 -> 2026-03-18 18:16:40.041456896+00:00` + +This is the hardest and most useful stress test because it removes the +recent-slice bias entirely. + +Results: + +- **old mirror-long** + - `N=1..6` win rate range: `44.95% -> 46.60%` + - best mean PnL at `N=6`: `-0.000163` per trade + - best threshold still compounds to `-100%` over the full archive +- **new stressed-unwind long** + - `N=1..6` win rate range: `44.16% -> 46.86%` + - best mean PnL at `N=6`: `-0.000218` per trade + - best threshold also compounds to `-100%` + +Interpretation: + +- the condition gate does not rescue either long theory at full-archive scale +- the old mirror-long is still the stronger of the two, but only marginally +- the long-side edge, if it exists, is too weak or too regime-dependent to + survive this archive-wide flip rule without additional filtering +- the full-tape result is a warning against over-trusting the favorable + recent-month slices + +Report: + +- [`flip_on_loss_condition_gate_stream_full_report.md`]() + +## Post-outlier-short-win long-flip probe + +Motivation: the May 8 live footer showed a familiar-looking pattern: + +- large 9x short win, e.g. `ALGOUSDT` `+$466` or `VETUSDT` `+$574` +- immediately followed by a somewhat larger-than-normal short loss, e.g. + `DASHUSDT -$191` or `STXUSDT -$54` + +The question was whether this is a real post-outlier rebound signature: + +```text +after a very large short win, +should the next trade, or next few trades, be treated as LONG candidates? +``` + +Dataset and hygiene: + +- source: BLUE only +- ClickHouse `dolphin.trade_events`: `1305` rows, `1296` unique trade IDs +- trader logs: `1712` exit rows, `1092` unique trade IDs +- merged near-duplicate-cleaned sequence: `1609` unique trade IDs +- analysis subset after excluding hibernate / subday ACB exits: `1321` trades +- span: `2026-03-31 01:10:34 UTC` to `2026-05-08 13:26:06 UTC` + +The log and warehouse streams overlap but do not have perfectly identical +timestamps, so the analysis de-duplicates by trade id where possible and by +near-time / asset / reason / realized PnL where the same exit was written by +both paths. This matters because a naive merge double-counts many recent exits. + +Counterfactual method: + +- keep the same entry/exit skeleton +- actual side is the live BLUE short +- counterfactual long return is approximated as `-short_return - 4 bps` +- this is not a separately selected long engine; it only tests whether the + immediate post-win tape direction would have favored the other side + +Baseline over the cleaned sequence: + +- always short: `1321` trades, WR `55.79%`, mean return/trade `+0.0781%`, + compounded return `+166.36%`, max DD `15.70%` +- always long on the same skeleton: WR `38.46%`, mean return/trade `-0.1181%`, + compounded return `-80.08%`, max DD `80.48%` + +So the full ledger does **not** support a broad long flip. The question only +survives as a narrow post-outlier condition. + +Primary post-outlier trigger: + +```text +trigger if prior trade: + pnl_abs >= $400 + leverage >= 8.5x + pnl_pct >= +0.50% +``` + +Immediate next-trade result: + +- triggers: `47` +- next trades affected: `47` +- actual next short subset: WR `53.19%`, mean return `-0.0821%`, + compounded return `-4.05%`, realized PnL `-$1,725.40` +- flipped-to-long subset: WR `40.43%`, mean return `+0.0421%`, + compounded return `+1.72%`, estimated PnL `+$409.47` +- estimated dollar delta: `+$2,134.88` +- whole-sequence policy if only those next trades are flipped: + compounded return improves from `+166.36%` to `+182.38%` + and max DD improves from `15.70%` to `13.33%` + +The stricter trigger `pnl_abs >= $400`, `leverage >= 8.5x`, +`pnl_pct >= +0.95%` is similar: + +- triggers: `46` +- actual next short subset: `-$1,534.21` +- flipped-to-long estimate: `+$276.64` +- estimated dollar delta: `+$1,810.85` +- whole-sequence compounded return: `+180.91%` + +The effect is strongest on the immediately following trade. It decays quickly: + +- next `2` trades after the primary trigger: affected `91`, actual `-$2,689.16`, + flipped estimate `+$555.98`, dollar delta `+$3,245.15` +- next `3` trades: affected `134`, actual `-$2,357.77`, flipped estimate + `-$588.02`, dollar delta still positive because the flip loses less +- next `5` trades: benefit becomes materially less clean + +Examples from the live tail: + +- `ALGOUSDT` `2026-05-08 09:55 UTC`, `+466.34`, `9x`, `+0.929%` + - next trade `DASHUSDT`: actual short `-191.19`; same-skeleton long would + have been directionally positive after fee +- `VETUSDT` `2026-05-08 12:37 UTC`, `+573.64`, `9x`, `+1.546%` + - next trade `STXUSDT`: actual short `-53.52`; same-skeleton long would + have been directionally positive after fee +- larger historic outlier `STXUSDT` `2026-05-05 20:29 UTC`, `+6796.86`, + `9x`, `+13.845%` + - the following trade was a small short loss, and the next several trades + were mixed rather than uniformly long-favorable + +Interpretation: + +- there is a real event-conditioned post-outlier rebound / exhaustion signal +- it is not a win-rate improvement; it is a dollar / drawdown improvement +- it should not be promoted as a general long engine +- it is best framed as a one-trade post-outlier **long probe** or short + cooldown candidate, not as a multi-trade regime flip + +Relationship to the long-system research: + +- this is different from both deterministic long theories already studied: + - old mirror-long: negative `vel_div` mean-reversion long + - new stressed-unwind long: high instability plus negative slow velocities +- the post-outlier signal is more local and path-conditioned: + - a violent short win likely means the chosen asset or local basket has + just completed an exhaustion leg + - the next trade may be more exposed to rebound / adverse short continuation + than to fresh downside continuation +- this should become a feature inside the dual-shadow side-selection sampler: + - `last_trade_was_outlier_short_win` + - `last_trade_leverage` + - `last_trade_realized_pnl_abs` + - `last_trade_return_pct` + - `bars_since_outlier_win` + - `same_asset_or_correlated_asset_followup` + +Research conclusion: + +- broad `SHORT -> LONG` inversion remains false on the full sequence +- immediate one-trade long probing after a large 9x short win is empirically + plausible and improved historical BLUE dollars in this cleaned replay +- the next test should condition this event trigger on the existing long gates + and market fingerprint state, rather than using it as a naked side switch + +## Leverage-as-conviction win-probe sweep + +Follow-up thesis: + +```text +leverage is a conviction expression + +if a high-conviction short probe wins: + make subsequent / next trades LONG + +if leverage is below roughly 0.69: + possibly do not trade +``` + +The initial test used: + +```text +trigger_lev = 0.70 +trade_min_lev = 0.69 +win = net PnL > 0 +``` + +Two side-selection forms were tested: + +- **persistent shadow probe**: the short engine continues to run as a shadow. + A high-lev short-shadow win turns the traded side LONG. A high-lev + short-shadow loss resets the traded side SHORT. +- **one-shot after win**: a high-lev short-shadow win arms only the next + eligible trade as LONG, then resets. + +The test used the same cleaned BLUE sequence as the post-outlier study, updated +through `2026-05-08 13:40:04 UTC`: + +- ClickHouse rows: `1307` +- ClickHouse unique trade IDs: `1298` +- trader-log exit rows: `1716` +- merged near-duplicate-cleaned trade IDs: `1612` +- analysis subset after excluding hibernate / subday ACB exits: `1324` + +Baselines: + +- always short: `1324` trades, WR `55.82%`, mean return/trade `+0.0784%`, + compounded return `+168.02%`, max DD `15.70%`, PnL `+$11,135.86` +- always long on the same skeleton: WR `38.44%`, compounded return `-80.23%`, + max DD `80.62%`, PnL `-$36,875.48` +- short-only with `trade_min_lev >= 0.69`: `1050` trades, compounded return + `+81.86%`, max DD `20.80%`, PnL `+$11,063.86` +- short-only with `trade_min_lev >= 5.0`: `565` trades, compounded return + `+88.08%`, max DD `8.94%`, PnL `+$11,980.01` +- short-only with `trade_min_lev >= 8.5`: `501` trades, compounded return + `+82.57%`, max DD `7.58%`, PnL `+$12,193.65` + +Initial `0.70 / 0.69` thesis result: + +- persistent shadow-probe switch: + - traded: `1050` + - LONG trades: `457` + - flips to LONG: `249` + - WR `37.08%` + - compounded return `-5.61%` + - max DD `26.60%` + - PnL `-$2,527.65` +- one-shot after high-lev win: + - traded: `1050` + - LONG trades: `455` + - flips to LONG: `456` + - WR `37.24%` + - compounded return `-3.56%` + - max DD `26.19%` + - PnL `-$2,113.83` + +So the literal initial thesis fails. `0.70` is too low as a +side-switch trigger. It arms hundreds of LONG trades and turns a strong +short-led ledger into a slightly losing one. + +Important evaluation frame: + +The goal is **not** to find a LONG overlay that beats the whole short-only +engine by itself. The goal is to find a side-selection overlay that adds +marginal value only on the subset where it intervenes. The correct comparison +is therefore: + +```text +overlay_delta = + pnl_if_intervened_long_on_triggered_trades + - pnl_if_original_short_was_left_unchanged_on_same_triggered_trades +``` + +The overlay is useful only if it satisfies all of the following: + +- it has positive `overlay_delta` after fees and conservative slippage +- it reduces realized drawdown or loss clustering on the intervention subset +- it does not cut too many profitable short trades +- it remains positive across time splits, assets, and neighboring thresholds +- it has enough triggers to be statistically more than a single accident + +Under that marginal-overlay framing, the broad leverage-win thesis still fails: + +- persistent `0.70 / 0.69` switch delta vs same `lev >= 0.69` short-only + baseline: about `-$13,591.51` +- one-shot `0.70 / 0.69` switch delta vs same `lev >= 0.69` short-only + baseline: about `-$13,177.69` +- best swept dollar switch delta vs same `lev >= 0.69` short-only baseline: + about `-$5,949.36` + +By contrast, the narrower post-outlier rule did show positive marginal overlay +value on its triggered subset: + +- triggered next-trade cases: `47` +- leaving the next trade SHORT: PnL `-$1,725.40` +- flipping only that next trade LONG: PnL `+$409.47` +- marginal overlay delta: `+$2,134.87` +- whole-sequence drawdown improved from about `15.70%` to `13.33%` + +That is the key distinction. The broad high-leverage-win rule is not reliable +enough. The narrow post-outlier rule is a legitimate candidate for guarded +shadow/live-probe research because it adds value exactly where it intervenes, +but the sample is still too small for unconditional deployment. + +### Lowered big-win threshold grid + +The phrase "sample too small" applies only to the original high-tail trigger +(`pnl_abs >= $400`, `lev >= 8.5`, immediate next trade). It does **not** mean +the BLUE ledger is small. The cleaned replay now spans: + +- `1328` non-hibernate / non-subday-ACB BLUE trades +- `1616` merged near-duplicate-cleaned trade IDs +- `2026-03-31 01:10:34 UTC` through `2026-05-08 14:21:31 UTC` + +To test whether the effect survives with more triggers, the post-win sweep was +expanded to: + +- dollar win thresholds: `$10`, `$25`, `$50`, `$75`, `$100`, `$150`, `$200`, + `$300`, `$400`, `$500`, `$750`, `$1000` +- leverage thresholds: `0`, `0.69`, `0.70`, `1`, `2`, `3`, `5`, `8.5`, `9` +- return thresholds: `0`, `0.10%`, `0.25%`, `0.50%`, `0.75%`, `0.95%`, + `1.25%` +- follow-on horizons: next `1`, `2`, `3`, and `5` trades + +Important result: + +- lowering **dollar threshold alone** does not work +- lowering dollar threshold **with a realized-return threshold** does work +- the effect is mostly next `1` to `2` trades +- by next `5` trades, flipping LONG is not positive; cooldown / abstain is + better than LONG if the horizon is that wide + +Grid-wide stability: + +- horizon `1`: `630` eligible threshold combinations, `60.0%` positive + marginal delta, `45.87%` positive LONG PnL +- horizon `2`: `630` eligible threshold combinations, `57.30%` positive + marginal delta, `39.52%` positive LONG PnL +- horizon `3`: `693` eligible threshold combinations, `59.60%` positive + marginal delta, `12.99%` positive LONG PnL +- horizon `5`: `693` eligible threshold combinations, `51.08%` positive + marginal delta, `0.0%` positive LONG PnL + +This says the post-win effect is a short-lived exhaustion / rebound artifact, +not a durable multi-trade LONG regime. + +Fixed dollar-only immediate-next-trade rows: + +| Trigger | Affected next trades | Leave SHORT | Flip LONG | Delta | Whole-policy compound | DD | +|---|---:|---:|---:|---:|---:|---:| +| `$10+`, no lev gate | 277 | `+$3,044` | `-$9,146` | `-$12,190` | `+24.74%` | `24.55%` | +| `$50+`, no lev gate | 181 | `+$4,495` | `-$8,870` | `-$13,365` | `+42.58%` | `22.18%` | +| `$100+`, no lev gate | 135 | `+$908` | `-$4,252` | `-$5,160` | `+97.78%` | `18.09%` | +| `$200+`, no lev gate | 89 | `-$947` | `-$1,496` | `-$549` | `+140.76%` | `14.96%` | +| `$300+`, no lev gate | 62 | `-$1,695` | `-$45` | `+$1,651` | `+174.25%` | `13.70%` | +| `$400+`, no lev gate | 48 | `-$1,725` | `+$407` | `+$2,133` | `+180.70%` | `13.33%` | +| `$500+`, no lev gate | 40 | `-$1,153` | `+$90` | `+$1,242` | `+173.51%` | `13.33%` | + +Dollar-only conclusion: + +- below about `$300`, the next short trade is still net-profitable or less bad + than the LONG flip +- around `$300`, the next short trade turns bad, but LONG is only near-flat +- around `$400` to `$500`, the next-trade LONG flip becomes positive + +Fixed immediate-next-trade rows with a `+0.75%` realized-return trigger: + +| Trigger | Affected next trades | Leave SHORT | Flip LONG | Delta | Whole-policy compound | DD | +|---|---:|---:|---:|---:|---:|---:| +| `$10+` and `+0.75%` | 99 | `-$1,735` | `-$409` | `+$1,326` | `+104.45%` | `14.03%` | +| `$50+` and `+0.75%` | 74 | `-$1,950` | `+$105` | `+$2,055` | `+155.62%` | `14.03%` | +| `$75+` and `+0.75%` | 70 | `-$2,028` | `+$194` | `+$2,223` | `+166.91%` | `13.95%` | +| `$100+` and `+0.75%` | 67 | `-$2,083` | `+$336` | `+$2,419` | `+168.60%` | `13.69%` | +| `$150+` and `+0.75%` | 63 | `-$2,082` | `+$344` | `+$2,426` | `+175.37%` | `13.69%` | +| `$300+` and `+0.75%` | 58 | `-$1,738` | `+$58` | `+$1,796` | `+173.61%` | `13.70%` | +| `$400+` and `+0.75%` | 48 | `-$1,725` | `+$407` | `+$2,133` | `+180.70%` | `13.33%` | + +Return-conditioned conclusion: + +- the effect becomes visible with more triggers when the dollar threshold is + lowered to `$50-$150` **and** the prior win is also at least `+0.75%` +- the best immediate-next-trade delta in this grid was around `$150+` and + `+0.75%`: `63` next trades, SHORT `-$2,081.81`, LONG `+$343.94`, delta + `+$2,425.75` +- the original `$400+`, high-leverage trigger remains good but is not the only + viable threshold; it is the cleaner high-tail version + +Two-trade horizon: + +| Trigger | Affected next trades | Leave SHORT | Flip LONG | Delta | Whole-policy compound | DD | +|---|---:|---:|---:|---:|---:|---:| +| `$300+`, `lev >= 8.5` | 115 | `-$3,201` | `+$511` | `+$3,712` | `+168.52%` | `14.27%` | +| `$400+`, `lev >= 8.5` | 91 | `-$2,689` | `+$556` | `+$3,245` | `+175.26%` | `13.71%` | +| `$500+`, `lev >= 8.5` | 75 | `-$2,237` | `+$509` | `+$2,747` | `+167.53%` | `14.71%` | + +Two-trade conclusion: + +- the high-leverage `$300-$500` zone supports a two-trade exhaustion rebound + more strongly than the original one-trade-only statement +- the best two-trade variant in this fixed grid was `$300+`, `lev >= 8.5`, + next two trades: delta `+$3,712`, estimated LONG PnL `+$511` +- the five-trade horizon should not be traded LONG; it is only a damage-control + / cooldown signal + +Reliability statement: + +The post-win overlay is more solid than initially stated. The robust form is +not "after any win"; that is false. The robust form is: + +```text +after a sufficiently large realized short win, +especially a high-return or high-leverage win, +the next 1-2 short-engine opportunities are often contaminated by rebound risk +and can be improved by LONG flip or, at minimum, cooldown/abstain. +``` + +The strongest candidates for shadow/live-probe research are: + +- immediate next trade after `$100-$200` win **and** prior return `>= +0.75%` +- immediate next trade after `$400+` win, especially `lev >= 8.5` +- next two trades after `$300-$500` win with `lev >= 8.5` + +Guardrail: + +The overlay should not optimize on WR. LONG WR remains lower than SHORT WR on +many triggered subsets. The edge is payoff asymmetry / loss-tail avoidance: +short wins become smaller or disappear after the exhaustion event, while short +losses on the next trade(s) become expensive. + +### Candidate codified overlay rule and EFSM + +Terminology: + +- **EFSM** means **Execution FSM** +- refer to this component as the post-win **EFSM**, not merely a generic + "state machine" + +Candidate rule proposed after the lowered-threshold sweep: + +```text +after a completed BLUE SHORT trade: + + if pnl_abs > $397: + tag next 1 trade as FLIP_LONG + + if pnl_abs > $397 and leverage > 8.6: + tag next 2 trades as FLIP_LONG + + if 0 < pnl_abs < $250 and pnl_pct >= +0.75%: + tag next 1 trade as FLIP_LONG + + after the armed slots are consumed: + reset to SHORT +``` + +EFSM semantics: + +- this is a **slot-based Execution FSM**, not a persistent regime switch +- each trigger arms an explicit number of future slots +- each future entry consumes exactly one slot +- when `slots_remaining == 0`, the state resets to SHORT +- while slots are active, new triggers are ignored by default +- a flipped LONG trade outcome is not allowed to re-arm the overlay +- this prevents the reset bug where one flipped trade recursively arms the next + and converts a bounded rebound probe into an unbounded side switch +- the implementation supports arbitrary future slot counts, not only `1` and + `2` + +Implementation location: + +- EFSM: `adaptive_exit/post_win_long_overlay.py` +- canonical class names: `PostWinExecutionFSM`, + `PostWinExecutionFSMConfig` +- compatibility aliases: `PostWinLongOverlay`, `PostWinLongOverlayConfig` +- tests: `prod/tests/test_post_win_long_overlay.py` + +Focused test coverage: + +- `$397+` non-high-leverage win arms one slot +- `$397+` and `lev > 8.6` arms two slots +- `< $250` and `pnl_pct >= +0.75%` arms one slot +- active arms consume deterministically and reset to SHORT +- re-arm attempts while active are ignored +- flipped LONG outcomes cannot re-arm +- optional TTL expiry works +- future `3+` slot rules work + +Focused verification: + +```text +python -m pytest -o cache_dir=/tmp/pytest-cache-post-win-overlay \ + prod/tests/test_post_win_long_overlay.py -q + +7 passed +``` + +Exact candidate replay, no re-arm during active flip slots: + +- input: `1333` cleaned BLUE trades through `2026-05-08 14:34:57 UTC` +- baseline short-only estimated PnL: `+$10,953.50` +- candidate policy estimated PnL: `+$12,464.30` +- marginal dollar delta: `+$1,510.80` +- baseline max DD: `15.70%` +- candidate max DD: `14.78%` +- long-flipped trades: `160` +- affected subset left SHORT: `-$2,415.46` +- affected subset flipped LONG: `-$904.67` +- affected subset marginal delta: `+$1,510.80` +- triggers armed: + - `small_dollar_high_return`: `77` + - `big_win_high_lev`: `41` + - `big_win`: `1` +- slots consumed: + - `small_dollar_high_return`: `77` + - `big_win_high_lev`: `82` + - `big_win`: `1` +- consumed arms: `119` +- dangling slots at end: `0` +- ignored re-arm attempts while active: `20` + +Reset sensitivity: + +Allowing active flipped trades / active arms to re-arm is harmful: + +- unsafe recursive re-arm variant long flips: `183` +- unsafe marginal delta: `-$5,425.32` +- safe no-rearm marginal delta: `+$1,510.80` + +Therefore the no-recursive-rearm reset invariant is not optional. It is part of +the edge definition. + +Compound-return caveat: + +- baseline short-only compound: `+164.89%` +- candidate compound: `+107.26%` + +This is why the overlay must be treated as a dollar-tail / drawdown-control +overlay first, not as a compounding optimizer. The current counterfactual uses +same entry/exit skeleton and estimated flipped LONG PnL, so the next validation +step must include actual LONG execution assumptions, long-side V7 behavior, and +time-to-next-entry gating. + +Time dependency: + +The replay showed material timing dependence: + +| Delay from trigger to flipped entry | n | SHORT PnL | LONG PnL | Delta | +|---|---:|---:|---:|---:| +| `<=15m` | 19 | `+$2,765.51` | `-$3,062.37` | `-$5,827.88` | +| `15-30m` | 67 | `-$3,588.76` | `+$2,381.96` | `+$5,970.72` | +| `30-60m` | 40 | `-$882.57` | `-$104.33` | `+$778.24` | +| `>60m` | 34 | `-$709.64` | `-$119.93` | `+$589.72` | + +This means the overlay may need a lower-bound delay, an upper-bound TTL, or +market-state confirmation. The current EFSM already supports TTL; +the exact timing gate remains research, not deployed doctrine. + +AdvancedExitManagerV7 / AlphaExitEngineV7 caveat: + +`AlphaExitEngineV7` is mechanically side-aware: + +- `side=0` means LONG +- `side=1` means SHORT +- PnL, MFE, MAE, trend direction, and adverse/favorable movement are signed by + `ctx.side` + +However, V7 calibration is SHORT-lineage: + +- bounce model labels were trained on BLUE SHORT adverse-bar samples +- pressure threshold `2.69` was selected on SHORT/GREEN-lineage replay +- MAE/MFE concepts are symmetric in code but not guaranteed symmetric in + fitted thresholds or bounce probabilities + +Before any live FLIP_LONG execution, V7 must be validated in one of these modes: + +- shadow-only LONG contexts using actual flipped LONG entries +- conservative LONG-specific V7 threshold override +- disable V7 live exits for overlay LONGs until enough shadow decisions show + it does not prematurely cut the rebound edge + +The rule can be codified, but production wiring must keep the EFSM, side +selection, and V7 exit policy explicitly separable. + +Sweep results: + +- best by compounded return: + - mode: one-shot after win + - `trigger_lev = 9.0` + - `trade_min_lev = 0.0` + - traded: `1324` + - LONG trades: `222` + - WR `50.91%` + - compounded return `+61.93%` + - max DD `19.36%` + - PnL `-$257.03` +- best by estimated dollars: + - mode: one-shot after win + - `trigger_lev = 2.0` + - `trade_min_lev = 0.69` + - traded: `1050` + - LONG trades: `297` + - WR `40.03%` + - compounded return `+27.71%` + - max DD `22.44%` + - PnL `+$5,114.50` + +Both sweep optima still underperform the relevant short-only baselines. In +particular, simply treating high leverage as a short-side quality filter is +stronger than using high-leverage short wins as a broad long-switch trigger: + +- `lev >= 8.5`, short-only: PnL `+$12,193.65`, max DD `7.58%` +- best long-switch dollar policy: PnL `+$5,114.50`, max DD `22.44%` + +Interpretation: + +- leverage does behave like conviction, but the first-order use is filtering / + sizing, not side inversion +- ordinary high-lev wins are too common to serve as a LONG regime switch +- the previous post-outlier result survives only because it was much narrower: + large dollar win, 9x, and immediate next trade +- high-lev wins may still be useful as **features** in the dual-shadow / + market-fingerprint layer: + - `last_high_lev_short_win` + - `last_high_lev_short_win_count` + - `last_high_lev_short_win_pnl_abs` + - `last_high_lev_short_win_return_pct` + - `bars_since_high_lev_short_win` + - `consecutive_high_lev_short_wins` + +Research conclusion: + +- do not implement the literal `lev > 0.70` long switch +- do preserve leverage as a strong conviction feature +- do keep the narrower post-outlier one-trade long probe in the research queue +- the strongest immediate operational lesson is that low-leverage trades may be + unnecessary, while high-leverage shorts remain the cleaner expression + +## AlphaExitEngineV7 LONG calibration replay + +Date: `2026-05-08` + +Scope: + +- system: BLUE only +- exit engine: `AlphaExitEngineV7` +- harness: `adaptive_exit/calibrate_v7_long_from_journal.py` +- source data: ClickHouse `dolphin.v7_decision_events` +- source rows: `6,812` +- reconstructed BLUE V7-tracked paths: `97` +- path side in source journal: SHORT +- replay side for calibration: synthetic LONG (`side=0`) +- fee assumption: `4 bps` +- natural exit comparator: final logged decision-row price for the same path +- V7 exit comparator: first replayed V7 `EXIT` on the same price path +- bounce model: disabled for this replay by intentionally using a missing model + path, because the current bounce model is trained on BLUE SHORT adverse-bar + samples and should not be treated as a validated LONG probability model + +This is a LONG-exit calibration proxy, not proof from exchange-filled LONG +trades. It answers a narrower question: if the post-win EFSM had flipped a +trade LONG on price paths that BLUE V7 actually observed, would a LONG-side V7 +cut/exit surface have improved or harmed the synthetic LONG outcome versus +holding to the path's natural end? + +### Original V7 SHORT calibration pattern + +The original V7 calibration was a pressure-threshold sweep over live shadow +decisions. V7 computes: + +```text +exit_pressure = clamp(directional_term + risk_term, -3.0, +3.0) +``` + +Then: + +```text +if exit_pressure > 2.69: + EXIT +elif exit_pressure > 1.0: + RETRACT +elif exit_pressure < -0.5 and pnl_pct > 0: + EXTEND +else: + HOLD +``` + +The documented SHORT lineage was: + +| Pressure threshold | Fires | Result | +|---:|---:|---:| +| `2.00` | `22/24` | `+$439`, ROI `+1.67%` | +| `2.35` | `17/24` | `+$891`, ROI `+3.38%` | +| `2.60` | `17/24` | `+$891`, ROI `+3.38%` | +| `3.00` | `14/24` | `+$796`, ROI `+3.02%` | +| base/no V7 | n/a | `+$784`, ROI `+2.98%` | + +The deployed threshold `2.69` was chosen as the high end of the useful +`2.35-2.70` band so V7 stayed closer to base behavior and avoided cutting +winners on transient pressure. + +### Threshold surface now explicit + +`AlphaExitEngineV7` now accepts an optional per-engine +`AlphaExitV7Config`. Defaults preserve the deployed SHORT-calibrated behavior. +This lets BLUE instantiate separate SHORT and LONG V7 engines later without +global mutation. + +V7-specific configurable fields: + +| Config field | Default | Meaning | +|---|---:|---| +| `rvol_w15` | `0.50` | realized-vol composite weight for 15-bar volatility | +| `rvol_w30` | `0.30` | realized-vol composite weight for 30-bar volatility | +| `rvol_w50` | `0.20` | realized-vol composite weight for 50-bar volatility | +| `rvol_floor` | `0.000001` | minimum realized-vol denominator | +| `mae_tier1_k` | `3.5` | MAE tier-1 multiplier on `rv_comp` | +| `mae_tier2_k` | `7.0` | MAE tier-2 multiplier on `rv_comp` | +| `mae_tier3_k` | `12.0` | MAE tier-3 multiplier on `rv_comp` | +| `mae_tier1_floor` | `0.005` | MAE tier-1 absolute floor | +| `mae_tier2_floor` | `0.012` | MAE tier-2 absolute floor | +| `mae_tier3_floor` | `0.025` | MAE tier-3 absolute floor | +| `mae_tier1_risk` | `0.5` | pressure contribution once tier 1 is breached | +| `mae_tier2_risk` | `0.8` | pressure contribution once tier 2 is breached | +| `mae_tier3_risk` | `1.2` | pressure contribution once tier 3 is breached | +| `mae_accel_min_bars` | `3` | minimum bars before adverse-acceleration gate can fire | +| `mae_accel_peak_floor` | `0.003` | adverse peak floor for MAE acceleration risk | +| `mae_accel_risk` | `0.6` | pressure contribution for MAE acceleration | +| `mae_recovery_peak_floor` | `0.004` | adverse peak floor for failed-recovery gate | +| `mae_recovery_prev_min` | `0.25` | prior recovery ratio required before snapback risk | +| `mae_recovery_snapback_max` | `0.10` | recovery ratio below which recovery is treated as failed | +| `mae_recovery_risk` | `1.0` | pressure contribution for failed recovery | +| `mae_late_floor` | `0.003` | MAE required before late adverse ramp applies | +| `mae_late_start_frac` | `0.60` | bars-held fraction where late adverse ramp starts | +| `mae_late_risk_max` | `0.4` | maximum late adverse pressure contribution | +| `max_hold_ref_mult_3m` | `3.0` | V7 internal max-hold reference multiplier | +| `mfe_slope_peak_floor` | `0.01` | peak favorable floor for convexity slope break | +| `mfe_convexity_decay_exit` | `0.35` | decay ratio for hard MFE giveback pressure | +| `mfe_convexity_decay_soft` | `0.20` | decay ratio for soft MFE giveback pressure | +| `mfe_convexity_exit_risk` | `1.5` | pressure contribution for hard MFE giveback | +| `mfe_convexity_soft_risk` | `0.3` | pressure contribution for soft MFE giveback | +| `mfe_accel_floor` | `-0.00001` | MFE acceleration floor for adverse convexity | +| `mfe_accel_peak_floor` | `0.005` | peak favorable floor for MFE acceleration risk | +| `mfe_accel_risk` | `0.2` | pressure contribution for MFE acceleration risk | +| `bounce_dir_w` | `0.15` | bounce score directional-term weight | +| `bounce_risk_w` | `0.35` | bounce risk-term weight | +| `bounce_rv_safe_floor` | `0.00001` | bounce feature volatility denominator floor | +| `exit_pressure_threshold` | `2.69` | live `EXIT` threshold | +| `retract_pressure_threshold` | `1.0` | `RETRACT` threshold | +| `extend_pressure_threshold` | `-0.5` | profitable `EXTEND` threshold | +| `pressure_min` | `-3.0` | pressure clamp lower bound | +| `pressure_max` | `3.0` | pressure clamp upper bound | + +Inherited V6 weight priors remain configurable through the existing +`WeightAdapter`/`WeightPriors` seam. The new config is specifically for V7 +threshold/gate surfaces and is init-time/per-engine configurable. + +### LONG replay results + +Baseline synthetic LONG natural exit across the 97 paths: + +- natural PnL: `-$328.84` +- natural WR: `59.79%` +- natural compound: `+3.50%` +- natural max DD: `2.28%` + +The dollar PnL and compound can diverge because path notionals differ. For this +exit calibration, dollar PnL is the more relevant metric because BLUE sizing is +not uniform. + +Top tested surfaces: + +| Candidate | V7 PnL | Delta vs natural | Exits | Exit rate | V7 WR | V7 max DD | +|---|---:|---:|---:|---:|---:|---:| +| `mfe_risk_scale_0.5` | `+$205.32` | `+$534.15` | `36` | `37.11%` | `50.52%` | `1.69%` | +| `mfe_risk_scale_0.75` | `+$205.32` | `+$534.15` | `36` | `37.11%` | `50.52%` | `1.69%` | +| `combo_p1.7_mae0.75` | `+$47.24` | `+$376.08` | `51` | `52.58%` | `47.42%` | `1.55%` | +| `exit_p1.7` | `+$36.88` | `+$365.72` | `51` | `52.58%` | `47.42%` | `1.53%` | +| `exit_p2.0` | `+$19.68` | `+$348.52` | `41` | `42.27%` | `49.48%` | `1.53%` | +| `short_default` / `exit_p2.69` | `+$1.43` | `+$330.26` | `38` | `39.18%` | `49.48%` | `1.81%` | +| `exit_p3.0` | `-$328.84` | `$0.00` | `0` | `0.00%` | `59.79%` | `2.28%` | + +Interpretation: + +- The deployed SHORT default is not mechanically broken for LONG. It improved + synthetic LONG dollar outcome by `+$330.26` versus natural exit on the 97 + replayed paths. +- The best tested LONG proxy did not come from lowering the pressure threshold. + It came from reducing MFE giveback/convexity pressure contribution + (`mfe_risk_scale_0.5` or `0.75`). +- Aggressively lowering `exit_pressure_threshold` to `1.4` over-fires: + `78/97` exits, V7 PnL `-$11.78`, and many negative deltas. That resembles the + original SHORT calibration failure at `2.0`: pressure that is too sensitive + cuts too much transient noise. +- A moderate pressure threshold around `1.7-2.0` is useful, but still inferior + to leaving pressure at `2.69` and reducing MFE-risk contributions in this + proxy. + +Recommended LONG overlay calibration candidate for shadow: + +```python +AlphaExitV7Config( + mfe_convexity_exit_risk=0.75, + mfe_convexity_soft_risk=0.15, + mfe_accel_risk=0.10, +) +``` + +This is the `mfe_risk_scale_0.5` surface. It keeps: + +- `exit_pressure_threshold = 2.69` +- all MAE vol-normalized loss-cut thresholds unchanged +- pressure clamp unchanged +- bounce disabled or neutral until a LONG-trained bounce model exists + +Why this candidate is preferable to simply lowering `exit_pressure_threshold`: + +- it preserved the useful loss-cut behavior while avoiding broad pressure + over-firing +- it improved dollar PnL more than all pressure-threshold sweeps tested +- it left MAE protection intact, which matters if the flipped LONG thesis is + wrong and the asset continues down +- it respects that the post-win EFSM edge is a rebound/cooldown edge, so the + exit manager should not over-penalize ordinary post-entry MFE shape + +Do not deploy this LONG config live yet. It should first be run in shadow on +actual EFSM-flipped candidate LONG contexts, because this replay uses SHORT +entries inverted to LONG and not real LONG fills. + +### Regression and safety notes + +Implemented code seams: + +- `nautilus_dolphin/nautilus_dolphin/nautilus/alpha_exit_v7_engine.py` + defines `AlphaExitV7Config` +- default `AlphaExitEngineV7()` behavior remains the SHORT-calibrated config +- a LONG-specific engine can be instantiated with `AlphaExitEngineV7(config=...)` +- the calibration harness writes full results to `/tmp/v7_long_calibration.json` + +Tests added: + +- default config equals the legacy SHORT threshold surface +- custom config is per-instance and does not mutate the default engine +- V7 remains mechanically side-aware for LONG and SHORT PnL/MFE/MAE +- BLUE live V7 provider wiring still records journal decisions and uses OB + signal input +- EFSM reset/no-recursive-rearm tests remain separate from V7 exit calibration + +Research caveats: + +- only `97` V7-tracked BLUE paths existed in the current decision journal +- this is enough to reject obviously bad LONG exit settings, but not enough to + canonize a live LONG exit policy +- bounce must remain neutral for LONG until trained or validated on LONG samples +- V7 `max_hold_ref_mult_3m` still uses an internal time reference rather than + the orchestrator's effective max hold; the system bible already tracks this + as a V7 TODO/bug because it can make adverse-ramp pressure too early diff --git a/prod/docs/SYSTEM_BIBLE_v7.md b/prod/docs/SYSTEM_BIBLE_v7.md new file mode 100644 index 0000000..47a961f --- /dev/null +++ b/prod/docs/SYSTEM_BIBLE_v7.md @@ -0,0 +1,3389 @@ +# DOLPHIN-NAUTILUS SYSTEM BIBLE +## Doctrinal Reference — As Running 2026-04-19 + +**Version**: v7.0 — ClickHouse Observability + Adaptive Exit Engine (Shadow) + TUI v9 + Full Path Audit +**Previous version**: v6.0 — NG8 Linux Scanner + TUI v3 Live Observability (2026-04-05) +**Previous version**: v5.0 — Supervisord-First Architecture + MHS v3 + OBF Universe (2026-03-30) +**CI gate (Nautilus)**: 46/46 tests green +**CI gate (MHS)**: 111/111 tests green (unit + E2E + race + Hypothesis) +**CI gate (ACB)**: 118/118 tests green +**Execution**: Binance Futures (USDT-M). Live trading active. +**Status**: Supervisord-managed. MHS v3 live. OBF universe 540 assets. RM_META=0.975–1.000 [GREEN]. ALGO=v2_gold_fix_v50-v750. Namespace split is strict: BLUE uses `dolphin`, `DOLPHIN_STATE_BLUE`, `DOLPHIN_PNL_BLUE`; PRODGREEN uses `dolphin_prodgreen`, `DOLPHIN_STATE_PRODGREEN`, `DOLPHIN_PNL_PRODGREEN`. +**NG8**: Linux-native eigenscan. Running. Fixes NG7 double-output bug. +**TUI v9**: `/mnt/dolphinng5_predict/Observability/TUI/dolphin_tui_v9.py` — live terminal with trades footer, AE shadow panel, and bucket performance panel. +**ClickHouse**: `http://localhost:8123/` (database: `dolphin`, user: `dolphin`, pass: `dolphin_ch_2026`). BLUE live writes go to `dolphin`. PRODGREEN live writes go to `dolphin_prodgreen`. Green-side readers must filter `strategy IN ('green','prodgreen')` and must not treat legacy BLUE rows in green-side tables as authoritative. +**Adaptive Exit Engine**: Shadow mode active. Per-bucket LR continuation model + SC threshold/gauge surfaces. Logs to `adaptive_exit_shadow` and `sc_*_shadow`. Zero impact on real exits. +**D_LIQ Gold Performance**: ROI=+189.48% | T=2155 | DD=21.31% (full backtest, post vel_div fix, post D_LIQ). + +### What changed since v6.0 (2026-04-19 — THIS VERSION) + +| Area | Change | +|---|---| +| **ALGO VERSION: v2_gold_fix_v50-v750** | `vel_div` corrected from `v50-v150` → `v50-v750` in `nautilus_event_trader.py`. Deployed 2026-04-10. See §29. | +| **ClickHouse — NEW** | `prod/ch_writer.py` — fire-and-forget HTTP/JSONEachRow writer. Tables: `trade_events`, `eigen_scans`, `adaptive_exit_shadow`. See §30. | +| **Adaptive Exit Engine — NEW** | `adaptive_exit/` package — per-bucket LR continuation model plus SC threshold / SC gauge shadow surfaces. Shadow mode only (no real exits). Trained on 5yr 1m klines. Integrated into `prod/nautilus_event_trader.py`. See §31. | +| **Asset Bucket System — NEW** | KMeans k=7 buckets by market characteristics. B3 best (WR≈61%, net+$3,858), B1/B4 worst. Live panel in TUI. See §33. | +| **TUI v9** | `Observability/TUI/dolphin_tui_v9.py` — complete rewrite from v3. Panels: header, trader, sys-health, alpha, scan, extf, obf, capital, prefect, acb, MC-forewarner (sparklines + bars), trades footer (live v7 exits vs AE shadow), bucket footer (live per-bucket stats), test footer. | +| **D_LIQ Performance Confirmed** | Post vel_div fix gold: ROI=+189.48%, T=2155, DD=21.31%. Supersedes v5 +54.67% reference (that was pre-D_LIQ-fix era). | +| **Full path audit** | All file references updated to absolute Linux paths. Windows paths removed from active sections. | +| **ExF NPZ Backfill** | 1658 daily NPZ files (2021-06-15 → 2026-01-12) confirmed on disk: `fng`, `fng_prev`, `funding_btc`, `dvol_btc`, `chg24_btc`. Used for AE training. | + +### What changed since v5.0 (2026-04-05 — PREVIOUS) + +| Area | Change | +|---|---| +| **NG8 Linux Scanner — NEW** | `- Dolphin NG8/ng8_scanner.py` — Linux-native eigenscan service replacing Windows NG7. Fixes double-output bug. Single `enhance()` call processes all 4 windows (w50/150/300/750) in one pass → exactly one Arrow file + one HZ write per scan_number. See §27. | +| **Arrow Writer Shim — NEW** | `- Dolphin NG8/arrow_writer.py` — thin re-export so `dolphin_correlation_arb512_with_eigen_tracking.py` imports correctly on Linux (Windows had this file natively). | +| **TUI v3 — NEW** | `Observability/TUI/dolphin_tui_v3.py` — full live observability terminal. All panels event-driven via HZ entry listeners. Zero origin-system load. Replaces mocked TUI v2. See §28. | +| **Test Footer CI Hook — NEW** | `run_logs/test_results_latest.json` + `write_test_results()` API in TUI v3. Test scripts push results; TUI footer displays live. See §28.4 and `TEST_REPORTING.md`. | +| **NG7 Double-Output — Root Cause Confirmed** | Windows NG7 ran two independent tracker cycles (fast w50/w150 + slow w300/w750) sharing the same scan_number counter → two Arrow files + two HZ writes per scan, second file arriving ~3 min late with stale prices. NG8 eliminates this by design. | + +--- + +### What changed since v4.1 (2026-03-30 — PREVIOUS) + +| Area | Change | +|---|---| +| **Process Manager: Systemd → Supervisord** | ALL dolphin services migrated exclusively to supervisord. No service is managed by both. `dolphin-supervisord.conf` is the single source of process truth. See §16, §26. | +| **"Random Killer" Root Cause Fixed** | `meta_health_daemon_v2.py` had been running under systemd for 4 days calling `systemctl restart` on supervisord-managed services every 5s. Dual-management race caused random service kills. Stopped + disabled. | +| **MHS v3 — Complete Rewrite** | `meta_health_service_v3.py` — product formula bug fixed (zero-collapse replaced by weighted sum), recovery via supervisorctl not systemctl, `RECOVERY_COOLDOWN_CRITICAL_S=10s` (was 600s), non-blocking daemon thread recovery. See §24.5. | +| **OBF Universe Service — NEW** | `obf_universe_service.py` — 540 USDT perp assets on 3 WebSocket connections, zero REST weight, 60s health snapshots → HZ `obf_universe_latest`. Supervisord `autostart=true`. See §26. | +| **OBF Retention Fix** | `obf_persistence.py` `MAX_FILE_AGE_DAYS = 0` (was 7 — was deleting all backtesting data). Data now accumulates indefinitely for backtesting. | +| **Test Suite: MHS** | NEW `prod/tests/test_mhs_v3.py` — 111 tests: unit, live integration, E2E kill/revive, race conditions, 13 Hypothesis property tests. | +| **HZ Schema additions** | `DOLPHIN_FEATURES["obf_universe_latest"]`, `DOLPHIN_META_HEALTH["latest"]`. See §15. | +| **Supervisord groups** | `dolphin_data` group: exf_fetcher, acb_processor, obf_universe, meta_health (all autostart=true). `dolphin` group: nautilus_trader, scan_bridge, clean_arch_trader (autostart=false). | + +### What changed since v4 (2026-03-24) + +| Area | Change | +|---|---| +| **Multi-Speed Architecture** | NEW multi-layer frequency isolation: OBF (0.1s), Scan (5s), ExtF (varied), Health (5s), Daily batch. See §24. | +| **Event-Driven Nautilus** | NEW `nautilus_event_trader.py` — Hz entry listener for <1ms scan-to-trade latency. Not a Prefect flow — long-running systemd daemon. See §24.2. | +| **MHS v2** | ENHANCED `meta_health_daemon_v2.py` — Full 5-sensor monitoring (M1-M5), per-subsystem data freshness tracking, automated recovery. See §24.3. | +| **Resource Safety** | NEW systemd resource limits: MemoryMax=2G, CPUQuota=200%, TasksMax=50 per service. Prevents process explosion. | +| **Scan Bridge Hardening** | Deployment concurrency limit=1, work pool concurrency=1, cgroups integration. See §24.1. | +| **Systemd Service Mesh** | NEW services: `dolphin-nautilus-trader.service`, updated `meta_health_daemon.service`. Systemd-managed, not Prefect-managed. | +| **Incident Response** | Post-mortem: 2026-03-24 kernel deadlock from 60+ uncontrolled Prefect processes. Fixed via concurrency controls. | + +### What changed since v3 (2026-03-22) + +| Area | Change | +|---|---| +| **Clean Architecture** | NEW hexagonal architecture in `prod/clean_arch/` — Ports, Adapters, Core separation. Adapter-agnostic business logic. | +| **Hazelcast DataFeed** | NEW `HazelcastDataFeed` adapter implementing `DataFeedPort` — reads from DolphinNG6 via Hazelcast (single source of truth). | +| **Scan Bridge Service** | NEW `scan_bridge_service.py` — Linux Arrow file watcher that pushes to Hazelcast. Uses file mtime (not scan #) to handle NG6 restarts. **Phase 2: Prefect daemon integration complete** — auto-restart, health monitoring, unified logging. **18 unit tests** in `tests/test_scan_bridge_prefect_daemon.py`. +| **Paper Trading Engine** | NEW `paper_trade.py` — Clean architecture trading CLI with 23 round-trip trades executed in testing. | +| **Market Data** | Live data flowing: 50 assets, BTC @ $71,281.03, velocity divergence signals active. | + +### What changed since v2 (2026-03-22) + +| Area | Change | +|---|---| +| **Binance Futures** | Switched system focus from Spot to Perpetuals; updated API endpoints (`fapi.binance.com`); added `recvWindow` for signature stability. | +| **Friction Management** | **SP Bypass Logic**: Alpha engines now support disabling internal fees/slippage to allow Nautilus to handle costs natively. Prevents double-counting. | +| **Paper Trading** | NEW `launch_paper_portfolio.py` — uses Sandbox matching with live Binance data; includes realistic Tier 0 friction (0.02/0.05). | +| **Session Logging** | NEW `TradeLoggerActor` — independent CSV/JSON audit trails for every session. | + +| Area | Change | +|---|---| +| **DolphinActor** | Refactored to step_bar() API (incremental, not batch); threading.Lock on ACB; _GateSnap stale-state detection; replay vs live mode; bar_idx tracking | +| **OBF Subsystem** | Sprint 1 hardening complete: circuit breaker, stall watchdog, crossed-book guard, dark streak, first flush 60s, fire-and-forget HZ pushes, dynamic asset discovery | +| **nautilus_prefect_flow.py** | NEW — Prefect-supervised BacktestEngine daily flow; champion SHA256 hash check; HZ heartbeats; capital continuity; HIBERNATE guard | +| **Test suite** | +35 DolphinActor tests (test_dolphin_actor.py); total 46 Nautilus + ~120 OBF | +| **prod/docs/** | All prod .md files consolidated; SYSTEM_FILE_MAP.md; NAUTILUS_DOLPHIN_SPEC.md added | +| **0.1s resolution** | Assessed: BLOCKED by 3 hard blockers (see §22) | +| **Capital Sync** | NEW — DolphinActor now syncs initial_capital with Nautilus Portfolio balance on_start. | +| **Verification** | NEW — `TODO_CHECK_SIGNAL_PATHS.md` systematic test spec for local agents. | +| **MC-Forewarner** | Now wired in `DolphinActor.on_start()` — both flows run full gold-performance stack; `_MC_BASE_CFG` + `_MC_MODELS_DIR_DEFAULT` as frozen module constants; empty-parquet early-return bug fixed in `on_bar` replay path | + +--- + +## TABLE OF CONTENTS + +1. [System Philosophy](#1-system-philosophy) +2. [Physical Architecture](#2-physical-architecture) +2a. [Clean Architecture Layer (NEW v4)](#2a-clean-architecture-layer) +3. [Data Layer](#3-data-layer) +4. [Signal Layer — vel_div & DC](#4-signal-layer) +5. [Asset Selection — IRP](#5-asset-selection-irp) +6. [Position Sizing — AlphaBetSizer](#6-position-sizing) +7. [Exit Management](#7-exit-management) +8. [Fee & Slippage Model](#8-fee--slippage-model) +9. [OB Intelligence Layer](#9-ob-intelligence-layer) +10. [ACB v6 — Adaptive Circuit Breaker](#10-acb-v6) +11. [Survival Stack — Posture Control](#11-survival-stack) +12. [MC-Forewarner Envelope Gate](#12-mc-forewarner-envelope-gate) +13. [NDAlphaEngine — Full Bar Loop](#13-ndalpha-engine-full-bar-loop) +14. [DolphinActor — Nautilus Integration](#14-dolphin-actor) +15. [Hazelcast — Full IMap Schema](#15-hazelcast-full-imap-schema) +16. [Production Daemon Topology & HZ Bridge](#16-production-daemon-topology) +17. [Prefect Orchestration Layer](#17-prefect-orchestration-layer) +18. [CI Test Suite](#18-ci-test-suite) +19. [Parameter Reference](#19-parameter-reference) +20. [OBF Sprint 1 Hardening](#20-obf-sprint-1-hardening) +21. [Known Research TODOs](#21-known-research-todos) +22. [0.1s Resolution — Readiness Assessment](#22-01s-resolution-readiness-assessment) +23. [Signal Path Verification Specification](#23-signal-path-verification) +24. [Multi-Speed Event-Driven Architecture (v4.1)](#24-multi-speed-event-driven-architecture) +25. [Numerical Precision Policy](#25-numerical-precision-policy) +26. [Supervisord Architecture & OBF Universe (v5.0)](#26-supervisord-architecture--obf-universe) +27. [NG8 Linux Eigenscan Service (v6.0)](#27-ng8-linux-eigenscan-service) +28. [TUI v9 — Live Observability Terminal (v7.0)](#28-tui-v9-live-observability-terminal) +29. [Algo Versioning & Lineage Tracking](#29-algo-versioning--lineage-tracking) +30. [ClickHouse Observability Layer (v7.0)](#30-clickhouse-observability-layer) +31. [Adaptive Exit Engine — Shadow Mode (v7.0)](#31-adaptive-exit-engine--shadow-mode) +31.11 [SC Gauge Surface — Shadow Bucketed Policy](#3111-sc-gauge-surface--shadow-bucketed-policy) +32. [Asset Bucket System (v7.0)](#32-asset-bucket-system) + +--- + +## 1. SYSTEM PHILOSOPHY + +DOLPHIN-NAUTILUS is a **SHORT-only** (champion configuration) systematic trading engine targeting crypto perpetual futures on Binance. + +**Core thesis**: When crypto market correlation matrices show accelerating eigenvalue-velocity divergence (`vel_div < -0.02`), the market is entering an instability regime. Shorting during early instability onset and exiting at fixed take-profit captures the mean-reversion from panic to normalization. + +**Design constraints**: +- Zero signal re-implementation in the Nautilus layer. All alpha logic lives in `NDAlphaEngine`. +- 512-bit arithmetic for correlation matrix processing (separate NG3 pipeline; not in hot path of this engine). +- Champion parameters are FROZEN. They were validated via exhaustive VBT backtest on `dolphin_vbt_real.py`. +- The Nautilus actor is a thin wire, not a strategy. It routes parquet data → NDAlphaEngine → HZ result. + +**Champion performance** (ACBv6 + IRP + DC + OB, full-stack 55-day Dec31–Feb25): +- ROI: +54.67% | PF: 1.141 | Sharpe: 2.84 | Max DD: 15.80% | WR: 49.5% | Trades: 2145 +- Log: `run_logs/summary_20260307_163401.json` + +> **Data correction note (2026-03-07)**: An earlier reference showed ROI=+57.18%, PF=1.149, +> Sharpe=3.00. Those figures came from a stale `vbt_cache/2026-02-25.parquet` that was built +> mid-day — missing 435 scans and carrying corrupt vel_div on 492 rows for the final day of the +> window. ALGO-3 parity testing caught the mismatch (max_diff=1.22 vs tolerance 1e-10). +> The parquet was rebuilt from live NG3 JSON (`build_parquet_cache(dates=['2026-02-25'], force=True)`). +> The stale file is preserved as `2026-02-25.parquet.STALE_20260307` for replicability. +> The corrected numbers above are the canonical reference. The ~2.5pp ROI drop reflects real +> late-day trades on Feb 25 that the stale parquet had silently omitted. + +--- + +## 2. PHYSICAL ARCHITECTURE + +``` +┌──────────────────────────────────────────────────────────────────────┐ +│ DATA SOURCES │ +│ NG3 Scanner (Win) → /mnt/ng6_data/eigenvalues/ (SMB DolphinNG6_Data)│ +│ Binance WS → 5s OHLCV bars + live order book (48+ USDT perpetuals) │ +│ VBT Cache → vbt_cache_klines/*.parquet (DOLPHIN-local + /mnt/dolphin)│ +└────────────────────────┬─────────────────────────────────────────────┘ + │ +┌────────────────────────▼─────────────────────────────────────────────┐ +│ HAZELCAST IN-MEMORY GRID (localhost:5701, cluster: "dolphin") │ +│ *** SYSTEM MEMORY — primary real-time data bus *** │ +│ DOLPHIN_SAFETY → posture + Rm (CP AtomicRef / IMap) │ +│ DOLPHIN_FEATURES → acb_boost {boost,beta}, latest_eigen_scan │ +│ DOLPHIN_PNL_BLUE/GREEN → per-date trade results │ +│ DOLPHIN_STATE_BLUE → capital continuity (latest + per-run) │ +│ DOLPHIN_HEARTBEAT → liveness pulses (nautilus_prefect_flow) │ +│ DOLPHIN_OB → order book snapshots │ +│ DOLPHIN_FEATURES_SHARD_00..09 → 400-asset OBF sharded store │ +└────────────────────────┬─────────────────────────────────────────────┘ + │ +┌────────────────────────▼─────────────────────────────────────────────┐ +│ PREFECT ORCHESTRATION (localhost:4200, work-pool: dolphin) │ +│ paper_trade_flow.py 00:05 UTC — NDAlphaEngine direct │ +│ nautilus_prefect_flow.py 00:10 UTC — BacktestEngine + DolphinActor│ +│ obf_prefect_flow.py Continuous ~500ms — OB ingestion │ +│ mc_forewarner_flow.py Daily — MC gate prediction │ +│ exf_fetcher_flow.py Periodic — ExF macro data fetch │ +└────────────────────────┬─────────────────────────────────────────────┘ + │ +┌────────────────────────▼─────────────────────────────────────────────┐ +│ SUPERVISORD (v5.0 — sole process manager) │ +│ Config: prod/supervisor/dolphin-supervisord.conf │ +│ Socket: /tmp/dolphin-supervisor.sock │ +│ │ +│ dolphin_data group (autostart=true): │ +│ ├── exf_fetcher_flow.py — ExF live daemon │ +│ ├── acb_processor_service.py — ACB boost + HZ write (CP lock) │ +│ ├── obf_universe_service.py — 540-asset OBF universe (NEW v5.0) │ +│ └── meta_health_service_v3.py — MHS watchdog (NEW v5.0) │ +│ │ +│ dolphin group (autostart=false): │ +│ ├── nautilus_event_trader.py — HZ entry listener trader │ +│ ├── scan_bridge_service.py — Arrow → HZ scan bridge │ +│ └── clean_arch/main.py — Clean architecture trader │ +└────────────────────────┬─────────────────────────────────────────────┘ + │ +┌────────────────────────▼─────────────────────────────────────────────┐ +│ NAUTILUS TRADING ENGINE (siloqy-env, nautilus_trader 1.219.0) │ +│ BacktestEngine + DolphinActor(Strategy) → NDAlphaEngine │ +│ on_bar() fires per date tick; step_bar() iterates parquet rows │ +│ HZ ACB listener → pending-flag → applied at top of next on_bar() │ +│ TradingNode (launcher.py) → future live exchange connectivity │ +└──────────────────────────────────────────────────────────────────────┘ +``` + +**Key invariant v2**: `DolphinActor.on_bar()` receives one synthetic bar per date in paper mode, which triggers `engine.begin_day()` then iterates through all parquet rows via `step_bar()`. In live mode, one real bar → one `step_bar()` call. The `_processed_dates` guard is replaced by date-boundary detection comparing `current_date` to the bar's timestamp date. + +--- + +## 2a. CLEAN ARCHITECTURE LAYER (NEW v4) + +### 2a.1 Overview + +The Clean Architecture layer provides a **hexagonal** (ports & adapters) implementation for paper trading, ensuring core business logic is independent of infrastructure concerns. + +``` +┌─────────────────────────────────────────────────────────────────────────┐ +│ CLEAN ARCHITECTURE (prod/clean_arch/) │ +├─────────────────────────────────────────────────────────────────────────┤ +│ PORTS (Interfaces) │ +│ ├── DataFeedPort → Abstract market data source │ +│ └── TradingPort → Abstract order execution │ +├─────────────────────────────────────────────────────────────────────────┤ +│ ADAPTERS (Infrastructure) │ +│ ├── HazelcastDataFeed → Reads from DOLPHIN_FEATURES map │ +│ └── PaperTradeExecutor → Simulated execution (no real orders) │ +├─────────────────────────────────────────────────────────────────────────┤ +│ CORE (Business Logic) │ +│ ├── TradingEngine → Position sizing, signal processing │ +│ ├── SignalProcessor → Eigenvalue-based signal generation │ +│ └── PortfolioManager → PnL tracking, position management │ +└─────────────────────────────────────────────────────────────────────────┘ +``` + +### 2a.2 Key Design Principles + +**Dependency Rule**: Dependencies only point inward. Core knows nothing about Hazelcast, Arrow files, or Binance. + +**Single Source of Truth**: All data comes from Hazelcast `DOLPHIN_FEATURES.latest_eigen_scan`, written atomically by DolphinNG6. + +**File Timestamp vs Scan Number**: The Scan Bridge uses file modification time (mtime) instead of scan numbers because DolphinNG6 resets counters on restarts. + +### 2a.3 Components + +| Component | File | Purpose | +|-----------|------|---------| +| `DataFeedPort` | `ports/data_feed.py` | Abstract interface for market data | +| `HazelcastDataFeed` | `adapters/hazelcast_feed.py` | Hz implementation of DataFeedPort | +| `TradingEngine` | `core/trading_engine.py` | Pure business logic | +| `Scan Bridge` | `../scan_bridge_service.py` | Arrow → Hazelcast bridge | +| `Paper Trader` | `paper_trade.py` | CLI trading session | + +### 2a.4 Data Flow + +``` +DolphinNG6 → Arrow Files (/mnt/ng6_data/arrow_scans/) → Scan Bridge → Hazelcast → HazelcastDataFeed → TradingEngine + (5s) (watchdog) (SSOT) (Adapter) (Core) + ↑ + (Prefect daemon + supervises) +``` + +**Management**: The scan bridge runs as a Prefect-supervised daemon (`scan_bridge_prefect_daemon.py`): +- Health checks every 30 seconds +- Automatic restart on crash or stale data (>60s) +- Centralized logging via Prefect UI +- Deployed to `dolphin-daemon-pool` + +### 2a.5 MarketSnapshot Structure + +```python +MarketSnapshot( + timestamp=datetime, + symbol="BTCUSDT", + price=71281.03, # From asset_prices[0] + eigenvalues=[...], # From asset_loadings (50 values) + velocity_divergence=-0.0058, # vel_div field + scan_number=7315 +) +``` + +### 2a.6 Current Status + +- **Assets Tracked**: 50 (BTC, ETH, BNB, etc.) +- **BTC Price**: $71,281.03 +- **Test Trades**: 23 round-trip trades executed +- **Strategy**: Mean reversion on velocity divergence +- **Data Latency**: ~5 seconds (DolphinNG6 pulse) +- **Bridge Management**: Prefect daemon (auto-restart, health checks every 30s) + +### 2a.7 Testing + +**Unit Tests:** `prod/tests/test_scan_bridge_prefect_daemon.py` (18 tests) + +| Test Category | Count | Description | +|--------------|-------|-------------| +| ScanBridgeProcess | 6 | Process lifecycle (start, stop, restart) | +| Hazelcast Freshness | 6 | Data age detection (fresh, stale, warning) | +| Health Check Task | 3 | Prefect task health validation | +| Integration | 3 | Real Hz connection, process lifecycle | + +**Run Tests:** +```bash +cd /mnt/dolphinng5_predict/prod +source /home/dolphin/siloqy_env/bin/activate +pytest tests/test_scan_bridge_prefect_daemon.py -v +``` + +**Test Coverage:** +- ✅ Process start/stop/restart +- ✅ Graceful and force kill +- ✅ Fresh/stale/warning data detection +- ✅ Hazelcast connection error handling +- ✅ Health check state transitions + +--- + +## 3. DATA LAYER + +### 3.1 vbt_cache_klines Parquet Schema + +Location: `/mnt/dolphinng5_predict/vbt_cache_klines/YYYY-MM-DD.parquet` +*(SMB: `//100.119.158.61/DolphinNG5_Predict/vbt_cache_klines/`)* + +| Column | Type | Description | +|--------|------|-------------| +| `vel_div` | float64 | Eigenvalue velocity divergence: `v50_vel − v750_vel` (primary signal) | +| `v50_lambda_max_velocity` | float64 | Short-window (50-bar) lambda_max rate of change | +| `v150_lambda_max_velocity` | float64 | 150-bar window lambda velocity | +| `v300_lambda_max_velocity` | float64 | 300-bar window lambda velocity | +| `v750_lambda_max_velocity` | float64 | Long-window (750-bar) macro eigenvalue velocity | +| `instability_50` | float64 | General market instability index (50-bar) | +| `instability_150` | float64 | General market instability index (150-bar) | +| `BTCUSDT` … `STXUSDT` | float64 | Per-asset close prices (48 assets in current dataset) | + +Each file: 1,439 rows (1 per 5-second bar over 24h), 57 columns. + +### 3.2 NG3/NG6 Eigenvalue Data + +Linux path: `/mnt/ng6_data/` (SMB share `DolphinNG6_Data`, server `100.119.158.61`) + +``` +/mnt/ng6_data/eigenvalues/ + YYYY-MM-DD/ + scan_NNNNNN__Indicators.npz ← ACBv6 external factors: funding_btc, dvol_btc, fng, taker + scan_NNNNNN__scan_global.npz ← lambda_vel_w750 for dynamic beta + extf_snapshot_*__Indicators.npz ← ExF snapshots (v2 format) +/mnt/ng6_data/matrices/ + YYYY-MM-DD/ + scan_NNNNNN_w50_HHMMSS.arb512.pkl.zst ← 512-bit correlation matrix (unused in hot path) +/mnt/ng6_data/ob_universe/ ← OBF Universe Parquet (Hive partitioned, never pruned) +/mnt/ng6_data/ob_features/ ← OBF shard Parquet archive +/mnt/ng6_data/arrow_scans/ ← NG8 Arrow IPC scan files + YYYY-MM-DD/ + scan_NNNNNN_HHMMSS.arrow +``` + +NPZ files loaded by `AdaptiveCircuitBreaker._load_external_factors()` (max 10 scans per date, median-aggregated). + +### 3.3 ExF NPZ Backfill (for AE Training) + +**1658 daily NPZ files** (2021-06-15 → 2026-01-12) confirmed on disk at `/mnt/ng6_data/eigenvalues/`. +Fields: `fng`, `fng_prev`, `funding_btc`, `dvol_btc`, `chg24_btc`. +Used by `adaptive_exit/data_pipeline.py` to inject ExF features into training trajectories. +NOT a look-ahead bias source — each file covers its own calendar date. + +### 3.4 1-Minute Klines (AE Training Dataset) + +**Full 5-year 1m klines** available at `/mnt/dolphinng5_predict/vbt_cache_klines/` (5s bars) and +`/mnt/dolphin/klines_1m/` (1m bars, used by AE training pipeline). +AE training uses `adaptive_exit/data_pipeline.py` with `max_samples_per_asset=50_000`. + +--- + +## 4. SIGNAL LAYER + +### 4.1 Primary Signal: vel_div Threshold Gate + +**Source**: `alpha_signal_generator.py`, `AlphaSignalGenerator.generate()` + +**SHORT signal condition**: +``` +vel_div < VEL_DIV_THRESHOLD (-0.02) +``` + +**LONG signal condition** (green posture, not champion): +``` +vel_div > LONG_THRESHOLD (0.01) +``` + +**Confidence calculation** (SHORT path): +```python +ratio = clamp((threshold - vel_div) / (threshold - extreme), 0, 1) + = clamp((-0.02 - vel_div) / (-0.02 - (-0.05)), 0, 1) + = clamp((-0.02 - vel_div) / 0.03, 0, 1) +confidence = 0.50 + ratio * 0.40 # range: [0.50, 0.90] +``` + +`is_extreme = (vel_div <= -0.05)` + +### 4.2 Direction Confirmation (DC) — Layer 6 + +**Source**: `alpha_signal_generator.py`, `check_dc_nb()` (numba JIT) + +```python +# Looks back dc_lookback_bars (default 7) bars on the selected asset price +p0 = price[n - lookback - 1] +p1 = price[n - 1] +chg_bps = (p1 - p0) / p0 * 10000.0 + +if chg_bps < -min_magnitude_bps (-0.75): return CONFIRM # falling price → SHORT OK +if chg_bps > min_magnitude_bps (+0.75): return CONTRADICT +else: return NEUTRAL +``` + +**dc_skip_contradicts = True** (champion): CONTRADICT returns null signal (skip entry). +**Effect on leverage**: DC has `dc_leverage_boost=1.0` (no boost in champion). CONTRADICT kills entry. + +### 4.3 OB Sub-2: Per-Asset Imbalance Confirmation + +When `ob_engine` is wired in (`use_ob_edge=True`): +```python +eff_imb = -ob_signal.imbalance_ma5 # For SHORT: sell pressure = positive eff_imb + +if eff_imb > 0.10: # OB confirms → confidence boost ≤+15% + ob_adj = 1 + min(0.15, eff_imb * persistence * 0.5) + confidence *= ob_adj +elif eff_imb < -0.15 and persistence > 0.60: # Strong persistent OB contradiction → HARD SKIP + return null_signal +elif eff_imb < -0.10: # Moderate → soft dampen confidence + ob_adj = max(0.85, 1 - |eff_imb| * persistence * 0.4) + confidence *= ob_adj +``` + +--- + +## 5. ASSET SELECTION — IRP + +### 5.1 Overview + +**Source**: `alpha_asset_selector.py`, `AlphaAssetSelector.rank_assets()` + numba kernels + +IRP = **Impulse Response Profiling**. Ranks all available assets by historical behavior over the last 50 bars in the regime direction. Selects the asset with the highest ARS (Asset Ranking Score) that passes all filters. + +**Enabled by**: `use_asset_selection=True` (production default). + +### 5.2 Numba Kernel: compute_irp_nb + +```python +# Input: price_segment (last 50 prices), direction (-1 or +1) +dir_returns[i] = (price[i+1] - price[i]) * direction # directional returns + +cumulative = cumsum(dir_returns) +mfe = max(cumulative) # Maximum Favorable Excursion +mae = abs(min(cumulative, 0)) # Maximum Adverse Excursion +efficiency = mfe / (mae + 1e-6) +alignment = count(dir_returns > 0) / n_ret +noise = variance(dir_returns) +latency = bars_to_reach_10pct_of_mfe # (default: 50 if mfe==0) +``` + +### 5.3 Numba Kernel: compute_ars_nb + +``` +ARS = 0.5 * log1p(efficiency) + 0.35 * alignment - 0.15 * noise * 1000 +``` + +### 5.4 Numba Kernel: rank_assets_irp_nb + +For each asset: +1. Compute IRP in DIRECT direction (regime_direction) +2. Compute IRP in INVERSE direction (-regime_direction) +3. Take whichever gives higher ARS (allows inverse selection) +4. Apply filter gates: + - `noise > 500` → skip + - `latency > 20` → skip (must reach 10% MFE within 20 bars) + - `alignment < 0.20` → skip +5. Bubble-sort by ARS descending (numba nopython) + +### 5.5 AlphaAssetSelector Python Wrapper + +```python +# Build 2D array (max_len × n_assets), right-aligned +valid = rank_assets_irp_nb(prices_2d, idx=max_len, regime_direction, ...) +# Walk ranked list: +for r in rankings: + if min_irp_alignment > 0 and r.metrics.alignment < min_irp_alignment: + continue # alignment gate (default 0.45) + if r.action != expected_action: + continue # direction gate + if ob_engine and ob_placement.depth_quality < 0.20: + continue # OB depth gate (try next asset) + trade_asset = r.asset + break +# No match → return None (no fallback to BTCUSDT when IRP enabled) +``` + +**OB Sub-1**: ARS adjusted ±5%/10% by per-asset OB depth quality before sorting. + +--- + +## 6. POSITION SIZING + +### 6.1 Signal Strength Score + +**Source**: `alpha_bet_sizer.py`, `compute_sizing_nb()` (numba JIT) + +```python +# SHORT path (vel_div < threshold): +if vel_div <= extreme (-0.05): + strength_score = 1.0 +else: + strength_score = (threshold - vel_div) / (threshold - extreme) + # = (-0.02 - vel_div) / 0.03 + strength_score = clamp(strength_score, 0.0, 1.0) +``` + +### 6.2 Dynamic Leverage (Cubic Convex) + +```python +scaled_score = strength_score ** leverage_convexity (3.0) +eff_leverage = min_leverage + scaled_score * (max_leverage - min_leverage) + = 0.5 + scaled_score³ * 4.5 # range: [0.5, 5.0] before ACB +``` + +### 6.3 Signal Bucket Classification + +```python +# Numba: get_signal_bucket_nb(vel_div, threshold=-0.02, extreme=-0.05) +if vel_div <= extreme * 1.5 (-0.075): bucket = 0 # "extreme" +elif vel_div <= extreme (-0.05): bucket = 1 # "strong" +elif vel_div <= (threshold+extreme)/2: bucket = 2 # "moderate" (-0.035) +else: bucket = 3 # "weak" +``` + +### 6.4 Alpha Layers (Layer 7) + +**Bucket Boost** — adaptive win-rate feedback: +```python +# get_bucket_boost_nb: per-bucket win rate → multiplier +wr > 0.60 → 1.3x | wr > 0.55 → 1.1x | wr < 0.40 → 0.7x | wr < 0.45 → 0.85x +``` + +**Streak Multiplier** — recent 5-trade loss streak: +```python +# get_streak_mult_nb +losses_in_last_5 >= 4 → 0.5x | >= 3 → 0.7x | <= 1 → 1.1x +``` + +**Trend Multiplier** — vel_div acceleration: +```python +# get_trend_mult_nb(vd_trend = vel_div_history[-1] - vel_div_history[-10]) +vd_trend < -0.01 → 1.3x (deepening instability) +vd_trend > 0.01 → 0.7x (recovering) +``` + +**Effective Fraction computation**: +```python +confidence = 0.70 if is_extreme else 0.55 +conf_mult = confidence / 0.95 +extreme_boost = 2.0 if is_extreme else 1.0 + +base_frac = 0.02 + strength_score * (base_fraction - 0.02) +eff_fraction = base_frac * conf_mult * extreme_boost * trend_mult * bucket_boost * streak_mult +eff_fraction = clamp(eff_fraction, 0.02, base_fraction=0.20) +``` + +**Final notional**: +```python +notional = capital * eff_fraction * final_leverage +``` + +### 6.5 ACB + MC Size Multiplier + +```python +# regime_size_mult is recomputed every bar via _update_regime_size_mult(vel_div) +if day_beta > 0: + strength_cubic = clamp((threshold - vel_div) / (threshold - extreme), 0, 1) ** convexity + regime_size_mult = day_base_boost * (1.0 + day_beta * strength_cubic) * day_mc_scale +else: + regime_size_mult = day_base_boost * day_mc_scale + +# Applied to leverage ceiling: +clamped_max_leverage = min(base_max_leverage * regime_size_mult * market_ob_mult, abs_max_leverage=6.0) +raw_leverage = size_result["leverage"] * dc_lev_mult * regime_size_mult * market_ob_mult + +# STALKER posture hard cap: +if posture == 'STALKER': clamped_max_leverage = min(clamped_max_leverage, 2.0) + +final_leverage = clamp(raw_leverage, min_leverage=0.5, clamped_max_leverage) +``` + +--- + +## 7. EXIT MANAGEMENT + +### 7.1 Exit Priority Order (champion) + +**Source**: `alpha_exit_manager.py`, `AlphaExitManager.evaluate()` + +1. **FIXED_TP**: `pnl_pct >= 0.0095` (95 basis points) +2. **STOP_LOSS**: `pnl_pct <= -1.0` (DISABLED in practice — 100% loss never triggers before TP/max_hold) +3. **OB DURESS exits** (when ob_engine != None): + - Cascade Detection: `cascade_count > 0` → widen TP ×1.40, halve max_hold + - Liquidity Withdrawal: `regime_signal == 1` → hard SL 10%, TP ×0.60 +4. **vel_div adverse-turn exits** (`vd_enabled=False` by default — disabled pending calibration) +5. **MAX_HOLD**: `bars_held >= 120` (= 600 seconds) + +### 7.2 OB Dynamic Exit Parameter Adjustment + +```python +if cascade_count > 0: + dynamic_tp_pct *= 1.40 + dynamic_max_hold = int(max_hold_bars * 0.50) # take profit fast before snap-back + +elif regime_signal == 1: # LIQUIDITY WITHDRAWAL STRESS + dynamic_sl_pct = 0.10 # hard 10% stop (tail protection) + if pnl_pct > 0.0: + dynamic_tp_pct *= 0.60 # take profit sooner under stress + if eff_imb < -0.10: # OB actively opposing + dynamic_max_hold = int(max_hold_bars * 0.40) + +elif regime_signal == -1 and eff_imb > 0.15: # CALM + FAVORABLE + dynamic_max_hold = int(max_hold_bars * 1.50) # let winners run + +# Per-asset withdrawal (micro-level): +if withdrawal_velocity < -0.20 and not in cascade/stress: + dynamic_max_hold = min(dynamic_max_hold, int(max_hold_bars * 0.40)) + if pnl_pct > 0.0: dynamic_tp_pct *= 0.75 +``` + +### 7.3 Sub-day ACB Force Exit + +When HZ listener fires an ACB update mid-day: +```python +# In update_acb_boost(boost, beta): +if old_boost >= 1.25 and boost < 1.10: + evaluate_subday_exits() # → _execute_exit("SUBDAY_ACB_NORMALIZATION", ...) +``` + +Threshold is ARBITRARY (not backtested). Marked research TODO. Safe under pending-flag pattern (fires on next bar, not mid-loop). + +### 7.4 Slippage on Exit + +```python +# SHORT position exit: +exit_price = current_price * (1.0 + slip) # slippage against us when covering short +# STOP_LOSS: slip = 0.0005 (5 bps — market order fill) +# FIXED_TP: slip = 0.0002 (2 bps — likely limit fill) +# All others: slip = 0.0002 +``` + +--- + +## 8. FEE & SLIPPAGE MODEL + +### 8.1 SmartPlacer Fee Model (Layer 3) + +**Source**: `esf_alpha_orchestrator.py`, `_execute_exit()` + +Blended taker/maker fee rates based on historical SP fill statistics. **IMPORTANT**: In production/paper sessions using Nautilus friction, these MUST be disabled via `use_sp_fees=False`. + +```python +# Entry fee (ONLY applied if use_sp_fees=True): +entry_fee = (0.0002 * sp_maker_entry_rate + 0.0005 * (1 - sp_maker_entry_rate)) * notional + = (0.0002 * 0.62 + 0.0005 * 0.38) * notional + = (0.0001240 + 0.0001900) * notional + = 0.000314 * notional (31.4 bps) +``` + +### 8.2 SP Slippage Refund (Layer 3) + +Also disabled when `use_sp_slippage=False` is passed to the engine. These were used to "re-approximate" fills in low-fidelity simulations. In paper/live trading, the matching engine provides the fill price directly. + +### 8.3 Production-Grade Native Friction (Nautilus) + +In `launch_paper_portfolio.py` and live production flows: +1. **Engine Bypass**: `use_sp_fees = False`, `use_sp_slippage = False`. +2. **Nautilus Node Side**: Commissions are applied by the kernel via `CommissionConfig`. +3. **Execution**: Slippage is realized via the spread in the Nautilus Sandbox (Paper) or on-chain (Live). + +### 8.4 Independent Session Logging + +Every high-fidelity session now deploys a `TradeLoggerActor` that independently captures: +- `logs/paper_trading/settings_.json`: Full configuration metadata. +- `logs/paper_trading/trades_.csv`: Every execution event. + +### 8.3 OB Edge (Layer 4) + +```python +# With real OB engine: +if ob_placement.depth_quality > 0.5: + pnl_pct_raw += ob_placement.fill_probability * ob_edge_bps * 1e-4 + +# Without OB engine (legacy Monte Carlo fallback): +if rng.random() < ob_confirm_rate (0.40): + pnl_pct_raw += ob_edge_bps * 1e-4 # default: +5 bps +``` + +**Net PnL**: +```python +gross_pnl = pnl_pct_raw * notional +net_pnl = gross_pnl - entry_fee - exit_fee +capital += net_pnl +``` + +--- + +## 9. OB INTELLIGENCE LAYER + +**Source**: `ob_features.py`, `ob_provider.py`, `hz_ob_provider.py` + +The OB layer is wired in via `engine.set_ob_engine(ob_engine)` which propagates to signal_gen, asset_selector, and exit_manager. It is OPTIONAL — the engine degrades gracefully to legacy Monte Carlo when `ob_engine=None`. + +### 9.1 OB Signals Per Asset + +```python +ob_signal = ob_engine.get_signal(asset, timestamp) +# Fields: +# imbalance_ma5 — 5-bar MA of bid/ask size imbalance ([-1, +1]) +# imbalance_persistence — fraction of last N bars sustaining sign +# withdrawal_velocity — rate of depth decay (negative = book thinning) +``` + +### 9.2 OB Macro (Market-Wide) + +```python +ob_macro = ob_engine.get_macro() +# Fields: +# cascade_count — number of assets in liquidation cascade +# regime_signal — (-1=calm/trending, 0=neutral, +1=withdrawal stress) +``` + +### 9.3 OB Placement Quality + +```python +ob_placement = ob_engine.get_placement(asset, timestamp) +# Fields: +# depth_quality — book depth score ([0, 2+]; >1 = deep book) +# fill_probability — probability of maker fill at entry price +``` + +### 9.4 OB Sub-Systems Summary + +| Sub | Location | Effect | +|-----|----------|--------| +| OB-1 | AlphaAssetSelector | ARS adjusted ±5%/10% by depth quality | +| OB-2 | AlphaSignalGenerator | Confidence boost/dampen by imbalance; hard skip if persistent contradiction | +| OB-3 | NDAlphaEngine._try_entry | Market-wide imbalance multiplier on final leverage (±20%/15%) | +| OB-4 | AdaptiveCircuitBreaker | Macro withdrawal stress modulates ACBv6 dynamic beta (+25% max) | +| OB-5 | AlphaExitManager | Dynamic TP/SL/max_hold based on cascade/withdrawal/calm regime | + +--- + +## 10. ACB v6 — ADAPTIVE CIRCUIT BREAKER + +### 10.1 Architecture (3-Scale Confirmation) + +**Source**: `adaptive_circuit_breaker.py`, `AdaptiveCircuitBreaker` + +``` +Scale 1 (Daily): External macro factors → log_0.5 base_boost +Scale 2 (Per-bar): vel_div signal strength → meta-boost multiplier +Scale 3 (Macro): w750 eigenvalue velocity → dynamic beta switch +``` + +### 10.2 Scale 1 — External Factor Signals + +Loaded from NG3 `scan_*__Indicators.npz` files (median of first 10 scans per date): + +| Factor | Source | Bearish Threshold | Signal Weight | +|--------|--------|------------------|---------------| +| `funding_btc` | Funding rate | < -0.0001 (very) / < 0.0 (mild) | 1.0 / 0.5 | +| `dvol_btc` | BTC implied vol | > 80 (extreme) / > 55 (elevated) | 1.0 / 0.5 | +| `fng` | Fear & Greed Index | < 25 (extreme fear) / < 40 (fear) | 1.0 / 0.5 (requires confirmation) | +| `taker` | Taker buy ratio | < 0.80 (selling) / < 0.90 (mild) | 1.0 / 0.5 | + +**Signal counting**: +```python +signals = sum(individual_signal_weights) # float, e.g. 2.5 +``` + +**Base boost formula**: +```python +if signals >= 1.0: + base_boost = 1.0 + 0.5 * log1p(signals) + # signals=1 → 1.347x | signals=2 → 1.549x | signals=3 → 1.693x +else: + base_boost = 1.0 +``` + +### 10.3 Scale 3 — Dynamic Beta + +```python +# Preloaded: w750_threshold = np.percentile(all_w750_vels, 60) +if w750_vel >= w750_threshold: + beta = BETA_HIGH = 0.8 # aggressive meta-boost during macro acceleration +else: + beta = BETA_LOW = 0.2 # conservative during calm macro +``` + +### 10.4 Scale 2 — Per-Bar Meta-Boost + +```python +# Computed every bar inside _update_regime_size_mult(vel_div): +strength_cubic = clamp((threshold - vel_div) / (threshold - extreme), 0, 1) ** 3 +# leverage_convexity = 3 → cubic + +if day_beta > 0: + regime_size_mult = base_boost * (1.0 + beta * strength_cubic) * mc_scale +else: + regime_size_mult = base_boost * mc_scale +``` + +### 10.5 Sub-Day ACB Update (HZ Listener) + +The `acb_processor_service.py` re-runs ACB computation mid-day when new NG3 scan data arrives and writes `{boost, beta}` to `DOLPHIN_FEATURES` IMap. + +`_on_acb_event()` in `DolphinActor` stores the payload in `self._pending_acb` (GIL-safe dict write). Applied at start of next `on_bar()` iteration: + +```python +# In on_bar() — BEFORE processing: +if _pending_acb is not None and engine is not None: + engine.update_acb_boost(pending_acb['boost'], pending_acb['beta']) + _pending_acb = None +``` + +--- + +## 11. SURVIVAL STACK — POSTURE CONTROL + +### 11.1 Overview + +**Source**: `survival_stack.py`, `SurvivalStack` + +Computes a continuous Risk Multiplier `Rm ∈ [0, 1]` from 5 sensor categories. Maps to discrete posture {APEX, STALKER, TURTLE, HIBERNATE}. + +### 11.2 Five Sensor Categories + +**Cat1 — Binary Invariant** (kill switch): +```python +if hz_nodes < 1 or heartbeat_age_s > 30: + return 0.0 # Total system failure → HIBERNATE immediately +return 1.0 +``` + +**Cat2 — Structural** (MC-Forewarner + data staleness): +```python +base = {OK: 1.0, ORANGE: 0.5, RED: 0.1}[mc_status] +decay = exp(-max(0, staleness_hours - 6) / 3) +f_structural = base * decay # Exponential decay after 6h stale +``` + +**Cat3 — Microstructure** (OB depth/fill quality): +```python +if ob_stale: + return 0.5 +score = min(depth_quality, fill_prob) +return clamp(0.3 + 0.7 * score, 0.3, 1.0) +``` + +**Cat4 — Environmental** (DVOL spike impulse): +```python +if dvol_spike and t_since_spike_min == 0: + return 0.3 # Instant degradation at spike +return 0.3 + 0.7 * (1 - exp(-t_since_spike_min / 60)) # 60-min recovery tau +``` + +**Cat5 — Capital** (sigmoid drawdown constraint): +```python +# Rm5 ≈ 1.0 at DD<5%, ≈ 0.5 at DD=12%, ≈ 0.1 at DD=20% +return 1 / (1 + exp(30 * (drawdown - 0.12))) +``` + +### 11.3 Hierarchical Combination + +```python +f_environment = min(f_structural, f_ext) # worst of Cat2/Cat4 +f_execution = f_micro # Cat3 +r_target = Cat1 * Cat5 * f_environment * f_execution + +# Correlated sensor collapse penalty: +degraded = count([f_structural < 0.8, f_micro < 0.8, f_ext < 0.8]) +if degraded >= 2: + r_target *= 0.5 +``` + +### 11.4 Bounded Recovery Dynamics + +```python +# Fast attack (instant degradation), slow recovery (5%/minute max): +if r_target < last_r_total: + r_final = r_target # immediate drop +else: + alpha = min(1.0, 0.02 * dt_min) + step = min(alpha * (r_target - last_r_total), 0.05 * dt_min) + r_final = last_r_total + step +``` + +### 11.5 Posture Mapping + +**NOTE: Thresholds are deliberately TIGHTER than mathematical spec (safety buffer).** + +```python +if Rm >= 0.90: APEX # Full trading, no constraints +if Rm >= 0.75: STALKER # Max leverage capped at 2.0x +if Rm >= 0.50: TURTLE # regime_dd_halt = True (no new entries) +else: HIBERNATE # Force-close open positions, no new entries +``` + +### 11.6 Hysteresis + +```python +# Down: requires hysteresis_down=2 consecutive bars at lower level +# Up: requires hysteresis_up=5 consecutive bars at higher level +# Prevents flip-flopping around thresholds +``` + +### 11.7 Posture → Engine Effect + +| Posture | Engine Effect | +|---------|--------------| +| APEX | No constraint (max leverage = abs_max=6.0 × regime_size_mult) | +| STALKER | `clamped_max_leverage = min(..., 2.0)` in `_try_entry` | +| TURTLE | `regime_dd_halt = True` → `process_bar` skips entry block | +| HIBERNATE | `_manage_position` forces EXIT("HIBERNATE_HALT"), `regime_dd_halt = True` | + +--- + +## 12. MC-FOREWARNER ENVELOPE GATE + +**Source**: Called via `engine.set_mc_forewarner(forewarner, mc_base_cfg)` + +Runs daily at start of `process_day()`: + +```python +mc_cfg = {**mc_base_cfg, 'max_leverage': base_max_leverage * day_base_boost} +mc_report = forewarner.assess_config_dict(mc_cfg) + +mc_red = mc_report.catastrophic_probability > 0.25 or mc_report.envelope_score < -1.0 +mc_orange = (not mc_red) and (envelope_score < 0 or catastrophic_probability > 0.10) + +day_mc_status = 'RED' | 'ORANGE' | 'OK' +day_mc_scale = 0.5 if ORANGE else 1.0 + +if mc_red or posture in ['TURTLE', 'HIBERNATE']: + regime_dd_halt = True # No new entries for the entire day +``` + +--- + +## 13. NDAlphaEngine — FULL BAR LOOP + +### 13.1 process_day() Flow + +```python +def process_day(date_str, df, asset_columns, vol_regime_ok, direction, posture): + # --- Per-day initialization --- + regime_direction = direction or -1 # -1=SHORT champion + regime_dd_halt = False + day_posture = posture + + # --- ACB: Scale 1 + Scale 3 (once per day) --- + if acb: + info = acb.get_dynamic_boost_for_date(date_str, ob_engine) + day_base_boost = info['boost'] # log_0.5 curve + day_beta = info['beta'] # dynamic beta (w750 percentile switch) + + # --- MC-Forewarner gate (once per day) --- + if forewarner: + mc_report = forewarner.assess_config_dict(...) + if mc_red or posture in [TURTLE, HIBERNATE]: + regime_dd_halt = True # kill new entries for day + + # === BAR LOOP === + for ri, row in df.iterrows(): + vd = row['vel_div'] # primary signal + v50_vel = row['v50_lambda_max_velocity'] + v750_vel = row['v750_lambda_max_velocity'] + prices = {col: row[col] for col in asset_columns if row[col] > 0} + + # Append to price_histories (capped at 200; sliding window) + for ac, p in prices.items(): + price_histories[ac].append(p) + if len(price_histories[ac]) > 500: + price_histories[ac] = price_histories[ac][-200:] + + vol_ok = vol_regime_ok[ri] if vol_regime_ok else (bid >= 100) + + # ACB Scale 2: regime_size_mult updated every bar + _update_regime_size_mult(vd) + + process_bar(global_bar_idx, vd, prices, vol_ok, price_histories, v50_vel, v750_vel) + global_bar_idx += 1 + + return {date, pnl, capital, boost, beta, mc_status, trades} +``` + +### 13.2 process_bar() Flow + +```python +def process_bar(bar_idx, vel_div, prices, vol_regime_ok, price_histories, v50_vel, v750_vel): + bar_count += 1 + vel_div_history.append(vel_div) # trimmed to 200 + + # === EXIT MANAGEMENT (always first) === + if position is not None: + exit_info = _manage_position(bar_idx, prices, vel_div, v50_vel, v750_vel) + # → AlphaExitManager.evaluate() → if EXIT: _execute_exit() + + # === ENTRY (only when no position) === + if position is None AND bar_idx > last_exit_bar AND NOT regime_dd_halt: + if bar_count >= lookback (100) AND vol_regime_ok: + entry_info = _try_entry(bar_idx, vel_div, prices, price_histories, v50_vel, v750_vel) +``` + +### 13.3 _try_entry() Flow + +```python +def _try_entry(bar_idx, vel_div, prices, price_histories, v50_vel, v750_vel): + if capital <= 0: return None + + # 1. IRP Asset Selection (Layer 2) + if use_asset_selection: + market_data = {a: history[-50:] for a, history in price_histories if len >= 50} + rankings = asset_selector.rank_assets(market_data, regime_direction) + trade_asset = first_asset_passing_all_gates(rankings) + if trade_asset is None: return None # strict: no fallback + else: + trade_asset = "BTCUSDT" # fallback when IRP disabled + + # 2. Signal Generation + DC (Layer 6) + signal = signal_gen.generate(vel_div, vel_div_history, + price_histories[trade_asset], + regime_direction, trade_asset) + if not signal.is_valid: return None # vel_div or DC killed it + + # 3. Position Sizing (Layers 7-8) + size = bet_sizer.calculate_size(capital, vel_div, signal.vel_div_trend, regime_direction) + + # 4. OB Sub-3: Cross-asset market multiplier + market_ob_mult = ob_engine.get_market_multiplier(...) # ±20% + + # 5. ACB leverage ceiling enforcement + clamped_max = min(base_max_leverage * regime_size_mult * market_ob_mult, abs_max_leverage=6.0) + if posture == STALKER: clamped_max = min(clamped_max, 2.0) + final_leverage = clamp(size.leverage * regime_size_mult * market_ob_mult, min_lev, clamped_max) + + # 6. Notional and entry + notional = capital * size.fraction * final_leverage + entry_price = prices[trade_asset] + + # 7. Create position + position = NDPosition(trade_asset, regime_direction, entry_price, + notional, final_leverage, ...) + exit_manager.setup_position(trade_id, entry_price, direction, bar_idx, v50_vel, v750_vel) +``` + +--- + +## 14. DOLPHIN ACTOR — NAUTILUS INTEGRATION + +**Source**: `nautilus_dolphin/nautilus_dolphin/nautilus/dolphin_actor.py` +**Base**: `nautilus_trader.trading.strategy.Strategy` (Rust/Cython core) +**Lines**: 338 + +### 14.1 Lifecycle (v2 — step_bar API) + +``` +__init__: + dolphin_config, engine=None, hz_client=None + current_date=None, posture='APEX', _processed_dates=set() + _pending_acb: dict|None = None + _acb_lock = threading.Lock() ← v2: explicit lock (not GIL reliance) + _stale_state_events = 0 + _day_data = None, _bar_idx_today = 0 + +on_start(): + 1. _connect_hz() → HazelcastClient(cluster="dolphin", members=["localhost:5701"]) + 2. _read_posture() → DOLPHIN_SAFETY (CP AtomicRef, map fallback) + 3. _setup_acb_listener() → add_entry_listener(DOLPHIN_FEATURES["acb_boost"]) + 4. create_boost_engine(mode=boost_mode, **engine_kwargs) → NDAlphaEngine + 5. MC-Forewarner injection (gold-performance stack — always active): + mc_models_dir = config.get('mc_models_dir', _MC_MODELS_DIR_DEFAULT) + if Path(mc_models_dir).exists(): + forewarner = DolphinForewarner(models_dir=mc_models_dir) + engine.set_mc_forewarner(forewarner, _MC_BASE_CFG) + ← graceful degradation: logs warning + continues if models missing + ← disable explicitly: set mc_models_dir=None/'' in config + +on_bar(bar): + ① Drain ACB under _acb_lock: + pending = _pending_acb; _pending_acb = None ← atomic swap + if pending: engine.update_acb_boost(boost, beta) + + ② Date boundary: + date_str = datetime.fromtimestamp(bar.ts_event/1e9, UTC).strftime('%Y-%m-%d') + if current_date != date_str: + if current_date: engine.end_day() + current_date = date_str + posture = _read_posture() + _bar_idx_today = 0 + engine.begin_day(date_str, posture=posture, direction=±1) + if not live_mode: _load_parquet_data(date_str) → _day_data + + ③ HIBERNATE guard: if posture=='HIBERNATE': return ← hard skip, no step_bar + + ④ Feature extraction: + live_mode=False → if _day_data empty: return ← early exit, no step_bar with zeros + elif _bar_idx_today >= len(df): return ← end-of-day + else: row = df.iloc[_bar_idx_today], vol_regime_ok = (idx>=100) + live_mode=True → _get_latest_hz_scan(), staleness check (>10s → warning), + dedup on scan_number + + ⑤ _GateSnap BEFORE: (acb_boost, acb_beta, posture, mc_gate_open) + + ⑥ engine.pre_bar_proxy_update(inst50, v750_vel) ← if ProxyBoostEngine + + ⑦ result = engine.step_bar(bar_idx, vel_div, prices, v50_vel, v750_vel, vol_regime_ok) + _bar_idx_today += 1 + + ⑧ _GateSnap AFTER: compare → if changed: stale_state_events++, result['stale_state']=True + + ⑨ _write_result_to_hz(date_str, result) + +on_stop(): + _processed_dates.clear() + _stale_state_events = 0 + if hz_client: hz_client.shutdown() +``` + +### 14.2 Thread Safety: ACB Pending-Flag Pattern (v2) + +**CRITICAL**: HZ entry listeners run on HZ client pool threads, NOT the Nautilus event loop. + +```python +# HZ listener thread — parse outside lock, assign inside lock: +def _on_acb_event(event): + try: + val = event.value + if val: + parsed = json.loads(val) # CPU work OUTSIDE lock + with self._acb_lock: + self._pending_acb = parsed # atomic write under lock + except Exception as e: + self.log.error(f"ACB event parse error: {e}") + +# Nautilus event loop — drain under lock, apply outside lock: +def on_bar(bar): + with self._acb_lock: + pending = self._pending_acb + self._pending_acb = None # atomic consume under lock + if pending is not None and self.engine is not None: + boost = float(pending.get('boost', 1.0)) + beta = float(pending.get('beta', 0.0)) + self.engine.update_acb_boost(boost, beta) +``` + +**v2 vs v1**: v1 relied on GIL for safety (bare dict assignment). v2 uses explicit `threading.Lock` — correct even if GIL is removed in future Python versions. Lock hold time is minimized to a single pointer swap. + +### 14.3 _GateSnap — Stale-State Detection + +New in v2. Detects when ACB boost, posture, or MC gate changes between the pre-step and post-step snapshot: + +```python +_GateSnap = namedtuple('_GateSnap', ['acb_boost', 'acb_beta', 'posture', 'mc_gate_open']) + +before = _GateSnap(engine._day_base_boost, engine._day_beta, posture, engine._mc_gate_open) +result = engine.step_bar(...) +after = _GateSnap(engine._day_base_boost, engine._day_beta, _read_posture(), engine._mc_gate_open) + +if before != after: + self._stale_state_events += 1 + self.log.warning(f"[STALE_STATE] gate changed mid-eval: {changed_fields}") + result['stale_state'] = True # flagged in HZ write — DO NOT use for live orders +``` + +### 14.4 Replay vs Live Mode + +| | Replay Mode (live_mode=False) | Live Mode (live_mode=True) | +|---|---|---| +| Data source | `vbt_cache_klines/YYYY-MM-DD.parquet` | `DOLPHIN_FEATURES["latest_eigen_scan"]` (HZ) | +| Per-bar iteration | `df.iloc[_bar_idx_today]` | One bar = one HZ scan fetch | +| vol_regime_ok | `bar_idx >= 100` (warmup) | From scan dict | +| Stale guard | — | `abs(now_ns - scan_ts_ns) > 10s` → warning | +| Dedup | — | `scan_num == last_scan_number` → skip | + +### 14.5 Data Loading (Replay) + +```python +def _load_parquet_data(date_str): + path = HCM_DIR / "vbt_cache_klines" / f"{date_str}.parquet" + df = pd.read_parquet(path) + meta_cols = {vel_div, scan_number, v50_..., v750_..., instability_50, instability_150} + asset_columns = [c for c in df.columns if c not in meta_cols] + return df, asset_columns, None # vol_regime_ok deferred to on_bar warmup check +``` + +### 14.6 Posture Reading + +Primary: `HZ CP Subsystem AtomicReference('DOLPHIN_SAFETY')` — linearizable. +Fallback: `HZ IMap('DOLPHIN_SAFETY').get('latest')` — eventually consistent. +Default when HZ unavailable: `'APEX'` (non-fatal degradation). + +### 14.7 Result Writing + +```python +def _write_result_to_hz(date_str, result): + if not self.hz_client: return # silent noop + imap_pnl = hz_client.get_map('DOLPHIN_PNL_BLUE').blocking() + imap_pnl.put(date_str, json.dumps(result)) + if result.get('stale_state'): + self.log.error("[STALE_STATE] DO NOT use for live order submission") + # result: {date, pnl, capital, boost, beta, mc_status, trades, stale_state?} +``` + +### 14.8 Important Notes for Callers + +- **`actor.log` is read-only** (Rust-backed Cython property). Never try to assign `actor.log = MagicMock()` in tests — use the real Nautilus logger instead. +- **`actor.posture`** is a regular Python attribute (writable in tests). +- **`actor.engine`** is set in `on_start()`. Tests can set directly after `__init__`. + +--- + +## 15. HAZELCAST — FULL IMAP SCHEMA + +Hazelcast is the **system memory**. All subsystem state flows through it. Every consumer must treat HZ maps as authoritative real-time sources. + +**Infrastructure**: Hazelcast 5.3, Docker (`prod/docker-compose.yml`), `localhost:5701`, cluster `"dolphin"`. +**CP Subsystem**: Enabled — required for ACB atomic operations. +**Management Center**: `http://localhost:8080`. +**Python client**: `hazelcast-python-client 5.6.0` (siloqy-env). + +### 15.1 Complete IMap Reference + +| Map | Key | Value | Writer | Reader(s) | Notes | +|---|---|---|---|---|---| +| `DOLPHIN_SAFETY` | `"latest"` | JSON `{posture, Rm, sensors, ...}` | `system_watchdog_service.py` | `DolphinActor`, `paper_trade_flow`, `nautilus_prefect_flow` | CP AtomicRef preferred; IMap fallback | +| `DOLPHIN_FEATURES` | `"acb_boost"` | JSON `{boost, beta}` | `acb_processor_service.py` | `DolphinActor` (HZ entry listener) | Triggers `_on_acb_event` | +| `DOLPHIN_FEATURES` | `"latest_eigen_scan"` | JSON `{vel_div, scan_number, asset_prices, timestamp_ns, w50_velocity, w750_velocity, instability_50}` | Eigenvalue scanner bridge | `DolphinActor` (live mode) | Dedup on scan_number | +| `DOLPHIN_PNL_BLUE` | `"YYYY-MM-DD"` | JSON daily result `{pnl, capital, trades, boost, beta, mc_status, posture, stale_state?}` | `paper_trade_flow`, `DolphinActor._write_result_to_hz`, `nautilus_prefect_flow` | Analytics | stale_state=True means DO NOT use for live orders | +| `DOLPHIN_PNL_GREEN` | `"YYYY-MM-DD"` | JSON daily result | `paper_trade_flow` (green) | Analytics | GREEN config only | +| `DOLPHIN_STATE_BLUE` | `"latest"` | JSON `{strategy, capital, date, pnl, trades, peak_capital, drawdown, engine_state, updated_at}` | `paper_trade_flow` | `paper_trade_flow` (capital restore) | Full engine_state for position continuity | +| `DOLPHIN_STATE_BLUE` | `"latest_nautilus"` | JSON `{strategy, capital, date, pnl, trades, posture, param_hash, engine, updated_at}` | `nautilus_prefect_flow` | `nautilus_prefect_flow` (capital restore) | param_hash = champion SHA256[:16] | +| `DOLPHIN_STATE_BLUE` | `"state_{strategy}_{date}"` | JSON per-run snapshot | `paper_trade_flow` | Recovery | Full historical per-run snapshots | +| `DOLPHIN_HEARTBEAT` | `"nautilus_flow_heartbeat"` | JSON `{ts, iso, run_date, phase, flow}` | `nautilus_prefect_flow` (heartbeat_task) | External monitoring | Written at flow_start, engine_start, flow_end | +| `DOLPHIN_HEARTBEAT` | `"probe_ts"` | Timestamp string | `nautilus_prefect_flow` (hz_probe_task) | Liveness check | Written at HZ probe time | +| `DOLPHIN_OB` | per-asset key | JSON OB snapshot | `obf_prefect_flow` | `HZOBProvider` | Raw OB map | +| `DOLPHIN_FEATURES_SHARD_00` | symbol | JSON OB feature dict `{imbalance, fill_probability, depth_quality, regime_signal, ...}` | `obf_prefect_flow` | `HZOBProvider` | shard routing (see §15.2) | +| `DOLPHIN_FEATURES_SHARD_01..09` | symbol | Same schema | `obf_prefect_flow` | `HZOBProvider` | — | +| `DOLPHIN_SIGNALS` | signal key | Signal distribution | `signal_bridge.py` | Strategy consumers | — | +| `DOLPHIN_FEATURES` | `"obf_universe_latest"` | JSON `{_snapshot_utc, _n_assets, assets: {symbol: {spread_bps, depth_1pct_usd, depth_quality, fill_probability, imbalance, best_bid, best_ask, n_bid_levels, n_ask_levels}}}` | `obf_universe_service.py` | MHS v3 (M5 coherence), Asset Picker | 540 USDT perps; 60s push cadence. NEW v5.0 | +| `DOLPHIN_META_HEALTH` | `"latest"` | JSON `{rm_meta, status, m4_control_plane, m1_data_infra, m1_trader, m2_heartbeat, m3_data_freshness, m5_coherence, service_status, hz_key_status, timestamp}` | `meta_health_service_v3.py` | External monitoring, MHS tests | GREEN/DEGRADED/CRITICAL/DEAD. NEW v5.0 | + +### 15.2 OBF Shard Routing + +```python +SHARD_COUNT = 10 +shard_idx = sum(ord(c) for c in symbol) % SHARD_COUNT +imap_name = f"DOLPHIN_FEATURES_SHARD_{shard_idx:02d}" # ..._00 through ..._09 +``` + +Routing is **stable** (sum-of-ord, not `hash()`) — deterministic across Python versions and process restarts. 400+ assets distribute evenly across 10 shards. + +### 15.3 ShardedFeatureStore API + +**Source**: `hz_sharded_feature_store.py`, `ShardedFeatureStore` + +```python +store = ShardedFeatureStore(hz_client) +store.put('BTCUSDT', 'vel_div', -0.03) # routes to shard based on symbol hash +val = store.get('BTCUSDT', 'vel_div') +store.delete('BTCUSDT', 'vel_div') +# Internal key format: "vel_div_BTCUSDT" +``` + +Near cache config: TTL=300s, invalidate_on_change=True, LRU eviction, max_size=5000 per shard. + +### 15.4 HZOBProvider — Dynamic Asset Discovery + +```python +# On connect (lazy), discovers which assets are present in any shard: +for shard_idx in range(SHARD_COUNT): + key_set = client.get_map(f"DOLPHIN_FEATURES_SHARD_{shard_idx:02d}").blocking().key_set() + discovered_assets.update(key_set) +``` + +No static asset list required — adapts automatically as OBF flow adds/removes assets. + +### 15.5 CP Subsystem (ACB Processor) + +`acb_processor_service.py` uses `HZ CP FencedLock` to prevent simultaneous ACB writes from multiple instances. CP Subsystem must be enabled in `docker-compose.yml`. All writers must use the same CP lock name to get protection. + +### 15.6 OBF Circuit Breaker (HZ Push) + +After 5 consecutive HZ push failures, OBF flow opens a circuit breaker and switches to file-only mode (`ob_cache/latest_ob_features.json`). Consumers should prefer the JSON file during HZ outages. + +--- + +## 16. PRODUCTION DAEMON TOPOLOGY + +> **v5.0 NOTE**: ALL services are managed exclusively by **supervisord**. No service is managed by systemd. The `meta_health_daemon.service`, `dolphin-nautilus-trader.service`, and `dolphin-scan-bridge.service` systemd units are stopped and disabled. Any attempt to re-enable them will create a dual-management race condition ("random killer" bug — see §26.1). + +### 16.1 Supervisord Config + +**File**: `/mnt/dolphinng5_predict/prod/supervisor/dolphin-supervisord.conf` +**Socket**: `/tmp/dolphin-supervisor.sock` +**PYTHONPATH** (dolphin_data group): `/mnt/dolphinng5_predict:/mnt/dolphinng5_predict/nautilus_dolphin:/mnt/dolphinng5_predict/prod` + +```bash +# Status check +supervisorctl -c /mnt/dolphinng5_predict/prod/supervisor/dolphin-supervisord.conf status + +# Restart a service +supervisorctl -c /mnt/dolphinng5_predict/prod/supervisor/dolphin-supervisord.conf restart dolphin_data:meta_health +``` + +### 16.2 dolphin_data Group (autostart=true — data pipeline) + +| Program | Full Path | Purpose | startsecs | +|---|---|---|---| +| `exf_fetcher` | `/mnt/dolphinng5_predict/prod/exf_fetcher_flow.py --warmup 15` | ExF live daemon: funding/dvol/fng/taker → HZ `exf_latest` | 20 | +| `acb_processor` | `/mnt/dolphinng5_predict/prod/acb_processor_service.py` | ACBv6 daily boost + dynamic beta → HZ `acb_boost` (CP FencedLock) | 10 | +| `obf_universe` | `/mnt/dolphinng5_predict/prod/obf_universe_service.py` | 540-asset OBF universe L2 health → HZ `obf_universe_latest` | 15 | +| `meta_health` | `/mnt/dolphinng5_predict/prod/meta_health_service_v3.py` | MHS v3 watchdog — monitors all data services, auto-restarts | 5 | + +### 16.3 dolphin Group (autostart=false — trading, started manually) + +| Program | Full Path | Purpose | Notes | +|---|---|---|---| +| `nautilus_trader` | `/mnt/dolphinng5_predict/prod/nautilus_event_trader.py` | HZ entry listener trader | Start only during trading hours | +| `scan_bridge` | `/mnt/dolphinng5_predict/prod/scan_bridge_service.py` | Arrow → HZ scan bridge | Start when NG8 active | +| `clean_arch_trader` | `/mnt/dolphinng5_predict/prod/clean_arch/main.py` | Clean architecture trader | Experimental | + +### 16.4 ACB Processor (`acb_processor_service.py`) + +**Purpose**: ACBv6 daily boost + dynamic beta from NG3 NPZ files → HZ `DOLPHIN_FEATURES["acb_boost"]`. +**HZ**: CP FencedLock prevents simultaneous writes. + +### 16.5 OBF Universe (`obf_universe_service.py`) — NEW v5.0 + +**Purpose**: L2 health monitor for all 540 USDT perpetuals → HZ `DOLPHIN_FEATURES["obf_universe_latest"]`. +**Coverage**: 540 active USDT perps, 3 WS connections (200/200/140 streams). +**Stream**: `{symbol}@depth5@500ms` — zero REST weight. +**Cadence**: 60s health snapshots; 300s Parquet flush. +**Storage**: `/mnt/ng6_data/ob_universe/` (Hive partitioned; `MAX_FILE_AGE_DAYS=0` — never pruned). +**See §26.2 for full schema.** + +### 16.6 Meta Health Service v3 (`meta_health_service_v3.py`) — NEW v5.0 + +**Purpose**: 5-sensor weighted health monitor + auto-recovery for all data pipeline services. +**Recovery**: `supervisorctl restart` via daemon thread. `RECOVERY_COOLDOWN_CRITICAL_S=10s`. +**Output**: `DOLPHIN_META_HEALTH["latest"]` + `/mnt/dolphinng5_predict/run_logs/meta_health.json`. +**Announcements (BLUE-first)**: posture transitions and sensor red-entry events are routed through +`prod/announcement_router.py`, spooled to `/mnt/dolphin_training/observability_announcements_blue.jsonl`, +and mirrored to `DOLPHIN_ANNOUNCEMENTS["latest"]` for the TUI. Telegram + SMTP targets are config-driven +placeholders in `prod/configs/observability_notifications_blue.json` and are disabled until populated. +**See §26.3 for full specification.** + +### 16.7 ExF Daemon (`exf_fetcher_flow.py`) + +**Purpose**: External factors — funding rate, DVOL, Fear&Greed, taker ratio → HZ `DOLPHIN_FEATURES["exf_latest"]`. +**Field**: `_pushed_at` (Unix timestamp) is the canonical freshness field. + +### 16.8 MC-Forewarner Flow (`mc_forewarner_flow.py`) + +**Purpose**: Prefect-orchestrated daily ML assessment. Outcome: OK / ORANGE / RED → HZ. +**Effect**: ORANGE → `day_mc_scale=0.5`. RED → `regime_dd_halt=True`. + +### 16.9 paper_trade_flow.py (Primary — 00:05 UTC) + +**Purpose**: Daily NDAlphaEngine run. Loads klines, wires ACB+OB+MC, runs `begin_day/step_bar/end_day`. +**Direction**: `direction = -1` (SHORT, blue). + +### 16.10 Daemon Start Sequence + +``` +1. docker-compose up -d ← Hazelcast 5701, ManCenter 8080, Prefect 4200 +2. supervisord (auto) ← starts dolphin_data group automatically on boot + └── exf_fetcher, acb_processor, obf_universe, meta_health start in parallel + +3. (Manual when needed): + supervisorctl start dolphin:nautilus_trader ← HZ entry listener + supervisorctl start dolphin:scan_bridge ← when DolphinNG6 active + +4. Prefect deployments (daily, scheduled): + paper_trade_flow.py ← 00:05 UTC + nautilus_prefect_flow.py ← 00:10 UTC + mc_forewarner_flow.py ← daily +``` + +### 16.11 Monitoring Endpoints + +| Service | URL / Command | +|---|---| +| Hazelcast Management Center | `http://localhost:8080` | +| Prefect UI | `http://localhost:4200` | +| Supervisord status | `supervisorctl -c /mnt/dolphinng5_predict/prod/supervisor/dolphin-supervisord.conf status` | +| MHS health JSON | `cat /mnt/dolphinng5_predict/run_logs/meta_health.json` | +| Daily PnL | `HZ IMap DOLPHIN_PNL_BLUE[YYYY-MM-DD]` | +| ACB State | `HZ IMap DOLPHIN_FEATURES["acb_boost"]` | +| OBF Universe | `HZ IMap DOLPHIN_FEATURES["obf_universe_latest"]` | + +--- + +## 17. PREFECT ORCHESTRATION LAYER + +**Version**: Prefect 3.6.22 (siloqy-env) +**Server**: `http://localhost:4200/api` +**Work pool**: `dolphin` (process type) +**Worker command**: `prefect worker start --pool dolphin --type process` + +### 17.1 Registered Deployments + +| Deployment | Flow | Schedule | Config | +|---|---|---|---| +| `dolphin-paper-blue` | `paper_trade_flow.py` | `0 0 * * *` (00:05 UTC) | `configs/blue.yml` | +| `dolphin-paper-green` | `paper_trade_flow.py` | `0 0 * * *` (00:05 UTC) | `configs/green.yml` | +| `dolphin-nautilus-blue` | `nautilus_prefect_flow.py` | `10 0 * * *` (00:10 UTC) | `configs/blue.yml` | + +### 17.2 nautilus_prefect_flow.py — Nautilus BacktestEngine Supervisor + +New in v2. Tasks in execution order: + +``` +hz_probe_task retries=3 timeout=30s — verify HZ reachable; abort on failure +validate_champion_params retries=0 timeout=10s — SHA256 hash vs FROZEN params; ValueError on drift +load_bar_data_task retries=2 timeout=120s — load vbt_cache_klines parquet; validate vel_div col +read_posture_task retries=2 timeout=20s — read DOLPHIN_SAFETY +restore_capital_task retries=2 timeout=20s — restore capital from DOLPHIN_STATE_BLUE + → HIBERNATE? skip engine, write result, heartbeat, return +run_nautilus_backtest_task retries=0 timeout=600s — BacktestEngine + DolphinActor full cycle +write_hz_result_task retries=3 timeout=30s — DOLPHIN_PNL_BLUE + DOLPHIN_STATE_BLUE write +heartbeat_task retries=0 timeout=15s — phase=flow_end +``` + +**Champion integrity**: `_CHAMPION_HASH = sha256(json.dumps(_CHAMPION_PARAMS, sort_keys=True))[:16]`. Computed at import time. Any config drift triggers `ValueError` before engine starts. + +**Capital continuity**: Restores from `DOLPHIN_STATE_BLUE["latest_nautilus"]`. Falls back to `initial_capital` (25,000 USDT) if absent. + +### 17.3 paper_trade_flow.py — Task Reference + +| Task | Retries | Purpose | +|---|---|---| +| `load_config` | 0 | YAML config load | +| `load_day_scans` | 2 | Parquet (preferred) or JSON fallback; vel_div validation | +| `run_engine_day` | 0 | begin_day/step_bar×N/end_day; returns daily stats | +| `write_hz_state` | 3 | DOLPHIN_STATE_BLUE + DOLPHIN_PNL_BLUE persist | +| `log_pnl` | 0 | Disk JSONL append (`paper_logs/{color}/`) | + +### 17.4 Registration Commands + +```bash +source /home/dolphin/siloqy_env/bin/activate +PREFECT_API_URL=http://localhost:4200/api + +python prod/paper_trade_flow.py --register # blue + green paper deployments +python prod/nautilus_prefect_flow.py --register # nautilus blue deployment +``` + +### 17.5 Manual Run + +```bash +# Paper trade: +python prod/paper_trade_flow.py --config prod/configs/blue.yml --date 2026-03-21 + +# Nautilus supervisor: +python prod/nautilus_prefect_flow.py --date 2026-03-21 + +# Dry-run (data + param validation, no engine): +python prod/nautilus_prefect_flow.py --date 2026-03-21 --dry-run +``` + +--- + +## 18. CI TEST SUITE + +### 18.1 Test Suites Overview + +| Suite | Location | Runner | Gate | +|-------|----------|--------|------| +| Nautilus bootstrap | `nautilus_dolphin/tests/test_0_nautilus_bootstrap.py` | `pytest nautilus_dolphin/tests/test_0_nautilus_bootstrap.py -v` | 11/11 | +| DolphinActor | `nautilus_dolphin/tests/test_dolphin_actor.py` | `pytest nautilus_dolphin/tests/test_dolphin_actor.py -v` | 35/35 | +| OBF unit tests | `tests/test_obf_unit.py` | `pytest tests/test_obf_unit.py -v` | ~120/~120 | +| Legacy CI | `ci/` directory | `pytest ci/ -v` | 14/14 | +| ACB + HZ status | `prod/tests/test_acb_hz_status_integrity.py` | `pytest prod/tests/test_acb_hz_status_integrity.py -v` | 118/118 | +| **MHS v3** | `prod/tests/test_mhs_v3.py` | `pytest prod/tests/test_mhs_v3.py -v` | **111/111** | + +**Total: 46 Nautilus + ~120 OBF + 14 legacy CI + 118 ACB/HZ + 111 MHS = ~409 tests green.** + +**Run all prod tests**: +```bash +source /home/dolphin/siloqy_env/bin/activate +cd /mnt/dolphinng5_predict +python -m pytest prod/tests/ -v --tb=short +``` + +### 18.2 Nautilus Bootstrap Tests (11 tests) + +`test_0_nautilus_bootstrap.py` — foundation sanity checks: +- Nautilus import, catalog construction, Bar/BarType creation +- DolphinActor instantiation without full kernel (uses `__new__` + `__init__` pattern) +- Champion config loading from blue.yml +- HZ connectivity probe (skip if HZ unavailable) +- BacktestEngine construction with DolphinActor registered + +### 18.3 DolphinActor Tests (35 tests, 8 classes) + +`test_dolphin_actor.py` — full behavioral coverage: + +| Class | Tests | What It Covers | +|-------|-------|----------------| +| `TestChampionParamInvariants` | 6 | Config loading, SHA256 hash stability, frozen param values, blue.yml parity | +| `TestACBPendingFlagThreadSafety` | 5 | Lock acquisition, JSON parse outside lock, dict assign inside lock, concurrent event safety | +| `TestHibernatePostureGuard` | 3 | HIBERNATE skips engine entirely, APEX/STALKER/TURTLE pass through, posture gate logic | +| `TestDateChangeHandling` | 5 | Date rollover triggers end_day/begin_day, once-per-date guard, bar_idx reset | +| `TestHZUnavailableDegradation` | 4 | HZ down → engine continues with stale OB features; heartbeat errors silenced; file fallback | +| `TestReplayModeBarTracking` | 3 | bar_idx increments per step_bar call; total_bars_processed correct; replay vs live mode flag | +| `TestOnStopCleanup` | 4 | on_stop writes final HZ result; HZ down on stop is non-fatal; engine state serialized | +| `TestStaleStateGuard` | 5 | _GateSnap detects mid-eval posture/acb changes; snap mismatch triggers abort; re-eval on next bar | + +**Critical implementation note**: `actor.log` is a Cython/Rust-backed read-only property on `Actor`. +Do NOT attempt `actor.log = MagicMock()` — raises `AttributeError: attribute 'log' of ... objects is not writable`. +The real Nautilus logger is initialized by `super().__init__()` and works in test context. + +### 18.4 Legacy CI Tests (14 tests) + +**Location**: `ci/` directory. Runner: `pytest ci/ -v` + +| File | Tests | What It Covers | +|------|-------|----------------| +| `test_13_nautilus_integration.py` | 6 | Actor import, instantiation, on_bar, HIBERNATE posture, once-per-day guard, ACB thread safety | +| `test_14_long_system.py` | 3 | Multi-day run, capital persistence, trade count | +| `test_15_acb_reactive.py` | 1 | ACB boost update applied correctly mid-day | +| `test_16_scaling.py` | 4 | Memory footprint <4GB (50 assets), shard routing (400 symbols), 400-asset no-crash, 400-asset with IRP | + +### 18.5 Key Test Patterns + +**ACB pending-flag pattern** (ThreadSafety test): +```python +# JSON parse OUTSIDE lock, dict assign INSIDE lock +with patch.object(actor.engine, 'update_acb_boost') as mock_update: + actor._on_acb_event(event) + assert actor._pending_acb['boost'] == 1.35 + mock_update.assert_not_called() # engine NOT called from listener thread +``` + +**Date rollover pattern** (DateChange test): +```python +# Fires 3 bars on same date → assert begin_day.call_count == 1 +# Fires 1 bar on next date → assert begin_day.call_count == 2, end_day.call_count == 1 +``` + +**_GateSnap stale-state detection**: +```python +# Snap taken at start of step_bar; posture changes mid-eval → abort, retry next bar +snap = actor._gate_snap # namedtuple(acb_boost, acb_beta, posture, mc_gate_open) +``` + +--- + +## 19. PARAMETER REFERENCE + +### 19.1 Champion Parameters (Frozen) + +| Parameter | Value | Layer | Notes | +|-----------|-------|-------|-------| +| `vel_div_threshold` | -0.02 | Signal gate | PRIMARY entry threshold | +| `vel_div_extreme` | -0.05 | Signal/sizing | Extreme regime: full size | +| `min_leverage` | 0.5 | Sizing | Floor leverage | +| `max_leverage` | 5.0 | Sizing | Base ceiling (before ACB) | +| `abs_max_leverage` | 6.0 | Sizing | Hard geometric ceiling | +| `leverage_convexity` | 3.0 | Sizing | Cubic convex curve | +| `fraction` | 0.20 | Sizing | Max capital fraction per trade | +| `fixed_tp_pct` | 0.0095 | Exit | 95 bps take-profit | +| `stop_pct` | 1.0 | Exit | Effectively disabled | +| `max_hold_bars` | 120 | Exit | 600 seconds | +| `dc_lookback_bars` | 7 | DC | 35 seconds price momentum | +| `dc_min_magnitude_bps` | 0.75 | DC | Minimum BTC momentum | +| `dc_skip_contradicts` | True | DC | Hard skip on contradiction | +| `min_irp_alignment` | 0.45 | IRP | Alignment gate | +| `sp_maker_entry_rate` | 0.62 | Fees | 62% maker fill at entry | +| `sp_maker_exit_rate` | 0.50 | Fees | 50% maker fill at exit | +| `ob_edge_bps` | 5.0 | OB | Legacy MC OB edge | +| `ob_confirm_rate` | 0.40 | OB | Legacy MC confirmation rate | +| `lookback` | 100 | Warmup | Bars before first entry allowed | +| `seed` | 42 | RNG | Deterministic numpy RandomState | + +### 19.2 ACBv6 Parameters (Frozen — Validated) + +| Parameter | Value | Notes | +|-----------|-------|-------| +| `BETA_HIGH` | 0.8 | w750 above p60 threshold | +| `BETA_LOW` | 0.2 | w750 below p60 threshold | +| `W750_THRESHOLD_PCT` | 60 | Percentile switch point | +| `FUNDING_VERY_BEARISH` | -0.0001 | 1.0 signal | +| `DVOL_EXTREME` | 80 | 1.0 signal | +| `FNG_EXTREME_FEAR` | 25 | 1.0 signal (needs confirmation) | +| `TAKER_SELLING` | 0.8 | 1.0 signal | + +### 19.3 Survival Stack Thresholds (Deliberately Tight) + +| Posture | Rm Threshold | vs. Math Spec | +|---------|-------------|---------------| +| APEX | ≥ 0.90 | Tighter — spec was 0.85 | +| STALKER | ≥ 0.75 | Tighter — spec was 0.70 | +| TURTLE | ≥ 0.50 | Tighter — spec was 0.45 | +| HIBERNATE | < 0.50 | — | + +**Do NOT loosen these without quantitative justification.** + +--- + +## 20. OBF SPRINT 1 HARDENING + +**Completed**: 2026-03-22. All 25 items in `AGENT_TODO_PRIORITY_FIXES_AND_TODOS.md` addressed. + +### 20.1 P0/P1/P2 Hardening (Production Safety) + +| Item | Change | Severity | +|------|--------|----------| +| Circuit breaker | 5 consecutive HZ push failures → exponential backoff + file-only fallback | P0 | +| Crossed-book guard | Ask ≤ bid on incoming feed → discard snapshot, log warning, continue | P0 | +| Dark streak detector | N consecutive zero-volume bars → emit STALE_DATA warning | P1 | +| First flush delay | No OB features published until 60s after startup (warmup) | P1 | +| Stall watchdog | No new bar for `STALL_TIMEOUT` seconds → alert + optional restart | P1 | +| Fire-and-forget HZ push | HZ write moved to background thread; hot loop never blocks on HZ | P2 | +| Dynamic asset discovery | `hzobprovider` discovers active symbols from HZ at runtime; no hardcoded list | P2 | +| Per-timestamp macro map | `latest_macro_at_ts` keyed by bar timestamp; resolves stale-read race on fast replays | P2 | + +### 20.2 P3 Infrastructure Items + +| Item | Status | +|------|--------| +| `scripts/verify_parquet_archive.py` — validates all daily parquet files for schema and row count | DONE | +| `ob_cache/SCHEMA.md` — authoritative JSON schema for `latest_ob_features.json` | DONE | +| P3-1 / P3-5 / P3-6 — out of scope for sprint 1, deferred | SKIPPED | + +### 20.3 OBF Architecture Post-Sprint + +``` +Binance WS feed + ↓ +obf_prefect_flow.py (hot loop, ~100ms cadence) + ├── Crossed-book guard → discard if ask ≤ bid + ├── Dark streak detector → N zero-vol bars + ├── First flush delay → 60s warmup + ├── Feature compute (depth imbalance, spread, vwap, pressure ratio) + ├── Per-timestamp macro map update + ├── Fire-and-forget HZ push (background thread) + │ └── Circuit breaker (5 failures → file-only) + └── ob_cache/latest_ob_features.json (local fallback) +``` + +### 20.4 OBF Live Data Gap — KNOWN LIMITATION (2026-03-26) + +> **CRITICAL DATA QUALITY CAVEAT**: `nautilus_event_trader.py` (live event trader) is currently wired to `MockOBProvider` with static per-asset imbalance biases (BTC=-0.086, ETH=-0.092, BNB=+0.05, SOL=+0.05). All four OBF functional dimensions compute and produce real outputs — but with frozen, market-unresponsive inputs. The OB cascade regime will always be CALM (no depth drain in mock data). +> +> `HZOBProvider` (`/mnt/dolphinng5_predict/nautilus_dolphin/nautilus_dolphin/nautilus/hz_ob_provider.py`) exists and is format-compatible with `obf_prefect_flow.py`'s HZ output, but `OBFeatureEngine` has no live streaming path — only `preload_date()` (batch/backtest). A `step_live()` method must be added before the switch. +> +> **Acceptable for**: paper trading +> **NOT acceptable for**: live capital deployment +> +> **Full spec**: `/mnt/dolphinng5_predict/prod/docs/AGENT_SPEC_OBF_LIVE_SWITCHOVER.md` + +### 20.5 Test Coverage + +`tests/test_obf_unit.py` — ~120 unit tests covering all hardening items: +- Circuit breaker state machine (CLOSED → OPEN → HALF-OPEN) +- Crossed-book guard triggers on malformed data +- Dark streak threshold detection +- Warmup period gating +- Background thread non-blocking behavior +- Asset discovery via HZ key scan + +--- + +## 21. KNOWN RESEARCH TODOs + +| ID | Description | Priority | +|----|-------------|----------| +| TODO-1 | Calibrate `vd_enabled` adverse-turn exits (currently disabled). Requires analysis of trade vel_div distribution at entry vs. subsequent bars. True invalidation threshold likely ~+0.02 sustained for N=3 bars. | MEDIUM | +| TODO-2 | Validate SUBDAY_ACB force-exit threshold (`old_boost >= 1.25 and boost < 1.10`). Currently ARBITRARY — agent-chosen, not backtest-derived. | MEDIUM | +| TODO-3 | MIG8: Binance live adapter (real order execution). OUT OF SCOPE until after 30-day paper trading validation. | LOW | +| TODO-4 | 48-hour chaos test with all daemons running simultaneously. Watch for: KeyError, stale-read anomalies, concurrent HZ writer collisions. | HIGH (before live capital) | +| TODO-5 | Memory profiler with IRP enabled at 400 assets (current 71 MB measurement was without IRP). Projected ~600 MB — verify. | LOW | +| TODO-6 | TF-spread recovery exits (`tf_enabled=False`). Requires sweep of tf_exhaust_ratio and tf_flip_ratio vs. champion backtest. | LOW | +| TODO-7 | GREEN (LONG) posture paper validation. LONG thresholds (long_threshold=0.01, long_extreme=0.04) not yet production-validated. | MEDIUM | +| TODO-8 | ~~ML-MC Forewarner injection into `nautilus_prefect_flow.py`.~~ **DONE 2026-03-22** — wired in `DolphinActor.on_start()` for both flows. | CLOSED | +| TODO-9 | Live TradingNode integration (launcher.py exists; Binance adapter config incomplete). Requires 30-day clean paper run first. | LOW | +| TODO-10 | BingX futures private-WS `SNAPSHOT` burst absorption. On connect/reconnect, absorb the initial futures `SNAPSHOT` flood into account/config caches, gate `ws_primed` readiness on snapshot drain, and suppress false drift / excess REST polling during the burst. Treat as startup/reconnect performance work, not fill-truth logic. | MEDIUM | +| TODO-11 | Dual-shadow regime sampler for side selection. Run two ultra-light shadow engines in real time over recent sample trades: (A) basal SHORT Alpha Engine posture and (B) basal LONG posture. Use their relative WR / ROI-per-trade / drawdown asymmetry as a regime probe: SHORT down + LONG up → LONG-favorable; LONG down + SHORT up → SHORT-favorable; both up → permissive; both down → likely choppy / abstain. Treat this initially as a shadow-only market-sampling / regime-detection layer. Later, cross the shadow streams with market fingerprints so a learner can predict or simplify the switch logic. The first persistence pass on extant trades found only mild short-loss clustering, so the live switch should be hysteresis-gated, not a raw flip-on-first-loss rule. See `LONG_DETERMINISTIC_RULE_RESEARCH.md` for the measured flip-after-loss counterfactual. | MEDIUM | +| BUG-1 | **V7 `_max_hold_ref` decoupled from actual MAX_HOLD.** `alpha_exit_v7_engine.py:491` computes `_max_hold_ref = self._3m_bars * 3 = 48 bars` (from `bar_duration_sec=11.0`). The MAE-D time-pressure ramps from bar 29 and saturates at bar 48 — only ~9 minutes into a trade whose real MAX_HOLD is 125 bars (OB-halved). Effect: V7 is **over-eager on adverse-excursion trades** (mae > 0.3% after bar 29 gets time-pressure that should belong at bar 75+). No effect on winning trades (mae too low to trigger the gate). Fix: derive `_max_hold_ref` from the orchestrator's effective `max_hold_bars` (post OB-halving) rather than `_3m_bars * 3`. | MEDIUM | +| BUG-2 | **OB dynamic max_hold adjustments discard per-trade `max_hold_override`.** `alpha_exit_manager.py:135,147,152,157` all multiply `self.max_hold_bars` (global default) instead of the per-trade `dynamic_max_hold`. If a `max_hold_override` is set via `setup_position()`, the cascade/withdrawal/convexity adjustments silently replace it with the global-based computation. Currently latent (no overrides in use), but will bite if per-trade hold tuning is ever deployed. Fix: multiply from `dynamic_max_hold` (already resolved from override at line 120) instead of `self.max_hold_bars`. | LOW | + +--- + +## 22. 0.1S RESOLUTION — READINESS ASSESSMENT + +**Assessment date**: 2026-03-22. **Status: BLOCKED — 3 hard blockers.** + +The current system processes 5s OHLCV bars. Upgrading to 0.1s tick resolution requires resolving all three blockers below before any code changes. + +### 22.1 Blocker 1 — Async HZ Push + +**Problem**: The OBF hot loop fires at ~100ms cadence. At 0.1s resolution, the per-bar HZ write latency (currently synchronous in feature compute path, despite fire-and-forget for the push itself) would exceed bar cadence, causing HZ write queue growth and eventual OOM. + +**Required**: Full async HZ client (`hazelcast-python-client` async API or aiohazelcast). Currently all HZ operations are synchronous blocking calls. Estimated effort: 2–3 days of refactor + regression testing. + +### 22.2 Blocker 2 — `get_depth` Timeout + +**Problem**: `get_depth()` in `HZOBProvider` issues a synchronous HZ `IMap.get()` call with a 500ms timeout. At 0.1s resolution, each bar would wait up to 500ms for OB depth data — 5× the bar cadence. This makes 0.1s resolution impossible without an in-process depth cache. + +**Required**: Pre-fetched depth cache (e.g., local dict refreshed by a background subscriber), making `get_depth()` a pure in-process read with <1µs latency. Estimated effort: 1–2 days. + +### 22.3 Blocker 3 — Lookback Recalibration + +**Problem**: All champion parameters that reference "bars" were validated against 5s bars: +- `lookback=100` (100 × 5s = 500s warmup) +- `max_hold_bars=120` (120 × 5s = 600s max hold) +- `dc_lookback_bars=7` (7 × 5s = 35s DC window) + +At 0.1s resolution, the same bar counts would mean 10s warmup, 12s max hold, 0.7s DC window — **completely invalidating champion params**. All params must be re-validated from scratch via VBT backtest at 0.1s resolution. + +**Required**: Full backtest sweep at 0.1s. Estimated effort: 1–2 weeks of compute + validation time. This is a research milestone, not an engineering task. + +### 22.4 Assessment Summary + +| Blocker | Effort | Dependency | +|---------|--------|------------| +| Async HZ push | 2–3 days engineering | None — can start now | +| `get_depth` cache | 1–2 days engineering | None — can start now | +| Lookback recalibration | 1–2 weeks research | Requires blockers 1+2 resolved first | + +**Recommendation**: Do NOT attempt 0.1s resolution until after 30-day paper trading validation at 5s. The engineering blockers can be prototyped in parallel, but champion params cannot be certified until post-paper-run stability is confirmed. + +## 23. SIGNAL PATH VERIFICATION SPECIFICATION + +Testing the asynchronous, multi-scale signal path requires systematic validation of the data bridge and cross-layer trigger logic. + +### 23.1 Verification Flow +A local agent (Prefect or standalone) should verify: +1. **Micro Ingestion**: 100ms OB features sharded across 10 HZ maps. +2. **Regime Bridge**: NG5 Arrow scan detection by `scan_hz_bridge.py` and push to `latest_eigen_scan`. +3. **Strategy Reactivity**: `DolphinActor.on_bar` (5s) pulling HZ data and verifying `scan_number` idempotency. +4. **Macro Safety**: Survival Stack Rm-computation pushing `APEX/STALKER/HIBERNATE` posture to `DOLPHIN_SAFETY`. + +### 23.2 Reference Document +Full test instructions, triggers, and expected values are defined in: +`TODO_CHECK_SIGNAL_PATHS.md` (Project Root) + +--- + +*End of DOLPHIN-NAUTILUS System Bible v3.0 — 2026-03-23* +*Champion: SHORT only (APEX posture, blue configuration)* +*Automation: Prefect-supervised paper trading active.* +*Status: Capital Sync enabled; Friction SP-bypass active; TradeLogger running.* +*Do NOT deploy real capital until 30-day paper run is clean.* + +## 24. MULTI-SPEED EVENT-DRIVEN ARCHITECTURE + +**Version**: v4.1 Addition — 2026-03-25 +**Status**: DEPLOYED (Production) +**Author**: Kimi Code CLI Agent +**Related**: `AGENT_READ_ARCHITECTURAL_CHANGES_SPEC.md` (detailed specification) + +### 24.1 Overview + +The DOLPHIN system has been re-architected from a **single-speed batch-oriented Prefect deployment** to a **multi-speed, event-driven, multi-worker architecture** with proper resource isolation and self-healing capabilities. + +**Problem Solved**: 2026-03-24 system outage caused by uncontrolled Prefect process explosion (60+ `prefect.engine` zombies → resource exhaustion → kernel deadlock). + +**Solution**: Frequency isolation + concurrency limits + systemd resource constraints + event-driven architecture. + +### 24.2 Architecture Layers + +| Layer | Frequency | Component | Pattern | Status | +|-------|-----------|-----------|---------|--------| +| L1 | <1ms | Nautilus Event Trader | Hz Entry Listener | ✅ Active (PID 159402) | +| L2 | 1-10s | Scan Bridge | File watcher → Hz | ✅ Active (PID 158929) | +| L3 | Varied | ExtF Indicators | Scheduled per-indicator | ⚠️ Not running (NG6 down) | +| L4 | ~5s | Meta Health Service | 5-sensor monitoring | ✅ Active (PID 160052) | +| L5 | Daily | Paper/Nautilus Flows | Prefect scheduled | ✅ Scheduled | + +### 24.3 Nautilus Event-Driven Trader + +**Purpose**: Millisecond-latency trading via Hazelcast event listener (not polling). + +**Implementation**: +```python +# Hz Entry Listener Pattern +features_map.add_entry_listener( + key='latest_eigen_scan', + updated_func=on_scan_update # Called per scan +) + +def on_scan_update(event): + scan = json.loads(event.value) + signal = compute_signal(scan, ob_data, extf_data) + if signal.valid: + execute_trade(signal) # <10ms total latency +``` + +**Service**: `dolphin-nautilus-trader.service` +**Resource Limits**: MemoryMax=2G, CPUQuota=200%, TasksMax=50 +**Hz Input**: `DOLPHIN_FEATURES["latest_eigen_scan"]` +**Hz Output**: `DOLPHIN_PNL_BLUE[YYYY-MM-DD]`, `DOLPHIN_STATE_BLUE` + +### 24.4 Scan Bridge Service + +**Purpose**: Detect Arrow scan files from DolphinNG6, push to Hz. + +**Deployment**: `scan-bridge-flow/scan-bridge` (Prefect) +**Concurrency**: Strictly limited to 1 +**Safety Mechanisms**: +- Work pool concurrency limit: 1 +- Deployment concurrency limit: 1 +- File mtime-based detection (handles NG6 restarts) + +**Current Status**: Running directly (PID 158929) due to Prefect worker scheduling issues. + +### 24.5 Meta Health Service v3 (MHS) — REWRITTEN v5.0 + +> **MHS v2 is retired.** `meta_health_daemon_v2.py` was calling `systemctl restart` on supervisord-managed processes — this was the "random killer" bug. v3 is the canonical implementation. + +**File**: `meta_health_service_v3.py` +**Supervisord**: `dolphin_data:meta_health` (`autostart=true`) + +#### 24.5.1 Five-Sensor Model (Weighted Sum — NOT product) + +| Sensor | Weight | Metric | Thresholds | +|--------|--------|--------|------------| +| M4 | 0.35 | Control Plane (HZ port 5701 + Prefect 4200) | HZ=0.8w, Prefect=0.2w | +| M1 | 0.35 | Process Integrity (supervisord status) | data services scored separately from trader | +| M3 | 0.20 | Data Freshness (HZ key timestamps) | >30s=stale(0.5), >120s=dead(0.0) | +| M5 | 0.10 | Data Coherence (boost range, OBF coverage) | OBF<200 assets=0.5 | +| M2 | — | Heartbeat (informational only) | Not in rm_meta | +| M1_trader | — | Trader process (informational only) | Not in rm_meta (may be intentionally stopped) | + +#### 24.5.2 Rm_meta Formula + +```python +# FIX-1: Weighted sum — no single sensor can zero rm_meta (v2 bug fixed) +rm_meta = (0.35*m4 + 0.35*m1_data + 0.20*m3 + 0.10*m5) / 1.0 + +# Thresholds +rm > 0.85: GREEN +rm > 0.60: DEGRADED +rm > 0.30: CRITICAL +rm ≤ 0.30: DEAD → Recovery triggered (only for STOPPED critical_data services) +``` + +#### 24.5.3 Recovery Policy + +```python +# FIX-2: supervisorctl restart, NOT systemctl (v2 bug fixed) +# FIX-3: 10s cooldown for critical services (was 600s) +# FIX-4: Non-blocking daemon thread (hung subprocess won't block check loop) +# FIX-5: Per-service cooldown (independent buckets per program) +# FIX-6: Only STOPPED critical_data services are restarted. Trader never auto-restarted. + +RECOVERY_COOLDOWN_CRITICAL_S = 10.0 # exf, acb, obf_universe +RECOVERY_COOLDOWN_DEFAULT_S = 300.0 # nautilus_trader, scan_bridge (informational only) +CHECK_INTERVAL_S = 10.0 +``` + +#### 24.5.4 Monitored Services + +| supervisord program | critical_data | Auto-restarted by MHS | +|---|---|---| +| `dolphin_data:exf_fetcher` | ✅ | ✅ (10s cooldown) | +| `dolphin_data:acb_processor` | ✅ | ✅ (10s cooldown) | +| `dolphin_data:obf_universe` | ✅ | ✅ (10s cooldown) | +| `dolphin:nautilus_trader` | ❌ | ❌ (informational) | +| `dolphin:scan_bridge` | ❌ | ❌ (informational) | + +#### 24.5.5 Monitored HZ Sources + +| Key | Map | Timestamp Field | Notes | +|---|---|---|---| +| `exf_latest` | `DOLPHIN_FEATURES` | `_pushed_at` | Unix float | +| `acb_boost` | `DOLPHIN_FEATURES` | (none — presence only) | — | +| `latest_eigen_scan` | `DOLPHIN_FEATURES` | `timestamp` | ISO string | +| `obf_universe_latest` | `DOLPHIN_FEATURES` | `_snapshot_utc` | Unix float | + +**Output**: `DOLPHIN_META_HEALTH["latest"]` — JSON health report, also written to `run_logs/meta_health.json` + +### 24.6 Safety Mechanisms + +#### 24.6.1 Concurrency Controls (Root Cause Fix) + +| Level | Mechanism | Value | Prevents | +|-------|-----------|-------|----------| +| Work Pool | `concurrency_limit` | 1 | Multiple simultaneous runs | +| Deployment | `prefect concurrency-limit` | 1 (tag-based) | Tag-based overflow | +| Systemd | `TasksMax` | 50 | Process fork bombs | +| Systemd | `MemoryMax` | 2G | OOM conditions | +| Systemd | `CPUQuota` | 200% | CPU starvation | + +#### 24.6.2 Recovery Procedures + +| Scenario | Trigger | Action | +|----------|---------|--------| +| Critical data service STOPPED | rm CRITICAL/DEAD + service STOPPED | `supervisorctl restart ` (async, 10s cooldown) | +| Data staleness | M3 < 0.5 | Alert only (external data dependency) | +| Control plane down | M4 < 0.5 | Alert (MHS can't self-heal HZ) | +| Trader stopped | m1_trader < 1.0 | Informational only — NEVER auto-restarted | + +### 24.7 Data Flow: Scan-to-Trade + +``` +DolphinNG6 → Arrow File → Scan Bridge → Hz → Entry Listener → Nautilus → Trade + (Win) (SMB) (5s poll) (μs) (<1ms) (<1ms) (<10ms) + +Target: <10ms from NG6 scan to trade execution +Current: Waiting for NG6 restart to validate +``` + +### 24.8 Service Status (v5.0 — As Running 2026-03-30) + +| supervisord program | Status | Notes | +|---|---|---| +| `dolphin_data:exf_fetcher` | ✅ RUNNING | Pushes exf_latest every ~60s | +| `dolphin_data:acb_processor` | ✅ RUNNING | Pushes acb_boost on NG3 data | +| `dolphin_data:obf_universe` | ✅ RUNNING | 512/540 assets healthy at launch | +| `dolphin_data:meta_health` | ✅ RUNNING | RM_META≈0.975 [GREEN] | +| `dolphin:nautilus_trader` | ⚙️ STOPPED (manual) | Start when trading | +| `dolphin:scan_bridge` | ⚙️ STOPPED (manual) | Start when DolphinNG6 active | +| hazelcast | ✅ (docker) | Port 5701 | +| prefect-server | ✅ (docker) | Port 4200 | + +**RETIRED (stopped + disabled)**: +- `dolphin-nautilus-trader.service` (systemd) — was causing dual-management +- `dolphin-scan-bridge.service` (systemd) — was causing dual-management +- `meta_health_daemon.service` (systemd) — was calling `systemctl restart` on supervisord processes (root cause of random killer bug) + +### 24.9 Known Issues (v5.0) + +| Issue | Status | Notes | +|-------|--------|-------| +| NG6 down (no scan data) | External dependency | `latest_eigen_scan` key absent; MHS reports this cleanly | +| OBF shard store (400 assets) vs universe (540) | Architecture gap | Shard store is used by trading engine; universe is health-only | + +### 24.10 Operational Commands + +```bash +CONF=/mnt/dolphinng5_predict/prod/supervisor/dolphin-supervisord.conf + +# Status +supervisorctl -c $CONF status + +# Restart a service +supervisorctl -c $CONF restart dolphin_data:exf_fetcher + +# Start the trader +supervisorctl -c $CONF start dolphin:nautilus_trader + +# View MHS health +cat /mnt/dolphinng5_predict/run_logs/meta_health.json + +# View supervisord logs +tail -f /mnt/dolphinng5_predict/prod/supervisor/logs/meta_health.log +``` + +### 24.11 File Locations + +| Component | Full Path | +|-----------|-----------| +| **Nautilus Trader** | `/mnt/dolphinng5_predict/prod/nautilus_event_trader.py` | +| **MHS v3** | `/mnt/dolphinng5_predict/prod/meta_health_service_v3.py` | +| MHS v2 (retired) | `/mnt/dolphinng5_predict/prod/meta_health_daemon_v2.py` | +| **OBF Universe Service** | `/mnt/dolphinng5_predict/prod/obf_universe_service.py` | +| **Scan Bridge** | `/mnt/dolphinng5_predict/prod/scan_bridge_service.py` | +| **ACB Processor** | `/mnt/dolphinng5_predict/prod/acb_processor_service.py` | +| **ExF Fetcher** | `/mnt/dolphinng5_predict/prod/exf_fetcher_flow.py` | +| **CH Writer** | `/mnt/dolphinng5_predict/prod/ch_writer.py` | +| **Supervisord Conf** | `/mnt/dolphinng5_predict/prod/supervisor/dolphin-supervisord.conf` | +| Supervisord Socket | `/tmp/dolphin-supervisor.sock` | +| MHS Logs | `/mnt/dolphinng5_predict/prod/supervisor/logs/meta_health.log` | +| MHS Health JSON | `/mnt/dolphinng5_predict/run_logs/meta_health.json` | +| Test Results JSON | `/mnt/dolphinng5_predict/run_logs/test_results_latest.json` | +| OBF Universe Data | `/mnt/ng6_data/ob_universe/` (Hive partitioned Parquet) | +| **TUI** | `/mnt/dolphinng5_predict/Observability/TUI/dolphin_tui_v9.py` | +| **Adaptive Exit Engine** | `/mnt/dolphinng5_predict/adaptive_exit/` (package) | +| AE Models | `/mnt/dolphinng5_predict/adaptive_exit/models/` | +| NG8 Scanner | `/mnt/dolphinng5_predict/- Dolphin NG8/ng8_scanner.py` | +| Arrow Writer Shim | `/mnt/dolphinng5_predict/- Dolphin NG8/arrow_writer.py` | +| Arrow Scans | `/mnt/ng6_data/arrow_scans/YYYY-MM-DD/scan_NNNNNN_HHMMSS.arrow` | +| Champion Config | `/mnt/dolphinng5_predict/prod/configs/blue.yml` | +| Green Config | `/mnt/dolphinng5_predict/prod/configs/green.yml` | +| Docker Compose | `/mnt/dolphinng5_predict/prod/docker-compose.yml` | +| vbt_cache_klines | `/mnt/dolphinng5_predict/vbt_cache_klines/YYYY-MM-DD.parquet` | + +--- + +## §25 Numerical Precision Policy + +### 25.1 Principle + +**Must use 512-bit native precision when possible, if performance allows.** + +This system has `python-flint` v0.8.0 installed on all compute nodes. All critical +financial math (vol computation, returns, rolling std) MUST use `flint.arb` at 512-bit +unless performance constraints make it prohibitive. + +### 25.2 Configuration + +```python +from flint import arb, ctx +ctx.prec = 512 # 512-bit mantissa; set once per process +``` + +### 25.3 Performance Benchmark (2026-03-28) + +| Mode | dvol/day | Total 56d | Overhead | +|--------------|-----------|-----------|---------- | +| numpy float64 | ~94 ms | ~5.3 s | baseline | +| arb 512-bit | ~295 ms | ~16.5 s | +11 s | + ++11 seconds on a ~718 second total run (1.5% overhead) — acceptable. + +### 25.4 Validated Equivalence + +Benchmark result against full 56-day window (6154 rows/day sampled): +- NaNs in arb result: **0** +- vol_ok=True bars match float64: **MATCH=True** +- Divergent bars: **0** + +float64 and 512-bit produce identical `vol_ok` decisions for this signal at current +BTC price magnitudes. The 512-bit path is used as the primary path to prevent +precision erosion from future edge cases (extreme micro-volatility, very large +or very small price moves). + +### 25.5 Implementation Pattern + +```python +def _compute_dvol_arb512(prices, n_rows, threshold): + """Primary: 512-bit arb. Returns None if flint unavailable (fall back to float64).""" + try: + from flint import arb, ctx + ctx.prec = 512 + except ImportError: + return None + # ... arb rolling std ... + +# Call site: +vol_ok_mask = _compute_dvol_arb512(btc, n_rows, VOL_P60_THRESHOLD) +if vol_ok_mask is None: + # float64 fallback — guards only; should not be reached on production nodes + ... +``` + +### 25.6 Scope + +| Computation | Precision | File | +|-------------|-----------|------| +| Rolling 50-bar dvol (vol_ok) | arb 512-bit | `nautilus_native_continuous.py` | +| All other paths | numpy float64 | — | + +Future additions (returns, leverage math, position sizing) should follow the same +pattern: 512-bit primary, float64 last-resort guard. + +--- + +## 26. SUPERVISORD ARCHITECTURE & OBF UNIVERSE (v5.0) + +### 26.1 The "Random Killer" Bug — Root Cause & Fix + +**Incident**: Services were being unexpectedly killed and restarted at seemingly random intervals. The system appeared healthy according to supervisord but processes would die without obvious cause. + +**Root cause** (diagnosed 2026-03-30): +1. `meta_health_daemon_v2.py` had been running under `meta_health_daemon.service` (systemd) for 4+ days. +2. MHS v2's process patterns (`exf_prefect_final`, `esof_prefect_flow`) did not match any running process → M1=0 → `rm_meta = M1*M2*M3*M4*M5 = 0` always → status="DEAD". +3. MHS v2 recovery action: `systemctl restart ` — called every 5s. +4. But the services were supervisord-managed, not systemd-managed. `systemctl restart` on a supervisord process: + - Sends SIGTERM to the process (it dies) + - Supervisord detects the death and autostarts a new instance + - Creates brief duplicate processes, interleaved with MHS v2's next kill cycle +5. Additionally, `dolphin-nautilus-trader.service` (systemd) AND supervisord were both managing `nautilus_event_trader.py` simultaneously — two PIDs running at once. + +**Fix applied**: +```bash +systemctl stop meta_health_daemon.service && systemctl disable meta_health_daemon.service +systemctl stop dolphin-nautilus-trader.service && systemctl disable dolphin-nautilus-trader.service +systemctl stop dolphin-scan-bridge.service && systemctl disable dolphin-scan-bridge.service +``` + +**Permanent guard**: `test_mhs_v3.py::TestKillAndRevive::test_no_systemd_units_active_for_managed_services` asserts no conflicting systemd units are active. + +### 26.2 OBF Universe Service + +**Purpose**: Lightweight L2 order book health monitor for ALL 540 active USDT perpetuals on Binance Futures. + +**Why**: Asset Picker needs OB health scores for the full universe (540 assets) to make informed selection decisions, not just the 400 assets covered by the existing OBF shard store. + +**Design**: Push streams (zero REST weight), no polling. + +``` +wss://fstream.binance.com/ws + Connection 1: 200 symbols × @depth5@500ms + Connection 2: 200 symbols × @depth5@500ms + Connection 3: 140 symbols × @depth5@500ms + (total: 540, Binance limit: 300/conn) +``` + +**Computed metrics per asset** (every 60s snapshot): + +| Field | Description | +|---|---| +| `spread_bps` | (ask - bid) / mid × 10000 | +| `depth_1pct_usd` | Total USD volume within 1% of mid on both sides | +| `depth_quality` | Normalized depth score [0,1] | +| `fill_probability` | Estimated probability of fill at mid | +| `imbalance` | (bid_vol - ask_vol) / (bid_vol + ask_vol) | +| `best_bid`, `best_ask` | L1 prices | +| `n_bid_levels`, `n_ask_levels` | Depth5 levels received | + +**HZ output** (`DOLPHIN_FEATURES["obf_universe_latest"]`): +```json +{ + "_snapshot_utc": 1743350400.0, + "_n_assets": 512, + "assets": { + "BTCUSDT": {"spread_bps": 0.42, "depth_quality": 0.91, ...}, + "ETHUSDT": {...}, + ... + } +} +``` + +**Parquet storage**: `/mnt/ng6_data/ob_universe/` (Hive: `date=YYYY-MM-DD/part-NNN.parquet`) +- `MAX_FILE_AGE_DAYS = 0` — never pruned, accumulates for backtesting +- Flush cadence: every 300s + +**Key constants**: +```python +SNAPSHOT_INTERVAL_S = 60 # HZ push cadence +MAX_STREAMS_PER_CONN = 200 # Binance limit respected +FLUSH_INTERVAL_S = 300 # Parquet write cadence +``` + +### 26.3 MHS v3 — Full Architecture Reference + +**File**: `prod/meta_health_service_v3.py` +**Tests**: `prod/tests/test_mhs_v3.py` (111 tests, including Hypothesis property tests) + +#### 26.3.1 Constants + +```python +CHECK_INTERVAL_S = 10.0 # main loop cadence +DATA_STALE_S = 30.0 # age threshold for stale (score=0.5) +DATA_DEAD_S = 120.0 # age threshold for dead (score=0.0) +RECOVERY_COOLDOWN_CRITICAL_S = 10.0 # critical data infra restart cooldown +RECOVERY_COOLDOWN_DEFAULT_S = 300.0 # informational services (never restarted) +``` + +#### 26.3.2 Weighted Sensor Formula + +```python +SENSOR_WEIGHTS = { + "m4_control_plane": 0.35, # HZ port 5701 (×0.8) + Prefect 4200 (×0.2) + "m1_data_infra": 0.35, # fraction of critical_data services RUNNING + "m3_data_freshness": 0.20, # average freshness score across HZ keys + "m5_coherence": 0.10, # ACB boost range validity + OBF coverage +} +# m1_trader and m2_heartbeat: emitted but NOT in rm_meta (may be intentionally stopped) + +rm_meta = sum(weight × sensor) / sum(weights) +``` + +#### 26.3.3 Recovery Logic + +```python +def _restart_via_supervisorctl(self, program: str): + """ + - Checks per-service cooldown (10s critical, 300s default) + - Commits timestamp BEFORE spawning thread (prevents double-fire) + - Runs in daemon thread — never blocks the check loop + - Uses: supervisorctl -c restart + - NEVER calls systemctl + """ +``` + +#### 26.3.4 Test Suite Summary + +| Class | Tests | Coverage | +|---|---|---| +| `TestSupervisordStatusParsing` | 7 | parseg all supervisorctl output variants | +| `TestM1ProcessIntegrity` | 7 | scoring with mocked sv_status, psutil fallback | +| `TestM3DataFreshnessScoring` | 7 | stale/dead thresholds, ISO timestamps | +| `TestRmMetaFormula` | 10 | weighted sum, product-formula regression guard | +| `TestRecoveryGating` | 5 | cooldown, thread isolation | +| `TestRecoveryNeverKillsRunning` | 6 | running services never restarted | +| `TestM4ControlPlane` | 4 | port checks with mocked socket | +| `TestM5Coherence` | 7 | boost range, OBF coverage thresholds | +| `TestLiveIntegration` | 10 | live HZ + supervisord (skip if unavailable) | +| `TestKillAndRevive` | 9 | E2E: stop service → MHS detects → restarts within 30s | +| `TestServiceRegistry` | 7 | invariants: cooldown ≤ 10s, check interval ≤ 15s | +| `TestRaceConditions` | 5 | 10 concurrent restarts same service → only 1 fires | +| `TestEdgeCases` | 14 | garbage JSON, future timestamps, NaN sensors | +| `TestHypothesisProperties` | 13 | 300–500 examples each: rm∈[0,1], monotone sensors, status valid | + +**Run**: +```bash +source /home/dolphin/siloqy_env/bin/activate +cd /mnt/dolphinng5_predict +python -m pytest prod/tests/test_mhs_v3.py -v --tb=short # ~5 minutes (E2E tests) +``` + +### 26.4 OBF Persistence Fix + +**File**: `prod/obf_persistence.py` + +**Bug (v4.1)**: `MAX_FILE_AGE_DAYS = 7` — every daily cleanup run deleted all OBF Parquet data older than 7 days, destroying the entire backtesting dataset. + +**Fix (v5.0)**: +```python +MAX_FILE_AGE_DAYS = 0 # 0 = disabled — never prune, accumulate for backtesting + +def _cleanup_old_partitions(self): + """0 = disabled.""" + if not MAX_FILE_AGE_DAYS or not self.base_dir.exists(): + return + ... +``` + +Data now accumulates indefinitely in `/mnt/ng6_data/ob_features/` (existing OBF) and `/mnt/ng6_data/ob_universe/` (new universe service). + +--- + +--- + +## 27. NG8 LINUX EIGENSCAN SERVICE + +**File**: `- Dolphin NG8/ng8_scanner.py` +**Status**: Built, smoke-tested. Replaces Windows NG7 eigenscan. +**Run**: `source /home/dolphin/siloqy_env/bin/activate && cd "/mnt/dolphinng5_predict/- Dolphin NG8" && python3 ng8_scanner.py` + +### 27.1 Root Cause: NG7 Double-Output Bug + +Windows NG7 maintained two independent tracker cycles: +- **Fast cycle** (w50, w150): completed ~11s after scan start → wrote Arrow file 1, HZ write 1 +- **Slow cycle** (w300, w750): completed ~3 min later with **stale BTC price** → wrote Arrow file 2, HZ write 2 + +Both cycles shared the same `scan_number` counter. Result: two Arrow files per logical scan, the second containing stale prices from 3 minutes earlier. The scan bridge de-duplicated by file mtime (file 1 is always the useful one). + +### 27.2 NG8 Fix: Single `enhance()` Pass + +`DolphinCorrelationEnhancerArb512.enhance()` processes all four windows (50, 150, 300, 750) in a single sequential loop. NG8 calls this once per scan cycle: + +```python +result = self.engine.enhance(price_data, PRIORITY_SYMBOLS, now) +# result.multi_window_results has all four windows populated +# Exactly one Arrow write + one HZ write follows +``` + +`use_arrow=False` is passed to the engine constructor so the engine does **not** perform its own internal Arrow write — `ng8_scanner.py` owns that write exclusively. + +### 27.3 Schema Contract (Doctrinal NG5) + +Arrow IPC schema is defined in `ng7_arrow_writer_original.py` → `SCAN_SCHEMA` (27 fields, `SCHEMA_VERSION="5.0.0"`). `arrow_writer.py` is a thin re-export shim: + +```python +# arrow_writer.py +from ng7_arrow_writer_original import ( + ArrowEigenvalueWriter, ArrowScanReader, write_scan_arrow, read_scan_arrow, +) +``` + +**NEVER** modify `arrow_writer.py` schema — edit `ng7_arrow_writer_original.py`. + +Key schema fields: +| Field | Type | Description | +|---|---|---| +| `scan_number` | int64 | monotonic counter, resumes from last Arrow file on restart | +| `timestamp_ns` | int64 | Unix nanoseconds at scan start | +| `w50_lambda_max` … `w750_instability` | float64 × 16 | per-window eigenstats | +| `vel_div` | float64 | velocity divergence (cross-window signal) | +| `regime_signal` | float64 | -1 / 0 / +1 | +| `instability_composite` | float64 | composite of w50…w750 instability | +| `assets` / `prices` / `loadings` | utf8 | JSON-serialised | +| `schema_version` | utf8 | "5.0.0" | + +### 27.4 Storage + +``` +Arrow files : /mnt/dolphinng6_data/arrow_scans/YYYY-MM-DD/scan_NNNNNN_HHMMSS.arrow +ArrowEigenvalueWriter storage_root = /mnt/dolphinng6_data # writer appends arrow_scans/ internally +``` + +**Critical**: pass `get_arrow_scans_path().parent` (= `/mnt/dolphinng6_data`) — NOT `get_arrow_scans_path()` — or the writer creates `arrow_scans/arrow_scans/` double-nesting. + +### 27.5 Hazelcast Output + +Map: `DOLPHIN_FEATURES` → key `latest_eigen_scan` + +**NG8 flat payload** (written by NG8, differs from NG7 nested payload): +```python +{ + "scan_number": int, + "timestamp": "ISO-8601", + "bridge_ts": float, # Unix epoch at HZ write + "vel_div": float, + "w50_velocity": float, + "w150_velocity": float, + "w300_velocity": float, + "w750_velocity": float, + "eigenvalue_gradients": {...}, + "multi_window_results": {...}, # full per-window stats +} +``` + +TUI v3 `_eigen_from_scan()` normalises both NG7 nested and NG8 flat formats transparently. + +### 27.6 Scan Number Continuity + +On startup, `_load_last_scan_number(arrow_scans_dir)` scans all `scan_NNNNNN_*.arrow` filenames for the highest N and resumes from N+1. Prevents counter reset gaps after service restart. + +### 27.7 Symbol List + +50 symbols matching doctrinal NG3/NG5/NG7 `PRIORITY_SYMBOLS`. Do NOT change this list without a full schema migration — historical correlation matrices are computed on this exact universe. + +### 27.8 Supervisord Integration (Pending) + +Add to `dolphin-supervisord.conf`: +```ini +[program:ng8_scanner] +command=/home/dolphin/siloqy_env/bin/python3 ng8_scanner.py +directory=/mnt/dolphinng5_predict/- Dolphin NG8 +autostart=false ; manual start until NG7 Windows is formally retired +autorestart=true +stderr_logfile=/var/log/dolphin/ng8_scanner.err.log +stdout_logfile=/var/log/dolphin/ng8_scanner.out.log +``` + +Set `autostart=true` only after confirming Windows NG7 is shut down — dual-write to the same HZ key is safe (last-write-wins) but creates confusing Arrow audit trails. + +--- + +## 28. TUI v9 — LIVE OBSERVABILITY TERMINAL + +**File**: `/mnt/dolphinng5_predict/Observability/TUI/dolphin_tui_v9.py` +**Run**: `source /home/dolphin/siloqy_env/bin/activate && cd /mnt/dolphinng5_predict/Observability/TUI && python3 dolphin_tui_v9.py` +**Framework**: Textual 8.1.1 (siloqy_env) +**Bindings**: `q` quit · `r` force-refresh · `l` log panel · `t` toggle test footer + +> v9 completely rewrites v3. All previous v3 panel descriptions remain valid but the filename is now `dolphin_tui_v9.py`. Version history: v3 (HZ listeners) → v4 (SysHealth labels) → v5 (capital panel) → v6 (ExF detail) → v7 (MC-Forewarner) → v8 (AE trades footer) → v9 (bucket performance panel). + +### 28.1 Architecture: Zero Load on Origin System + +All data flows via **Hazelcast entry listeners** (push model): + +``` +HZ maps ──push──► _State (thread-safe dict) ──call_from_thread──► Textual asyncio loop + │ + set_interval(1s) ────────┘ +``` + +`IMap.add_entry_listener(include_value=True, updated=fn, added=fn)` fires callbacks from the HZ internal thread pool on any map change. No polling of origin systems. + +Prefect is the **only** polled source — 60s interval via `run_worker(prefect_poll_loop())`. + +### 28.2 Panel Map (v9) + +| Panel | Source | Update Trigger | CSS id | +|---|---|---|---| +| **Header** | `DOLPHIN_HEARTBEAT` | HZ listener | `#header` | +| **Trader** | `DOLPHIN_STATE_BLUE`, `DOLPHIN_FEATURES/latest_eigen_scan`, `DOLPHIN_HEARTBEAT` | HZ listener | `#p_trader` | +| **SysHealth (M1–M5)** | `DOLPHIN_META_HEALTH/latest` | HZ listener | `#p_health` | +| **AlphaEngine** | `DOLPHIN_FEATURES/latest_eigen_scan`, `DOLPHIN_SAFETY` | HZ listener | `#p_alpha` | +| **Scan** | `DOLPHIN_FEATURES/latest_eigen_scan` | HZ listener | `#p_scan` | +| **ExtF** | `DOLPHIN_FEATURES/exf_latest` | HZ listener | `#p_extf` | +| **OBF** | `DOLPHIN_FEATURES/obf_universe_latest` | HZ listener | `#p_obf` | +| **Capital** | `DOLPHIN_STATE_BLUE`, `DOLPHIN_SAFETY`, `DOLPHIN_HEARTBEAT` | HZ listener | `#p_capital` | +| **Prefect** | Prefect SDK | 60s poll | `#p_prefect` | +| **ACB** | `DOLPHIN_FEATURES/acb_boost` | HZ listener | `#p_acb` | +| **MC-Forewarner** | `DOLPHIN_FEATURES/mc_forewarner_latest` | HZ listener | `#mc_outer` | +| **Trades Footer** | CH `trade_events` (live v7 exit reasons) + `adaptive_exit_shadow` (CLOSED) | 30s CH poll | `#trades_footer` | +| **Bucket Footer** | CH `adaptive_exit_shadow` (CLOSED rows grouped by bucket_id) | 60s CH poll | `#bucket_footer` | +| **Test Footer** | `/mnt/dolphinng5_predict/run_logs/test_results_latest.json` | File read + `t` toggle | `#test_footer` | + +### 28.3 HZ Maps Listened + +```python +DOLPHIN_FEATURES: latest_eigen_scan, exf_latest, obf_universe_latest, + acb_boost, mc_forewarner_latest +DOLPHIN_META_HEALTH: latest +DOLPHIN_SAFETY: latest +DOLPHIN_STATE_BLUE: capital_checkpoint, engine_snapshot +DOLPHIN_HEARTBEAT: nautilus_flow_heartbeat +DOLPHIN_PNL_BLUE: session_perf +``` + +### 28.4 CH Poll Threads + +Two background poll threads (daemon, named): + +| Thread | Name | Cadence | CH Query | +|---|---|---|---| +| `_start_trades_poll()` | `ch-trades-poll` | 30s | `trade_events` last 8 today (live v7 exit reasons) + `adaptive_exit_shadow` CLOSED rows today | +| `_start_bucket_poll()` | `ch-bucket-poll` | 60s | `adaptive_exit_shadow` CLOSED rows, grouped by `bucket_id`, all-time, excl HIBERNATE/ACB | + +### 28.5 Test Results Footer + +The footer reads `/mnt/dolphinng5_predict/run_logs/test_results_latest.json`. + +**Schema**: +```json +{ + "_run_at": "2026-04-05T12:00:00", + "data_integrity": {"passed": 15, "total": 15, "status": "PASS"}, + "finance_fuzz": {"passed": null, "total": null, "status": "N/A"}, + "signal_fill": {"passed": null, "total": null, "status": "N/A"}, + "degradation": {"passed": 12, "total": 12, "status": "PASS"}, + "actor": {"passed": null, "total": null, "status": "N/A"} +} +``` + +**Write API** (exported from `dolphin_tui_v9.py`): +```python +from dolphin_tui_v9 import write_test_results + +write_test_results({ + "data_integrity": {"passed": 15, "total": 15, "status": "PASS"}, + "finance_fuzz": {"passed": 8, "total": 8, "status": "PASS"}, + ... +}) +``` + +`write_test_results()` atomically writes `_run_at` (current UTC ISO timestamp) + the provided category dict. The TUI footer auto-refreshes on next mount or `t` keypress. + +Full integration documentation: `prod/docs/TEST_REPORTING.md`. + +### 28.6 NG7 / NG8 Dual Format Normalisation + +`_eigen_from_scan(scan)` handles both live HZ formats: + +```python +def _eigen_from_scan(scan): + # NG7 nested: scan["result"]["multi_window_results"]["50"]["velocity"] + # NG8 flat: scan["multi_window_results"]["50"]["velocity"] + result = scan.get("result", scan) + mwr = result.get("multi_window_results", {}) + for w in (50, 150, 300, 750): + row = mwr.get(w) or mwr.get(str(w)) or {} + ... +``` + +### 28.7 MC-Forewarner Integration + +**Status: DEPLOYED AND RUNNING** — `prod/mc_forewarner_flow.py`, Prefect schedule `0 */4 * * *` (every 4 hours UTC). + +MC-Forewarner writes to `DOLPHIN_FEATURES` key `mc_forewarner_latest`. The TUI entry listener fires on each write and populates the full MC footer panel: `catastrophic_prob` Digits + ProgressBar, `envelope_score` bar, prob sparkline history, `source` label (`REAL_MODEL` / `FALLBACK_NO_DATA` / `FALLBACK_ERROR`). + +If the TUI starts between 4-hour runs and HZ has never been written to (e.g., fresh HZ instance), the footer shows `"awaiting HZ data (runs every 4h via Prefect)"` in yellow. This is a cold-start state only — once the first Prefect run completes the key persists in HZ indefinitely (no TTL). + +**MC payload schema**: +```json +{ + "status": "GREEN | ORANGE | RED", + "catastrophic_prob": 0.07, + "envelope_score": 0.91, + "source": "REAL_MODEL | FALLBACK_NO_DATA | FALLBACK_ERROR", + "timestamp": "2026-04-05T14:00:00+00:00" +} +``` + +**Thresholds**: GREEN `prob < 0.10` · ORANGE `0.10–0.30` · RED `≥ 0.30` + +**Models path**: `nautilus_dolphin/mc_results/models/*.pkl` — if absent, falls back to `FALLBACK_NO_DATA` (ORANGE, prob=0.20, env=0.80) which is a safe conservative posture, never random. + +### 28.8 DOLPHIN_PNL_BLUE + +`DOLPHIN_PNL_BLUE["session_perf"]` is now wired in TUI v9. Displays WR, PF, Sharpe, Calmar live. + +--- + +*End of DOLPHIN-NAUTILUS System Bible v6.0 — 2026-04-05 (see v7.0 below for all updates)* + + +--- + +## 29. ALGO VERSIONING & LINEAGE TRACKING + +### v2_gold_fix_v50-v750 (Deployed 2026-04-10) + +**Context:** During the initial 3.5-day "shakedown cruise" (`v1_shakedown`), the system +executed ~179 trades matching the Gold Spec frequency (~50/day) but suffered a **-6% +drawdown** via MAX_HOLD fee-bleed. + +**Root Cause:** In `nautilus_event_trader.py -> _normalize_ng7()`, the live `vel_div` +calculation was `v50 - v150`. The Gold Spec backtest (181% ROI) strictly used `v50 - v750`. +Subtracting medium-term `v150` instead of macro `v750` resulted in a high-noise signal +that triggered on micro-jitters rather than structural macro instability, eliminating the +mean-reversion snap-back required for the 95 bps FIXED_TP exit. + +**Fix:** Corrected `_normalize_ng7()` to `'vel_div': v50 - v750`. + +**Lineage Tag:** All trades executed after this fix are tagged in `nautilus_trader.log` +and `DOLPHIN_STATE_BLUE` with `[v2_gold_fix_v50-v750]`. Data science queries against +ClickHouse/PnL logs should split analysis at this tag to isolate true Gold Spec performance. + +### Version Registry + +| Tag | Deployed | vel_div Formula | Notes | +|-----|----------|-----------------|-------| +| `v1_shakedown` | 2026-04-02 | `v50 - v150` | Noise bug — medium-term subtraction. ~179 trades, -6% drawdown. | +| `v2_gold_fix_v50-v750` | 2026-04-10 | `v50 - v750` | **Gold Spec corrected.** Macro divergence signal. Matches 181% ROI backtest. | + +### How Versioning Works in Code + +- **Constant:** `ALGO_VERSION` defined near the top of `nautilus_event_trader.py` directly + after `VOL_P60_THRESHOLD`. +- **Startup log:** Emitted on launch as `ALGO_VERSION: ` — impossible to miss. +- **HZ Snapshot:** Written into `DOLPHIN_STATE_BLUE` engine snapshot as `algo_version` key + on every heartbeat cycle. +- **Trade logs:** Every `ENTRY:` and `EXIT:` log line in `nautilus_trader.log` carries the + `[]` suffix for post-hoc query filtering. + +--- + +## 30. CLICKHOUSE OBSERVABILITY LAYER + +**Version**: v7.0 Addition — 2026-04-19 +**Status**: DEPLOYED. All BLUE trades logged in real time. + +### 30.1 Infrastructure + +| Parameter | Value | +|---|---| +| **URL** | `http://localhost:8123/` | +| **Database** | `dolphin` | +| **User** | `dolphin` | +| **Password** | `dolphin_ch_2026` | +| **Docker** | Managed via `/mnt/dolphinng5_predict/prod/docker-compose.yml` | +| **HTTP Headers** | `X-ClickHouse-User: dolphin`, `X-ClickHouse-Key: dolphin_ch_2026` | + +### 30.2 Writer Module + +**File**: `/mnt/dolphinng5_predict/prod/ch_writer.py` + +```python +from prod.ch_writer import ch_put, ch_put_green, ch_put_prodgreen +ch_put("trade_events", {...}) # BLUE -> dolphin.* +ch_put_green("trade_events", {...}) # legacy GREEN -> dolphin_green.* +ch_put_prodgreen("trade_events", {...}) # PRODGREEN -> dolphin_prodgreen.* +``` + +- Durable local spool first, then background replay to ClickHouse +- Rows survive process restarts via SQLite spool under `/mnt/dolphin_training/ch_spool` +- Zero external dependencies for the network path: pure `urllib.request` HTTP/JSONEachRow +- Env overrides: `CH_URL`, `CH_USER`, `CH_PASS`, `CH_DB`, `CH_SPOOL_DIR` +- Switch to OTel/Uptrace: replace replay internals only — callers unchanged + +### 30.3 Table Reference + +#### `dolphin.trade_events` +One row per closed trade. Written by `prod/nautilus_event_trader.py` on every EXIT. + +| Column | Type | Description | +|---|---|---| +| `ts` | DateTime64(6, 'UTC') | Microsecond close timestamp | +| `date` | Date | UTC date of entry | +| `strategy` | LowCardinality(String) | `"blue"` or `"green"` | +| `asset` | LowCardinality(String) | e.g. `"BTCUSDT"` | +| `side` | LowCardinality(String) | `"SHORT"` or `"LONG"` | +| `entry_price` | Float64 | Entry price | +| `exit_price` | Float64 | Exit price | +| `quantity` | Float64 | Contract quantity | +| `pnl` | Float64 | Net PnL in USDT | +| `pnl_pct` | Float32 | PnL as fraction (0.01 = 1%) | +| `exit_reason` | LowCardinality(String) | `FIXED_TP`, `MAX_HOLD`, `V7_MAE_STOP`, `HIBERNATE_HALT`, `SUBDAY_ACB_NORMALIZATION`, … | +| `vel_div_entry` | Float32 | vel_div at entry | +| `boost_at_entry` | Float32 | ACB boost factor at entry | +| `beta_at_entry` | Float32 | ACB beta at entry | +| `posture` | LowCardinality(String) | `APEX` / `STALKER` / `TURTLE` / `HIBERNATE` | +| `leverage` | Float32 | Final leverage used | +| `bars_held` | UInt16 | Bars held until exit | +| `regime_signal` | Int8 | OB macro regime at exit (-1/0/+1) | + +**Bucket analysis query** (exclude forced exits): +```sql +SELECT exit_reason, count(), avg(pnl_pct)*100 +FROM dolphin.trade_events +WHERE exit_reason NOT IN ('HIBERNATE_HALT', 'SUBDAY_ACB_NORMALIZATION') +GROUP BY exit_reason ORDER BY count() DESC +``` + +#### `dolphin.eigen_scans` +One row per processed eigenvalue scan. Written by `prod/nautilus_event_trader.py` on every scan. + +| Column | Notes | +|---|---| +| `ts` | Microsecond timestamp | +| `scan_number` | Monotonic counter from NG8 | +| `vel_div` | Primary signal | +| `w50_velocity`, `w750_velocity` | Window velocities | +| `instability_50` | 50-bar instability index | +| `bridge_ts` | HZ write timestamp (latency = ts - bridge_ts) | + +#### `dolphin.adaptive_exit_shadow` +One row per bar where AE evaluated an open trade, PLUS one CLOSED row per trade close. +Created by `adaptive_exit/adaptive_exit_engine.py` `_ensure_shadow_table()`. + +| Column | Type | Description | +|---|---|---| +| `ts` | DateTime64(6, 'UTC') | Evaluation timestamp | +| `ts_day` | Date MATERIALIZED | Partition key (TTL 90 days) | +| `trade_id` | String | Trade identifier | +| `asset` | LowCardinality(String) | Asset symbol | +| `bucket_id` | UInt8 | KMeans bucket (0–6) | +| `bars_held` | UInt16 | Bars held at evaluation time | +| `mae_norm` | Float32 | MAE / ATR (normalised) | +| `mfe_norm` | Float32 | MFE / ATR (normalised) | +| `tau_norm` | Float32 | bars_held / max_hold (0→1) | +| `p_cont` | Float32 | P(continuation) from LR model | +| `vel_div_entry` | Float32 | vel_div at trade entry | +| `vel_div_now` | Float32 | vel_div at evaluation bar | +| `action` | LowCardinality(String) | `HOLD` / `EXIT` / `CLOSED` | +| `exit_reason` | LowCardinality(String) | AE shadow reason (not executed) | +| `actual_exit` | LowCardinality(String) | Real exit reason (CLOSED rows only) | +| `pnl_pct` | Float32 | Final PnL (CLOSED rows only) | + +**CLOSED rows** (`action='CLOSED'`): written once per trade at real close. Use these for comparing real exits vs AE assessment. + +### 30.4 Monitoring Commands + +```bash +# Recent trades today +clickhouse-client --query "SELECT asset, pnl_pct, exit_reason, bars_held FROM dolphin.trade_events WHERE date=today() ORDER BY ts DESC LIMIT 10" + +# Bucket performance (all-time, excl forced exits) +clickhouse-client --query " +SELECT s.bucket_id, count() n, countIf(s.pnl_pct>0) wins, avg(s.pnl_pct)*100 avg_pct +FROM dolphin.adaptive_exit_shadow s +WHERE s.action='CLOSED' AND s.actual_exit NOT IN ('HIBERNATE_HALT','SUBDAY_ACB_NORMALIZATION') +GROUP BY s.bucket_id ORDER BY s.bucket_id" +``` + +--- + +## 31. ADAPTIVE EXIT ENGINE — SHADOW MODE + +**Version**: v7.0 Addition — 2026-04-19 +**Status**: Shadow mode active. No real exits influenced. Logging to `adaptive_exit_shadow`. +**Integration**: `prod/nautilus_event_trader.py` — per-bar daemon thread evaluation. + +### 31.1 Overview + +The Adaptive Exit Engine (AE) is a per-bucket logistic regression model that estimates `P(continuation)` — the probability that holding a trade further will yield positive outcome — given the current trade state features. + +In shadow mode, it: +- **Evaluates** every active trade every bar +- **Logs** its decision to `dolphin.adaptive_exit_shadow` in ClickHouse +- **Never** interferes with real exits + +Decision logic (mirrors doctrinal spec): +``` +EXIT if: + mae_norm > MAE_MULT_TIER1 × ATR [hard stop, MAE_MULT_TIER1=3.5] + giveback: mfe < 0.50 × peak_mfe AND p_cont < 0.40 + tau_norm > 1.0 [time cap] +else: + HOLD +``` + +### 31.2 File Layout + +| File | Full Path | Purpose | +|---|---|---| +| **Engine** | `/mnt/dolphinng5_predict/adaptive_exit/adaptive_exit_engine.py` | Main evaluation + shadow logging. `AdaptiveExitEngine.evaluate()`, `on_entry()`, `on_exit()`, `log_shadow()` | +| **Continuation Model** | `/mnt/dolphinng5_predict/adaptive_exit/continuation_model.py` | `ContinuationModelBank` — per-bucket LR + online SGD update. 15-feature vector | +| **Bucket Engine** | `/mnt/dolphinng5_predict/adaptive_exit/bucket_engine.py` | `build_buckets()`, `get_bucket()` — KMeans k=7 asset assignment | +| **Data Pipeline** | `/mnt/dolphinng5_predict/adaptive_exit/data_pipeline.py` | `build_training_data()` — 5yr 1m klines → trajectory simulation → training DataFrame | +| **Train Script** | `/mnt/dolphinng5_predict/adaptive_exit/train.py` | CLI training runner. `use_obf_ch=False` (OBF CH only 13 days; backfill pending) | +| **Models** | `/mnt/dolphinng5_predict/adaptive_exit/models/` | `continuation_models.pkl`, `bucket_assignments.pkl`, `training_data.parquet` | + +### 31.3 15-Feature Vector (FEATURE_COLS) + +``` +mae_norm — MAE / ATR +mfe_norm — MFE / ATR +tau_norm — bars_held / max_hold +ret_1 — log return, last 1 bar +ret_3 — log return, last 3 bars +vel_div_entry — vel_div at trade entry (distribution match: BLUE entries ≈ vel_div < -0.02) +vel_div_now — vel_div at evaluation bar (real-time) +spread_bps — OBF spread in basis points (0.0 if OBF unavailable) +depth_usd — OBF 1% depth in USD +fill_prob — OBF fill probability +exf_fng — Fear & Greed Index +exf_fng_delta — FnG change (day-over-day) +exf_funding_btc — BTC funding rate +exf_dvol_btc — BTC implied volatility +exf_chg24_btc — BTC 24h price change +``` + +**OBF features**: set to 0.0 in training (OBF CH only 13 days; will bolt on in Phase 2). +**ExF features**: backfilled from NPZ files (2021–2026), joined by calendar date. + +### 31.4 Training Data Pipeline + +```python +# Universal sampling: ALL price bars as candidate entries +# Memory bound: pre-select ceil(max_samples / MAX_HOLD) entry bars per asset +# Full k-trajectories (MAX_HOLD=120 bars) preserved per selected entry +# OBF: use_obf_ch=False ← CH only has 13 days live data (Apr 6-19 2026) +# ExF: loaded from NPZ backfill ← 1658 daily files, joined by date + +python -m adaptive_exit.train # runs from /mnt/dolphinng5_predict/ +# Output: adaptive_exit/models/continuation_models.pkl + bucket_assignments.pkl +``` + +Training time: ~15–30 min for 48 assets × 5yr 1m klines. + +### 31.5 Integration in nautilus_event_trader.py + +```python +# Init (in _build_engine): +self._ae = AdaptiveExitEngine.load() # loads models from adaptive_exit/models/ + +# On entry: +self._ae.on_entry(trade_id, asset, direction, entry_price, vel_div_entry=vel_div) + +# Per-bar (daemon thread, non-blocking): +shadow = self._ae.evaluate(trade_id=tid, asset=asset, direction=dir, + entry_price=entry, current_price=cur, + bars_held=bars, max_hold=120, + recent_prices=price_buf, vel_div_now=vel_div) +self._ae.log_shadow(shadow) # async CH insert + +# On exit: +self._ae.on_exit(trade_id, actual_exit_reason, pnl_pct) +# → triggers online_update() for live model refinement +# → inserts CLOSED row with actual_exit + p_cont at close +``` + +**Thread safety**: `evaluate()` uses internal `threading.Lock`. Daemon thread is fire-and-forget. Zero impact on main scan loop latency. + +### 31.6 Online Learning + +After each trade closes, `ContinuationModelBank.online_update()` feeds the outcome back via **SGD** (partial fit). Natural exits only — `HIBERNATE_HALT` and `SUBDAY_ACB_NORMALIZATION` are filtered to prevent regime artifacts from biasing the continuation distribution. + +### 31.7 Promote to Live (Prerequisites) + +AE remains shadow-only until: +1. 500+ closed trades per bucket (for statistical significance) +2. Shadow accuracy > 60% (p_cont > 0.50 → WIN correctly) +3. Explicit shadow vs real exit comparison query confirms AE would have improved net$ +4. Code review of exit integration (requires new `exit_reason` codes: `AE_MAE_STOP`, `AE_GIVEBACK_LOW_CONT`, `AE_TIME`) + +### 31.8 v2 Training / Replay Spec + +The next revision is documented in [`AdaptiveExitManager_v2_SPEC.md`](AdaptiveExitManager_v2_SPEC.md). + +Key requirements for v2: + +- preserve the current shadow-only safety boundary +- replay against the NG7 scan tape / eigenfile prices, not just terminal trade rows +- penalize early winner clipping more heavily than loser-saving +- write all v2 artifacts to versioned paths under `adaptive_exit/models/v2/` +- keep `adaptive_exit/models/continuation_models.pkl` and `adaptive_exit/models/bucket_assignments.pkl` intact + +The v2 spec is intentionally non-destructive and must not overwrite the current model artifacts in place. + +### 31.9 EsoF Value Gate — Live Exposure-Only Haircut + +The live engine now consumes the continuous EsoF advisory score as a **size-only** gate. +This is a conservative haircut on leverage / notional only. It does **not** alter: + +- asset selection +- direction choice +- exit logic +- trade accounting +- HZ / CH observability paths + +Implementation details: + +- Live score source: `DOLPHIN_FEATURES['esof_latest']` with fallback to `esof_advisor_latest` +- Freshness rule: stale / missing payloads are neutral and leave sizing unchanged +- The live haircut is label-aligned: `NEUTRAL` receives the main haircut, `UNFAVORABLE` receives the deepest haircut +- `MILD_POSITIVE` and `MILD_NEGATIVE` remain mostly full-size apart from narrow transition shoulders around the label boundaries +- The gate is intentionally conservative to avoid overfit + +Scope note: +- The live gate is allowed to be non-monotonic only around the `NEUTRAL` and `UNFAVORABLE` label boundaries. +- Positive `sc` and `MILD_NEGATIVE` remain outside the haircut experiment space and should stay at `1.0x` unless a new documented study explicitly changes that contract. + +Amendment note: +- Earlier draft notes and helper comments temporarily kept BLUE neutral until the `UNFAVORABLE` boundary. +- The live helper was revised on 2026-05-06 to make the haircut label-aware with small transition shoulders around `NEUTRAL` and `UNFAVORABLE`. + +Replay note, BLUE closed-trade history: + +- Sample: `1217` BLUE closed trades from `2026-03-31` through `2026-04-29` +- Entry timestamp proxy used for replay: `entry_ts ≈ exit_ts - bars_held × 11s` +- Outcome with current gate: realized net PnL `+3191.91` → counterfactual `+4964.63` (`+1772.73` uplift) +- Normal exits only (`MAX_HOLD` / `FIXED_TP` / `STOP_LOSS`): `+8268.07` → `+8866.59` (`+598.53` uplift) +- The gate is exposure-only, so the trade sign / win-rate is unchanged by construction; the benefit comes from reducing notional in weaker `sc` regimes + +### 31.10 SC Threshold Advisor — Shadow ML Overlay + +The `sc` gate now has an advisory-only learning overlay that observes live context and logs a recommended size-multiplier / implied threshold. It is a shadow artifact only and must **never** override the deterministic live gate. + +File locations: + +- Runtime advisor: `adaptive_exit/sc_threshold_advisor.py` +- Benchmark harness: `adaptive_exit/sc_threshold_benchmark.py` +- Future / optional offline bootstrap path: `adaptive_exit/train_sc_threshold_advisor.py` +- Shadow table: `dolphin.sc_threshold_advisor_shadow` or `dolphin_prodgreen.sc_threshold_advisor_shadow` depending on the live node +- HZ latest snapshot: `DOLPHIN_FEATURES['sc_threshold_advisor_latest']` + +Design constraints: + +- advisory only, no execution authority +- deterministic `esof` / `sc` size gate remains the live source of truth +- learns from trade outcomes, ExF context, recent trade performance, and current `sc` +- logs every evaluation so replay and threshold search can be audited later +- existing model artifacts are respected and not overwritten in place + +The live wiring is intentionally narrow: + +- it reads the current `sc` state and ExF context +- it emits a shadow recommendation and confidence +- it records the recommendation in ClickHouse / Hazelcast +- it observes realized outcomes only after the real trade closes +- it does **not** alter order selection, exit logic, or capital accounting + +### 31.11 SC Gauge Surface — Shadow Bucketed Policy + +The `sc` gate now also has a second advisory layer: a bucket-aware action-surface gauge. It is still shadow-only and must never alter live execution, but it learns a richer policy than the threshold advisor: + +- size multiplier +- take-profit multiplier +- max-hold multiplier +- per-bucket action selection + +File locations: + +- Runtime advisor: `adaptive_exit/sc_gauge_advisor.py` +- Replay benchmark: `adaptive_exit/sc_gauge_benchmark.py` +- Shadow table: `dolphin.sc_bucket_gauge_shadow` or `dolphin_prodgreen.sc_bucket_gauge_shadow` +- HZ latest snapshot: `DOLPHIN_FEATURES['sc_bucket_gauge_latest']` + +Design constraints: + +- advisory only, no execution authority +- deterministic EsoF / `sc` size gate remains the live source of truth +- learns from executed outcomes plus replayed price paths +- uses point-in-time OBF placement/signal/market/macro context and ExF context +- bucket-aware via `adaptive_exit/bucket_engine.py` +- logs every evaluation so replay, OOS benchmarking, and online-learning drift can be audited +- existing model artifacts are respected and not overwritten in place + +Anti-degradation rules: + +- online updates pause if replay quality regresses materially +- frozen-vs-online walk-forward benchmark is mandatory before promotion +- the replay harness must reconstruct paths only from data available at trade time or earlier +- OBF used for benchmarking must be point-in-time, not end-of-day aggregated + +Benchmark outputs: + +- actual vs policy PnL +- ROI, win rate, PF, Sharpe, Sortino, max drawdown +- average recommended size / TP / hold multipliers +- regret vs actual trade +- frozen vs online OOS comparison + +The gauge is complementary to the threshold advisor. The threshold advisor decides how much to scale the trade from `sc`; the gauge learns whether, within that regime, smaller or larger size/TP/hold actions are better on a per-bucket basis. + +--- + +## 32. ALPHA EXIT ENGINE V7 — GREEN LIVE VALIDATION + +**Version**: v7.1 Addition — 2026-04-20 +**Source**: `nautilus_dolphin/nautilus/alpha_exit_v7_engine.py` +**Status**: Active in GREEN (paper-traded Nautilus node). Threshold calibrated from live shadow data. +**NOT active in BLUE** — BLUE uses base `AlphaExitManager` only (FIXED_TP / STOP_LOSS / MAX_HOLD). + +### 32.1 Overview + +AlphaExitEngineV7 extends V6 with two improvements validated by Monte Carlo and backtest: + +- **V7-1**: Vol-normalized MAE thresholds — adaptive SL tiers based on `rv_composite = 0.50·rv15 + 0.30·rv30 + 0.20·rv50`. At high vol, thresholds widen → fewer false-positive SL exits. +- **V7-2**: Bounce-probability soft injection — trained on 838 adverse-bar samples. `bounce_score ∈ [-1,+1]` modulates directional/risk terms. + +The engine computes a composite `exit_pressure` score and decides: EXIT / RETRACT / EXTEND / HOLD. + +### 32.2 Exit Pressure Formula + +```python +exit_pressure = 2.0 * mae_risk + 2.5 * mfe_risk + directional_term + risk_term +``` + +Where: +- `mae_risk`: tiered vol-normalized MAE contribution (0..3+) +- `mfe_risk`: MFE convexity decay (giveback detection) +- `directional_term`: bounce_score modulation, OB imbalance +- `risk_term`: bounce_risk, late-stage time weighting + +Decision thresholds: +| Pressure | Action | Reason | +|----------|--------|--------| +| `> 2.69` | **EXIT** | `V7_MAE_SL_VOL_NORM` or `V7_COMPOSITE_PRESSURE` | +| `> 1.0` | RETRACT | `V7_RISK_DOMINANT` | +| `< -0.5` & pnl>0 | EXTEND | `V7_DIRECTIONAL_EDGE` | +| else | HOLD | — | + +### 32.3 Pressure Threshold Calibration — Live Research (2026-04-19/20) + +**Dataset**: 24 completed GREEN trades, live eigen_scan data, V7 shadow evaluations at 100ms cadence via `NautilusCachePriceFeed` (live Binance WebSocket bid/ask). + +**Methodology**: For each trade, V7 emitted shadow EXIT signals at various pressure levels. We replayed with different pressure thresholds to find the level where V7 cuts genuine losers while letting winners run. + +**Results by threshold**: + +| Threshold | Trades Cut | Total PnL | ROI | vs Base ($784) | +|-----------|-----------|-----------|-----|----------------| +| ≥ 2.00 (original) | 22/24 | +$439 | +1.67% | **-$345** | +| ≥ 2.35 (data-optimal) | 17/24 | +$891 | +3.38% | +$107 | +| ≥ 2.60 (+buffer) | 17/24 | +$891 | +3.38% | +$107 | +| **≥ 2.69 (chosen)** | **17/24** | **+$891** | **+3.38%** | **+$107** | +| ≥ 3.00 | 14/24 | +$796 | +3.02% | +$12 | +| ≥ 3.25 | 0/24 | +$784 | +2.98% | $0 (identical to base) | + +### 32.4 Inverse-ARS Bounce Detector — Shadow Research + +**Version**: v1 shadow research addition — 2026-05-02 +**Status**: Shadow-only advisory. Not wired into BLUE live execution. + +This detector is a separate research overlay for bounce-trap detection and +inverse-long viability scoring. It uses bounded direct-vs-inverse ARS features +plus pre-entry tape shape to answer a single question: + +- "Is this short likely to be a bounce trap, and therefore also a stronger + inverse-long candidate?" + +Implementation notes: + +- trained on historical closed BLUE trades from `dolphin.trade_events` +- uses a bounded feature set to reduce overfit: + - direct ARS + - inverse ARS + - ARS gap/share + - pre-entry return / trend / range + - `vel_div` at entry +- online learning capable via buffered `partial_fit` +- model artifact stored on the DOLPHIN local volume: + - `/mnt/dolphin_training/models/inverse_ars_bounce_detector_blue.pkl` +- benchmark/report path defaults to `/tmp/inverse_ars_bounce_benchmark_blue.json` + +Operational rule: + +- a high score is interpreted as higher short bounce-risk +- the same score is also the inverse-long viability score on that same window +- live BLUE execution remains unchanged and must continue to ignore this model + +### 32.4.1 Inverse-ARS Bounce Detector - TA/OBF Ablation + +**Version**: shadow ablation note — 2026-05-02 +**Status**: Shadow-only research. No live BLUE wiring. + +After adding optional technical-analysis proximity features and point-in-time +OBF context to the detector, the full available replay window was re-run on: + +- scan cache: `2025-12-31` to `2026-03-05` +- BLUE closed trades: 947 rows +- PRODGREEN closed trades: 131 rows + +The full-history result did **not** justify promoting the richer feature stack: + +- baseline remained the best overall variant on the full run +- TA alone was marginal and did not improve the composite score +- OBF alone improved some narrower smoke slices, but not the full replay +- TA+OBF won the small smoke slice, but not the full-history benchmark + +Saved shadow artifacts: + +- full ablation report: `/tmp/inverse_ars_bounce_ablation_full.json` +- full best-shadow model: `/mnt/dolphin_training/models/inverse_ars_bounce_detector_blue_best_shadow_full.pkl` +- smoke best-shadow model: `/mnt/dolphin_training/models/inverse_ars_bounce_detector_blue_best_shadow_smoke.pkl` + +Interpretation: + +- keep OBF and TA support in the shadow harness +- do not treat them as a live promotion signal yet +- the detector remains most defensible as a selection warning / haircut aid, + not a hard veto + +### 32.4.2 Inverse-ARS Bounce Advisor - Live Shadow Wiring + +**Version**: live shadow wiring note — 2026-05-03 +**Status**: Shadow-only advisory. No order placement or exit control. + +The BLUE-trained inverse-ARS bounce detector is now wrapped in a live +advisory layer for PRODGREEN observability. The wrapper keeps the execution +path unchanged and is used only for audit, TUI display, and cautious online +learning. + +Live wiring: + +- ClickHouse shadow table: `dolphin_prodgreen.inverse_ars_bounce_shadow` +- Hazelcast latest key: `DOLPHIN_FEATURES["bounce_advisor_latest"]` +- live trader hook: `prod/nautilus_event_trader.py` +- TUI display: `Observability/dolphin_TUI_prod_green.py` + +Operational behavior: + +- every entry and open-trade scan can emit a bounce score row to ClickHouse +- the TUI shows the latest v7 exit row and the latest bounce advisor row + directly underneath it when a trade is open +- online updates only happen after close, and forced exits are skipped +- the underlying detector still uses buffered, guarded `partial_fit` + +Traceability: + +- base detector artifact: `/mnt/dolphin_training/models/inverse_ars_bounce_detector_blue.pkl` +- advisor artifact: `/mnt/dolphin_training/models/inverse_ars_bounce_advisor_blue.pkl` +- live output is advisory only and must not influence order placement unless a + separate ablation later proves a net benefit + +### 32.4.3 BLUE V7 Exit Wiring - Live Control + +**Version**: live wiring note — 2026-05-04 +**Status**: BLUE live exits now follow `AlphaExitEngineV7`; the AE shadow engine remains observational. +**Activated**: 2026-05-04 UTC + +BLUE trade exits now use the v7 pressure decision surface in practice: + +- live trader hook: `prod/nautilus_event_trader.py` +- orchestrator hook: `nautilus_dolphin/nautilus_dolphin/nautilus/esf_alpha_orchestrator.py` +- live v7 reason codes: + - `V7_MAE_SL_VOL_NORM` + - `V7_COMPOSITE_PRESSURE` + - `V7_RISK_DOMINANT` + - `V7_DIRECTIONAL_EDGE` + +Operational behavior: + +- the live BLUE orchestrator consults `AlphaExitEngineV7` before the base exit manager +- when v7 returns `EXIT`, the actual `trade_events.exit_reason` records the v7 reason string +- `v7_decision_events` remains the authoritative observability journal for v7 actions and is now populated by live BLUE decisions, not just shadow replays +- HIBERNATE remains a hard override and still forces `HIBERNATE_HALT` +- AE shadow logging stays unchanged and continues to be purely observational + +This promotion is limited to BLUE exit control and logging. It does not promote the +separate AE shadow engine to live exit authority. + +#### 32.4.3.1 V7 Exit Audit Snapshot (2026-05-08) + +Post-activation audit of the live V7 journal and trade ledger (`2026-05-01` through +`2026-05-06`) showed: + +- `V7_COMPOSITE_PRESSURE` was net positive in the live ledger: + - `35` total exits across `blue` + `prodgreen` + - total PnL `+734.30` + - average `pnl_pct` `+0.000319` + - `5` winners, `30` losers +- `V7_MAE_SL_VOL_NORM` was net negative: + - `7` total exits across `blue` + `prodgreen` + - total PnL `-2234.97` + - average `pnl_pct` `-0.003306` + - `0` winners, `7` losers +- `FIXED_TP` was not the main failure mode in this audit window. +- The primary live-exit concern is therefore the MAE/pressure branch calibration, not a blanket TP miscalibration. + +Audit artifacts: + +- [adaptive_exit/audit_v7_exit_quality.py](/mnt/dolphinng5_predict/adaptive_exit/audit_v7_exit_quality.py) +- [run_logs/v7_exit_quality_audit.md](/mnt/dolphinng5_predict/run_logs/v7_exit_quality_audit.md) +- [run_logs/v7_exit_quality_audit.json](/mnt/dolphinng5_predict/run_logs/v7_exit_quality_audit.json) + +**Key insight — why 2.0 was too low**: +V7 at 2.0 cut 9 winners early (missing +$868 collectively) while saving +$245 on 5 losers. The low threshold reacted to transient adverse pressure that reversed into profitable exits. + +**Key false positives at low pressure** (prevented by raising to 2.69): +| Trade | Pressure | V7@2.0 PnL | Base PnL | Issue | +|-------|----------|------------|----------|-------| +| BNBUSDT #20 | 2.31 | -$177 | +$218 | V7 panicked on noise; base held for big win | +| FETUSDT #11 | 2.00 | -$142 | +$68 | V7 misread vol spike; price recovered | +| LTCUSDT #2 | 3.00 | +$9 | +$276 | V7 cut at tiny profit; base rode to full TP | + +**Key true positives at high pressure** (still caught at 2.69): +| Trade | Pressure | Base PnL | V7 PnL | Saved | +|-------|----------|----------|--------|-------| +| ENJUSDT #24 | 3.00 | -$342 | +$26 | +$368 | +| ENJUSDT #19 | 2.04 | -$375 | -$254 | +$121 | +| ENJUSDT #16 | 2.73 | -$36 | +$10 | +$45 | +| ONTUSDT #9 | 3.00 | +$273 | +$297 | +$24 | + +**Chosen threshold: 2.69** — in the optimal plateau (2.35–2.70), with a small buffer above the data-optimal 2.35 to stay closer to base-engine "let winners run" behaviour while still catching high-pressure adverse exits. + +### 32.3.1 Underwater Recovery Shadow Replay + +After the above live-threshold calibration, we added a stricter replay question: if a trade has already been underwater, then later recovers into profit, should V7 exit on the first positive `RETRACT` rather than waiting for the later terminal `EXIT`? + +**Replay source**: `dolphin_green.v7_decision_events` + +**Replay policy tested** +- Policy A: exit on the first positive `RETRACT` after the trade has been underwater +- Policy B: same as A, but only when `vel_div_now > 0` as a momentum confirmation + +**Replay sample** +- `30` V7-tracked trades total +- `20` trades with terminal `EXIT` +- `9` trades had an eligible underwater-to-positive `RETRACT` candidate +- `8` of those also had positive `vel_div_now` + +**Results** + +| Policy | Eligible trades | Avg terminal V7 exit | Avg shadow exit | Improvement | +|--------|-----------------|----------------------|-----------------|-------------| +| A: first positive `RETRACT` | 9 | `-23.186%` | `+15.799%` | `+38.985 pp` | +| B: positive `RETRACT` + `vel_div_now > 0` | 8 | `-24.526%` | `+17.173%` | `+41.699 pp` | + +**Best examples** +- `ENJUSDT` `1a6f39f1`: terminal `-80.991%` vs shadow `+77.815%` +- `ENJUSDT` `3d23043c`: terminal `-47.499%` vs shadow `+9.779%` +- `DASHUSDT` `afd67d96`: terminal `-32.354%` vs shadow `+16.177%` + +### 32.5 Full-Tape Long Signal Characterization + +**Version**: research pass — 2026-05-06 +**Status**: offline characterization only. No live BLUE wiring. + +The raw daily scan tape was characterized directly from +`/mnt/dolphin_training/share_offload/vbt_cache_klines/*.parquet` via +`adaptive_exit/characterize_long_signals.py`. + +Artifacts: + +- summary dataset: + `/mnt/dolphin_training/long_signal_research/long_signal_scan_summary_h24.parquet` +- machine report: + `/mnt/dolphin_training/long_signal_research/long_signal_characterization_report.json` +- human summary: + `/mnt/dolphin_training/long_signal_research/long_signal_characterization_report.md` + +Definitions used: + +- horizon: `24` forward rows +- TP reference: `0.95%` +- `strong_long`: best-asset MFE `>= 2.0%` and top-3 mean end return `>= 0.8%` +- `broad_long`: top-3 mean end return `>= 1.0%` and positive breadth `>= 50%` + +Key result: + +- `vel_div > 0.01` alone is **not** the main long edge. +- The stronger long regimes are **stressed unwind / squeeze** states: + high `instability_50`, negative slower-window velocity (`v300 < 0`, + `v750 < 0`), and often a previously negative `vel_div` state. +- Best recent-HQ rule (`2025-12-31` onward): + `inst50_q90 & v300_neg & v750_neg` + - support: `6305` rows (`3.55%`) + - `strong_long`: `0.341` vs base `0.165` (`2.07x` lift) + - `broad_long`: `0.354` vs base `0.137` (`2.59x` lift) + +Implication: + +- the long counterpart to the short dislocation thesis is **not** a generic + bullish breakout detector +- it is a **reversal / squeeze detector on the same instability manifold** +- market fingerprint should therefore query an asset-fingerprint layer for the + assets most likely to express that unwind, rather than assuming the market + signal alone identifies the tradeable long +- `BNBUSDT` `9bf88b81`: terminal `-12.469%` vs shadow `+0.789%` + +**Interpretation** +- The replay strongly supports a momentum-aware profit-lock rule on rebound trades. +- The current terminal V7 logic is good at cutting some losers, but still gives back recoveries on a subset of trades. +- This is still shadow research. It should not be promoted to live execution without a separate ablation on the same day/window and a dedicated false-positive review. + +### 32.4 GREEN vs BLUE Exit Architecture + +| Aspect | BLUE | GREEN | +|--------|------|-------| +| Exit engine | `AlphaExitManager` (V6 base) | `AlphaExitEngineV7` (V7 + base) | +| Exit eval cadence | Scan cadence (~10-13s) | Scan cadence + **100ms RT timer** | +| Exit eval price | `prices[pos.asset]` from eigen_scan | Same + **live WebSocket bid/ask** via `NautilusCachePriceFeed` | +| V7 active exits | No | Yes — `V7_COMPOSITE_PRESSURE`, `V7_MAE_SL_VOL_NORM` | +| RT TP/SL monitoring | No | Yes — `RealTimeExitManager` at 100ms | +| V7 pressure threshold | N/A | **2.69** (raised from 2.0) | + +**Important**: GREEN's RT/V7 exits are **observability-only** — they write to CH with live exit prices but do NOT close the engine position. The base `AlphaExitManager` remains authoritative for position lifecycle (FIXED_TP / MAX_HOLD / STOP_LOSS). The engine fires its own EXIT on the next scan cycle regardless. + +### 32.5 Per-Trade Comparison (24 trades, 2026-04-19/20) + +| # | Asset | Base PnL | V7@2.69 PnL | Δ | V7 Reason | Pressure | +|---|-------|----------|-------------|---|-----------|----------| +| 1 | HBARUSDT | +$33 | +$81 | +$48 | COMPOSITE | 2.87 | +| 2 | LTCUSDT | +$276 | +$9 | -$268 | COMPOSITE | 3.00 | +| 3 | LINKUSDT | -$10 | +$4 | +$13 | COMPOSITE | 3.00 | +| 4 | ENJUSDT | +$10 | +$10 | $0 | (no V7 signal) | — | +| 5 | XLMUSDT | +$8 | +$8 | $0 | (no V7 signal) | — | +| 6 | ENJUSDT | +$4 | -$42 | -$46 | COMPOSITE | 3.00 | +| 7 | TRXUSDT | -$7 | -$7 | $0 | (< 2.69, base kept) | 2.01 | +| 8 | ONGUSDT | -$5 | +$1 | +$6 | COMPOSITE | 3.00 | +| 9 | ONTUSDT | +$273 | +$297 | +$24 | MAE_SL_VOL | 3.00 | +| 10 | FUNUSDT | +$687 | +$590 | -$97 | MAE_SL_VOL | 3.00 | +| 11 | FETUSDT | +$68 | +$68 | $0 | (< 2.69, base kept) | 2.00 | +| 12 | FETUSDT | +$2 | $0 | -$2 | COMPOSITE | 3.00 | +| 13 | DASHUSDT | -$3 | +$1 | +$4 | COMPOSITE | 3.00 | +| 14 | XRPUSDT | -$14 | -$14 | $0 | (< 2.69, base kept) | 2.01 | +| 15 | CELRUSDT | +$3 | +$1 | -$3 | COMPOSITE | 3.00 | +| 16 | ENJUSDT | -$36 | +$10 | +$45 | MAE_SL_VOL | 2.73 | +| 17 | FETUSDT | +$0 | +$2 | +$2 | COMPOSITE | 2.83 | +| 18 | DASHUSDT | $0 | $0 | $0 | COMPOSITE | 3.00 | +| 19 | ENJUSDT | -$375 | -$375 | $0 | (< 2.69, base kept) | 2.04 | +| 20 | BNBUSDT | +$218 | +$218 | $0 | (< 2.69, base kept) | 2.31 | +| 21 | LINKUSDT | $0 | +$1 | +$1 | COMPOSITE | 3.00 | +| 22 | ONTUSDT | +$6 | +$2 | -$5 | COMPOSITE | 3.00 | +| 23 | LTCUSDT | -$13 | +$2 | +$15 | COMPOSITE | 3.00 | +| 24 | ENJUSDT | -$342 | +$26 | +$368 | COMPOSITE | 3.00 | + +### 32.6 Promote to Live (Next Steps) + +1. Collect 100+ trades at 2.69 threshold before considering promotion to BLUE +2. Verify the optimal plateau (2.35–2.70) holds across different market regimes (trending vs ranging) +3. Evaluate whether V7 MAE_SL_VOL_NORM exits should use a lower threshold than COMPOSITE_PRESSURE (tiered thresholds) +4. Compare V7@2.69 against Adaptive Exit Engine (§31) shadow results on the same trade set + +--- + +## 33. ASSET BUCKET SYSTEM + +**Version**: v7.0 Addition — 2026-04-19 +**Status**: Live. Bucket assignments in `adaptive_exit/models/bucket_assignments.pkl`. + +### 32.1 Overview + +Assets are clustered into **7 buckets** (KMeans k=7) using 5-year market characteristics. Buckets represent asset archetypes — how predictably an asset behaves under instability regimes. + +**Purpose**: Trade selection (pick assets from high-ROI buckets) and AE model stratification (separate continuation distributions per bucket). + +### 32.2 Clustering Features + +| Feature | Description | +|---|---| +| `vol_daily_pct` | Daily return volatility | +| `corr_btc` | Pearson correlation with BTCUSDT | +| `log_price` | log(mean price) — proxy for market cap tier | +| `btc_relevance` | `corr_btc × log_price` — interaction term | +| `vov` | Volatility-of-volatility | + +Features computed from 5yr 1m klines. Buckets are PRICE/VOLATILITY characteristics, **not** OBF features. OBF is an overlay-phase add-on, not a bucket driver. + +### 32.2.1 Extended Asset-Feature Sweep + +`adaptive_exit/asset_feature_sweep.py` builds a higher-dimensional per-asset feature vector from the 5 bucket seed features plus 100+ derived TA / path-shape features, using local kline caches first and public Binance futures klines as fallback. It is a research-only pipeline used to discover regime-discriminative asset-feature ranges for later vector retrieval. Default outputs are written under `/mnt/dolphin_training/asset_feature_sweep/` so the SMB repo mount is not used for large derived artifacts. See also [REGIME_ASSET_FINGERPRINT_WORKLOG.md](). + +### 32.2.2 Regime-to-Asset Prototype Retrieval + +`adaptive_exit/regime_asset_retriever.py` is the next research layer on top of the expanded asset vectors. It builds a trade-time regime fingerprint from `vel_div`, `sc`, and the EXF snapshot fields, joins that with the sweep vectors, clusters the regime states, and stores the best-performing asset prototypes per regime cluster. Those prototypes are then used as the query object for a live asset-vector lookup. The retriever is research/offline only and writes its model/report under `/mnt/dolphin_training/regime_asset_prototypes/`. The trade-only baseline is explicitly stored as the **pure performed-trade** model at `/mnt/dolphin_training/regime_asset_prototypes/pure_performed_trade_regime_asset_model.pkl`. + +### 32.2.3 Scan-Tape Backrunner + +`adaptive_exit/scan_tape_backrunner.py` extends the pure performed-trade +baseline into a scan-tape backrunner. It walks the historical scan parquet tape +row by row, reconstructs market fingerprints from the tape itself, derives +candidate asset feature vectors from trailing scan windows, and binds forward +path labels from the tape as counterfactual short outcomes. It is the research +bridge from the trade-conditioned model to the enlarged market-regime model. + +### 32.3 Bucket Performance (Live — 400+ BLUE trades, 2026-04-19) + +Dollar-weighted analysis excluding HIBERNATE/ACB exits: + +| Bucket | n trades | WR% | Net$ | Interpretation | +|---|---|---|---|---| +| B3 | ~89 | ~61% | +$3,858 | **Best alpha** — medium-corr, mid-vol assets | +| B2 | ~31 | ~55% | ~+$800 | Positive, smaller sample | +| B0 | ~23 | ~52% | ~-$200 | Leverage anti-correlation masks WR% | +| B5 | ~18 | ~50% | ~+$300 | Small sample | +| B4 | ~41 | ~38% | -$1,392 | **Avoid** — leverage anti-correlated to outcomes | +| B1 | ~109 | ~41% | -$1,787 | **Worst** — high-corr BTC-like, leverage mis-applied | +| B6 | ~12 | ~67% | small | Sample too small | + +> **B1/B4 leverage problem**: ACB assigns higher leverage to trades that happen to be in B1/B4 — anti-correlation between leverage and outcome. This is an ENTRY selection issue, not fixable by AE. +> **B3 dominance**: B3 assets have moderate BTC correlation + medium volatility → cleaner mean-reversion behavior. + +### 32.4 Live TUI Panel + +`#bucket_footer` in TUI v9 shows per-bucket n/WR%/avg-pnl% updated every 60s from ClickHouse. +Query: `adaptive_exit_shadow` CLOSED rows, `GROUP BY bucket_id`, all-time, excl HIBERNATE/ACB. + +### 32.5 Future: Active Asset Modulation + +Modulating asset picking by live bucket performance is the intended next step once we have 200+ trades per bucket. Hold threshold for current data: B1 (109 trades, -$1,787) is the clearest cut candidate. +**Do NOT modulate until**: per-bucket sample ≥ 200 trades AND bucket performance is stable over 30 days. + +### 32.6 File Locations + +| File | Full Path | +|---|---| +| Bucket assignments | `/mnt/dolphinng5_predict/adaptive_exit/models/bucket_assignments.pkl` | +| Bucket engine | `/mnt/dolphinng5_predict/adaptive_exit/bucket_engine.py` | +| Training data | `/mnt/dolphinng5_predict/adaptive_exit/models/training_data.parquet` | + +--- + +## 34. CRITICAL OPERATIONAL WARNINGS + +### 34.1 SMB Disk-Full Silent Truncation + +**CRITICAL**: The SMB mount at `/mnt/dolphinng5_predict/` (96% full as of 2026-04-19, 42GB free) can silently truncate files to 0 bytes if disk space runs out during a write. + +**The Edit tool opens files with `O_TRUNC` before writing. If ENOSPC is hit, the file becomes 0 bytes with no error message.** + +**Rule**: Always write large files to `/tmp/` first, verify content, then `cp` to the SMB mount. Never use direct Edit/Write on SMB for files > 50KB when disk usage is > 95%. + +```bash +# Check disk space before large writes +df -h /mnt/dolphinng5_predict +# Always verify after writing +wc -l /mnt/dolphinng5_predict/prod/nautilus_event_trader.py +``` + +### 34.2 Hazelcast Volatile State + +HZ is **RAM-only**. Every restart wipes all state. If Hz restarts: +- `DOLPHIN_SAFETY` → reverts to `APEX` (engine will re-read on next bar) +- `acb_boost` → lost (engine uses yesterday's file-based ACB until next ACB push) +- `latest_eigen_scan` → empty (no trades until NG8 produces next scan) +- `capital_checkpoint` → lost (engine falls back to `initial_capital`) + +Never restart HZ unless you know what you're losing. + +### 34.3 Supervisord vs Systemd + +All services are exclusively supervisord-managed. Never use `systemctl start/restart` on dolphin services — creates dual-management race ("random killer" bug). See §26.1. + +### 34.4 vel_div Formula + +The canonical vel_div is `v50_lambda_max_velocity − v750_lambda_max_velocity` (50-window minus 750-window). **Never** compute as `v50 − v150`. The v150 formula was the v1 shakedown bug that caused -6% drawdown on 179 trades. See §29. + +--- + +*End of DOLPHIN-NAUTILUS System Bible v7.0 — 2026-04-19* +*Champion: SHORT only (APEX posture, blue configuration). ALGO=v2_gold_fix_v50-v750.* +*Process manager: Supervisord exclusively (systemd units retired).* +*MHS v3: Active, RM_META≈0.975 [GREEN], 10s critical recovery cooldown.* +*OBF Universe: 540 assets live, zero REST weight WS push streams.* +*NG8 Scanner: Running. Arrow IPC → /mnt/ng6_data/arrow_scans/. NG7 Windows retired.* +*TUI v9: `/mnt/dolphinng5_predict/Observability/TUI/dolphin_tui_v9.py`. All panels live.* +*ClickHouse: `http://localhost:8123/` database=dolphin. Tables: trade_events, eigen_scans, adaptive_exit_shadow.* +*Adaptive Exit Engine: Shadow mode active. 400+ trades logged. Per-bucket LR models trained.* +*D_LIQ Gold Performance: ROI=+189.48% | T=2155 | DD=21.31%.* +*Test gates: 409+ tests green across all suites.* +*Do NOT deploy live capital without review of AE promote-to-live prerequisites (§31.7).* + +> 2026-05-03 bucket update: BLUE now has 1,217 closed trades. Current live ranking is still B3 best and B4 worst; B5, B6, and B1 are net-positive on this larger sample. Keep the old bucket prior as a soft routing prior, not a universal blacklist. +> 2026-05-05 regime-fingerprint addendum: historical backfill artifact now exists on the Dolphin machine at `/mnt/dolphin_training/regime_fingerprint_backfill/regime_fingerprint_backfill.parquet` (416 rows, 161 cols, 556 KB) with report at `/mnt/dolphin_training/regime_fingerprint_backfill/regime_fingerprint_backfill_report.json`. It merges CH trades, recent live log trades, ExF/EsoF, price-path signatures, and matrix overlays. +> +> 2026-05-05 asset-fingerprint addendum: the candidate-asset spec now explicitly includes recency-biased exhaustion / continuation features (local overextension, short continuation quality, bounce susceptibility, OBF symmetry, path persistence / entropy, reversal pressure, vol-normalized stretch) as point-in-time asset features, not as a market-state substitute. +> 2026-05-05 recency-gating addendum: the asset fingerprint is now defined as a multi-window bank plus a query-time recency gate. This is intentional so the market fingerprint can tune short-vs-long lookback emphasis at inference time without retraining the raw asset history. The bank must be preserved in storage, along with the gate metadata, so alternative recency profiles can be replayed later. +> 2026-05-05 implementation-map addendum: `ASSET_FINGERPRINT_CANDIDATE_SYSTEM.md` now includes a concrete implementation map for dev agents. It defines query-side entry objects, asset-side builder responsibilities, window-bank storage, gate modes, retrieval outputs, suggested file boundaries, and acceptance tests. The map is designed to stay compatible with the market-fingerprint → asset-fingerprint query flow and future universe enlargement. +> 2026-05-05 scan-backrunner addendum: the pure performed-trade baseline is now named `/mnt/dolphin_training/regime_asset_prototypes/pure_performed_trade_regime_asset_model.pkl`, and the enlarged scan-tape layer lives in `adaptive_exit/scan_tape_backrunner.py`. +> 2026-05-05 full-sweep addendum: the canonical asset feature store is now `/mnt/dolphin_training/asset_feature_sweep/asset_feature_vectors.parquet` (948 rows, 37 assets, 109 features). The pure-performed-trade retriever was retrained on the full expanded feature table and now lives at `/mnt/dolphin_training/regime_asset_prototypes/pure_performed_trade_regime_asset_model.pkl` (421,500 merged rows, 4 clusters, 12 prototypes, ~434 MB). The scan-tape backrunner remains the larger regime-enlarged layer with OOS/OOD validation. +> 2026-05-05 market-state split addendum: the deterministic market-state statistics now live in `adaptive_exit/market_state_outputs.py` as fingerprint inputs (`market_fingerprint_*`). The learned output bundle has two families: (1) the exit-policy head emits `market_state_tp_pct` and `market_state_max_hold_bars` with a BLUE base policy of `0.95%` TP and `120` bars hold; (2) the asset-target head surfaces historically favorable asset fingerprints for the given market fingerprint / regime-state pairing. The original design also reserved `SIZE(x)` as a learned output, but that remains downstream work. +> 2026-05-05 market-state trainer addendum: the bundle trainer now defaults to full available scan history (`--days 0`) and emits a progress file at `/mnt/dolphin_training/market_state_bundle/market_state_bundle_progress.json` plus a learning log at `/mnt/dolphin_training/market_state_bundle/learning_log.jsonl`. This makes the retrain observable without changing the learned semantics. +> 2026-05-05 market-state asset-head addendum: the learned bundle now predicts a direct asset-fingerprint vector, then uses nearest-neighbor lookup to surface candidate assets. This is the correct shape for later universe enlargement because the fingerprint can be compared against new assets without retraining the exit policy head. +> 2026-05-06 market-state runtime addendum: the live trader now routes scan snapshots and natural trade closes through `adaptive_exit/market_state_runtime.py`, a thin runtime adapter that caches the latest market-state bundle, keeps the rolling scan window, and calls the bundle's online update path. This is the abstraction seam for future batch automation of data refresh, retraining, and post-trade learning. +> 2026-05-06 storage-format addendum: large market-state / asset-fingerprint tabular artifacts should use Arrow IPC / Feather first, Parquet second, and JSON only for small control metadata. The runtime reads Arrow IPC / Feather or Parquet transparently and writes the latest live bundle snapshot to `/mnt/dolphin_training/market_state_bundle/latest_market_state_bundle.feather`. The trainer accepts the same format family for backfill and asset-lookup inputs. +> 2026-05-06 model-storage addendum: the learned market-state bundle is persisted as a gzip-compressed pickle at `/mnt/dolphin_training/market_state_bundle/market_state_bundle_model.pkl` for space efficiency, with backward-compatible load support for the older plain-pickle form. The estimator state is still the canonical learned object; the surrounding tabular snapshots remain Arrow IPC / Feather or Parquet. +> 2026-05-08 post-outlier-win side-selection addendum: BLUE trade/log replay found a narrow event-conditioned long probe after large 9x short wins. On the cleaned BLUE sequence (`1321` non-hibernate/non-ACB trades), flipping only the immediate next trade after `pnl_abs >= $400`, `leverage >= 8.5x`, and `pnl_pct >= +0.50%` improved estimated dollars and drawdown but did not improve WR. This is a one-trade post-exhaustion/cooldown candidate, not a broad long engine. Details are in `prod/docs/LONG_DETERMINISTIC_RULE_RESEARCH.md`. +> 2026-05-08 leverage-as-conviction sweep addendum: BLUE replay does **not** support a broad rule that turns trades LONG after ordinary high-leverage short wins. The correct criterion is marginal overlay value on the intervened subset, not replacement of the whole short engine. Even under that criterion, the literal `trigger_lev >= 0.70`, `trade_min_lev >= 0.69` thesis degraded the cleaned sequence badly (`WR ~37%`, negative compound return, negative estimated dollars, strongly negative overlay delta). The best swept long-switch variants still failed to add value over leaving the same triggered trades short. The useful signal is that leverage is a conviction / quality feature for filtering and sizing; it is not, by itself, a side-inversion trigger. Keep the narrower post-outlier one-trade long probe as research because it showed positive marginal overlay delta, but do not deploy the broad leverage-win LONG switch. +> 2026-05-08 lowered post-win-threshold addendum: the post-win overlay is stronger than the original narrow sample implied, but only when conditioned on realized exhaustion. Dollar-only thresholds below about `$300` are harmful. With a prior-return filter (`pnl_pct >= +0.75%`), lower thresholds become useful: e.g. `$100-$150` prior wins produced `63-67` immediate next-trade cases with about `+$2.4k` marginal delta and positive flipped-LONG PnL. High-leverage `$300-$500` wins support a next-`2`-trade rebound/cooldown signal (`+$2.7k` to `+$3.7k` marginal delta). The edge is payoff-asymmetry / loss-tail avoidance, not WR improvement, and should be researched as a guarded next-1/next-2 overlay or abstain gate. +> 2026-05-08 post-win EFSM implementation addendum: the candidate BLUE overlay is now the post-win **EFSM** (**Execution FSM**) at `adaptive_exit/post_win_long_overlay.py` with tests in `prod/tests/test_post_win_long_overlay.py`. Canonical class names are `PostWinExecutionFSM` and `PostWinExecutionFSMConfig` (`PostWinLongOverlay` names remain compatibility aliases). Codified rule: `pnl_abs > $397` arms next `1` FLIP_LONG slot; `pnl_abs > $397 and lev > 8.6` arms next `2`; `0 < pnl_abs < $250 and pnl_pct >= +0.75%` arms next `1`; consumed slots reset to SHORT. Active slots cannot re-arm and overlay-flipped LONG outcomes cannot re-arm. This reset invariant is mandatory: unsafe recursive re-arm replay turned `+$1.51k` marginal delta into `-$5.43k`. V7 is side-aware but SHORT-calibrated; validate LONG overlay exits in shadow or with conservative LONG-specific settings before live use. +> 2026-05-08 AlphaExitEngineV7 LONG calibration addendum: V7 threshold/gate constants are now surfaced as `AlphaExitV7Config` in `nautilus_dolphin/nautilus_dolphin/nautilus/alpha_exit_v7_engine.py`. Default `AlphaExitEngineV7()` remains the deployed SHORT-calibrated surface: `exit_pressure_threshold=2.69`, `retract_pressure_threshold=1.0`, `extend_pressure_threshold=-0.5`, vol-normalized MAE tiers `max(floor, k * rv_comp)` with `k=(3.5,7.0,12.0)` and floors `(0.005,0.012,0.025)`, and bounce soft weights `(0.15,0.35)`. A separate LONG engine can now be initialized with a different `AlphaExitV7Config` without mutating BLUE SHORT defaults. Synthetic LONG replay over BLUE V7 journal paths (`97` paths, `6,812` rows, bounce disabled because the current bounce model is SHORT-trained) found natural LONG PnL `-$328.84`; deployed default V7 improved this to `+$1.43` (`+$330.26` delta); best tested LONG proxy was reducing MFE-risk contributions by half while keeping pressure threshold `2.69`, yielding `+$205.32` (`+$534.15` delta), `36/97` exits, and `1.69%` max DD. Do not deploy this live from proxy alone; first shadow it on actual EFSM-flipped LONG contexts. Detailed method/results: `prod/docs/LONG_DETERMINISTIC_RULE_RESEARCH.md`. +> The retrieval spec is documented in `prod/docs/ASSET_FINGERPRINT_CANDIDATE_SYSTEM.md`. diff --git a/prod/tests/test_post_win_long_overlay.py b/prod/tests/test_post_win_long_overlay.py new file mode 100644 index 0000000..c2edc2e --- /dev/null +++ b/prod/tests/test_post_win_long_overlay.py @@ -0,0 +1,195 @@ +from datetime import datetime, timedelta, timezone + +from adaptive_exit.post_win_long_overlay import ( + PostWinExecutionFSM, + PostWinExecutionFSMConfig, + PostWinFlipTrigger, +) + + +def _ts(seconds: int = 0) -> datetime: + return datetime(2026, 5, 8, 12, 0, tzinfo=timezone.utc) + timedelta(seconds=seconds) + + +def test_big_win_arms_one_slot_and_resets_after_consumption(): + overlay = PostWinExecutionFSM() + + armed = overlay.observe_closed_trade( + trade_id="t1", + asset="ALGOUSDT", + side="SHORT", + pnl=398.0, + pnl_pct=0.004, + leverage=2.0, + closed_ts=_ts(), + ) + + assert armed.action == "ARMED" + assert armed.reason == "big_win" + assert overlay.pending_slots == 1 + + tag = overlay.tag_next_entry(asset="DASHUSDT", entry_ts=_ts(30)) + assert tag.action == "TAG" + assert tag.side == "LONG" + assert tag.consumed_slot == 1 + assert tag.reset is True + assert overlay.pending_slots == 0 + + after = overlay.tag_next_entry(asset="TRXUSDT", entry_ts=_ts(60)) + assert after.action == "PASS" + assert after.side == "SHORT" + + +def test_big_win_high_lev_arms_two_slots_then_resets(): + overlay = PostWinExecutionFSM() + + armed = overlay.observe_closed_trade( + trade_id="t2", + asset="VETUSDT", + side="SHORT", + pnl=573.0, + pnl_pct=0.0148, + leverage=9.0, + closed_ts=_ts(), + ) + + assert armed.action == "ARMED" + assert armed.reason == "big_win_high_lev" + assert overlay.pending_slots == 2 + + first = overlay.tag_next_entry(asset="STXUSDT", entry_ts=_ts(10)) + assert first.side == "LONG" + assert first.consumed_slot == 1 + assert first.reset is False + assert overlay.pending_slots == 1 + + second = overlay.tag_next_entry(asset="TRXUSDT", entry_ts=_ts(20)) + assert second.side == "LONG" + assert second.consumed_slot == 2 + assert second.reset is True + assert overlay.pending_slots == 0 + + third = overlay.tag_next_entry(asset="ATOMUSDT", entry_ts=_ts(30)) + assert third.side == "SHORT" + + +def test_small_dollar_high_return_arms_one_slot(): + overlay = PostWinExecutionFSM() + + armed = overlay.observe_closed_trade( + trade_id="t3", + asset="ETCUSDT", + side="SHORT", + pnl=149.0, + pnl_pct=0.0075, + leverage=0.8, + closed_ts=_ts(), + ) + + assert armed.action == "ARMED" + assert armed.reason == "small_dollar_high_return" + assert overlay.tag_next_entry(asset="LTCUSDT", entry_ts=_ts(10)).side == "LONG" + assert overlay.tag_next_entry(asset="BNBUSDT", entry_ts=_ts(20)).side == "SHORT" + + +def test_rearm_attempt_while_slots_active_is_ignored_and_does_not_extend_counter(): + overlay = PostWinExecutionFSM() + + overlay.observe_closed_trade( + trade_id="first", + asset="ALGOUSDT", + side="SHORT", + pnl=500.0, + pnl_pct=0.010, + leverage=9.0, + closed_ts=_ts(), + ) + ignored = overlay.observe_closed_trade( + trade_id="second", + asset="VETUSDT", + side="SHORT", + pnl=900.0, + pnl_pct=0.020, + leverage=9.0, + closed_ts=_ts(5), + ) + + assert ignored.action == "IGNORED" + assert ignored.reason == "active_arm_no_rearm" + assert overlay.ignored_rearm_attempts == 1 + assert overlay.pending_slots == 2 + + assert overlay.tag_next_entry(asset="A", entry_ts=_ts(10)).side == "LONG" + assert overlay.tag_next_entry(asset="B", entry_ts=_ts(20)).side == "LONG" + assert overlay.tag_next_entry(asset="C", entry_ts=_ts(30)).side == "SHORT" + + +def test_overlay_flipped_trade_outcome_cannot_rearm(): + overlay = PostWinExecutionFSM() + + ignored = overlay.observe_closed_trade( + trade_id="long-flip", + asset="DASHUSDT", + side="LONG", + pnl=1000.0, + pnl_pct=0.03, + leverage=9.0, + closed_ts=_ts(), + was_overlay_flip=True, + ) + + assert ignored.action == "IGNORED" + assert ignored.reason == "overlay_flip_outcome" + assert overlay.pending_slots == 0 + + +def test_arm_expires_by_optional_ttl_without_consuming_slot(): + overlay = PostWinExecutionFSM(PostWinExecutionFSMConfig(max_arm_age_sec=60.0)) + + overlay.observe_closed_trade( + trade_id="ttl", + asset="VETUSDT", + side="SHORT", + pnl=500.0, + pnl_pct=0.01, + leverage=9.0, + closed_ts=_ts(), + ) + + tag = overlay.tag_next_entry(asset="LATEUSDT", entry_ts=_ts(61)) + assert tag.action == "PASS" + assert tag.side == "SHORT" + assert overlay.pending_slots == 0 + assert overlay.expired_arms == 1 + + +def test_future_expansion_supports_more_than_two_slots(): + overlay = PostWinExecutionFSM( + PostWinExecutionFSMConfig( + rules=( + PostWinFlipTrigger( + name="future_three_slot_rule", + slots=3, + min_pnl_abs=100.0, + strict_min_pnl_abs=True, + ), + ) + ) + ) + + overlay.observe_closed_trade( + trade_id="three", + asset="XRPUSDT", + side="SHORT", + pnl=101.0, + pnl_pct=0.001, + leverage=1.0, + closed_ts=_ts(), + ) + + assert [overlay.tag_next_entry(asset=str(i), entry_ts=_ts(i)).side for i in range(1, 5)] == [ + "LONG", + "LONG", + "LONG", + "SHORT", + ] diff --git a/prod/tests/test_v7_live_exit_wiring.py b/prod/tests/test_v7_live_exit_wiring.py new file mode 100644 index 0000000..8fcb81b --- /dev/null +++ b/prod/tests/test_v7_live_exit_wiring.py @@ -0,0 +1,295 @@ +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +ROOT = Path("/mnt/dolphinng5_predict") +sys.path.insert(0, str(ROOT / "nautilus_dolphin")) +sys.path.insert(1, str(ROOT)) +if "nautilus_dolphin" in sys.modules: + pkg = sys.modules["nautilus_dolphin"] + pkg_file = str(getattr(pkg, "__file__", "") or "") + if not pkg_file.endswith("nautilus_dolphin/nautilus_dolphin/__init__.py"): + del sys.modules["nautilus_dolphin"] + +from nautilus_dolphin.nautilus.esf_alpha_orchestrator import NDAlphaEngine, NDPosition +from nautilus_dolphin.nautilus.alpha_exit_v7_engine import AlphaExitEngineV7, AlphaExitV7Config +from prod.nautilus_event_trader import DolphinLiveTrader + + +class _DummyCtx: + def __init__(self, entry_price: float, entry_bar: int, side: int) -> None: + self.entry_price = entry_price + self.entry_bar = entry_bar + self.side = side + self.exf = None + + def set_exf(self, funding: float = 0.0, dvol: float = 0.0, fear_greed: float = 0.0, taker: float = 0.0) -> None: + self.exf = { + "funding": funding, + "dvol": dvol, + "fear_greed": fear_greed, + "taker": taker, + } + + +class _DummyV7Engine: + def __init__(self) -> None: + self.make_calls = [] + self.evaluate_calls = [] + + def make_context(self, entry_price: float, entry_bar: int, side: int) -> _DummyCtx: + self.make_calls.append((entry_price, entry_bar, side)) + return _DummyCtx(entry_price=entry_price, entry_bar=entry_bar, side=side) + + def evaluate(self, ctx, current_price: float, current_bar: int, ob_imbalance: float, asset: str = "default") -> dict: + self.evaluate_calls.append( + { + "ctx": ctx, + "current_price": current_price, + "current_bar": current_bar, + "ob_imbalance": ob_imbalance, + "asset": asset, + } + ) + return { + "action": "EXIT", + "reason": "V7_COMPOSITE_PRESSURE", + "pnl_pct": 1.25, + "bars_held": current_bar - ctx.entry_bar, + "mfe": 0.02, + "mae": 0.01, + "mfe_risk": 0.0, + "mae_risk": 0.0, + "exit_pressure": 2.81, + "rv_comp": 0.001, + "mae_thresh1": 0.002, + "bounce_score": 0.1, + "bounce_risk": 0.2, + } + + +class _DummyOBSignal: + def __init__(self, imbalance_ma5: float) -> None: + self.imbalance_ma5 = imbalance_ma5 + + +class _DummyOBEngine: + def __init__(self) -> None: + self.calls = [] + + def get_signal(self, asset: str, bar_idx: float): + self.calls.append((asset, bar_idx)) + return _DummyOBSignal(0.42) + + +def test_ndalphaengine_prefers_exit_decision_provider_before_base_manager(): + engine = NDAlphaEngine( + initial_capital=1000.0, + use_sp_fees=False, + use_sp_slippage=False, + use_ob_edge=False, + use_asset_selection=False, + use_direction_confirm=False, + use_alpha_layers=False, + use_dynamic_leverage=False, + ) + pos = NDPosition( + trade_id="tid-1", + asset="DASHUSDT", + direction=-1, + entry_price=100.0, + entry_bar=0, + notional=100.0, + leverage=1.0, + fraction=0.2, + entry_vel_div=-0.03, + bucket_idx=4, + current_price=90.0, + ) + engine.position = pos + engine._day_posture = "APEX" + engine.regime_dd_halt = False + provider_called = {} + + def provider(**kwargs): + provider_called.update(kwargs) + return { + "action": "EXIT", + "reason": "V7_COMPOSITE_PRESSURE", + "pnl_pct": 1.25, + "bars_held": 7, + } + + engine.exit_decision_provider = provider + + def _should_not_run(*args, **kwargs): + raise AssertionError("base exit_manager should not be consulted when provider returns a decision") + + engine.exit_manager.evaluate = _should_not_run + executed = {} + + def _fake_execute_exit(reason: str, bar_idx: int, pnl_pct_raw: float = 0.0, bars_held: int = 0): + executed.update( + { + "reason": reason, + "bar_idx": bar_idx, + "pnl_pct_raw": pnl_pct_raw, + "bars_held": bars_held, + } + ) + engine.position = None + return executed + + engine._execute_exit = _fake_execute_exit + + out = engine._manage_position( + bar_idx=17, + prices={"DASHUSDT": 89.0}, + vel_div=-0.12, + v50_vel=0.03, + v750_vel=0.01, + ) + + assert provider_called["pos"] is pos + assert provider_called["bar_idx"] == 17 + assert out["reason"] == "V7_COMPOSITE_PRESSURE" + assert executed["reason"] == "V7_COMPOSITE_PRESSURE" + assert executed["bar_idx"] == 17 + + +def test_blue_live_v7_provider_records_journal_and_uses_ob_signal(): + trader = DolphinLiveTrader.__new__(DolphinLiveTrader) + trader._v7_exit_engine = _DummyV7Engine() + trader._pending_entries = { + "tid-2": { + "entry_price": 100.0, + "entry_bar": 4, + "side": "SHORT", + "quantity": 2.0, + "leverage": 3.0, + "notional": 200.0, + } + } + trader._v7_contexts = {} + trader._v7_decision_seq = {} + trader._v7_decisions = {} + trader._last_exf = { + "funding": 1.0, + "dvol": 2.0, + "fear_greed": 3.0, + "taker": 4.0, + } + trader.ob_eng = _DummyOBEngine() + captured = {} + + def _capture_record(**kwargs): + captured.update(kwargs) + + trader._record_v7_decision = _capture_record + + pos = SimpleNamespace(trade_id="tid-2", asset="DASHUSDT", current_price=97.0) + decision = trader._v7_live_exit_decision( + pos=pos, + bar_idx=10, + prices={"DASHUSDT": 97.5}, + vel_div=-0.3, + v50_vel=0.1, + v750_vel=0.2, + ) + + assert decision["action"] == "EXIT" + assert trader._v7_contexts["tid-2"].exf == { + "funding": 1.0, + "dvol": 2.0, + "fear_greed": 3.0, + "taker": 4.0, + } + assert trader.ob_eng.calls == [("DASHUSDT", 9.0)] + assert trader._v7_exit_engine.evaluate_calls[0]["current_bar"] == 9 + assert trader._v7_exit_engine.evaluate_calls[0]["ob_imbalance"] == pytest.approx(0.42) + assert captured["source"] == "live_exit" + assert captured["bar_idx"] == 9 + assert captured["trade_id"] == "tid-2" + assert captured["asset"] == "DASHUSDT" + + +def test_alpha_exit_v7_is_mechanically_side_aware_for_long_and_short(): + engine = AlphaExitEngineV7(bar_duration_sec=11.0, bounce_model_path="/tmp/nonexistent-bounce-model.pkl") + + long_ctx = engine.make_context(entry_price=100.0, entry_bar=0, side=0) + long_favorable = engine.evaluate(long_ctx, current_price=101.0, current_bar=1, ob_imbalance=0.0) + assert long_favorable["pnl_pct"] == pytest.approx(1.0) + assert long_favorable["mfe"] == pytest.approx(0.01) + assert long_favorable["mae"] == pytest.approx(0.0) + + long_adverse = engine.make_context(entry_price=100.0, entry_bar=0, side=0) + long_adverse_out = engine.evaluate(long_adverse, current_price=99.0, current_bar=1, ob_imbalance=0.0) + assert long_adverse_out["pnl_pct"] == pytest.approx(-1.0) + assert long_adverse_out["mfe"] == pytest.approx(0.0) + assert long_adverse_out["mae"] == pytest.approx(0.01) + + short_ctx = engine.make_context(entry_price=100.0, entry_bar=0, side=1) + short_favorable = engine.evaluate(short_ctx, current_price=99.0, current_bar=1, ob_imbalance=0.0) + assert short_favorable["pnl_pct"] == pytest.approx(1.0) + assert short_favorable["mfe"] == pytest.approx(0.01) + assert short_favorable["mae"] == pytest.approx(0.0) + + short_adverse = engine.make_context(entry_price=100.0, entry_bar=0, side=1) + short_adverse_out = engine.evaluate(short_adverse, current_price=101.0, current_bar=1, ob_imbalance=0.0) + assert short_adverse_out["pnl_pct"] == pytest.approx(-1.0) + assert short_adverse_out["mfe"] == pytest.approx(0.0) + assert short_adverse_out["mae"] == pytest.approx(0.01) + + +def test_alpha_exit_v7_default_config_matches_legacy_threshold_surface(): + engine = AlphaExitEngineV7(bar_duration_sec=11.0, bounce_model_path="/tmp/nonexistent-bounce-model.pkl") + cfg = engine.config + + assert cfg.rvol_w15 == pytest.approx(0.50) + assert cfg.rvol_w30 == pytest.approx(0.30) + assert cfg.rvol_w50 == pytest.approx(0.20) + assert cfg.mae_tier1_k == pytest.approx(3.5) + assert cfg.mae_tier2_k == pytest.approx(7.0) + assert cfg.mae_tier3_k == pytest.approx(12.0) + assert cfg.mae_tier1_floor == pytest.approx(0.005) + assert cfg.mae_tier2_floor == pytest.approx(0.012) + assert cfg.mae_tier3_floor == pytest.approx(0.025) + assert cfg.mae_tier1_risk == pytest.approx(0.5) + assert cfg.mae_tier2_risk == pytest.approx(0.8) + assert cfg.mae_tier3_risk == pytest.approx(1.2) + assert cfg.mae_accel_min_bars == 3 + assert cfg.mae_accel_peak_floor == pytest.approx(0.003) + assert cfg.mae_recovery_peak_floor == pytest.approx(0.004) + assert cfg.mae_recovery_prev_min == pytest.approx(0.25) + assert cfg.mae_recovery_snapback_max == pytest.approx(0.10) + assert cfg.mae_late_floor == pytest.approx(0.003) + assert cfg.mae_late_start_frac == pytest.approx(0.60) + assert cfg.mae_late_risk_max == pytest.approx(0.4) + assert cfg.mfe_convexity_decay_exit == pytest.approx(0.35) + assert cfg.mfe_convexity_decay_soft == pytest.approx(0.20) + assert cfg.bounce_dir_w == pytest.approx(0.15) + assert cfg.bounce_risk_w == pytest.approx(0.35) + assert cfg.exit_pressure_threshold == pytest.approx(2.69) + assert cfg.retract_pressure_threshold == pytest.approx(1.0) + assert cfg.extend_pressure_threshold == pytest.approx(-0.5) + + +def test_alpha_exit_v7_custom_threshold_config_is_per_instance(): + strict = AlphaExitEngineV7( + bar_duration_sec=11.0, + bounce_model_path="/tmp/nonexistent-bounce-model.pkl", + config=AlphaExitV7Config(exit_pressure_threshold=0.1, retract_pressure_threshold=0.05), + ) + default = AlphaExitEngineV7(bar_duration_sec=11.0, bounce_model_path="/tmp/nonexistent-bounce-model.pkl") + + strict_ctx = strict.make_context(entry_price=100.0, entry_bar=0, side=0) + default_ctx = default.make_context(entry_price=100.0, entry_bar=0, side=0) + strict_decision = strict.evaluate(strict_ctx, current_price=100.0, current_bar=1, ob_imbalance=0.0) + default_decision = default.evaluate(default_ctx, current_price=100.0, current_bar=1, ob_imbalance=0.0) + + assert strict_decision["action"] == "EXIT" + assert default_decision["action"] == "HOLD" + assert strict.config.exit_pressure_threshold == pytest.approx(0.1) + assert default.config.exit_pressure_threshold == pytest.approx(2.69)