initial: import DOLPHIN baseline 2026-04-21 from dolphinng5_predict working tree
Includes core prod + GREEN/BLUE subsystems: - prod/ (BLUE harness, configs, scripts, docs) - nautilus_dolphin/ (GREEN Nautilus-native impl + dvae/ preserved) - adaptive_exit/ (AEM engine + models/bucket_assignments.pkl) - Observability/ (EsoF advisor, TUI, dashboards) - external_factors/ (EsoF producer) - mc_forewarning_qlabs_fork/ (MC regime/envelope) Excludes runtime caches, logs, backups, and reproducible artifacts per .gitignore.
This commit is contained in:
hjnormey
207
external_factors/meta_adaptive_optimizer.py
Executable file
207
external_factors/meta_adaptive_optimizer.py
Executable file
@@ -0,0 +1,207 @@
|
||||
"""
|
||||
Meta-Adaptive ExF Optimizer
|
||||
===========================
|
||||
Runs nightly (or on-demand) to calculate dynamic lag configurations and
|
||||
active indicator thresholds for the Adaptive Circuit Breaker (ACB).
|
||||
|
||||
Implementation of the "Meta-Adaptive" Blueprint:
|
||||
1. Pulls up to the last 90 days of market returns and indicator values.
|
||||
2. Runs lag hypothesis testing (0-7 days) on all tracked ExF indicators.
|
||||
3. Uses strict Point-Biserial correlation (p < 0.05) against market stress (< -1% daily drop).
|
||||
4. Persists the active, statistically verified JSON configuration for realtime_exf_service.py.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import json
|
||||
import time
|
||||
import logging
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
from pathlib import Path
|
||||
from collections import defaultdict
|
||||
import threading
|
||||
from scipy import stats
|
||||
from datetime import datetime, timezone
|
||||
|
||||
PROJECT_ROOT = Path(__file__).resolve().parent.parent
|
||||
sys.path.insert(0, str(PROJECT_ROOT))
|
||||
sys.path.insert(0, str(PROJECT_ROOT / 'nautilus_dolphin'))
|
||||
|
||||
try:
|
||||
from realtime_exf_service import INDICATORS, OPTIMAL_LAGS
|
||||
from dolphin_paper_trade_adaptive_cb_v2 import EIGENVALUES_BASE_PATH
|
||||
from dolphin_vbt_real import load_all_data, run_full_backtest, STRATEGIES, INIT_CAPITAL
|
||||
except ImportError:
|
||||
pass
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
CONFIG_PATH = Path(__file__).parent / "meta_adaptive_config.json"
|
||||
|
||||
class MetaAdaptiveOptimizer:
|
||||
def __init__(self, days_lookback=90, max_lags=6, p_value_gate=0.05):
|
||||
self.days_lookback = days_lookback
|
||||
self.max_lags = max_lags
|
||||
self.p_value_gate = p_value_gate
|
||||
self.indicators = list(INDICATORS.keys()) if 'INDICATORS' in globals() else []
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def _build_history_cache(self, dates, limit_days):
|
||||
"""Build daily feature cache from NPZ files."""
|
||||
logger.info(f"Building cache for last {limit_days} days...")
|
||||
cache = {}
|
||||
target_dates = dates[-limit_days:] if len(dates) > limit_days else dates
|
||||
|
||||
for date_str in target_dates:
|
||||
date_path = EIGENVALUES_BASE_PATH / date_str
|
||||
if not date_path.exists(): continue
|
||||
|
||||
npz_files = list(date_path.glob('scan_*__Indicators.npz'))
|
||||
if not npz_files: continue
|
||||
|
||||
accum = defaultdict(list)
|
||||
for f in npz_files:
|
||||
try:
|
||||
data = dict(np.load(f, allow_pickle=True))
|
||||
names = [str(n) for n in data.get('api_names', [])]
|
||||
vals = data.get('api_indicators', [])
|
||||
succ = data.get('api_success', [])
|
||||
for n, v, s in zip(names, vals, succ):
|
||||
if s and not np.isnan(v):
|
||||
accum[n].append(float(v))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
if accum:
|
||||
cache[date_str] = {k: np.mean(v) for k, v in accum.items()}
|
||||
|
||||
return cache, target_dates
|
||||
|
||||
def _get_daily_returns(self, df, target_dates):
|
||||
"""Derive daily returns proxy from the champion strategy logic."""
|
||||
logger.info("Computing proxy returns for the time window...")
|
||||
champion = STRATEGIES['champion_5x_f20']
|
||||
returns = []
|
||||
cap = INIT_CAPITAL
|
||||
|
||||
valid_dates = []
|
||||
for d in target_dates:
|
||||
day_df = df[df['date_str'] == d]
|
||||
if len(day_df) < 200:
|
||||
returns.append(np.nan)
|
||||
valid_dates.append(d)
|
||||
continue
|
||||
|
||||
res = run_full_backtest(day_df, champion, init_cash=cap, seed=42, verbose=False)
|
||||
ret = (res['capital'] - cap) / cap
|
||||
returns.append(ret)
|
||||
cap = res['capital']
|
||||
valid_dates.append(d)
|
||||
|
||||
return np.array(returns), valid_dates
|
||||
|
||||
def run_optimization(self) -> dict:
|
||||
"""Run the full meta-adaptive optimization routine and return new config."""
|
||||
with self._lock:
|
||||
logger.info("Starting META-ADAPTIVE optimization loop.")
|
||||
t0 = time.time()
|
||||
|
||||
df = load_all_data()
|
||||
if 'date_str' not in df.columns:
|
||||
df['date_str'] = df['timestamp'].dt.date.astype(str)
|
||||
all_dates = sorted(df['date_str'].unique())
|
||||
|
||||
cache, target_dates = self._build_history_cache(all_dates, self.days_lookback + self.max_lags)
|
||||
daily_returns, target_dates = self._get_daily_returns(df, target_dates)
|
||||
|
||||
# Predict market stress dropping by more than 1%
|
||||
stress_arr = (daily_returns < -0.01).astype(float)
|
||||
|
||||
candidate_lags = {}
|
||||
active_thresholds = {}
|
||||
candidate_count = 0
|
||||
|
||||
for key in self.indicators:
|
||||
ind_arr = np.array([cache.get(d, {}).get(key, np.nan) for d in target_dates])
|
||||
|
||||
corrs = []; pvals = []; sc_corrs = []
|
||||
for lag in range(self.max_lags + 1):
|
||||
if lag == 0: x, y, y_stress = ind_arr, daily_returns, stress_arr
|
||||
else: x, y, y_stress = ind_arr[:-lag], daily_returns[lag:], stress_arr[lag:]
|
||||
|
||||
mask = ~np.isnan(x) & ~np.isnan(y)
|
||||
if mask.sum() < 20: # Need at least 20 viable days
|
||||
corrs.append(0); pvals.append(1); sc_corrs.append(0)
|
||||
continue
|
||||
|
||||
# Pearson to price returns
|
||||
r, p = stats.pearsonr(x[mask], y[mask])
|
||||
corrs.append(r); pvals.append(p)
|
||||
|
||||
# Point-Biserial to stress events
|
||||
# We capture the relation to binary stress to figure out threshold direction
|
||||
if y_stress[mask].sum() > 2: # At least a few stress days required
|
||||
sc = stats.pointbiserialr(y_stress[mask], x[mask])[0]
|
||||
else:
|
||||
sc = 0
|
||||
sc_corrs.append(sc)
|
||||
|
||||
if not corrs: continue
|
||||
|
||||
# Find lag with highest correlation strength
|
||||
best_lag = int(np.argmax(np.abs(corrs)))
|
||||
best_p = pvals[best_lag]
|
||||
|
||||
# Check gate
|
||||
if best_p <= self.p_value_gate:
|
||||
direction = ">" if sc_corrs[best_lag] > 0 else "<"
|
||||
|
||||
# Compute a stress threshold logic (e.g. 15th / 85th percentile of historical)
|
||||
valid_vals = ind_arr[~np.isnan(ind_arr)]
|
||||
thresh = np.percentile(valid_vals, 85 if direction == '>' else 15)
|
||||
|
||||
candidate_lags[key] = best_lag
|
||||
active_thresholds[key] = {
|
||||
'threshold': float(thresh),
|
||||
'direction': direction,
|
||||
'p_value': float(best_p),
|
||||
'r_value': float(corrs[best_lag])
|
||||
}
|
||||
candidate_count += 1
|
||||
|
||||
# Fallback checks mapping to V4 baseline if things drift too far
|
||||
logger.info(f"Optimization complete ({time.time() - t0:.1f}s). {candidate_count} indicators passed P < {self.p_value_gate}.")
|
||||
|
||||
output_config = {
|
||||
'timestamp': datetime.now(timezone.utc).isoformat(),
|
||||
'days_lookback': self.days_lookback,
|
||||
'lags': candidate_lags,
|
||||
'thresholds': active_thresholds
|
||||
}
|
||||
|
||||
# Atomic save
|
||||
temp_path = CONFIG_PATH.with_suffix('.tmp')
|
||||
with open(temp_path, 'w', encoding='utf-8') as f:
|
||||
json.dump(output_config, f, indent=2)
|
||||
temp_path.replace(CONFIG_PATH)
|
||||
|
||||
return output_config
|
||||
|
||||
def get_current_meta_config() -> dict:
|
||||
"""Read the latest meta-adaptive config, or return empty/default dict."""
|
||||
if not CONFIG_PATH.exists():
|
||||
return {}
|
||||
try:
|
||||
with open(CONFIG_PATH, 'r', encoding='utf-8') as f:
|
||||
return json.load(f)
|
||||
except Exception as e:
|
||||
logger.error(f"Failed to read meta-adaptive config: {e}")
|
||||
return {}
|
||||
|
||||
if __name__ == '__main__':
|
||||
logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
|
||||
optimizer = MetaAdaptiveOptimizer(days_lookback=90)
|
||||
config = optimizer.run_optimization()
|
||||
print(f"\nSaved config to: {CONFIG_PATH}")
|
||||
for k, v in config['lags'].items():
|
||||
print(f" {k}: lag={v} days, dir={config['thresholds'][k]['direction']} thresh={config['thresholds'][k]['threshold']:.4g}")
|
||||
Reference in New Issue
Block a user