DOLPHIN/external_factors/meta_adaptive_optimizer.py

"""
Meta-Adaptive ExF Optimizer
===========================
Runs nightly (or on-demand) to calculate dynamic lag configurations and
active indicator thresholds for the Adaptive Circuit Breaker (ACB).

Implementation of the "Meta-Adaptive" Blueprint:
1. Pulls up to the last 90 days of market returns and indicator values.
2. Runs lag hypothesis testing (0-7 days) on all tracked ExF indicators.
3. Uses strict Point-Biserial correlation (p < 0.05) against market stress (< -1% daily drop).
4. Persists the active, statistically verified JSON configuration for realtime_exf_service.py.
"""

import sys
import json
import time
import logging
import numpy as np
import pandas as pd
from pathlib import Path
from collections import defaultdict
import threading
from scipy import stats
from datetime import datetime, timezone

PROJECT_ROOT = Path(__file__).resolve().parent.parent
sys.path.insert(0, str(PROJECT_ROOT))
sys.path.insert(0, str(PROJECT_ROOT / 'nautilus_dolphin'))

try:
    from realtime_exf_service import INDICATORS, OPTIMAL_LAGS
    from dolphin_paper_trade_adaptive_cb_v2 import EIGENVALUES_BASE_PATH
    from dolphin_vbt_real import load_all_data, run_full_backtest, STRATEGIES, INIT_CAPITAL
except ImportError:
    pass

logger = logging.getLogger(__name__)

CONFIG_PATH = Path(__file__).parent / "meta_adaptive_config.json"

class MetaAdaptiveOptimizer:
    def __init__(self, days_lookback=90, max_lags=6, p_value_gate=0.05):
        self.days_lookback = days_lookback
        self.max_lags = max_lags
        self.p_value_gate = p_value_gate
        self.indicators = list(INDICATORS.keys()) if 'INDICATORS' in globals() else []
        self._lock = threading.Lock()

    def _build_history_cache(self, dates, limit_days):
        """Build daily feature cache from NPZ files."""
        logger.info(f"Building cache for last {limit_days} days...")
        cache = {}
        target_dates = dates[-limit_days:] if len(dates) > limit_days else dates
        
        for date_str in target_dates:
            date_path = EIGENVALUES_BASE_PATH / date_str
            if not date_path.exists(): continue
            
            npz_files = list(date_path.glob('scan_*__Indicators.npz'))
            if not npz_files: continue
            
            accum = defaultdict(list)
            for f in npz_files:
                try:
                    data = dict(np.load(f, allow_pickle=True))
                    names = [str(n) for n in data.get('api_names', [])]
                    vals = data.get('api_indicators', [])
                    succ = data.get('api_success', [])
                    for n, v, s in zip(names, vals, succ):
                        if s and not np.isnan(v):
                            accum[n].append(float(v))
                except Exception:
                    pass
            
            if accum:
                cache[date_str] = {k: np.mean(v) for k, v in accum.items()}
                
        return cache, target_dates

    def _get_daily_returns(self, df, target_dates):
        """Derive daily returns proxy from the champion strategy logic."""
        logger.info("Computing proxy returns for the time window...")
        champion = STRATEGIES['champion_5x_f20']
        returns = []
        cap = INIT_CAPITAL
        
        valid_dates = []
        for d in target_dates:
            day_df = df[df['date_str'] == d]
            if len(day_df) < 200:
                returns.append(np.nan)
                valid_dates.append(d)
                continue
            
            res = run_full_backtest(day_df, champion, init_cash=cap, seed=42, verbose=False)
            ret = (res['capital'] - cap) / cap
            returns.append(ret)
            cap = res['capital']
            valid_dates.append(d)
            
        return np.array(returns), valid_dates

    def run_optimization(self) -> dict:
        """Run the full meta-adaptive optimization routine and return new config."""
        with self._lock:
            logger.info("Starting META-ADAPTIVE optimization loop.")
            t0 = time.time()
            
            df = load_all_data()
            if 'date_str' not in df.columns:
                df['date_str'] = df['timestamp'].dt.date.astype(str)
            all_dates = sorted(df['date_str'].unique())
            
            cache, target_dates = self._build_history_cache(all_dates, self.days_lookback + self.max_lags)
            daily_returns, target_dates = self._get_daily_returns(df, target_dates)
            
            # Predict market stress dropping by more than 1%
            stress_arr = (daily_returns < -0.01).astype(float)
            
            candidate_lags = {}
            active_thresholds = {}
            candidate_count = 0
            
            for key in self.indicators:
                ind_arr = np.array([cache.get(d, {}).get(key, np.nan) for d in target_dates])
                
                corrs = []; pvals = []; sc_corrs = []
                for lag in range(self.max_lags + 1):
                    if lag == 0: x, y, y_stress = ind_arr, daily_returns, stress_arr
                    else: x, y, y_stress = ind_arr[:-lag], daily_returns[lag:], stress_arr[lag:]
                    
                    mask = ~np.isnan(x) & ~np.isnan(y)
                    if mask.sum() < 20: # Need at least 20 viable days
                        corrs.append(0); pvals.append(1); sc_corrs.append(0)
                        continue
                        
                    # Pearson to price returns
                    r, p = stats.pearsonr(x[mask], y[mask])
                    corrs.append(r); pvals.append(p)
                    
                    # Point-Biserial to stress events
                    # We capture the relation to binary stress to figure out threshold direction
                    if y_stress[mask].sum() > 2: # At least a few stress days required
                        sc = stats.pointbiserialr(y_stress[mask], x[mask])[0]
                    else:
                        sc = 0
                    sc_corrs.append(sc)

                if not corrs: continue

                # Find lag with highest correlation strength
                best_lag = int(np.argmax(np.abs(corrs)))
                best_p = pvals[best_lag]
                
                # Check gate
                if best_p <= self.p_value_gate:
                    direction = ">" if sc_corrs[best_lag] > 0 else "<"
                    
                    # Compute a stress threshold logic (e.g. 15th / 85th percentile of historical)
                    valid_vals = ind_arr[~np.isnan(ind_arr)]
                    thresh = np.percentile(valid_vals, 85 if direction == '>' else 15)
                    
                    candidate_lags[key] = best_lag
                    active_thresholds[key] = {
                        'threshold': float(thresh),
                        'direction': direction,
                        'p_value': float(best_p),
                        'r_value': float(corrs[best_lag])
                    }
                    candidate_count += 1
                    
            # Fallback checks mapping to V4 baseline if things drift too far
            logger.info(f"Optimization complete ({time.time() - t0:.1f}s). {candidate_count} indicators passed P < {self.p_value_gate}.")
            
            output_config = {
                'timestamp': datetime.now(timezone.utc).isoformat(),
                'days_lookback': self.days_lookback,
                'lags': candidate_lags,
                'thresholds': active_thresholds
            }
            
            # Atomic save
            temp_path = CONFIG_PATH.with_suffix('.tmp')
            with open(temp_path, 'w', encoding='utf-8') as f:
                json.dump(output_config, f, indent=2)
            temp_path.replace(CONFIG_PATH)
            
            return output_config

def get_current_meta_config() -> dict:
    """Read the latest meta-adaptive config, or return empty/default dict."""
    if not CONFIG_PATH.exists():
        return {}
    try:
        with open(CONFIG_PATH, 'r', encoding='utf-8') as f:
            return json.load(f)
    except Exception as e:
        logger.error(f"Failed to read meta-adaptive config: {e}")
        return {}

if __name__ == '__main__':
    logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s')
    optimizer = MetaAdaptiveOptimizer(days_lookback=90)
    config = optimizer.run_optimization()
    print(f"\nSaved config to: {CONFIG_PATH}")
    for k, v in config['lags'].items():
        print(f"  {k}: lag={v} days, dir={config['thresholds'][k]['direction']} thresh={config['thresholds'][k]['threshold']:.4g}")