01c19662cb
Includes core prod + GREEN/BLUE subsystems: - prod/ (BLUE harness, configs, scripts, docs) - nautilus_dolphin/ (GREEN Nautilus-native impl + dvae/ preserved) - adaptive_exit/ (AEM engine + models/bucket_assignments.pkl) - Observability/ (EsoF advisor, TUI, dashboards) - external_factors/ (EsoF producer) - mc_forewarning_qlabs_fork/ (MC regime/envelope) Excludes runtime caches, logs, backups, and reproducible artifacts per .gitignore.
313 lines
13 KiB
Python
Executable File
313 lines
13 KiB
Python
Executable File
import sys, time, datetime, zoneinfo, json, math
|
|
from pathlib import Path
|
|
import numpy as np
|
|
import pandas as pd
|
|
|
|
sys.path.insert(0, str(Path(__file__).parent))
|
|
|
|
from astropy.time import Time
|
|
import astropy.coordinates as coord
|
|
import astropy.units as u
|
|
from astropy.coordinates import solar_system_ephemeris, get_body, EarthLocation
|
|
|
|
from nautilus_dolphin.nautilus.alpha_orchestrator import NDAlphaEngine
|
|
from nautilus_dolphin.nautilus.adaptive_circuit_breaker import AdaptiveCircuitBreaker
|
|
from nautilus_dolphin.nautilus.ob_features import OBFeatureEngine
|
|
from nautilus_dolphin.nautilus.ob_provider import MockOBProvider
|
|
|
|
class MarketIndicators:
|
|
def __init__(self):
|
|
self.regions = [
|
|
{'name': 'Americas', 'tz': 'America/New_York', 'pop': 1000, 'liq_weight': 0.35},
|
|
{'name': 'EMEA', 'tz': 'Europe/London', 'pop': 2200, 'liq_weight': 0.30},
|
|
{'name': 'South_Asia', 'tz': 'Asia/Kolkata', 'pop': 1400, 'liq_weight': 0.05},
|
|
{'name': 'East_Asia', 'tz': 'Asia/Shanghai', 'pop': 1600, 'liq_weight': 0.20},
|
|
{'name': 'Oceania_SEA', 'tz': 'Asia/Singapore', 'pop': 800, 'liq_weight': 0.10}
|
|
]
|
|
self.cycle_length_days = 1460
|
|
self.last_halving = datetime.datetime(2024, 4, 20, tzinfo=datetime.timezone.utc)
|
|
|
|
def get_calendar_items(self, now):
|
|
return {
|
|
'year': now.year,
|
|
'month': now.month,
|
|
'day_of_month': now.day,
|
|
'hour': now.hour,
|
|
'minute': now.minute,
|
|
'day_of_week': now.weekday(),
|
|
'week_of_year': now.isocalendar().week
|
|
}
|
|
|
|
def get_market_cycle_position(self, now_utc):
|
|
days_since_halving = (now_utc - self.last_halving).days
|
|
position = (days_since_halving % self.cycle_length_days) / self.cycle_length_days
|
|
return position
|
|
|
|
def get_moon_phase(self, now_utc):
|
|
t = Time(now_utc)
|
|
with solar_system_ephemeris.set('builtin'):
|
|
moon = get_body('moon', t)
|
|
sun = get_body('sun', t)
|
|
elongation = sun.separation(moon)
|
|
phase_angle = np.arctan2(sun.distance * np.sin(elongation),
|
|
moon.distance - sun.distance * np.cos(elongation))
|
|
illumination = (1 + np.cos(phase_angle)) / 2.0
|
|
|
|
phase_name = "WAXING"
|
|
if illumination < 0.03: phase_name = "NEW_MOON"
|
|
elif illumination > 0.97: phase_name = "FULL_MOON"
|
|
elif illumination < 0.5: phase_name = "WAXING_CRESCENT" if moon.dec.deg > sun.dec.deg else "WANING_CRESCENT"
|
|
else: phase_name = "WAXING_GIBBOUS" if moon.dec.deg > sun.dec.deg else "WANING_GIBBOUS"
|
|
|
|
return {'illumination': float(illumination), 'phase_name': phase_name}
|
|
|
|
def is_mercury_retrograde(self, now_utc):
|
|
t = Time(now_utc)
|
|
is_retro = False
|
|
try:
|
|
with solar_system_ephemeris.set('builtin'):
|
|
loc = EarthLocation.of_site('greenwich')
|
|
merc_now = get_body('mercury', t, loc)
|
|
merc_later = get_body('mercury', t + 1 * u.day, loc)
|
|
|
|
lon_now = merc_now.geometrictrueecliptic.lon.deg
|
|
lon_later = merc_later.geometrictrueecliptic.lon.deg
|
|
|
|
diff = (lon_later - lon_now) % 360
|
|
is_retro = diff > 180
|
|
except Exception as e:
|
|
pass
|
|
return is_retro
|
|
|
|
def get_indicators(self, now_utc):
|
|
# We drop hour-specific ones since we evaluate at a fixed time daily
|
|
moon_data = self.get_moon_phase(now_utc)
|
|
calendar = self.get_calendar_items(now_utc)
|
|
return {
|
|
'timestamp': now_utc.isoformat(),
|
|
'day_of_week': calendar['day_of_week'],
|
|
'week_of_year': calendar['week_of_year'],
|
|
'market_cycle_position': round(self.get_market_cycle_position(now_utc), 4),
|
|
'moon_illumination': moon_data['illumination'],
|
|
'moon_phase_name': moon_data['phase_name'],
|
|
'mercury_retrograde': int(self.is_mercury_retrograde(now_utc)),
|
|
}
|
|
|
|
VBT_DIR = Path(r"C:\Users\Lenovo\Documents\- DOLPHIN NG HD HCM TSF Predict\vbt_cache")
|
|
parquet_files = sorted(VBT_DIR.glob("*.parquet"))
|
|
parquet_files = [p for p in parquet_files if 'catalog' not in str(p)]
|
|
|
|
print("Loading data & extracting daily precursor AND ESOTERIC metrics...")
|
|
|
|
daily_metrics = []
|
|
mi = MarketIndicators()
|
|
|
|
for pf in parquet_files:
|
|
ds = pf.stem
|
|
df = pd.read_parquet(pf)
|
|
|
|
# 1. Structural Precursors
|
|
vd = df['vel_div'].fillna(0).values
|
|
vol_accel = np.diff(vd, prepend=vd[0])
|
|
daily_vol_accel_max = np.max(np.abs(vol_accel))
|
|
|
|
assets = ['BTCUSDT', 'ETHUSDT', 'BNBUSDT', 'SOLUSDT']
|
|
valid_assets = [a for a in assets if a in df.columns]
|
|
if len(valid_assets) > 1:
|
|
rets = df[valid_assets].pct_change().fillna(0)
|
|
corr_matrix = rets.corr().values
|
|
cross_corr = corr_matrix[np.triu_indices_from(corr_matrix, k=1)]
|
|
mean_cross_corr = np.nanmean(cross_corr)
|
|
max_cross_corr = np.nanmax(cross_corr)
|
|
else:
|
|
mean_cross_corr = 0; max_cross_corr = 0
|
|
|
|
entropy_max = df['instability_50'].max() if 'instability_50' in df.columns else 0
|
|
v750_max = df['v750_lambda_max_velocity'].max() if 'v750_lambda_max_velocity' in df.columns else 0
|
|
v50_max = df['v50_lambda_max_velocity'].max() if 'v50_lambda_max_velocity' in df.columns else 0
|
|
|
|
# 2. Esoteric Factors
|
|
y, m, d = map(int, ds.split('-'))
|
|
dt_utc = datetime.datetime(y, m, d, 12, 0, 0, tzinfo=datetime.timezone.utc)
|
|
eso = mi.get_indicators(dt_utc)
|
|
|
|
# Optionally save to disk as requested
|
|
eso_path = VBT_DIR / f"ESOTERIC_data_{ds}.json"
|
|
if not eso_path.exists():
|
|
with open(eso_path, 'w') as f:
|
|
json.dump(eso, f, indent=2)
|
|
|
|
daily_metrics.append({
|
|
'Date': ds,
|
|
'vol_accel_max': daily_vol_accel_max,
|
|
'cross_corr_mean': mean_cross_corr,
|
|
'cross_corr_max': max_cross_corr,
|
|
'entropy_max': entropy_max,
|
|
'v750_max': v750_max,
|
|
'v50_max': v50_max,
|
|
'day_of_week': eso['day_of_week'],
|
|
'week_of_year': eso['week_of_year'],
|
|
'market_cycle_position': eso['market_cycle_position'],
|
|
'moon_illumination': eso['moon_illumination'],
|
|
'mercury_retrograde': eso['mercury_retrograde']
|
|
})
|
|
|
|
metrics_df = pd.DataFrame(daily_metrics).set_index('Date')
|
|
precursor_df = metrics_df.shift(1).dropna() # Shift 1 day to ensure it's a T-1 precursor
|
|
|
|
print("Running fast 6.0x trajectory to isolate daily PnL...")
|
|
|
|
pq_data = {}
|
|
all_vols = []
|
|
for pf in parquet_files:
|
|
df = pd.read_parquet(pf)
|
|
ac = [c for c in df.columns if c not in {'timestamp', 'scan_number', 'v50_lambda_max_velocity', 'v150_lambda_max_velocity',
|
|
'v300_lambda_max_velocity', 'v750_lambda_max_velocity', 'vel_div',
|
|
'instability_50', 'instability_150'}]
|
|
dv = df['vel_div'].values if 'vel_div' in df.columns else np.zeros(len(df))
|
|
pq_data[pf.stem] = (df, ac, dv)
|
|
|
|
if 'BTCUSDT' in df.columns:
|
|
pr = df['BTCUSDT'].values
|
|
for i in range(60, len(pr)):
|
|
seg = pr[max(0,i-50):i]
|
|
if len(seg)<10: continue
|
|
v = float(np.std(np.diff(seg)/seg[:-1]))
|
|
if v > 0: all_vols.append(v)
|
|
|
|
vol_p60 = float(np.percentile(all_vols, 60))
|
|
|
|
acb = AdaptiveCircuitBreaker()
|
|
acb.preload_w750([pf.stem for pf in parquet_files])
|
|
|
|
mock = MockOBProvider(imbalance_bias=-0.09, depth_scale=1.0,
|
|
assets=["BTCUSDT", "ETHUSDT", "BNBUSDT", "SOLUSDT"],
|
|
imbalance_biases={"BNBUSDT": 0.20, "SOLUSDT": 0.20})
|
|
ob_engine_inst = OBFeatureEngine(mock)
|
|
ob_engine_inst.preload_date("mock", mock.get_assets())
|
|
|
|
ENGINE_KWARGS = dict(
|
|
initial_capital=25000.0, vel_div_threshold=-0.02, vel_div_extreme=-0.05,
|
|
min_leverage=0.5, max_leverage=6.0, leverage_convexity=3.0,
|
|
fraction=0.20, fixed_tp_pct=0.0099, stop_pct=1.0, max_hold_bars=120,
|
|
use_direction_confirm=True, dc_lookback_bars=7, dc_min_magnitude_bps=0.75,
|
|
dc_skip_contradicts=True, dc_leverage_boost=1.0, dc_leverage_reduce=0.5,
|
|
use_asset_selection=True, min_irp_alignment=0.45,
|
|
use_sp_fees=True, use_sp_slippage=True,
|
|
use_ob_edge=True, ob_edge_bps=5.0, ob_confirm_rate=0.40,
|
|
lookback=100, use_alpha_layers=True, use_dynamic_leverage=True, seed=42,
|
|
)
|
|
|
|
engine = NDAlphaEngine(**ENGINE_KWARGS)
|
|
engine.set_ob_engine(ob_engine_inst)
|
|
|
|
daily_returns = {}
|
|
bar_idx = 0
|
|
all_vols_engine = []
|
|
|
|
for pf in parquet_files:
|
|
ds = pf.stem
|
|
cs = engine.capital
|
|
|
|
acb_info = acb.get_dynamic_boost_for_date(ds, ob_engine=ob_engine_inst)
|
|
base_boost = acb_info['boost']
|
|
beta = acb_info['beta']
|
|
|
|
df, acols, dvol_raw = pq_data[ds]
|
|
ph = {}
|
|
|
|
for ri in range(len(df)):
|
|
row = df.iloc[ri]
|
|
vd = dvol_raw[ri]
|
|
if not np.isfinite(vd): bar_idx+=1; continue
|
|
|
|
prices = {}
|
|
for ac in acols:
|
|
p = row[ac]
|
|
if p and p > 0 and np.isfinite(p):
|
|
prices[ac] = float(p)
|
|
if ac not in ph: ph[ac] = []
|
|
ph[ac].append(float(p))
|
|
if len(ph[ac]) > 500: ph[ac] = ph[ac][-200:]
|
|
if not prices: bar_idx+=1; continue
|
|
|
|
btc_hist = ph.get("BTCUSDT", [])
|
|
engine_vrok = False
|
|
if len(btc_hist) >= 50:
|
|
seg = btc_hist[-50:]
|
|
vd_eng = float(np.std(np.diff(seg)/np.array(seg[:-1])))
|
|
all_vols_engine.append(vd_eng)
|
|
if len(all_vols_engine) > 100:
|
|
engine_vrok = vd_eng > np.percentile(all_vols_engine, 60)
|
|
|
|
if beta > 0:
|
|
ss = 0.0
|
|
if vd < -0.02:
|
|
raw = (-0.02 - float(vd)) / (-0.02 - -0.05)
|
|
ss = min(1.0, max(0.0, raw)) ** 3.0
|
|
engine.regime_size_mult = base_boost * (1.0 + beta * ss)
|
|
else:
|
|
engine.regime_size_mult = base_boost
|
|
|
|
engine.process_bar(bar_idx=bar_idx, vel_div=float(vd), prices=prices, vol_regime_ok=engine_vrok, price_histories=ph)
|
|
bar_idx += 1
|
|
|
|
daily_returns[ds] = (engine.capital - cs) / cs if cs > 0 else 0
|
|
|
|
returns_df = pd.DataFrame.from_dict(daily_returns, orient='index', columns=['Return'])
|
|
merged = precursor_df.join(returns_df, how='inner')
|
|
|
|
threshold_pnl = merged['Return'].quantile(0.10)
|
|
merged['Is_Tail'] = merged['Return'] <= threshold_pnl
|
|
base_p_tail = merged['Is_Tail'].mean()
|
|
|
|
print(f"\n==================================================================================================")
|
|
print(f" CONDITIONAL PROBABILITY ANALYSIS: P(Tail | Precursor Spike)")
|
|
print(f" Baseline P(Tail) = {base_p_tail:.1%}")
|
|
print(f"==================================================================================================")
|
|
print(f"{'Feature':<25} | {'P(Tail|X > 75th)':<20} | {'P(Tail|X > 90th)':<20} | {'P(Tail|X > 95th)':<20}")
|
|
print("-" * 98)
|
|
|
|
features = ['vol_accel_max', 'v750_max', 'v50_max', 'entropy_max', 'cross_corr_max']
|
|
|
|
for f in features:
|
|
p75 = merged[f].quantile(0.75)
|
|
p90 = merged[f].quantile(0.90)
|
|
p95 = merged[f].quantile(0.95)
|
|
|
|
cond_75 = merged[merged[f] > p75]['Is_Tail'].mean() if len(merged[merged[f]>p75]) > 0 else 0
|
|
cond_90 = merged[merged[f] > p90]['Is_Tail'].mean() if len(merged[merged[f]>p90]) > 0 else 0
|
|
cond_95 = merged[merged[f] > p95]['Is_Tail'].mean() if len(merged[merged[f]>p95]) > 0 else 0
|
|
|
|
print(f"{f:<25} | {cond_75:>19.1%} | {cond_90:>19.1%} | {cond_95:>19.1%}")
|
|
|
|
print(f"\n==================================================================================================")
|
|
print(f" ESOTERIC FACTORS CORRELATION WITH EXTREME LEFT TAILS")
|
|
print(f"==================================================================================================")
|
|
|
|
esoteric_features = ['day_of_week', 'moon_illumination', 'market_cycle_position', 'mercury_retrograde']
|
|
for f in esoteric_features:
|
|
# See if Esoteric factors map to higher chance of tail events
|
|
if f == 'day_of_week':
|
|
print("\nDay of Week P(Tail):")
|
|
for d in range(7):
|
|
subset = merged[merged[f] == d]
|
|
if len(subset) > 0:
|
|
print(f" Day {d}: {subset['Is_Tail'].mean():.1%} (N={len(subset)})")
|
|
elif f == 'mercury_retrograde':
|
|
print("\nMercury Retrograde P(Tail):")
|
|
for d in [0, 1]:
|
|
subset = merged[merged[f] == d]
|
|
if len(subset) > 0:
|
|
print(f" Retrograde={d}: {subset['Is_Tail'].mean():.1%} (N={len(subset)})")
|
|
else:
|
|
# Continuous values
|
|
p75 = merged[f].quantile(0.75)
|
|
cond_75 = merged[merged[f] > p75]['Is_Tail'].mean() if len(merged[merged[f]>p75]) > 0 else 0
|
|
p25 = merged[f].quantile(0.25)
|
|
cond_25 = merged[merged[f] < p25]['Is_Tail'].mean() if len(merged[merged[f]<p25]) > 0 else 0
|
|
print(f"\n{f}:")
|
|
print(f" P(Tail | Top 25% {f}): {cond_75:.1%}")
|
|
print(f" P(Tail | Bot 25% {f}): {cond_25:.1%}")
|
|
|