initial: import DOLPHIN baseline 2026-04-21 from dolphinng5_predict working tree
Includes core prod + GREEN/BLUE subsystems: - prod/ (BLUE harness, configs, scripts, docs) - nautilus_dolphin/ (GREEN Nautilus-native impl + dvae/ preserved) - adaptive_exit/ (AEM engine + models/bucket_assignments.pkl) - Observability/ (EsoF advisor, TUI, dashboards) - external_factors/ (EsoF producer) - mc_forewarning_qlabs_fork/ (MC regime/envelope) Excludes runtime caches, logs, backups, and reproducible artifacts per .gitignore.
This commit is contained in:
hjnormey
365
prod/D-VAE-RegimeMonitor-MC-F_imrovement.py
Executable file
365
prod/D-VAE-RegimeMonitor-MC-F_imrovement.py
Executable file
@@ -0,0 +1,365 @@
|
||||
"""
|
||||
HD Disentangled Regime Monitor
|
||||
A concrete architecture for:
|
||||
1. Numba-optimized HD feature extraction.
|
||||
2. Disentangled Temporal VAE for regime encoding.
|
||||
3. FLINT-backed 512-bit Latent Optimizer for "Most Likely Future" projection.
|
||||
"""
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from numba import jit, float64
|
||||
import math
|
||||
|
||||
# --- High Precision Imports ---
|
||||
# We wrap this in a try-except block to allow the module to load for viewing
|
||||
# even if python-flint isn't installed, though it won't run without it.
|
||||
try:
|
||||
from flint import arb, acb_mat, ctx
|
||||
FLINT_ENABLED = True
|
||||
except ImportError:
|
||||
FLINT_ENABLED = False
|
||||
print("Warning: python-flint not found. High-precision optimizer will run in mock mode.")
|
||||
|
||||
# =============================================================================
|
||||
# SECTION 1: NUMBA-OPTIMIZED FEATURE ENGINEERING
|
||||
# =============================================================================
|
||||
|
||||
@jit(nopython=True, fastmath=True, cache=True)
|
||||
def construct_hd_features(prices, volumes, window_size):
|
||||
"""
|
||||
Constructs High-Dimensional feature tensors from raw market data.
|
||||
Optimized with Numba to handle high-throughput data streams.
|
||||
|
||||
Args:
|
||||
prices (np.array): 1D array of price points.
|
||||
volumes (np.array): 1D array of volume points.
|
||||
window_size (int): Lookback window size.
|
||||
|
||||
Returns:
|
||||
np.array: Feature tensor of shape (n_samples, window_size, n_features).
|
||||
"""
|
||||
n_samples = len(prices) - window_size
|
||||
# Features: LogRet, Volatility, RelVolume, Acceleration
|
||||
n_features = 4
|
||||
features = np.empty((n_samples, window_size, n_features), dtype=np.float64)
|
||||
|
||||
for i in range(n_samples):
|
||||
window_p = prices[i : i + window_size]
|
||||
window_v = volumes[i : i + window_size]
|
||||
|
||||
# 1. Log Returns
|
||||
log_p = np.log(window_p)
|
||||
rets = np.diff(log_p)
|
||||
features[i, 1:, 0] = rets
|
||||
features[i, 0, 0] = 0.0
|
||||
|
||||
# 2. Rolling Volatility (Standard Deviation of returns over mini-windows)
|
||||
# Simplified for numba: rolling std dev
|
||||
std_val = np.std(rets)
|
||||
features[i, :, 1] = std_val
|
||||
|
||||
# 3. Relative Volume
|
||||
mean_vol = np.mean(window_v)
|
||||
if mean_vol > 0:
|
||||
features[i, :, 2] = window_v / mean_vol
|
||||
else:
|
||||
features[i, :, 2] = 0.0
|
||||
|
||||
# 4. Acceleration (Second derivative of price)
|
||||
if window_size > 2:
|
||||
acc = np.diff(log_p, n=2)
|
||||
features[i, 2:, 3] = acc
|
||||
features[i, :2, 3] = 0.0
|
||||
|
||||
return features
|
||||
|
||||
# =============================================================================
|
||||
# SECTION 2: DISENTANGLED TEMPORAL VAE (PyTorch)
|
||||
# =============================================================================
|
||||
|
||||
class BetaTCVAE(nn.Module):
|
||||
"""
|
||||
Temporal VAE with Total Correlation disentanglement.
|
||||
Uses a Bidirectional GRU encoder and a Generative GRU decoder.
|
||||
Forces latent space to separate "Knobs" (Trend, Vol, etc).
|
||||
"""
|
||||
def __init__(self, input_dim, hidden_dim, latent_dim, num_layers=1, beta=5.0):
|
||||
super(BetaTCVAE, self).__init__()
|
||||
self.latent_dim = latent_dim
|
||||
self.beta = beta
|
||||
|
||||
# Encoder: Bidirectional GRU
|
||||
self.encoder_rnn = nn.GRU(input_dim, hidden_dim, num_layers,
|
||||
batch_first=True, bidirectional=True)
|
||||
self.fc_mu = nn.Linear(hidden_dim * 2, latent_dim)
|
||||
self.fc_var = nn.Linear(hidden_dim * 2, latent_dim)
|
||||
|
||||
# Decoder: Generative GRU
|
||||
# Input to decoder is latent vector z repeated across time steps
|
||||
self.decoder_rnn = nn.GRU(latent_dim + input_dim, hidden_dim, num_layers, batch_first=True)
|
||||
self.decoder_out = nn.Linear(hidden_dim, input_dim)
|
||||
|
||||
def encode(self, x):
|
||||
# x: (batch, seq, feat)
|
||||
_, h = self.encoder_rnn(x)
|
||||
# Concatenate forward and backward final hidden states
|
||||
h_cat = torch.cat((h[0], h[1]), dim=1)
|
||||
return self.fc_mu(h_cat), self.fc_var(h_cat)
|
||||
|
||||
def reparameterize(self, mu, logvar):
|
||||
std = torch.exp(0.5 * logvar)
|
||||
eps = torch.randn_like(std)
|
||||
return mu + eps * std
|
||||
|
||||
def decode(self, z, seq_len):
|
||||
# Repeat z for seq_len steps to seed the generator
|
||||
z_rep = z.unsqueeze(1).repeat(1, seq_len, 1)
|
||||
# Auto-regressive logic simplified: we feed z and noise/placeholder
|
||||
# In a real system, this would be teacher-forcing or iterative sampling
|
||||
placeholder = torch.zeros(z_rep.shape[0], seq_len, z_rep.shape[2]).to(z.device)
|
||||
dec_input = torch.cat((z_rep, placeholder), dim=-1)
|
||||
|
||||
out, _ = self.decoder_rnn(dec_input)
|
||||
return self.decoder_out(out)
|
||||
|
||||
def forward(self, x):
|
||||
mu, logvar = self.encode(x)
|
||||
z = self.reparameterize(mu, logvar)
|
||||
recon_x = self.decode(z, x.shape[1])
|
||||
return recon_x, mu, logvar, z
|
||||
|
||||
def loss_function(self, recon_x, x, mu, logvar, z):
|
||||
# Reconstruction Loss
|
||||
BCE = F.mse_loss(recon_x, x, reduction='sum')
|
||||
|
||||
# KL Divergence
|
||||
# 0.5 * sum(1 + log(sigma^2) - mu^2 - sigma^2)
|
||||
KLD = -0.5 * torch.sum(1 + logvar - mu.pow(2) - logvar.exp())
|
||||
|
||||
# Total Correlation (TC) Minimization (Simplified for Beta-VAE style)
|
||||
# In a full TC-VAE, we would calculate minibatch TC. Here we use
|
||||
# a high Beta to force independence statistically.
|
||||
|
||||
return BCE + self.beta * KLD
|
||||
|
||||
# =============================================================================
|
||||
# SECTION 3: FLINT 512-BIT LATENT OPTIMIZER
|
||||
# =============================================================================
|
||||
|
||||
class FlintLatentOptimizer:
|
||||
"""
|
||||
Navigates the latent space using FLINT 512-bit precision.
|
||||
Solves for the 'Most Likely Future' by descending the potential energy landscape.
|
||||
"""
|
||||
def __init__(self, precision=512):
|
||||
if not FLINT_ENABLED:
|
||||
raise RuntimeError("FLINT library not installed.")
|
||||
|
||||
self.precision = precision
|
||||
ctx.precision = precision
|
||||
|
||||
# In a real scenario, these dynamics W are learned.
|
||||
# Here we initialize a random transition matrix for demonstration.
|
||||
# We assume z_t+1 = W * tanh(z_t) + noise
|
||||
self.W_dim = 10 # Example latent dim
|
||||
self.W_np = np.eye(self.W_dim) * 0.9 + np.random.randn(self.W_dim, self.W_dim) * 0.1
|
||||
|
||||
# Convert W to FLINT matrix
|
||||
self.W_flint = self._to_flint_matrix(self.W_np)
|
||||
|
||||
def _to_flint_matrix(self, arr):
|
||||
rows, cols = arr.shape
|
||||
mat = acb_mat(rows, cols)
|
||||
for r in range(rows):
|
||||
for c in range(cols):
|
||||
# Convert float to string for exact arb initialization
|
||||
mat[r, c] = arb(str(arr[r, c]))
|
||||
return mat
|
||||
|
||||
def _to_flint_vec(self, arr):
|
||||
vec = acb_mat(len(arr), 1)
|
||||
for i in range(len(arr)):
|
||||
vec[i, 0] = arb(str(arr[i]))
|
||||
return vec
|
||||
|
||||
def solve_most_likely_future(self, z_current_numpy, steps=10):
|
||||
"""
|
||||
Projects the current latent state forward to find the stable attractor.
|
||||
Uses 512-bit arithmetic to avoid "noise" hallucination.
|
||||
"""
|
||||
# 1. Convert numpy latent vector to FLINT arb vector
|
||||
z_state = self._to_flint_vec(z_current_numpy)
|
||||
|
||||
# 2. Iterate Dynamics
|
||||
# We look for a stable state (Fixed Point).
|
||||
# z_{t+1} = W @ z_t
|
||||
# We stop when ||z_{t+1} - z_t|| < epsilon (512-bit epsilon)
|
||||
|
||||
for _ in range(steps):
|
||||
# Matrix Multiply in High Precision
|
||||
z_next = self.W_flint * z_state
|
||||
|
||||
# Convergence Check (Simplified)
|
||||
# In a real app, we compute delta norm here using arb functions
|
||||
z_state = z_next
|
||||
|
||||
# 3. Return result as string (to preserve precision) and numpy array
|
||||
# We extract the real part (midpoint of the ball)
|
||||
result_np = np.zeros(self.W_dim)
|
||||
for i in range(self.W_dim):
|
||||
# arb to string to float (lossy, but for the sizing logic)
|
||||
result_np[i] = float(str(z_state[i, 0].real()))
|
||||
|
||||
return result_np
|
||||
|
||||
# =============================================================================
|
||||
# SECTION 4: MAIN MONITOR CLASS
|
||||
# =============================================================================
|
||||
|
||||
class AdaptiveRegimeMonitor:
|
||||
"""
|
||||
The main controller integrating Feature Engineering, VAE, and High-Precision Optimization.
|
||||
"""
|
||||
def __init__(self, input_dim=4, hidden_dim=32, latent_dim=10, precision=512):
|
||||
self.vae = BetaTCVAE(input_dim, hidden_dim, latent_dim)
|
||||
self.latent_optimizer = None
|
||||
|
||||
if FLINT_ENABLED:
|
||||
try:
|
||||
self.latent_optimizer = FlintLatentOptimizer(precision=precision)
|
||||
print(f"Initialized FLINT Optimizer at {precision}-bit precision.")
|
||||
except Exception as e:
|
||||
print(f"Failed to init FLINT: {e}. Using fallback.")
|
||||
|
||||
# Latent Mapping (Mock semantic labels for "Knobs")
|
||||
self.regime_knobs = {
|
||||
'trend_strength': 0, # Index in latent vector
|
||||
'volatility_regime': 1,
|
||||
'fragility': 2
|
||||
}
|
||||
|
||||
def train_vae(self, data_loader, epochs=5):
|
||||
"""Trains the VAE to learn the market 'physics'."""
|
||||
optimizer = torch.optim.Adam(self.vae.parameters(), lr=1e-3)
|
||||
self.vae.train()
|
||||
|
||||
print("Training VAE...")
|
||||
for epoch in range(epochs):
|
||||
total_loss = 0
|
||||
for batch in data_loader:
|
||||
x = batch[0] # assuming dataloader returns (x, y)
|
||||
optimizer.zero_grad()
|
||||
recon_x, mu, logvar, z = self.vae(x)
|
||||
loss = self.vae.loss_function(recon_x, x, mu, logvar, z)
|
||||
loss.backward()
|
||||
optimizer.step()
|
||||
total_loss += loss.item()
|
||||
print(f"Epoch {epoch+1}, Loss: {total_loss:.4f}")
|
||||
print("Training Complete.")
|
||||
|
||||
def compute_regime_score(self, raw_window, raw_volume):
|
||||
"""
|
||||
Full pipeline execution:
|
||||
1. Features (Numba)
|
||||
2. Latent State (VAE)
|
||||
3. Future Projection (FLINT)
|
||||
4. Sizing Output
|
||||
"""
|
||||
# 1. Features
|
||||
# Note: Numba returns (N, Win, Feat). We take the last sample for live trading
|
||||
# or simulate a batch. Here we expand dims to simulate batch=1
|
||||
features = construct_hd_features(raw_window, raw_volume, len(raw_window))
|
||||
last_feature_window = features[-1:] # Shape (1, Window, Feat)
|
||||
|
||||
# 2. Encode
|
||||
self.vae.eval()
|
||||
with torch.no_grad():
|
||||
x_tensor = torch.tensor(last_feature_window, dtype=torch.float32)
|
||||
_, mu, _, z = self.vae(x_tensor)
|
||||
|
||||
# Current Latent State (Numpy)
|
||||
z_current = mu.squeeze().numpy()
|
||||
|
||||
# 3. Optimize Future (FLINT)
|
||||
if self.latent_optimizer:
|
||||
z_future = self.latent_optimizer.solve_most_likely_future(z_current)
|
||||
else:
|
||||
# Fallback if FLINT missing
|
||||
z_future = z_current * 0.95 # Dummy decay
|
||||
|
||||
# 4. Disentangled Interpretation ("Knobs")
|
||||
# We apply sigmoid to normalize the score between 0 and 1
|
||||
trend_score = torch.sigmoid(torch.tensor(z_future[self.regime_knobs['trend_strength']]))
|
||||
fragility_score = torch.sigmoid(torch.tensor(z_future[self.regime_knobs['fragility']]))
|
||||
|
||||
# Regime Quality Indicator
|
||||
# High Quality = High Trend + Low Fragility
|
||||
regime_quality = (trend_score * (1 - fragility_score)).item()
|
||||
|
||||
return regime_quality, z_future
|
||||
|
||||
# =============================================================================
|
||||
# SECTION 5: BUILT-IN TESTS
|
||||
# =============================================================================
|
||||
|
||||
def run_tests():
|
||||
print("-" * 40)
|
||||
print("RUNNING MODULE TESTS")
|
||||
print("-" * 40)
|
||||
|
||||
# 1. Test Numba Features
|
||||
print("\n[1/3] Testing Numba Feature Engineering...")
|
||||
try:
|
||||
dummy_prices = np.random.rand(1000) * 100 + 50
|
||||
dummy_vols = np.random.rand(1000) * 1000
|
||||
feats = construct_hd_features(dummy_prices, dummy_vols, 64)
|
||||
assert feats.shape == (936, 64, 4) # 1000 - 64 = 936 samples
|
||||
print(" SUCCESS: Feature shape correct.")
|
||||
except Exception as e:
|
||||
print(f" FAILED: {e}")
|
||||
|
||||
# 2. Test VAE Structure
|
||||
print("\n[2/3] Testing VAE Architecture...")
|
||||
try:
|
||||
vae = BetaTCVAE(input_dim=4, hidden_dim=16, latent_dim=6)
|
||||
dummy_input = torch.randn(10, 64, 4) # Batch 10, Seq 64, Feat 4
|
||||
recon, mu, logvar, z = vae(dummy_input)
|
||||
loss = vae.loss_function(recon, dummy_input, mu, logvar, z)
|
||||
assert z.shape == (10, 6)
|
||||
print(f" SUCCESS: VAE Forward pass OK, Latent Dim: 6.")
|
||||
except Exception as e:
|
||||
print(f" FAILED: {e}")
|
||||
|
||||
# 3. Test Full Pipeline (with Mock Data)
|
||||
print("\n[3/3] Testing Full Pipeline (Mock Market Data)...")
|
||||
try:
|
||||
# Generate synthetic data
|
||||
prices = np.cumsum(np.random.randn(200)) + 100
|
||||
vols = np.abs(np.random.randn(200)) * 10
|
||||
|
||||
# Initialize Monitor
|
||||
monitor = AdaptiveRegimeMonitor(input_dim=4, hidden_dim=16, latent_dim=10)
|
||||
|
||||
# We skip training for the test and just run inference
|
||||
|
||||
# Run compute
|
||||
score, future_z = monitor.compute_regime_score(prices, vols)
|
||||
|
||||
print(f" Regime Quality Score: {score:.5f}")
|
||||
print(f" Future Latent Vector (First 3): {future_z[:3]}")
|
||||
|
||||
assert 0.0 <= score <= 1.0
|
||||
print(" SUCCESS: Full pipeline integration OK.")
|
||||
|
||||
except Exception as e:
|
||||
print(f" FAILED: {e}")
|
||||
|
||||
print("\n" + "="*40)
|
||||
print("TESTS COMPLETE")
|
||||
print("="*40)
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_tests()
|
||||
Reference in New Issue
Block a user