initial: import DOLPHIN baseline 2026-04-21 from dolphinng5_predict working tree
Includes core prod + GREEN/BLUE subsystems: - prod/ (BLUE harness, configs, scripts, docs) - nautilus_dolphin/ (GREEN Nautilus-native impl + dvae/ preserved) - adaptive_exit/ (AEM engine + models/bucket_assignments.pkl) - Observability/ (EsoF advisor, TUI, dashboards) - external_factors/ (EsoF producer) - mc_forewarning_qlabs_fork/ (MC regime/envelope) Excludes runtime caches, logs, backups, and reproducible artifacts per .gitignore.
This commit is contained in:
hjnormey
233
nautilus_dolphin/dvae/titan_sensor.py
Executable file
233
nautilus_dolphin/dvae/titan_sensor.py
Executable file
@@ -0,0 +1,233 @@
|
||||
"""
|
||||
TitanSensor — Non-invasive latent state extractor (Stage 1, read-only).
|
||||
|
||||
Architecture notes
|
||||
------------------
|
||||
The LSTM weights (W_ih, W_hh) are fixed random projections used as the
|
||||
encoder basis during training. W_mu/W_logvar/W_dec/W_out were trained to
|
||||
map the output of THESE specific W_ih/W_hh matrices. Loading the correct
|
||||
W_ih/W_hh is therefore mandatory for valid reconstruction errors.
|
||||
|
||||
Models saved before 2026-03-15 did not persist W_ih/W_hh — those fall back
|
||||
to legacy seed=42 re-init (meaningless recon_err ~10^14) and are flagged.
|
||||
|
||||
T5 oracle-leakage fix
|
||||
---------------------
|
||||
Training included T5 dims [111:261] = spectral path coefficients derived from
|
||||
FUTURE prices [t+1, t+51]. At inference we zero those dims unconditionally.
|
||||
|
||||
Normalisation (GD-v2 models, training >= 2026-03-15)
|
||||
------------------------------------------------------
|
||||
GD-v2 models store per-feature z-score stats (norm_mean, norm_std) computed
|
||||
from the raw training corpus. encode() applies v = (v - norm_mean)/norm_std
|
||||
before the LSTM step so that training and inference see the same distribution.
|
||||
Models without norm_mean (legacy) skip this step — recon_err is in raw-feature
|
||||
space and remains valid for relative comparison within that model.
|
||||
"""
|
||||
|
||||
import json
|
||||
import numpy as np
|
||||
|
||||
|
||||
class TitanSensor:
|
||||
INPUT_DIM = 261
|
||||
HIDDEN_DIM = 128
|
||||
LATENT_DIM = 32
|
||||
T5_START = 111 # dims [111:261] = spectral (oracle) → always zero
|
||||
|
||||
def __init__(self, model_path: str, lstm_seed: int = 42):
|
||||
with open(model_path) as f:
|
||||
m = json.load(f)
|
||||
|
||||
def _arr(key, default_shape):
|
||||
return np.array(m[key]) if key in m else np.zeros(default_shape)
|
||||
|
||||
self.W_mu = _arr('W_mu', (self.HIDDEN_DIM, self.LATENT_DIM))
|
||||
self.W_logvar = _arr('W_logvar', (self.HIDDEN_DIM, self.LATENT_DIM))
|
||||
self.W_dec = _arr('W_dec', (self.LATENT_DIM, self.HIDDEN_DIM))
|
||||
self.W_out = _arr('W_out', (self.HIDDEN_DIM, self.INPUT_DIM))
|
||||
self.b_mu = _arr('b_mu', (self.LATENT_DIM,))
|
||||
self.b_logvar = _arr('b_logvar', (self.LATENT_DIM,))
|
||||
self.b_dec = _arr('b_dec', (self.HIDDEN_DIM,))
|
||||
self.b_out = _arr('b_out', (self.INPUT_DIM,))
|
||||
|
||||
# LSTM weights: load from JSON if present (fixed since 2026-03-15).
|
||||
# Legacy models (no W_ih in JSON) fall back to seed=42 re-init —
|
||||
# recon_err will be ~10^14 for those; a warning is emitted.
|
||||
if 'W_ih' in m:
|
||||
self.W_ih = np.array(m['W_ih'], dtype=np.float64)
|
||||
self.W_hh = np.array(m['W_hh'], dtype=np.float64)
|
||||
self.b_h = np.array(m['b_h'], dtype=np.float64)
|
||||
self.lstm_weights_valid = True
|
||||
else:
|
||||
import warnings
|
||||
warnings.warn(
|
||||
f"TitanSensor: model at '{model_path}' does not contain LSTM weights "
|
||||
"(W_ih/W_hh/b_h). Falling back to seed=42 re-init. "
|
||||
"recon_err will be ~10^14 (meaningless). Retrain with fixed save_model().",
|
||||
RuntimeWarning, stacklevel=2,
|
||||
)
|
||||
rng = np.random.RandomState(lstm_seed)
|
||||
s = 0.1
|
||||
self.W_ih = rng.randn(self.INPUT_DIM, self.HIDDEN_DIM * 4) * s
|
||||
self.W_hh = rng.randn(self.HIDDEN_DIM, self.HIDDEN_DIM * 4) * s
|
||||
self.b_h = np.zeros(self.HIDDEN_DIM * 4)
|
||||
self.lstm_weights_valid = False
|
||||
|
||||
self.latent_names = {int(k): v for k, v in m.get('latent_names', {}).items()}
|
||||
|
||||
# Per-feature normalisation stats (present in GD-v2 models trained 2026-03-15+).
|
||||
# If absent (legacy models): no normalization applied — recon_err will be in
|
||||
# raw-feature space and is still valid for relative comparison within that model.
|
||||
if 'norm_mean' in m:
|
||||
self.norm_mean = np.array(m['norm_mean'], dtype=np.float64)
|
||||
self.norm_std = np.array(m['norm_std'], dtype=np.float64)
|
||||
else:
|
||||
self.norm_mean = None
|
||||
self.norm_std = None
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def _sigmoid(self, x):
|
||||
return 1.0 / (1.0 + np.exp(-np.clip(x, -500, 500)))
|
||||
|
||||
def _lstm_step(self, x, h, c):
|
||||
g = x @ self.W_ih + h @ self.W_hh + self.b_h
|
||||
i_, f_, g_, o_ = np.split(g, 4, axis=-1)
|
||||
i_ = self._sigmoid(i_); f_ = self._sigmoid(f_); o_ = self._sigmoid(o_)
|
||||
g_ = np.tanh(g_)
|
||||
c2 = f_ * c + i_ * g_
|
||||
h2 = o_ * np.tanh(c2)
|
||||
return h2, c2
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
def encode(self, x: np.ndarray):
|
||||
"""
|
||||
Encode one 261-dim feature row.
|
||||
|
||||
Returns
|
||||
-------
|
||||
z_mu : np.ndarray (32,) — latent mean
|
||||
recon_err : float — MSE on T0-T4 reconstruction (in normalised space)
|
||||
z_logvar : np.ndarray (32,) — per-dim log-variance
|
||||
"""
|
||||
v = np.array(x, dtype=np.float64).ravel()
|
||||
v = np.resize(v, self.INPUT_DIM) # pad/truncate
|
||||
# Stage-1 zero: T3 (ExF, 78-102), T4 (esoteric, 103-110) and T5 (oracle, 111+)
|
||||
# are always zero in the training corpus — enforce this here so that
|
||||
# callers not using build_feature_vector() don't blow up normalization.
|
||||
v[78:] = 0.0
|
||||
v[self.T5_START:] = 0.0 # T5 oracle fix (redundant but explicit)
|
||||
v = np.nan_to_num(v, nan=0.0, posinf=0.0, neginf=0.0)
|
||||
|
||||
# Apply stored per-feature normalisation (GD-v2 models only).
|
||||
# This reproduces the same transform applied during training so that
|
||||
# the LSTM sees the same input distribution and recon_err is O(1).
|
||||
if self.norm_mean is not None:
|
||||
v = (v - self.norm_mean) / self.norm_std
|
||||
v[78:] = 0.0 # re-zero Stage-1 dims after norm
|
||||
# Clip to [-50, 50] for safety — training post-norm max was ~37
|
||||
v = np.clip(v, -50.0, 50.0)
|
||||
|
||||
h = np.zeros((1, self.HIDDEN_DIM))
|
||||
c = np.zeros((1, self.HIDDEN_DIM))
|
||||
h, c = self._lstm_step(v.reshape(1, -1), h, c)
|
||||
|
||||
z_mu = (h @ self.W_mu + self.b_mu)[0]
|
||||
z_logvar = (h @ self.W_logvar + self.b_logvar)[0]
|
||||
|
||||
h_dec = np.tanh(z_mu @ self.W_dec + self.b_dec)
|
||||
recon = h_dec @ self.W_out + self.b_out
|
||||
recon_err = float(np.mean((recon[:self.T5_START] - v[:self.T5_START]) ** 2))
|
||||
|
||||
return z_mu, recon_err, z_logvar
|
||||
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Feature builder — constructs a 261-dim vector from a parquet row.
|
||||
# T3 (ExF, 78-102), T4 (esoteric, 103-110), T5 (111-260) are zeroed.
|
||||
# ------------------------------------------------------------------
|
||||
_ASSET_CACHE = {} # parquet-path → ordered asset list
|
||||
|
||||
|
||||
def build_feature_vector(df, ri: int, assets: list) -> np.ndarray:
|
||||
"""
|
||||
Build 261-dim T0-T4 feature vector (T5 always zero).
|
||||
|
||||
T0 (0-7) : time encoding + rolling breadth
|
||||
T1 (8-27) : eigenvalue velocity features (4 windows × 5 dims)
|
||||
T2 (28-77) : per-asset return z-scores (up to 50 assets, rolling 50 bars)
|
||||
T3 (78-102): ExF macro — zeroed (Stage 1)
|
||||
T4 (103-110): esoteric — zeroed (Stage 1)
|
||||
T5 (111-260): spectral — zeroed (oracle fix)
|
||||
"""
|
||||
x = np.zeros(261, dtype=np.float64)
|
||||
row = df.iloc[ri]
|
||||
|
||||
# --- T0: time encoding ---
|
||||
try:
|
||||
ts = row['timestamp']
|
||||
if hasattr(ts, 'hour'):
|
||||
h = ts.hour + ts.minute / 60.0
|
||||
d = ts.dayofweek
|
||||
else:
|
||||
import pandas as pd
|
||||
ts = pd.Timestamp(ts)
|
||||
h = ts.hour + ts.minute / 60.0
|
||||
d = ts.dayofweek
|
||||
x[0] = np.sin(2 * np.pi * h / 24)
|
||||
x[1] = np.cos(2 * np.pi * h / 24)
|
||||
x[2] = np.sin(2 * np.pi * d / 7)
|
||||
x[3] = np.cos(2 * np.pi * d / 7)
|
||||
except Exception:
|
||||
pass
|
||||
x[4] = 1.0 # has_eigen always true for NG3
|
||||
|
||||
# rolling BTC breadth (T0 dims 5-7)
|
||||
if ri >= 20 and 'BTCUSDT' in df.columns:
|
||||
diffs = np.diff(df['BTCUSDT'].values[ri - 20:ri + 1])
|
||||
x[5] = float(np.mean(diffs > 0))
|
||||
x[6] = float(np.mean(diffs < 0))
|
||||
x[7] = float(np.mean(diffs == 0))
|
||||
|
||||
# --- T1: eigenvalue velocity features ---
|
||||
def _g(col):
|
||||
v = row.get(col, 0.0)
|
||||
return float(v) if v is not None and v == v else 0.0
|
||||
|
||||
v50 = _g('v50_lambda_max_velocity')
|
||||
v150 = _g('v150_lambda_max_velocity')
|
||||
v300 = _g('v300_lambda_max_velocity')
|
||||
v750 = _g('v750_lambda_max_velocity')
|
||||
i50 = _g('instability_50')
|
||||
i150 = _g('instability_150')
|
||||
vd = _g('vel_div')
|
||||
|
||||
# window-0 (8-12): v50 group
|
||||
x[8] = v50; x[9] = i50; x[10] = v50 - v150
|
||||
x[11] = v50 - v300; x[12] = v50 - v750
|
||||
# window-1 (13-17): v150 group
|
||||
x[13] = v150; x[14] = i150; x[15] = v150 - v300
|
||||
x[16] = v150 - v750; x[17] = abs(v150)
|
||||
# window-2 (18-22): v300 group
|
||||
x[18] = v300; x[19] = abs(v300); x[20] = v300 - v750
|
||||
x[21] = v50 * v300; x[22] = i50 - v750 # proxy_B equivalent
|
||||
# window-3 (23-27): v750 + composite
|
||||
x[23] = v750; x[24] = abs(v750)
|
||||
x[25] = v50 / (abs(v750) + 1e-8)
|
||||
x[26] = i50 - i150; x[27] = vd
|
||||
|
||||
# --- T2: per-asset return z-scores ---
|
||||
if ri >= 50:
|
||||
prices_cache = {a: df[a].values for a in assets if a in df.columns}
|
||||
for j, asset in enumerate(assets[:50]):
|
||||
if asset not in prices_cache:
|
||||
continue
|
||||
seg = prices_cache[asset][ri - 50:ri + 1]
|
||||
if np.any(seg <= 0) or np.any(~np.isfinite(seg)):
|
||||
continue
|
||||
rets = np.diff(seg) / seg[:-1]
|
||||
std = np.std(rets)
|
||||
if std > 0:
|
||||
x[28 + j] = float(np.clip((rets[-1] - np.mean(rets)) / std, -5.0, 5.0))
|
||||
|
||||
return x
|
||||
Reference in New Issue
Block a user