6990ff3bee
Three independently toggleable features: 1. Asset-faithful depth/spread: levels sized by OB study power-law per asset 2. Intraday volume clock: depth scales by time-of-day (peak/trough) 3. Realistic spread: per-asset spread from OB study + Flight7 Composable via BookGenerationConfig toggles: use_asset_faithful_depth, use_asset_faithful_spread, use_intraday_clock, use_weekend_mode, use_stress_mode, use_fragility, worst_case_mode worst_case_mode overrides everything for max adversarial learning: spread * stress_mult, depth * fragility, no intraday/weekend. DuckDB registry for online updates: AssetRegistry: upsert/get/list/delete/query RuntimeProfileCache: hot-reload during CWM runs upsert_from_csv/export_csv: pipeline support upsert_all_from_asset_behaviors(): seed from OB study Results (8 assets): BTC: spread 0.031 bps, depth $350M (normal) / $4.9M (worst) DOGE: spread 2.86 bps, depth $2M (normal) / $132K (worst) ADA: spread 11.8 bps, depth $10M (normal) / $511K (worst) Intraday: BTC peak/trough = 2.8x depth ratio All 99 tests green (31 new + 68 existing).
291 lines
11 KiB
Python
291 lines
11 KiB
Python
"""
|
|
Asset-Faithful Book Generation — composable, per-asset order book simulation.
|
|
|
|
Three independently toggleable features:
|
|
1. Asset-faithful depth/spread: levels sized by OB study power-law per asset
|
|
2. Intraday volume clock: depth scales by time-of-day volume profile
|
|
3. Realistic spread: per-asset spread from Flight7 calibration
|
|
|
|
All features composed via BookGenerationConfig toggles.
|
|
worst_case_mode overrides everything for max adversarial learning.
|
|
|
|
Usage:
|
|
from malkhut.training.asset_book_profile import (
|
|
AssetBookProfile, BookGenerationConfig, BookGenerator,
|
|
build_profile_from_behavior,
|
|
)
|
|
profile = build_profile_from_behavior("BTCUSDT")
|
|
config = BookGenerationConfig(use_intraday_clock=True, intraday_hour=14)
|
|
gen = BookGenerator(profile, config)
|
|
book = gen.generate_initial_book(mid_price=64000.0, tick_size=0.1)
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import math
|
|
from dataclasses import dataclass, field, asdict
|
|
from typing import Dict, List, Optional
|
|
|
|
from malkhut.state import OrderBookState, PriceLevel
|
|
|
|
|
|
@dataclass(slots=True)
|
|
class AssetBookProfile:
|
|
symbol: str
|
|
depth_amplitude_usd: float
|
|
depth_alpha: float
|
|
depth_fragility: float
|
|
depth_at_10bps_usd: float
|
|
depth_at_100bps_usd: float
|
|
spread_normal_bps: float
|
|
spread_stress_mult: float
|
|
flow_orders_per_sec: float
|
|
flow_cancel_fill_ratio: float
|
|
flow_median_order_usd: float
|
|
flow_avg_trade_usd: float
|
|
vol_annualized_normal: float
|
|
vol_annualized_crisis: float
|
|
vol_garch_alpha: float
|
|
vol_garch_beta: float
|
|
vol_half_life_hours: float
|
|
intraday_peak_hour_utc: int
|
|
intraday_trough_hour_utc: int
|
|
intraday_ratio: float
|
|
weekend_vol_mult: float
|
|
weekend_volume_mult: float
|
|
weekend_spread_mult: float
|
|
mm_max_inventory_usd: float
|
|
mm_pull_speed_ms: float
|
|
mm_margin_bps: float
|
|
avg_level_size_usd: float
|
|
typical_num_levels: int
|
|
reference_price: float
|
|
|
|
def to_dict(self) -> dict:
|
|
return asdict(self)
|
|
|
|
@classmethod
|
|
def from_dict(cls, d: dict) -> AssetBookProfile:
|
|
return cls(**{k: v for k, v in d.items() if k in cls.__slots__})
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class BookGenerationConfig:
|
|
use_asset_faithful_depth: bool = True
|
|
use_asset_faithful_spread: bool = True
|
|
use_intraday_clock: bool = True
|
|
use_weekend_mode: bool = True
|
|
use_stress_mode: bool = False
|
|
use_fragility: bool = False
|
|
use_asset_faithful_flow: bool = True
|
|
intraday_hour: int = 15
|
|
is_weekend: bool = False
|
|
stress_depth_mult: float = 1.0
|
|
worst_case_mode: bool = False
|
|
|
|
|
|
def _intraday_multiplier(hour_utc: int, peak_hour: int, trough_hour: int, ratio: float) -> float:
|
|
"""Sinusoidal intraday volume profile. Returns multiplier in [1/ratio, ratio]."""
|
|
hours = list(range(24))
|
|
trough_dist = [min(abs(h - trough_hour), 24 - abs(h - trough_hour)) for h in hours]
|
|
peak_dist = [min(abs(h - peak_hour), 24 - abs(h - peak_hour)) for h in hours]
|
|
max_dist = max(max(trough_dist), max(peak_dist), 1)
|
|
if hour_utc == peak_hour:
|
|
return ratio
|
|
if hour_utc == trough_hour:
|
|
return 1.0
|
|
t = 1.0 - trough_dist[hour_utc] / max_dist
|
|
return 1.0 + (ratio - 1.0) * t
|
|
|
|
|
|
def build_profile_from_behavior(symbol: str) -> AssetBookProfile:
|
|
"""Build AssetBookProfile from existing AssetBehavior data."""
|
|
from malkhut.training.asset_behavior import get_behavior
|
|
b = get_behavior(symbol)
|
|
if b is None:
|
|
raise ValueError(f"No AssetBehavior for {symbol}")
|
|
|
|
ref_price = b.reference_price if b.reference_price > 0 else 1.0
|
|
typical_levels = 50 if b.depth.amplitude_usd > 200_000 else 30 if b.depth.amplitude_usd > 50_000 else 20
|
|
avg_level = b.depth.amplitude_usd / typical_levels
|
|
|
|
return AssetBookProfile(
|
|
symbol=symbol,
|
|
depth_amplitude_usd=b.depth.amplitude_usd,
|
|
depth_alpha=b.depth.alpha,
|
|
depth_fragility=b.depth.fragility_factor,
|
|
depth_at_10bps_usd=b.depth.depth_at_10bps_usd,
|
|
depth_at_100bps_usd=b.depth.depth_at_100bps_usd,
|
|
spread_normal_bps=b.spread.normal_bps,
|
|
spread_stress_mult=b.spread.stress_multiplier,
|
|
flow_orders_per_sec=b.flow.orders_per_sec_normal,
|
|
flow_cancel_fill_ratio=b.flow.cancel_fill_ratio,
|
|
flow_median_order_usd=b.flow.median_order_usd,
|
|
flow_avg_trade_usd=b.flow.avg_trade_usd,
|
|
vol_annualized_normal=b.vol.annualized_normal,
|
|
vol_annualized_crisis=b.vol.annualized_crisis,
|
|
vol_garch_alpha=b.vol.garch_alpha,
|
|
vol_garch_beta=b.vol.garch_beta,
|
|
vol_half_life_hours=b.vol.half_life_hours,
|
|
intraday_peak_hour_utc=b.intraday.peak_hour_utc,
|
|
intraday_trough_hour_utc=b.intraday.trough_hour_utc,
|
|
intraday_ratio=b.intraday.ratio,
|
|
weekend_vol_mult=b.weekend.vol_mult,
|
|
weekend_volume_mult=b.weekend.volume_mult,
|
|
weekend_spread_mult=b.weekend.spread_mult,
|
|
mm_max_inventory_usd=b.market_maker.max_inventory_usd,
|
|
mm_pull_speed_ms=b.market_maker.pull_speed_ms,
|
|
mm_margin_bps=b.market_maker.margin_bps,
|
|
avg_level_size_usd=avg_level,
|
|
typical_num_levels=typical_levels,
|
|
reference_price=ref_price,
|
|
)
|
|
|
|
|
|
class BookGenerator:
|
|
"""Generates and refreshes order books faithful to a specific asset's characteristics."""
|
|
|
|
def __init__(self, profile: AssetBookProfile, config: BookGenerationConfig) -> None:
|
|
self._p = profile
|
|
self._c = config
|
|
|
|
def _effective_spread_bps(self) -> float:
|
|
s = self._p.spread_normal_bps
|
|
if self._c.worst_case_mode:
|
|
return s * self._p.spread_stress_mult
|
|
if self._c.use_stress_mode:
|
|
s *= self._p.spread_stress_mult
|
|
if self._c.use_weekend_mode and self._c.is_weekend:
|
|
s *= self._p.weekend_spread_mult
|
|
return s
|
|
|
|
def _effective_depth_multiplier(self) -> float:
|
|
m = 1.0
|
|
if self._c.worst_case_mode:
|
|
return self._p.depth_fragility
|
|
if self._c.use_intraday_clock:
|
|
m *= _intraday_multiplier(
|
|
self._c.intraday_hour,
|
|
self._p.intraday_peak_hour_utc,
|
|
self._p.intraday_trough_hour_utc,
|
|
self._p.intraday_ratio,
|
|
)
|
|
if self._c.use_weekend_mode and self._c.is_weekend:
|
|
m *= self._p.weekend_volume_mult
|
|
if self._c.use_stress_mode:
|
|
m *= self._c.stress_depth_mult
|
|
return m
|
|
|
|
def _level_qty_usd(self, distance_bps: float) -> float:
|
|
A = self._p.depth_amplitude_usd
|
|
alpha = self._p.depth_alpha
|
|
depth_usd = A * (distance_bps ** (1.0 - alpha))
|
|
return depth_usd
|
|
|
|
def generate_initial_book(self, mid_price: float, tick_size: float, ts_ns: int = 0) -> OrderBookState:
|
|
if mid_price <= 0:
|
|
return OrderBookState(ts_ns=ts_ns, symbol=self._p.symbol, bids=(), asks=(),
|
|
last_trade_price=0.0, last_trade_qty=0.0, last_trade_side=None)
|
|
|
|
spread_bps = self._effective_spread_bps()
|
|
depth_mult = self._effective_depth_multiplier()
|
|
half_spread = mid_price * spread_bps / 20_000.0
|
|
half_spread = max(half_spread, tick_size)
|
|
|
|
best_bid = mid_price - half_spread
|
|
best_ask = mid_price + half_spread
|
|
|
|
ref_price = self._p.reference_price if self._p.reference_price > 0 else mid_price
|
|
n_levels = self._p.typical_num_levels
|
|
flow_mult = self._p.flow_avg_trade_usd / max(ref_price, 1e-12)
|
|
|
|
bids = []
|
|
asks = []
|
|
for i in range(n_levels):
|
|
dist_bps = spread_bps / 2 + (i + 1) * 0.1
|
|
level_usd = self._level_qty_usd(dist_bps) * depth_mult
|
|
level_qty = level_usd / max(mid_price, 1e-12)
|
|
level_qty = max(level_qty, 1e-8)
|
|
bid_price = best_bid - i * tick_size
|
|
ask_price = best_ask + i * tick_size
|
|
if bid_price > 0:
|
|
bids.append(PriceLevel(round(bid_price, 10), level_qty))
|
|
asks.append(PriceLevel(round(ask_price, 10), level_qty))
|
|
|
|
return OrderBookState(
|
|
ts_ns=ts_ns, symbol=self._p.symbol,
|
|
bids=tuple(bids), asks=tuple(asks),
|
|
last_trade_price=mid_price, last_trade_qty=flow_mult,
|
|
last_trade_side=None,
|
|
)
|
|
|
|
def refresh_book(self, book: OrderBookState, tick_size: float, rng) -> OrderBookState:
|
|
if not book.bids or not book.asks:
|
|
return book
|
|
|
|
mid = book.mid
|
|
if mid <= 0:
|
|
return book
|
|
|
|
vol_ann = self._p.vol_annualized_normal
|
|
vol_per_step = vol_ann / math.sqrt(252 * 6.5 * 3600) * 0.1
|
|
if self._c.worst_case_mode:
|
|
vol_per_step *= 2.0
|
|
elif self._c.use_stress_mode:
|
|
vol_per_step *= math.sqrt(self._p.vol_annualized_crisis / max(self._p.vol_annualized_normal, 1e-12))
|
|
|
|
drift_bps = rng.gauss(0, vol_per_step * 100)
|
|
drift_price = mid * drift_bps / 10_000.0
|
|
|
|
cancel_ratio = self._p.flow_cancel_fill_ratio
|
|
qty_noise_frac = min(0.15, 1.0 / max(cancel_ratio, 1.0))
|
|
|
|
fragility = 1.0
|
|
if self._c.use_fragility and not self._c.worst_case_mode:
|
|
if rng.random() < 0.01:
|
|
fragility = self._p.depth_fragility
|
|
|
|
new_bids = []
|
|
for level in book.bids:
|
|
new_qty = level.qty * fragility
|
|
noise = rng.gauss(0, new_qty * qty_noise_frac)
|
|
new_qty = max(1e-8, new_qty + noise)
|
|
new_price = level.price + drift_price
|
|
if new_price > 0:
|
|
new_bids.append(PriceLevel(round(new_price, 10), new_qty))
|
|
|
|
new_asks = []
|
|
for level in book.asks:
|
|
new_qty = level.qty * fragility
|
|
noise = rng.gauss(0, new_qty * qty_noise_frac)
|
|
new_qty = max(1e-8, new_qty + noise)
|
|
new_price = level.price + drift_price
|
|
if new_price > 0:
|
|
new_asks.append(PriceLevel(round(new_price, 10), new_qty))
|
|
|
|
if not new_bids or not new_asks:
|
|
return book
|
|
|
|
if new_bids[0].price >= new_asks[0].price:
|
|
spread_bps = self._effective_spread_bps()
|
|
half_spread = mid * spread_bps / 20_000.0
|
|
half_spread = max(half_spread, tick_size)
|
|
new_bids = [PriceLevel(round(mid - half_spread, 10), new_bids[0].qty)]
|
|
new_asks = [PriceLevel(round(mid + half_spread, 10), new_asks[0].qty)]
|
|
for i in range(1, min(len(book.bids), self._p.typical_num_levels)):
|
|
dist_bps = spread_bps / 2 + (i + 1) * 0.1
|
|
lq = self._level_qty_usd(dist_bps) * self._effective_depth_multiplier() / max(mid, 1e-12)
|
|
new_bids.append(PriceLevel(round(mid - half_spread - i * tick_size, 10), max(lq, 1e-8)))
|
|
for i in range(1, min(len(book.asks), self._p.typical_num_levels)):
|
|
dist_bps = spread_bps / 2 + (i + 1) * 0.1
|
|
lq = self._level_qty_usd(dist_bps) * self._effective_depth_multiplier() / max(mid, 1e-12)
|
|
new_asks.append(PriceLevel(round(mid + half_spread + i * tick_size, 10), max(lq, 1e-8)))
|
|
|
|
n = min(len(new_bids), len(new_asks))
|
|
return OrderBookState(
|
|
ts_ns=book.ts_ns, symbol=book.symbol,
|
|
bids=tuple(new_bids[:n]), asks=tuple(new_asks[:n]),
|
|
last_trade_price=book.last_trade_price,
|
|
last_trade_qty=book.last_trade_qty,
|
|
last_trade_side=book.last_trade_side,
|
|
)
|