""" Asset Compiler — auto-fetches from Binance/BingX APIs and produces system-ready AssetProfile + AssetBehavior for any tradeable symbol. Rate-limited (1 req/sec), cached, resumable. Usage: compiler = AssetCompiler() result = compiler.compile("XRPUSDT") # result.asset_profile → ready for ScenarioFactory # result.asset_behavior → ready for behavior-driven scenarios # Batch compile results = compiler.compile_batch(["XRPUSDT", "HBARUSDT", "APTUSDT"]) """ from __future__ import annotations import json import math import time import urllib.request import urllib.error from dataclasses import dataclass, field from typing import Dict, List, Optional, Tuple from malkhut.training.asset_classification import ( AssetProfile, ASSET_PROFILES, Sector, TokenRole, SupplyModel, ConsensusFamily, SmartContractCapability, MarketCapTier, DerivativeAccess, VolatilityProfile, LiquidityProfile, ) from malkhut.training.asset_behavior import ( AssetBehavior, BehaviorTemplate, TEMPLATES, ASSET_BEHAVIORS, DepthProfile, SpreadProfile, FlowProfile, VolatilityProfile as BehVol, IntradayProfile, WeekendProfile, CorrelationProfile, MarketMakerProfile, LiquidationProfile, FundingProfile, RetailProfile, BingxProfile, ) # ============================================================================== # Binance API client — rate-limited, cached # ============================================================================== class BinanceClient: """Rate-limited Binance REST API client with response caching.""" BASE = "https://api.binance.com" FAPI = "https://fapi.binance.com" MIN_INTERVAL_S = 1.0 # 1 req/sec = well under Binance 1200/min limit def __init__(self) -> None: self._last_request_s: float = 0.0 self._cache: Dict[str, dict] = {} def _throttle(self) -> None: elapsed = time.time() - self._last_request_s if elapsed < self.MIN_INTERVAL_S: time.sleep(self.MIN_INTERVAL_S - elapsed) self._last_request_s = time.time() def _get(self, url: str) -> dict: if url in self._cache: return self._cache[url] self._throttle() try: req = urllib.request.Request(url, headers={"User-Agent": "MalkhutCompiler/1.0"}) with urllib.request.urlopen(req, timeout=10) as resp: data = json.loads(resp.read()) self._cache[url] = data return data except (urllib.error.URLError, json.JSONDecodeError, OSError) as e: return {"error": str(e)} def ticker_24h(self, symbol: str) -> dict: return self._get(f"{self.BASE}/api/v3/ticker/24hr?symbol={symbol}") def depth(self, symbol: str, limit: int = 100) -> dict: return self._get(f"{self.BASE}/api/v3/depth?symbol={symbol}&limit={limit}") def klines(self, symbol: str, interval: str = "1h", limit: int = 168) -> list: url = f"{self.BASE}/api/v3/klines?symbol={symbol}&interval={interval}&limit={limit}" return self._get(url) def exchange_info(self, symbol: str) -> dict: data = self._get(f"{self.BASE}/api/v3/exchangeInfo") if "symbols" in data: for s in data["symbols"]: if s.get("symbol") == symbol: return s return {} def funding_rate(self, symbol: str, limit: int = 20) -> list: fapi_symbol = symbol.replace("USDT", "-USDT") return self._get(f"{self.FAPI}/fapi/v1/fundingRate?symbol={fapi_symbol}&limit={limit}") def open_interest(self, symbol: str) -> dict: fapi_symbol = symbol.replace("USDT", "-USDT") return self._get(f"{self.FAPI}/fapi/v1/openInterest?symbol={fapi_symbol}") def ticker_24h_perp(self, symbol: str) -> dict: fapi_symbol = symbol.replace("USDT", "-USDT") return self._get(f"{self.FAPI}/fapi/v1/ticker/24hr?symbol={fapi_symbol}") # ============================================================================== # Statistical computation helpers # ============================================================================== def _compute_annualized_vol(klines: list) -> float: """Compute annualized volatility from hourly klines.""" if not klines or len(klines) < 10: return 80.0 # default mid-cap returns = [] for i in range(1, len(klines)): o = float(klines[i][1]) c = float(klines[i][4]) if o > 0: returns.append(math.log(c / o)) if len(returns) < 5: return 80.0 mean_r = sum(returns) / len(returns) var_r = sum((r - mean_r) ** 2 for r in returns) / (len(returns) - 1) hourly_vol = math.sqrt(var_r) return hourly_vol * math.sqrt(8760) * 100 # annualize (8760 hours/year) def _compute_spread_bps(depth_data: dict) -> float: """Compute spread in bps from depth snapshot.""" bids = depth_data.get("bids", []) asks = depth_data.get("asks", []) if not bids or not asks: return 5.0 best_bid = float(bids[0][0]) best_ask = float(asks[0][0]) mid = (best_bid + best_ask) / 2 if mid <= 0: return 5.0 return ((best_ask - best_bid) / mid) * 10000 def _compute_depth_profile(depth_data: dict, mid_price: float) -> Tuple[float, float]: """Fit depth amplitude and alpha from depth snapshot. Returns (amplitude_usd, alpha).""" bids = depth_data.get("bids", []) asks = depth_data.get("asks", []) if not bids or not asks or mid_price <= 0: return 50_000.0, 0.9 cumulative_usd = 0.0 for level in bids[:50]: price = float(level[0]) qty = float(level[1]) dist_bps = abs(price - mid_price) / mid_price * 10000 if dist_bps < 1: cumulative_usd += qty * price amplitude = max(cumulative_usd, 1_000) bid_depths = [] for level in bids[:50]: price = float(level[0]) qty = float(level[1]) dist_bps = max(abs(price - mid_price) / mid_price * 10000, 0.5) bid_depths.append((dist_bps, qty * price)) if len(bid_depths) < 5: return amplitude, 0.9 log_dists = [math.log(d) for d, _ in bid_depths if d > 0] log_depths = [math.log(max(v, 1)) for d, v in bid_depths if d > 0] if len(log_dists) < 5: return amplitude, 0.9 n = len(log_dists) sum_x = sum(log_dists) sum_y = sum(log_depths) sum_xy = sum(x * y for x, y in zip(log_dists, log_depths)) sum_x2 = sum(x * x for x in log_dists) denom = n * sum_x2 - sum_x * sum_x if abs(denom) < 1e-10: return amplitude, 0.9 slope = (n * sum_xy - sum_x * sum_y) / denom alpha = max(0.5, min(1.5, -slope + 1.0)) return amplitude, alpha def _compute_order_flow_stats(klines: list) -> dict: """Compute order flow statistics from klines.""" if not klines or len(klines) < 10: return {"median_usd": 500, "p99_usd": 100_000, "avg_usd": 2_000} volumes_usd = [] for k in klines: vol = float(k[5]) # quote volume trades = float(k[8]) # number of trades if trades > 0: volumes_usd.append(vol / trades) if not volumes_usd: return {"median_usd": 500, "p99_usd": 100_000, "avg_usd": 2_000} volumes_usd.sort() n = len(volumes_usd) median = volumes_usd[n // 2] p99_idx = min(int(n * 0.99), n - 1) avg = sum(volumes_usd) / n return {"median_usd": median, "p99_usd": volumes_usd[p99_idx], "avg_usd": avg} def _compute_funding_stats(funding_data: list) -> Tuple[float, float, float]: """Compute funding rate statistics. Returns (mean_bps, std_bps, positive_pct).""" if not funding_data or isinstance(funding_data, dict): return 0.10, 0.20, 60.0 rates = [] for entry in funding_data: r = float(entry.get("fundingRate", 0)) rates.append(r * 10000) # convert to bps if not rates: return 0.10, 0.20, 60.0 mean_r = sum(rates) / len(rates) var_r = sum((r - mean_r) ** 2 for r in rates) / max(len(rates) - 1, 1) std_r = math.sqrt(var_r) pos_pct = sum(1 for r in rates if r > 0) / len(rates) * 100 return mean_r, std_r, pos_pct def _classify_market_cap(mcap_usd: float) -> MarketCapTier: if mcap_usd > 500e9: return MarketCapTier.MEGA if mcap_usd > 50e9: return MarketCapTier.LARGE if mcap_usd > 5e9: return MarketCapTier.MID if mcap_usd > 500e6: return MarketCapTier.SMALL return MarketCapTier.MICRO def _classify_volatility(ann_vol: float) -> VolatilityProfile: if ann_vol < 30: return VolatilityProfile.LOW if ann_vol < 80: return VolatilityProfile.MEDIUM if ann_vol < 150: return VolatilityProfile.HIGH return VolatilityProfile.EXTREME def _classify_liquidity(vol_usd: float) -> LiquidityProfile: if vol_usd > 100e6: return LiquidityProfile.DEEP if vol_usd > 10e6: return LiquidityProfile.NORMAL if vol_usd > 1e6: return LiquidityProfile.THIN return LiquidityProfile.ILLIQUID # ============================================================================== # Heuristic asset classification (for unknown assets) # ============================================================================== _KNOWN_CLASSIFICATIONS: Dict[str, dict] = { "BTCUSDT": {"sector": "currency", "role": "store_of_value", "supply": "fixed_cap", "consensus": "pow", "sc": "none", "deriv": "perps_and_options"}, "ETHUSDT": {"sector": "layer1", "role": "gas", "supply": "disinflationary", "consensus": "pos", "sc": "full", "deriv": "perps_and_options"}, "SOLUSDT": {"sector": "layer1", "role": "gas", "supply": "inflationary", "consensus": "pos", "sc": "full", "deriv": "perps_only"}, "DOGEUSDT": {"sector": "meme", "role": "meme", "supply": "inflationary", "consensus": "pow", "sc": "none", "deriv": "perps_only"}, "ADAUSDT": {"sector": "layer1", "role": "gas", "supply": "inflationary", "consensus": "dpos", "sc": "full", "deriv": "perps_only"}, "AVAXUSDT": {"sector": "layer1", "role": "gas", "supply": "inflationary", "consensus": "pos", "sc": "full", "deriv": "perps_only"}, "UNIUSDT": {"sector": "defi", "role": "governance", "supply": "inflationary", "consensus": "pos", "sc": "full", "deriv": "perps_only"}, "LINKUSDT": {"sector": "oracle", "role": "utility", "supply": "inflationary", "consensus": "pos", "sc": "full", "deriv": "perps_only"}, "BNBUSDT": {"sector": "exchange", "role": "exchange_fee", "supply": "burn_mechanism", "consensus": "dpos", "sc": "partial", "deriv": "perps_only"}, "MATICUSDT": {"sector": "layer2", "role": "gas", "supply": "inflationary", "consensus": "pos", "sc": "full", "deriv": "perps_only"}, "AAVEUSDT": {"sector": "defi", "role": "governance", "supply": "fixed_cap", "consensus": "pos", "sc": "full", "deriv": "perps_only"}, "DOTUSDT": {"sector": "layer1", "role": "gas", "supply": "inflationary", "consensus": "dpos", "sc": "full", "deriv": "perps_only"}, "ATOMUSDT": {"sector": "layer1", "role": "gas", "supply": "inflationary", "consensus": "pos", "sc": "full", "deriv": "perps_only"}, "XRPUSDT": {"sector": "currency", "role": "utility", "supply": "inflationary", "consensus": "bft", "sc": "partial", "deriv": "perps_only"}, "TRXUSDT": {"sector": "layer1", "role": "gas", "supply": "inflationary", "consensus": "dpos", "sc": "full", "deriv": "perps_only"}, "LTCUSDT": {"sector": "currency", "role": "store_of_value", "supply": "fixed_cap", "consensus": "pow", "sc": "none", "deriv": "perps_only"}, "NEARUSDT": {"sector": "layer1", "role": "gas", "supply": "inflationary", "consensus": "pos", "sc": "full", "deriv": "perps_only"}, "APTUSDT": {"sector": "layer1", "role": "gas", "supply": "inflationary", "consensus": "bft", "sc": "full", "deriv": "perps_only"}, "OPUSDT": {"sector": "layer2", "role": "gas", "supply": "inflationary", "consensus": "pos", "sc": "full", "deriv": "perps_only"}, "ARBUSDT": {"sector": "layer2", "role": "gas", "supply": "inflationary", "consensus": "pos", "sc": "full", "deriv": "perps_only"}, "SUIUSDT": {"sector": "layer1", "role": "gas", "supply": "inflationary", "consensus": "bft", "sc": "full", "deriv": "perps_only"}, "PEPEUSDT": {"sector": "meme", "role": "meme", "supply": "fixed_cap", "consensus": "pos", "sc": "none", "deriv": "perps_only"}, "WIFUSDT": {"sector": "meme", "role": "meme", "supply": "inflationary", "consensus": "pos", "sc": "none", "deriv": "perps_only"}, "SEIUSDT": {"sector": "layer1", "role": "gas", "supply": "inflationary", "consensus": "bft", "sc": "full", "deriv": "perps_only"}, "INJUSDT": {"sector": "defi", "role": "utility", "supply": "inflationary", "consensus": "pos", "sc": "full", "deriv": "perps_only"}, "FILUSDT": {"sector": "storage", "role": "utility", "supply": "inflationary", "consensus": "pos", "sc": "full", "deriv": "perps_only"}, "HBARUSDT": {"sector": "layer1", "role": "gas", "supply": "inflationary", "consensus": "bft", "sc": "full", "deriv": "perps_only"}, "IMXUSDT": {"sector": "gaming_nft", "role": "utility", "supply": "fixed_cap", "consensus": "pos", "sc": "full", "deriv": "perps_only"}, } # ============================================================================== # Compile result # ============================================================================== @dataclass class CompileResult: symbol: str asset_profile: Optional[AssetProfile] asset_behavior: Optional[AssetBehavior] raw_data: dict warnings: List[str] = field(default_factory=list) # ============================================================================== # Asset Compiler # ============================================================================== class AssetCompiler: """Auto-fetch from Binance/BingX, compute params, produce profiles.""" def __init__(self) -> None: self.client = BinanceClient() def compile(self, symbol: str) -> CompileResult: """Compile a single asset from live API data.""" warnings = [] raw = {} ticker = self.client.ticker_24h(symbol) if "error" in ticker or "lastPrice" not in ticker: return CompileResult(symbol=symbol, asset_profile=None, asset_behavior=None, raw_data=ticker, warnings=[f"Failed to fetch ticker: {ticker}"]) price = float(ticker["lastPrice"]) vol_usd_24h = float(ticker.get("quoteVolume", 0)) raw["ticker"] = {"price": price, "vol_usd_24h": vol_usd_24h} depth_data = self.client.depth(symbol, limit=100) spread_bps = _compute_spread_bps(depth_data) depth_amp, depth_alpha = _compute_depth_profile(depth_data, price) raw["depth"] = {"spread_bps": spread_bps, "amplitude": depth_amp, "alpha": depth_alpha} klines = self.client.klines(symbol, "1h", 168) ann_vol = _compute_annualized_vol(klines) if klines else 80.0 flow = _compute_order_flow_stats(klines) raw["volatility"] = {"annualized": ann_vol} raw["flow"] = flow exch_info = self.client.exchange_info(symbol) tick_size = 0.01 lot_size = 0.01 price_decimals = 2 if "filters" in exch_info: for f in exch_info["filters"]: if f["filterType"] == "PRICE_FILTER": tick_size = float(f["tickSize"]) price_decimals = max(0, -int(math.log10(tick_size)) if tick_size > 0 else 2) elif f["filterType"] == "LOT_SIZE": lot_size = float(f["stepSize"]) raw["exchange"] = {"tick_size": tick_size, "lot_size": lot_size} funding_data = self.client.funding_rate(symbol, limit=20) funding_mean, funding_std, funding_pos = _compute_funding_stats(funding_data) raw["funding"] = {"mean_bps": funding_mean, "std_bps": funding_std, "positive_pct": funding_pos} oi_data = self.client.open_interest(symbol) oi_value = float(oi_data.get("openInterest", 0)) * price if "openInterest" in oi_data else 0 raw["oi"] = {"value_usd": oi_value} klass = _KNOWN_CLASSIFICATIONS.get(symbol, {}) if not klass: warnings.append(f"No classification for {symbol} — using defaults") klass = {"sector": "layer1", "role": "gas", "supply": "inflationary", "consensus": "pos", "sc": "full", "deriv": "perps_only"} sector_map = {"currency": Sector.CURRENCY, "layer1": Sector.LAYER1, "layer2": Sector.LAYER2, "defi": Sector.DEFI, "oracle": Sector.ORACLE, "exchange": Sector.EXCHANGE, "meme": Sector.MEME, "privacy": Sector.PRIVACY, "storage": Sector.STORAGE, "gaming_nft": Sector.GAMING_NFT} role_map = {"gas": TokenRole.GAS, "store_of_value": TokenRole.STORE_OF_VALUE, "governance": TokenRole.GOVERNANCE, "utility": TokenRole.UTILITY, "meme": TokenRole.MEME, "exchange_fee": TokenRole.EXCHANGE_FEE} supply_map = {"fixed_cap": SupplyModel.FIXED_CAP, "disinflationary": SupplyModel.DISINFLATIONARY, "inflationary": SupplyModel.INFLATIONARY, "burn_mechanism": SupplyModel.BURN_MECHANISM} consensus_map = {"pow": ConsensusFamily.POW, "pos": ConsensusFamily.POS, "dpos": ConsensusFamily.DPOS, "bft": ConsensusFamily.POS} sc_map = {"full": SmartContractCapability.FULL, "partial": SmartContractCapability.PARTIAL, "none": SmartContractCapability.NONE} deriv_map = {"perps_and_options": DerivativeAccess.PERPS_AND_OPTIONS, "perps_only": DerivativeAccess.PERPS_ONLY, "none": DerivativeAccess.NONE} mcap_usd = vol_usd_24h * 100 # rough estimate from volume if vol_usd_24h > 1e9: mcap_est = vol_usd_24h * 2 elif vol_usd_24h > 100e6: mcap_est = vol_usd_24h * 5 else: mcap_est = vol_usd_24h * 10 asset_profile = AssetProfile( symbol=symbol, sectors=(sector_map.get(klass["sector"], Sector.LAYER1),), token_roles=(role_map.get(klass["role"], TokenRole.GAS),), supply_model=supply_map.get(klass["supply"], SupplyModel.INFLATIONARY), consensus=consensus_map.get(klass["consensus"], ConsensusFamily.POS), smart_contracts=sc_map.get(klass["sc"], SmartContractCapability.FULL), market_cap_tier=_classify_market_cap(mcap_est), volatility_profile=_classify_volatility(ann_vol), liquidity_profile=_classify_liquidity(vol_usd_24h), derivative_access=deriv_map.get(klass["deriv"], DerivativeAccess.PERPS_ONLY), tick_size=tick_size, lot_size=lot_size, price_decimals=price_decimals, maker_fee_bps=-0.2, taker_fee_bps=0.5, typical_spread_bps=spread_bps, typical_depth_usd=depth_amp, typical_daily_volume_usd=vol_usd_24h, has_funding=True, has_options=klass["deriv"] == "perps_and_options", ) template_name = "mid_cap_l1" if klass["sector"] in ("meme",): template_name = "retail_meme" elif klass["sector"] in ("currency",) and klass["consensus"] == "pow": template_name = "institutional_blue_chip" elif mcap_est > 50e9: template_name = "institutional_blue_chip" tmpl = TEMPLATES[template_name] bingx_spread = spread_bps * 5.0 # conservative estimate bingx_depth_ratio = 0.30 asset_behavior = AssetBehavior.from_template( symbol, tmpl, { "depth": DepthProfile( amplitude_usd=depth_amp, alpha=depth_alpha, fragility_factor=tmpl.depth.fragility_factor, depth_at_10bps_usd=depth_amp * (10 ** (1 - depth_alpha)), depth_at_100bps_usd=depth_amp * (100 ** (1 - depth_alpha)), ), "spread": SpreadProfile(normal_bps=spread_bps, stress_multiplier=tmpl.spread.stress_multiplier), "flow": FlowProfile( orders_per_sec_normal=tmpl.flow.orders_per_sec_normal, orders_per_sec_stress=tmpl.flow.orders_per_sec_stress, cancel_fill_ratio=tmpl.flow.cancel_fill_ratio, median_order_usd=flow["median_usd"], p99_order_usd=flow["p99_usd"], avg_trade_usd=flow["avg_usd"], ), "vol": BehVol( annualized_normal=ann_vol, annualized_crisis=ann_vol * 2.5, garch_alpha=tmpl.vol.garch_alpha, garch_beta=tmpl.vol.garch_beta, half_life_hours=tmpl.vol.half_life_hours, ), "funding": FundingProfile( mean_bps_8h=funding_mean, std_bps_8h=funding_std, positive_pct=funding_pos, basis_typical_bps=abs(funding_mean) * 5, ), "bingx": BingxProfile( spread_mult=bingx_spread / max(spread_bps, 0.01), depth_ratio=bingx_depth_ratio, latency_ms=100, taker_fee_bps=5.0, maker_fee_bps=2.0, funding_lag_hours=4, ), "liquidation": LiquidationProfile( oi_mcap_ratio=oi_value / max(mcap_est, 1), trigger_pct=tmpl.liquidation.trigger_pct, speed=tmpl.liquidation.speed, recovery=tmpl.liquidation.recovery, ), }, reference_price=price, ) return CompileResult( symbol=symbol, asset_profile=asset_profile, asset_behavior=asset_behavior, raw_data=raw, warnings=warnings, ) def compile_batch(self, symbols: List[str]) -> List[CompileResult]: """Compile multiple assets sequentially (rate-limited).""" results = [] for sym in symbols: results.append(self.compile(sym)) return results def register(self, result: CompileResult) -> bool: """Register compiled results into the global registries.""" if not result.asset_profile or not result.asset_behavior: return False ASSET_PROFILES[result.symbol] = result.asset_profile ASSET_BEHAVIORS[result.symbol] = result.asset_behavior return True def compile_and_register(self, symbol: str) -> CompileResult: """Compile and register in one step.""" result = self.compile(symbol) self.register(result) return result