diff --git a/MALKHUT/README.md b/MALKHUT/README.md index fb51242..9dd5c75 100644 --- a/MALKHUT/README.md +++ b/MALKHUT/README.md @@ -136,7 +136,7 @@ MALKHUT/ │ │ ├── dsl.py # Strategy DSL v2 (40+ primitives, 40+ sensors) │ │ ├── generator.py # Genetic programming strategy evolution │ │ ├── selector.py # Regime → strategy mapping + performance matrix -│ │ ├── asset_classification.py # Multi-label invariant asset taxonomy +│ │ ├── asset_classification.py # Multi-label taxonomy + exchange registry (system-wide) │ │ ├── asset_behavior.py # 10-dimension behavior DSL, research-validated │ │ ├── asset_compiler.py # Auto-fetch from Binance/BingX, compile profiles │ │ ├── parallel_eval.py # ProcessPoolExecutor episode runner, 9x speedup @@ -422,7 +422,7 @@ simple doctrinal tick-exits (C11) ship first via T19 step 3; MALKHUT supersedes | **Strategy DSL v2** | `training/dsl.py` | 69 | 40+ primitives, 40+ sensors, 16 builtins | | **Strategy Generator** | `training/generator.py` | 20 | Genetic programming: crossover, mutation, tournament | | **Strategy Selector** | `training/selector.py` | 24 | Regime → strategy mapping, performance matrix | -| **Asset Classification** | `training/asset_classification.py` | 190 | Multi-label invariant taxonomy, 13 assets, cross-dimensional consistency | +| **Asset Classification** | `training/asset_classification.py` | 190 | Multi-label taxonomy + exchange registry, 13 assets, system-wide store | | **Asset Behavior DSL** | `training/asset_behavior.py` | (in classification) | 10 orthogonal dimensions, 3 templates, 13 behaviors, research-validated | | **Asset Compiler** | `training/asset_compiler.py` | (new) | Auto-fetch from Binance/BingX API, compile profiles, rate-limited | | **Cognition Pipeline** | `training/cognition.py` | 29 | Rate-limited, 8 sources, dedup, perm-run | @@ -690,6 +690,63 @@ btc.sector # Sector.CURRENCY btc.token_role # TokenRole.STORE_OF_VALUE ``` +### Exchange Registry (Multi-Exchange Asset Universe) + +MALKHUT serves as the **system-wide asset universe store** for BLUE, VIOLET, UV, and all +downstream systems. Each asset knows which exchanges trade it; each exchange has a +standardized profile. + +#### ExchangeProfile + +| Field | Type | Purpose | +|-------|------|---------| +| `exchange_id` | str | Canonical key: `"binance"`, `"bingx"`, `"bybit"` | +| `display_name` | str | Human-readable name | +| `has_spot` | bool | Spot trading available | +| `has_perps` | bool | Perpetual futures available | +| `has_options` | bool | Options available | +| `api_base_url` | str | REST API root URL | +| `ws_base_url` | str | WebSocket root URL | +| `default_taker_fee_bps` | float | Default taker fee | +| `default_maker_fee_bps` | float | Default maker fee | +| `typical_latency_ms` | float | Typical API latency | + +#### Pre-defined Exchanges + +| Exchange | Spot | Perps | Options | Taker Fee | Latency | +|----------|------|-------|---------|-----------|---------| +| **Binance** | ✓ | ✓ | ✓ | 0.4 bps | 40 ms | +| **BingX** | ✓ | ✓ | ✗ | 0.5 bps | 100 ms | +| **Bybit** | ✓ | ✓ | ✓ | 0.06 bps | 50 ms | + +#### Exchange-Asset Mapping + +Each `AssetProfile` carries an `exchanges: tuple[str, ...]` field listing which venues +trade the asset. Default: `("binance",)`. Other systems (BLUE/VIOLET/UV) import their +asset universes into this store. + +```python +from malkhut.training.asset_classification import * + +# Which exchanges trade BTC? +btc = get_asset_profile("BTCUSDT") +print(btc.exchanges) # ('binance',) + +# All assets on Binance +binance_assets = get_assets_on_exchange("binance") + +# Assets traded on BOTH Binance and BingX +common = get_common_assets("binance", "bingx") + +# Which exchanges trade a given asset? +venues = get_exchange_for_asset("ETHUSDT") + +# Exchange metadata +ex = get_exchange("binance") +print(ex.default_taker_fee_bps) # 0.4 +print(ex.typical_latency_ms) # 40 +``` + ### Asset Behavior DSL (10-Dimension Research-Validated Model) The Asset Behavior DSL decomposes each asset's **trading behavior** into 10 orthogonal diff --git a/MALKHUT/malkhut/training/_DATA_STORAGE_SCHEMA_FORMATS.md b/MALKHUT/malkhut/training/_DATA_STORAGE_SCHEMA_FORMATS.md new file mode 100644 index 0000000..43c82d3 --- /dev/null +++ b/MALKHUT/malkhut/training/_DATA_STORAGE_SCHEMA_FORMATS.md @@ -0,0 +1,300 @@ +# MALKHUT Asset Store — Data Storage Schema & Formats + +**System-wide asset universe.** Used by BLUE, VIOLET, UV, and all downstream systems. + +This document defines the data model, storage formats, and query interfaces for the +MALKHUT asset classification and exchange registry. Other agents use this to: + +- Understand what data is stored and where +- Import assets from other systems (e.g., BLUE's Binance universe) +- Query the asset universe by any dimension +- Extend the store with new exchanges or assets + +## Quick Reference + +``` +asset_classification.py → AssetProfile, ExchangeProfile, query functions +asset_behavior.py → AssetBehavior (10-dimension behavior model) +asset_compiler.py → Auto-fetch from Binance/BingX API +parallel_eval.py → Parallel episode evaluation +cma_trainer.py → ScenarioFactory (uses both stores) +``` + +--- + +## 1. ExchangeProfile — Exchange Metadata + +**Frozen dataclass. One entry per exchange.** + +```python +@dataclass(frozen=True, slots=True) +class ExchangeProfile: + exchange_id: str # "binance", "bingx", "bybit" + display_name: str # "Binance" + has_spot: bool + has_perps: bool + has_options: bool + api_base_url: str # REST root + ws_base_url: str # WebSocket root ("" if N/A) + default_taker_fee_bps: float + default_maker_fee_bps: float + typical_latency_ms: float +``` + +**Storage:** `EXCHANGE_PROFILES: Dict[str, ExchangeProfile]` in `asset_classification.py`. + +**Pre-defined:** `binance`, `bingx`, `bybit`. + +**How to add a new exchange:** + +```python +from malkhut.training.asset_classification import EXCHANGE_PROFILES, ExchangeProfile + +EXCHANGE_PROFILES["okx"] = ExchangeProfile( + exchange_id="okx", display_name="OKX", + has_spot=True, has_perps=True, has_options=True, + api_base_url="https://www.okx.com", + ws_base_url="wss://ws.okx.com:8443/ws/v5/public", + default_taker_fee_bps=0.1, default_maker_fee_bps=-0.02, + typical_latency_ms=60, +) +``` + +--- + +## 2. AssetProfile — Per-Asset Classification + +**Frozen dataclass. One entry per symbol. Multi-label on Sector and TokenRole.** + +```python +@dataclass(frozen=True, slots=True) +class AssetProfile: + # Identity + symbol: str # "BTCUSDT" + + # Fundamental (intrinsic, never change) + sectors: tuple[Sector, ...] # ("CURRENCY",) + token_roles: tuple[TokenRole, ...] # ("STORE_OF_VALUE",) + supply_model: SupplyModel # FIXED_CAP | DISINFLATIONARY | INFLATIONARY | BURN_MECHANISM + consensus: ConsensusFamily # POW | POS | DPOS + smart_contracts: SmartContractCapability # FULL | PARTIAL | NONE + + # Technical (invariant market-structure) + market_cap_tier: MarketCapTier # MEGA | LARGE | MID | SMALL | MICRO + volatility_profile: VolatilityProfile # LOW | MEDIUM | HIGH | EXTREME + liquidity_profile: LiquidityProfile # DEEP | NORMAL | THIN | ILLIQUID + derivative_access: DerivativeAccess # PERPS_AND_OPTIONS | PERPS_ONLY | NONE + + # Execution parameters (exchange-set) + tick_size: float + lot_size: float + price_decimals: int + maker_fee_bps: float + taker_fee_bps: float + + # Order-book fingerprint (long-run averages) + typical_spread_bps: float + typical_depth_usd: float + typical_daily_volume_usd: float + + # Structural flags + has_funding: bool = False + has_options: bool = False + + # Exchange membership + exchanges: tuple[str, ...] = ("binance",) # which venues trade this +``` + +**Storage:** `ASSET_PROFILES: Dict[str, AssetProfile]` in `asset_classification.py`. + +**Multi-label rules:** +- `sectors` and `token_roles` are **tuples** (ordered). First element = primary label. +- `supply_model`, `consensus`, `smart_contracts` = **single enum** (inherently singular). +- `exchanges` = **tuple of strings** (which venues list the asset). + +**Query functions:** + +| Function | Returns | +|----------|---------| +| `get_asset_profile(symbol)` | Single profile or None | +| `list_assets()` | All symbols | +| `get_assets_by_sector(sector)` | Assets in ANY of the queried sector | +| `get_assets_by_token_role(role)` | Assets with ANY of the queried role | +| `get_assets_by_supply(model)` | Assets with given supply model | +| `get_assets_by_consensus(family)` | Assets with given consensus | +| `get_assets_by_market_cap(tier)` | Assets in market cap band | +| `get_assets_by_volatility(vol)` | Assets in vol band | +| `get_assets_by_liquidity(liq)` | Assets in liquidity band | +| `get_assets_by_derivatives(access)` | Assets with given derivative access | +| `get_gas_tokens()` | All gas tokens | +| `get_pov_assets()` | PoW assets (forced selling) | +| `get_shortable_assets()` | All shortable assets | +| `get_multi_sector_assets()` | Assets in >1 sector | +| `get_multi_role_assets()` | Assets with >1 role | +| `get_assets_on_exchange(exchange_id)` | Assets traded on given exchange | +| `get_common_assets(ex_a, ex_b)` | Assets on BOTH exchanges | +| `get_exchange_for_asset(symbol)` | Which exchanges trade this asset | + +--- + +## 3. AssetBehavior — Per-Asset Behavior Model + +**Frozen dataclass. 10 orthogonal dimensions. Research-validated.** + +```python +@dataclass(frozen=True, slots=True) +class AssetBehavior: + symbol: str + depth: DepthProfile # book shape: amplitude, alpha, fragility + spread: SpreadProfile # normal spread, stress multiplier + flow: FlowProfile # order rate, sizes, cancel ratio + vol: VolatilityProfile # ann vol, GARCH params, half-life + intraday: IntradayProfile # peak/trough hours, ratio + weekend: WeekendProfile # vol/volume/spread multipliers + correlation: CorrelationProfile # ETH beta, BTC corr (normal vs crash) + market_maker: MarketMakerProfile # inventory, pull speed, margins + liquidation: LiquidationProfile # OI/MCap, trigger %, cascade + funding: FundingProfile # mean/std, positive %, basis + retail: RetailProfile # retail ratio, inst gap + bingx: BingxProfile # BingX-specific multiplier, latency + template_name: str = "" # which template this came from + reference_price: float = 0.0 # last known mid-price +``` + +**Storage:** `ASSET_BEHAVIORS: Dict[str, AssetBehavior]` in `asset_behavior.py`. + +**3 templates:** `institutional_blue_chip`, `mid_cap_l1`, `retail_meme`. + +**Query functions:** same pattern as AssetProfile — `get_behavior()`, `get_behaviors_by_template()`, etc. + +--- + +## 4. Auto-Compilation (AssetCompiler) + +**Auto-fetches from Binance/BingX public API, computes profiles.** + +```python +compiler = AssetCompiler() +result = compiler.compile("XRPUSDT") # ~6s, rate-limited +compiler.register(result) # adds to ASSET_PROFILES + ASSET_BEHAVIORS +``` + +**What it auto-fetches:** + +| Endpoint | Computes | +|----------|----------| +| `/api/v3/ticker/24hr` | Price reference, daily volume | +| `/api/v3/depth?limit=100` | Spread, depth amplitude, decay α | +| `/api/v3/klines?interval=1h&limit=168` | Annualized vol, order flow stats | +| `/api/v3/exchangeInfo` | Tick size, lot size, price decimals | +| `/fapi/v1/fundingRate` | Funding rate mean/std | +| `/fapi/v1/openInterest` | OI/MCap ratio | + +**Known classifications:** 28 pre-defined assets. Unknowns get heuristic defaults. + +--- + +## 5. Exchange-Asset Mapping Pattern + +The mapping follows a many-to-many relationship: + +``` +Asset (BTCUSDT) ──exchanges──> (binance, bingx, bybit) +Exchange (binance) ──assets──> (BTCUSDT, ETHUSDT, SOLUSDT, ...) +``` + +**In code:** `AssetProfile.exchanges` is a tuple of `exchange_id` strings. +**For querying:** `get_assets_on_exchange(id)`, `get_common_assets(a, b)`. + +**When importing from BLUE/VIOLET/UV:** + +1. Get the full symbol list from the source system +2. For each symbol, check if it already exists in `ASSET_PROFILES` + - If yes: add the new exchange_id to the `exchanges` tuple + - If no: create a minimal profile with the exchange's default fees +3. The `_profile()` helper and `AssetProfile.from_template()` handle creation + +--- + +## 6. Data Flow Diagram + +``` +┌──────────────────┐ ┌──────────────────┐ +│ Binance API │ │ BingX API │ +│ (public, R/O) │ │ (public, R/O) │ +└────────┬─────────┘ └────────┬─────────┘ + │ │ + ▼ ▼ +┌────────────────────────────────────────────┐ +│ AssetCompiler │ +│ auto-fetch → compute → CompileResult │ +└────────────────────┬───────────────────────┘ + │ + ▼ +┌────────────────────────────────────────────┐ +│ ASSET_PROFILES (Dict[str, AssetProfile])│ +│ ASSET_BEHAVIORS (Dict[str, AssetBehavior])│ +│ EXCHANGE_PROFILES (Dict[str, ExchangeProfile])│ +│ │ +│ System-wide store: │ +│ BLUE ──imports──→ this store │ +│ VIOLET ──imports──→ this store │ +│ UV ──imports──→ this store │ +│ MALKHUT ──uses──→ this store │ +│ ScenarioFactory ──reads──→ this store │ +└────────────────────┬───────────────────────┘ + │ + ▼ +┌────────────────────────────────────────────┐ +│ ScenarioFactory │ +│ behavior-driven scenarios │ +│ auto-compile unknown assets │ +│ label queries (sector/role/template) │ +│ exchange-aware scenario generation │ +└────────────────────────────────────────────┘ +``` + +--- + +## 7. File Locations + +| File | Purpose | Lines | +|------|---------|-------| +| `malkhut/training/asset_classification.py` | Enums, AssetProfile, ExchangeProfile, queries | ~600 | +| `malkhut/training/asset_behavior.py` | 10-dimension behavior model, templates | ~400 | +| `malkhut/training/asset_compiler.py` | Binance/BingX auto-fetch, compile | ~520 | +| `malkhut/training/parallel_eval.py` | Parallel episode evaluation | ~91 | +| `malkhut/training/cma_trainer.py` | ScenarioFactory, CMA-ES trainer | ~1340 | +| `malkhut/tests/test_asset_classification.py` | 190 classification tests | ~355 | +| `malkhut/tests/test_parallel_eval.py` | 16 parallel eval tests | ~225 | + +--- + +## 8. Constants and Enums Reference + +### Sector (multi-label) +`CURRENCY`, `LAYER1`, `LAYER2`, `DEFI`, `ORACLE`, `EXCHANGE`, `MEME`, `PRIVACY`, `STORAGE`, `GAMING_NFT` + +### TokenRole (multi-label) +`GAS`, `STORE_OF_VALUE`, `GOVERNANCE`, `UTILITY`, `MEME`, `EXCHANGE_FEE` + +### SupplyModel (single) +`FIXED_CAP`, `DISINFLATIONARY`, `INFLATIONARY`, `BURN_MECHANISM` + +### ConsensusFamily (single) +`POW`, `POS`, `DPOS` + +### SmartContractCapability (single) +`FULL`, `PARTIAL`, `NONE` + +### MarketCapTier (single) +`MEGA` (>$500B), `LARGE` ($50-500B), `MID` ($5-50B), `SMALL` ($500M-5B), `MICRO` (<$500M) + +### VolatilityProfile (single) +`LOW` (<30%), `MEDIUM` (30-80%), `HIGH` (80-150%), `EXTREME` (>150%) + +### LiquidityProfile (single) +`DEEP` (>$100M), `NORMAL` ($10-100M), `THIN` ($1-10M), `ILLIQUID` (<$1M) + +### DerivativeAccess (single) +`PERPS_AND_OPTIONS`, `PERPS_ONLY`, `NONE` diff --git a/MALKHUT/malkhut/training/asset_classification.py b/MALKHUT/malkhut/training/asset_classification.py index e87cf38..d57f592 100644 --- a/MALKHUT/malkhut/training/asset_classification.py +++ b/MALKHUT/malkhut/training/asset_classification.py @@ -1,5 +1,5 @@ """ -Asset Classification — INVARIANT characteristics only. +Asset Classification — INVARIANT characteristics only. System-wide asset universe. Split into two axes: FUNDAMENTAL — intrinsic to the token's design (never changes): @@ -8,6 +8,11 @@ Split into two axes: TECHNICAL — invariant market-structure properties (set at listing, rarely change): MarketCapTier, TypicalSpread/Depth, TickSize/LotSize, FeeStructure, + +Plus: + EXCHANGE — multi-exchange support: + ExchangeProfile (per-exchange metadata), exchange-asset mapping + (which exchanges trade which assets, with exchange-specific parameters). DerivativeAccess, PriceDecimals, TypicalVolume Overlap handling (industry standard, per CoinGecko/CMC/Messari): @@ -120,6 +125,66 @@ class LiquidityProfile(str, Enum): ILLIQUID = "illiquid" # <$1M +# ============================================================================== +# ExchangeProfile — multi-exchange support +# ============================================================================== + +@dataclass(frozen=True, slots=True) +class ExchangeProfile: + """Metadata for a trading venue. System-wide — used by BLUE/VIOLET/UV/MALKHUT.""" + exchange_id: str # canonical key: "binance", "bingx", "bybit", etc. + display_name: str # human-readable: "Binance", "BingX" + has_spot: bool + has_perps: bool + has_options: bool + api_base_url: str # REST API root + ws_base_url: str # WebSocket root ("" if not applicable) + default_taker_fee_bps: float + default_maker_fee_bps: float + typical_latency_ms: float + + +EXCHANGE_PROFILES: Dict[str, ExchangeProfile] = {} + +BINANCE = ExchangeProfile( + exchange_id="binance", display_name="Binance", + has_spot=True, has_perps=True, has_options=True, + api_base_url="https://api.binance.com", + ws_base_url="wss://stream.binance.com:9443", + default_taker_fee_bps=0.4, default_maker_fee_bps=0.2, + typical_latency_ms=40, +) +EXCHANGE_PROFILES["binance"] = BINANCE + +BINGX = ExchangeProfile( + exchange_id="bingx", display_name="BingX", + has_spot=True, has_perps=True, has_options=False, + api_base_url="https://open-api.bingx.com", + ws_base_url="wss://open-api.bingx.com/swapMarket", + default_taker_fee_bps=0.5, default_maker_fee_bps=0.2, + typical_latency_ms=100, +) +EXCHANGE_PROFILES["bingx"] = BINGX + +BYBIT = ExchangeProfile( + exchange_id="bybit", display_name="Bybit", + has_spot=True, has_perps=True, has_options=True, + api_base_url="https://api.bybit.com", + ws_base_url="wss://stream.bybit.com/v5/public/linear", + default_taker_fee_bps=0.06, default_maker_fee_bps=-0.01, + typical_latency_ms=50, +) +EXCHANGE_PROFILES["bybit"] = BYBIT + + +def get_exchange(exchange_id: str) -> Optional[ExchangeProfile]: + return EXCHANGE_PROFILES.get(exchange_id) + + +def list_exchanges() -> List[str]: + return list(EXCHANGE_PROFILES.keys()) + + # ============================================================================== # AssetProfile — frozen, all-invariant, multi-label where appropriate # ============================================================================== @@ -160,6 +225,9 @@ class AssetProfile: has_funding: bool = False has_options: bool = False + # --- Exchange membership (which venues trade this asset) --- + exchanges: tuple[str, ...] = ("binance",) # tuple of exchange_ids + @property def sector(self) -> Sector: """Primary sector (first in tuple). For single-label consumers.""" @@ -224,6 +292,7 @@ def _profile( typical_daily_volume_usd: float, has_funding: bool = False, has_options: bool = False, + exchanges: Sequence[str] = ("binance",), ) -> AssetProfile: return AssetProfile( symbol=symbol, @@ -238,10 +307,10 @@ def _profile( derivative_access=derivative_access, tick_size=tick_size, lot_size=lot_size, price_decimals=price_decimals, maker_fee_bps=maker_fee_bps, taker_fee_bps=taker_fee_bps, - typical_spread_bps=typical_spread_bps, - typical_depth_usd=typical_depth_usd, + typical_spread_bps=typical_spread_bps, typical_depth_usd=typical_depth_usd, typical_daily_volume_usd=typical_daily_volume_usd, has_funding=has_funding, has_options=has_options, + exchanges=tuple(exchanges), ) @@ -528,3 +597,20 @@ def get_multi_sector_assets() -> List[AssetProfile]: def get_multi_role_assets() -> List[AssetProfile]: """Assets serving more than one token role.""" return [p for p in ASSET_PROFILES.values() if len(p.token_roles) > 1] + + +def get_assets_on_exchange(exchange_id: str) -> List[AssetProfile]: + """Assets traded on a given exchange.""" + return [p for p in ASSET_PROFILES.values() if exchange_id in p.exchanges] + + +def get_common_assets(exchange_a: str, exchange_b: str) -> List[AssetProfile]: + """Assets traded on BOTH exchanges (intersection).""" + return [p for p in ASSET_PROFILES.values() + if exchange_a in p.exchanges and exchange_b in p.exchanges] + + +def get_exchange_for_asset(symbol: str) -> tuple[str, ...]: + """Return which exchanges trade a given asset.""" + p = ASSET_PROFILES.get(symbol) + return p.exchanges if p else ()