#!/usr/bin/env python3 """ MALKHUT 3.5-Hour Instrumented E2E — 1K opponents, fill quality + slippage tracking. Runs for ~3.5 hours with: - 1,000 diverse opponents (randomized params) - 9 assets × 30 scenarios = 270 scenarios per cycle - HftBacktestCWM (PowerProbQueueModel + calibrated slippage) - All order types exercised - FILL QUALITY tracked per cycle (the core metric) - SLIPPAGE REDUCTION tracked over time - Improvement trends: does fill quality improve as CMA-ES learns? - Periodic reports every 5 minutes with improvement deltas Key questions answered: Q1: Can we improve fill quality over cycles? Q2: Can we reduce slippage over time? Q3: How does 1K-opponent swarm affect fill dynamics vs 100-opponent? Usage: python -m malkhut.long_e2e_35h """ from __future__ import annotations import json import math import os import random import sys import time from collections import defaultdict from typing import Any, Dict, List, Optional _HERE = os.path.dirname(os.path.abspath(__file__)) if _HERE not in sys.path: sys.path.insert(0, _HERE) from malkhut.state import ( AccountState, ActionKind, FulfilmentPolicyParams, MarketWorldState, OrderType, PositionState, Side, ) from malkhut.actions import FulfilmentAction, PlannedPolicy from malkhut.cwm.hft_cwm import HftBacktestCWM from malkhut.risk.gate import RiskGate from malkhut.training.cma_trainer import ( CMAESTrainer, CMAParameterCodec, PolicyEvaluator, ScenarioFactory, SelfPlayPool, PolicySnapshot, ) from malkhut.training.selector import PerformanceMatrix, MarketRegime from malkhut.counterparties import ( ToxicTakerPolicy, PassiveMakerPolicy, LatencyArbPolicy, NoiseTraderPolicy, ) from malkhut.counterparties_extended import ( MomentumTakerPolicy, MeanReversionTakerPolicy, InventoryMarketMakerPolicy, LiquidationFlowPolicy, StaleQuoteAttackerPolicy, ) DURATION_S = int(3.5 * 3600) # 3.5 hours ASSETS = ["BTCUSDT", "ETHUSDT", "SOLUSDT", "DOGEUSDT", "ADAUSDT", "AVAXUSDT", "UNIUSDT", "LINKUSDT", "BNBUSDT"] STEPS_PER_EPISODE = 20 SEED = 42 # ── 1K Opponent Swarm ──────────────────────────────────────────────────────── def _build_swarm(n: int = 1000) -> tuple: """Build a swarm of n diverse opponents with randomized parameters.""" pool = [ lambda: ToxicTakerPolicy(sensitivity=random.uniform(0.1, 0.8)), lambda: PassiveMakerPolicy(join_probability=random.uniform(0.3, 0.9)), lambda: LatencyArbPolicy(lead_threshold=random.uniform(0.3, 0.7)), lambda: NoiseTraderPolicy(), lambda: MomentumTakerPolicy(threshold=random.uniform(0.1, 0.5)), lambda: MeanReversionTakerPolicy(threshold=random.uniform(0.2, 0.8)), lambda: InventoryMarketMakerPolicy(max_inventory=random.uniform(0.02, 0.15)), lambda: LiquidationFlowPolicy(trigger_bps=random.uniform(20, 80)), lambda: StaleQuoteAttackerPolicy(stale_threshold_s=random.uniform(2, 10)), ] rng = random.Random(99) return tuple(rng.choice(pool)() for _ in range(n)) SWARM = _build_swarm(1000) print(f"Built 1K opponent swarm: {len(SWARM)} agents") # ── Action generator ────────────────────────────────────────────────────────── def _generate_action(state: MarketWorldState, rng: random.Random) -> FulfilmentAction: r = rng.random() if r < 0.12: return FulfilmentAction(ActionKind.NOOP, None, None, 0, 0.0, 0) elif r < 0.28: side = Side.BUY if rng.random() < 0.5 else Side.SELL return FulfilmentAction(ActionKind.CROSS_SPREAD, side, OrderType.LIMIT, 0, rng.uniform(0.01, 0.10), 50, time_in_force="IOC") elif r < 0.48: side = Side.BUY if rng.random() < 0.55 else Side.SELL return FulfilmentAction(ActionKind.PLACE, side, OrderType.LIMIT, rng.randint(0, 5), rng.uniform(0.05, 0.25), 200, post_only=True) elif r < 0.62: side = Side.BUY if rng.random() < 0.5 else Side.SELL return FulfilmentAction(ActionKind.PLACE, side, OrderType.LIMIT, rng.randint(0, 3), rng.uniform(0.05, 0.20), 200) elif r < 0.72: if state.open_orders: oo = rng.choice(state.open_orders) return FulfilmentAction(ActionKind.CANCEL, None, None, 0, 0.0, 0, cancel_order_id=oo.client_order_id) return FulfilmentAction(ActionKind.NOOP, None, None, 0, 0.0, 0) elif r < 0.82: pos = state.account.positions.get(state.venue.symbol) if pos and abs(pos.qty) > 0.001: side = Side.SELL if pos.qty > 0 else Side.BUY return FulfilmentAction(ActionKind.REDUCE, side, OrderType.MARKET, 0, rng.uniform(0.1, 0.5), 0, reduce_only=True) return FulfilmentAction(ActionKind.NOOP, None, None, 0, 0.0, 0) elif r < 0.92: pos = state.account.positions.get(state.venue.symbol) if pos and abs(pos.qty) > 0.001: side = Side.SELL if pos.qty > 0 else Side.BUY return FulfilmentAction(ActionKind.FULL_EXIT, side, OrderType.MARKET, 0, 1.0, 0, reduce_only=True) return FulfilmentAction(ActionKind.NOOP, None, None, 0, 0.0, 0) else: side = Side.BUY if rng.random() < 0.5 else Side.SELL return FulfilmentAction(ActionKind.PLACE, side, OrderType.STOP_MARKET, rng.randint(-5, 5), rng.uniform(0.01, 0.05), 200) # ── Episode runner with fill quality tracking ───────────────────────────────── def run_episode(cwm, scenario, params, steps, seed, rng, risk_gate): state = scenario.initial_state cp_policies = scenario.counterparties fills = 0; noops = 0; cancels = 0; post_onlys = 0; reduce_onlys = 0 aggressive = 0; passive = 0; peak_eq = state.account.equity; max_dd = 0.0 ot_counts = defaultdict(int); tif_counts = defaultdict(int) spreads = []; equities = [] # Fill quality tracking fq_slippage_sum = 0.0 fq_expected_slippage_sum = 0.0 fq_price_improve_sum = 0.0 fq_adverse_sum = 0.0 fq_value_sum = 0.0 fq_filled_count = 0 fq_total_count = 0 for step in range(steps): spread_bps = state.book.spread_bps if state.book.bids and state.book.asks else 0.0 spreads.append(spread_bps) equities.append(state.account.equity) action = _generate_action(state, rng) cp_actions = tuple(cp.rollout_action(state, rng) for cp in cp_policies) if risk_gate and action.kind != ActionKind.NOOP: planned = PlannedPolicy(actions=(action,), probabilities=(1.0,), selected_action=action, diagnostics={}) decision = risk_gate.validate(state, planned, params) if not decision.approved: action = FulfilmentAction(ActionKind.NOOP, None, None, 0, 0.0, 0) fq_total_count += 1 prev_eq = state.account.equity state = cwm.transition(state, (action, *cp_actions)) # Fill quality accumulation if state.fill_quality: fq = state.fill_quality fq_slippage_sum += fq.slippage_bps fq_expected_slippage_sum += fq.expected_slippage_bps fq_price_improve_sum += fq.price_improvement_bps fq_adverse_sum += fq.post_fill_adverse_bps fq_value_sum += fq.fill_value_score if fq.filled: fq_filled_count += 1 eq = state.account.equity peak_eq = max(peak_eq, eq) dd = (peak_eq - eq) / max(peak_eq, 1e-12) * 10_000 max_dd = max(max_dd, dd) ot = action.order_type.value if action.order_type else "NONE" ot_counts[ot] += 1 tif_counts[getattr(action, 'time_in_force', 'GTC')] += 1 if action.kind == ActionKind.NOOP: noops += 1 elif action.kind in (ActionKind.CANCEL, ActionKind.CANCEL_REPLACE): cancels += 1 elif action.kind == ActionKind.CROSS_SPREAD: aggressive += 1 else: passive += 1 if action.post_only: post_onlys += 1 if action.reduce_only: reduce_onlys += 1 if eq != prev_eq and action.kind != ActionKind.NOOP: fills += 1 fq_n = max(fq_total_count, 1) pnl = (state.account.equity - 10000.0) / 10000.0 * 10_000 pos = state.account.positions.get(state.venue.symbol, PositionState("", 0, 0, 0, 0, None, 0, None)) return { "pnl_bps": pnl, "max_dd_bps": max_dd, "fills": fills, "noops": noops, "cancels": cancels, "aggressive": aggressive, "passive": passive, "post_onlys": post_onlys, "reduce_onlys": reduce_onlys, "order_types": dict(ot_counts), "tifs": dict(tif_counts), "final_pos": pos.qty, "steps": steps, # Fill quality metrics "avg_slippage_bps": fq_slippage_sum / fq_n, "avg_expected_slippage_bps": fq_expected_slippage_sum / fq_n, "slippage_surprise": (fq_slippage_sum - fq_expected_slippage_sum) / fq_n, "avg_price_improvement_bps": fq_price_improve_sum / fq_n, "avg_adverse_bps": fq_adverse_sum / fq_n, "avg_fill_value_score": fq_value_sum / fq_n, "fill_rate": fq_filled_count / max(fq_total_count, 1), } def _baseline(): return FulfilmentPolicyParams( version="long_e2e_35h", ucb_c=1.414, max_sims=64, max_depth=2, rollout_depth=2, root_temperature=0.5, min_root_entropy=0.25, quote_offsets_ticks=(0, 1, 2), quote_size_fractions=(0.10, 0.25, 0.50), passive_ttl_ms=200, aggressive_ttl_ms=50, maker_edge_min_bps=0.5, cross_spread_edge_min_bps=5.0, adverse_toxicity_cancel_threshold=0.5, queue_churn_cancel_threshold=0.5, mae_tail_cut_bps=50.0, mfe_giveback_cut_fraction=0.5, max_time_in_loss_s=300.0, failed_recovery_cut_count=3, recovery_velocity_min_bps_per_s=0.0, max_symbol_notional_fraction=0.20, max_single_order_notional_fraction=0.05, reduce_when_global_up_fraction=0.30, session_profit_lock_fraction=0.02, w_expected_pnl=1.0, w_fill_probability=0.5, w_adverse_selection=2.0, w_queue_priority=0.5, w_inventory_risk=1.5, w_tail_loss=5.0, w_fee_quality=0.5, w_time_decay=0.3, w_policy_entropy=0.5, robust_tail_weight=2.0, toxic_counterparty_weight=3.0, low_liquidity_weight=2.0, latency_stress_weight=1.0, ) def main(): t_start = time.time() t_end = t_start + DURATION_S print("=" * 80) print("MALKHUT 3.5-HOUR INSTRUMENTED E2E") print(f" Duration: 3.5h ({DURATION_S}s)") print(f" CWM: HftBacktestCWM (calibrated slippage)") print(f" Swarm: {len(SWARM)} opponents (1K)") print(f" Assets: {', '.join(ASSETS)}") print(f" Steps/episode: {STEPS_PER_EPISODE}") print(f" Tracking: fill quality, slippage, improvement trends") print("=" * 80) print(flush=True) cwm_factory = lambda: HftBacktestCWM(use_queue_model=True) risk_gate = RiskGate() params = _baseline() factory = ScenarioFactory(exchange_id="bingx") rng = random.Random(SEED) # Build all scenarios print("Building scenarios...", flush=True) all_scenarios = [] for sym in ASSETS: scenarios = factory.build_suite(symbols=[sym], steps_per_scenario=STEPS_PER_EPISODE, seed=SEED) all_scenarios.extend(scenarios) print(f" Total scenarios: {len(all_scenarios)} ({len(ASSETS)} assets)", flush=True) # Rolling stats recent_pnls = [] recent_fills = 0; recent_noops = 0; recent_cancels = 0 recent_aggressive = 0; recent_passive = 0 recent_post_onlys = 0; recent_reduce_onlys = 0 recent_ot_counts = defaultdict(int); recent_tif_counts = defaultdict(int) recent_actions_total = 0; recent_episodes = 0 all_pnls = [] peak_pnl = -float("inf"); worst_pnl = float("inf") # Fill quality rolling stats fq_slippage_sum = 0.0; fq_expected_sum = 0.0; fq_surprise_sum = 0.0 fq_improve_sum = 0.0; fq_adverse_sum = 0.0; fq_value_sum = 0.0 fq_filled_count = 0; fq_total_count = 0 fq_n_reports = 0 # Historical fill quality per report window (for trend analysis) fq_history = [] # list of (timestamp, avg_slippage, avg_expected, avg_surprise, avg_value, fill_rate) # CMA-ES cma_bests = [] codec = CMAParameterCodec() pool = SelfPlayPool(max_size=20) print(f"\nStarting 3.5-hour run...", flush=True) while time.time() < t_end: cycle = int((time.time() - t_start) / 0.1) + 1 # estimate remaining = t_end - time.time() if remaining < 60: break try: n_episodes = min(len(all_scenarios), 20) selected = rng.sample(all_scenarios, n_episodes) for i, scenario in enumerate(selected): if time.time() > t_end - 30: break ep_rng = random.Random(SEED + int(time.time() * 1000) + i) ep = run_episode( cwm=cwm_factory(), scenario=scenario, params=params, steps=STEPS_PER_EPISODE, seed=SEED + int(time.time() * 1000) + i, rng=ep_rng, risk_gate=risk_gate, ) # Accumulate stats recent_pnls.append(ep["pnl_bps"]) all_pnls.append(ep["pnl_bps"]) peak_pnl = max(peak_pnl, ep["pnl_bps"]) worst_pnl = min(worst_pnl, ep["pnl_bps"]) recent_fills += ep["fills"] recent_noops += ep["noops"] recent_cancels += ep["cancels"] recent_aggressive += ep["aggressive"] recent_passive += ep["passive"] recent_post_onlys += ep["post_onlys"] recent_reduce_onlys += ep["reduce_onlys"] recent_episodes += 1 recent_actions_total += ep["steps"] for ot, c in ep["order_types"].items(): recent_ot_counts[ot] += c for t, c in ep["tifs"].items(): recent_tif_counts[t] += c if len(recent_pnls) > 200: recent_pnls = recent_pnls[-200:] # Fill quality accumulation fq_slippage_sum += ep["avg_slippage_bps"] fq_expected_sum += ep["avg_expected_slippage_bps"] fq_surprise_sum += ep["slippage_surprise"] fq_improve_sum += ep["avg_price_improvement_bps"] fq_adverse_sum += ep["avg_adverse_bps"] fq_value_sum += ep["avg_fill_value_score"] fq_filled_count += ep["fills"] fq_total_count += ep["steps"] fq_n_reports += 1 except Exception as e: print(f" Error: {e}", flush=True) # CMA-ES every 20 reports if fq_n_reports % 20 == 0 and remaining > 600: cma_scenarios = rng.sample(all_scenarios, min(3, len(all_scenarios))) try: evaluator = PolicyEvaluator(cwm_factory=cwm_factory, scoring_mode="fast") trainer = CMAESTrainer(codec=codec, evaluator=evaluator, pool=pool, workers=0) best = trainer.train(incumbent=params, scenarios=cma_scenarios, budget_evals=2, seed=SEED + fq_n_reports) cma_bests.append({"cycle": fq_n_reports, "score": best.score, "fill_value": ep.get("avg_fill_value_score", 0)}) if best.score > params.w_expected_pnl * 10: params = best.params except Exception as e: print(f" CMA-ES error: {e}", flush=True) # Periodic report with improvement tracking elapsed = time.time() - t_start if fq_n_reports > 0 and fq_n_reports % 10 == 0: n = fq_n_reports h = elapsed / 3600; m = (elapsed % 3600) / 60 avg_slip = fq_slippage_sum / n avg_expected = fq_expected_sum / n avg_surprise = fq_surprise_sum / n avg_improve = fq_improve_sum / n avg_adverse = fq_adverse_sum / n avg_value = fq_value_sum / n avg_fill_rate = fq_filled_count / max(fq_total_count, 1) # Trend: compare first half to second half fq_history.append({ "t": elapsed, "slippage": avg_slip, "expected": avg_expected, "surprise": avg_surprise, "improvement": avg_improve, "adverse": avg_adverse, "value": avg_value, "fill_rate": avg_fill_rate, }) trend_slip = "IMPROVING" if len(fq_history) >= 4 and fq_history[-1]["slippage"] < fq_history[-4]["slippage"] else "STABLE" trend_value = "IMPROVING" if len(fq_history) >= 4 and fq_history[-1]["value"] > fq_history[-4]["value"] else "STABLE" print(f"\n [{h:.1f}h{m:.0f}m] Cycle ~{n} | " f"slip={avg_slip:.2f}bpx({trend_slip}) | " f"expected={avg_expected:.2f} | " f"surprise={avg_surprise:+.2f} | " f"improve={avg_improve:.2f} | " f"adverse={avg_adverse:.2f} | " f"fill_value={avg_value:.2f}({trend_value}) | " f"fill_rate={avg_fill_rate:.1%} | " f"agg/pass={recent_aggressive/max(recent_passive,1):.2f}", flush=True) # Reset window fq_slippage_sum = 0.0; fq_expected_sum = 0.0; fq_surprise_sum = 0.0 fq_improve_sum = 0.0; fq_adverse_sum = 0.0; fq_value_sum = 0.0 fq_filled_count = 0; fq_total_count = 0; fq_n_reports = 0 recent_pnls.clear() recent_fills = recent_noops = recent_cancels = 0 recent_aggressive = recent_passive = 0 recent_post_onlys = recent_reduce_onlys = 0 recent_ot_counts.clear(); recent_tif_counts.clear() recent_actions_total = 0; recent_episodes = 0 # Final report elapsed = time.time() - t_start n = len(all_pnls) print(f"\n{'='*80}") print(f" FINAL REPORT — 3.5H INSTRUMENTED E2E") print(f"{'='*80}") if n > 0: total_actions = sum(recent_ot_counts.values()) + recent_actions_total print(f"\n Duration: {elapsed/3600:.1f}h ({elapsed:.0f}s)") print(f" Total episodes: {n}") print(f"\n PERFORMANCE") print(f" Avg PnL: {sum(all_pnls)/n:+.1f} bps") print(f" Median PnL: {sorted(all_pnls)[n//2]:+.1f} bps") print(f" Best: {max(all_pnls):+.1f} bps") print(f" Worst: {min(all_pnls):+.1f} bps") print(f" Win rate: {sum(1 for p in all_pnls if p > 0)/n*100:.1f}%") print(f"\n FILL QUALITY (CORE)") if fq_history: first = fq_history[0]; last = fq_history[-1] print(f" Avg fill value score: {last['value']:.2f}") print(f" Fill value trend: {first['value']:.2f} -> {last['value']:.2f} ({(last['value']-first['value'])/max(abs(first['value']),0.01)*100:+.1f}%)") print(f" Avg fill rate: {last['fill_rate']:.1%}") print(f" Fill rate trend: {first['fill_rate']:.1%} -> {last['fill_rate']:.1%}") print(f"\n SLIPPAGE (CORE)") if fq_history: print(f" Avg actual slippage: {last['slippage']:.2f} bps") print(f" Avg expected slip: {last['expected']:.2f} bps") print(f" Avg slippage surprise:{last['surprise']:+.2f} bps (neg=good: actual < expected)") print(f" Slippage trend: {first['slippage']:.2f} -> {last['slippage']:.2f} ({(last['slippage']-first['slippage'])/max(abs(first['slippage']),0.01)*100:+.1f}%)") print(f" Avg price improvement:{last['improvement']:.2f} bps") print(f" Avg post-fill adverse:{last['adverse']:.2f} bps") print(f"\n 1K OPPONENT SWARM") print(f" Aggressive: {sum(r['aggressive'] for r in fq_history):.0f}") print(f" Passive: {sum(r['passive'] for r in fq_history):.0f}") print(f" Post-only: {sum(r['post_onlys'] for r in fq_history):.0f}") print(f" CMA-ES: {len(cma_bests)} cycles") print(f"{'='*80}", flush=True) os.makedirs("malkhut/results", exist_ok=True) report = { "duration_s": round(elapsed, 1), "n_episodes": n, "n_scenarios": len(all_scenarios), "n_opponents": len(SWARM), "avg_pnl_bps": round(sum(all_pnls)/n, 1) if n else 0, "win_rate_pct": round(sum(1 for p in all_pnls if p > 0)/n*100, 1) if n else 0, "fq_history": fq_history, "cma_bests": cma_bests, } path = f"malkhut/results/e2e_35h_{int(time.time())}.json" with open(path, "w") as f: json.dump(report, f, indent=2) print(f"\nReport: {path}", flush=True) if __name__ == "__main__": try: main() except Exception as e: import traceback print(f"\nFATAL: {e}", flush=True) traceback.print_exc() sys.exit(1)