Files
sentiment-engine/MALKHUT/malkhut/long_e2e_35h.py

470 lines
21 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

#!/usr/bin/env python3
"""
MALKHUT 3.5-Hour Instrumented E2E — 1K opponents, fill quality + slippage tracking.
Runs for ~3.5 hours with:
- 1,000 diverse opponents (randomized params)
- 9 assets × 30 scenarios = 270 scenarios per cycle
- HftBacktestCWM (PowerProbQueueModel + calibrated slippage)
- All order types exercised
- FILL QUALITY tracked per cycle (the core metric)
- SLIPPAGE REDUCTION tracked over time
- Improvement trends: does fill quality improve as CMA-ES learns?
- Periodic reports every 5 minutes with improvement deltas
Key questions answered:
Q1: Can we improve fill quality over cycles?
Q2: Can we reduce slippage over time?
Q3: How does 1K-opponent swarm affect fill dynamics vs 100-opponent?
Usage:
python -m malkhut.long_e2e_35h
"""
from __future__ import annotations
import json
import math
import os
import random
import sys
import time
from collections import defaultdict
from typing import Any, Dict, List, Optional
_HERE = os.path.dirname(os.path.abspath(__file__))
if _HERE not in sys.path:
sys.path.insert(0, _HERE)
from malkhut.state import (
AccountState, ActionKind, FulfilmentPolicyParams, MarketWorldState,
OrderType, PositionState, Side,
)
from malkhut.actions import FulfilmentAction, PlannedPolicy
from malkhut.cwm.hft_cwm import HftBacktestCWM
from malkhut.risk.gate import RiskGate
from malkhut.training.cma_trainer import (
CMAESTrainer, CMAParameterCodec, PolicyEvaluator,
ScenarioFactory, SelfPlayPool, PolicySnapshot,
)
from malkhut.training.selector import PerformanceMatrix, MarketRegime
from malkhut.counterparties import (
ToxicTakerPolicy, PassiveMakerPolicy, LatencyArbPolicy, NoiseTraderPolicy,
)
from malkhut.counterparties_extended import (
MomentumTakerPolicy, MeanReversionTakerPolicy, InventoryMarketMakerPolicy,
LiquidationFlowPolicy, StaleQuoteAttackerPolicy,
)
DURATION_S = int(3.5 * 3600) # 3.5 hours
ASSETS = ["BTCUSDT", "ETHUSDT", "SOLUSDT", "DOGEUSDT", "ADAUSDT",
"AVAXUSDT", "UNIUSDT", "LINKUSDT", "BNBUSDT"]
STEPS_PER_EPISODE = 20
SEED = 42
# ── 1K Opponent Swarm ────────────────────────────────────────────────────────
def _build_swarm(n: int = 1000) -> tuple:
"""Build a swarm of n diverse opponents with randomized parameters."""
pool = [
lambda: ToxicTakerPolicy(sensitivity=random.uniform(0.1, 0.8)),
lambda: PassiveMakerPolicy(join_probability=random.uniform(0.3, 0.9)),
lambda: LatencyArbPolicy(lead_threshold=random.uniform(0.3, 0.7)),
lambda: NoiseTraderPolicy(),
lambda: MomentumTakerPolicy(threshold=random.uniform(0.1, 0.5)),
lambda: MeanReversionTakerPolicy(threshold=random.uniform(0.2, 0.8)),
lambda: InventoryMarketMakerPolicy(max_inventory=random.uniform(0.02, 0.15)),
lambda: LiquidationFlowPolicy(trigger_bps=random.uniform(20, 80)),
lambda: StaleQuoteAttackerPolicy(stale_threshold_s=random.uniform(2, 10)),
]
rng = random.Random(99)
return tuple(rng.choice(pool)() for _ in range(n))
SWARM = _build_swarm(1000)
print(f"Built 1K opponent swarm: {len(SWARM)} agents")
# ── Action generator ──────────────────────────────────────────────────────────
def _generate_action(state: MarketWorldState, rng: random.Random) -> FulfilmentAction:
r = rng.random()
if r < 0.12:
return FulfilmentAction(ActionKind.NOOP, None, None, 0, 0.0, 0)
elif r < 0.28:
side = Side.BUY if rng.random() < 0.5 else Side.SELL
return FulfilmentAction(ActionKind.CROSS_SPREAD, side, OrderType.LIMIT,
0, rng.uniform(0.01, 0.10), 50, time_in_force="IOC")
elif r < 0.48:
side = Side.BUY if rng.random() < 0.55 else Side.SELL
return FulfilmentAction(ActionKind.PLACE, side, OrderType.LIMIT,
rng.randint(0, 5), rng.uniform(0.05, 0.25), 200, post_only=True)
elif r < 0.62:
side = Side.BUY if rng.random() < 0.5 else Side.SELL
return FulfilmentAction(ActionKind.PLACE, side, OrderType.LIMIT,
rng.randint(0, 3), rng.uniform(0.05, 0.20), 200)
elif r < 0.72:
if state.open_orders:
oo = rng.choice(state.open_orders)
return FulfilmentAction(ActionKind.CANCEL, None, None, 0, 0.0, 0,
cancel_order_id=oo.client_order_id)
return FulfilmentAction(ActionKind.NOOP, None, None, 0, 0.0, 0)
elif r < 0.82:
pos = state.account.positions.get(state.venue.symbol)
if pos and abs(pos.qty) > 0.001:
side = Side.SELL if pos.qty > 0 else Side.BUY
return FulfilmentAction(ActionKind.REDUCE, side, OrderType.MARKET,
0, rng.uniform(0.1, 0.5), 0, reduce_only=True)
return FulfilmentAction(ActionKind.NOOP, None, None, 0, 0.0, 0)
elif r < 0.92:
pos = state.account.positions.get(state.venue.symbol)
if pos and abs(pos.qty) > 0.001:
side = Side.SELL if pos.qty > 0 else Side.BUY
return FulfilmentAction(ActionKind.FULL_EXIT, side, OrderType.MARKET,
0, 1.0, 0, reduce_only=True)
return FulfilmentAction(ActionKind.NOOP, None, None, 0, 0.0, 0)
else:
side = Side.BUY if rng.random() < 0.5 else Side.SELL
return FulfilmentAction(ActionKind.PLACE, side, OrderType.STOP_MARKET,
rng.randint(-5, 5), rng.uniform(0.01, 0.05), 200)
# ── Episode runner with fill quality tracking ─────────────────────────────────
def run_episode(cwm, scenario, params, steps, seed, rng, risk_gate):
state = scenario.initial_state
cp_policies = scenario.counterparties
fills = 0; noops = 0; cancels = 0; post_onlys = 0; reduce_onlys = 0
aggressive = 0; passive = 0; peak_eq = state.account.equity; max_dd = 0.0
ot_counts = defaultdict(int); tif_counts = defaultdict(int)
spreads = []; equities = []
# Fill quality tracking
fq_slippage_sum = 0.0
fq_expected_slippage_sum = 0.0
fq_price_improve_sum = 0.0
fq_adverse_sum = 0.0
fq_value_sum = 0.0
fq_filled_count = 0
fq_total_count = 0
for step in range(steps):
spread_bps = state.book.spread_bps if state.book.bids and state.book.asks else 0.0
spreads.append(spread_bps)
equities.append(state.account.equity)
action = _generate_action(state, rng)
cp_actions = tuple(cp.rollout_action(state, rng) for cp in cp_policies)
if risk_gate and action.kind != ActionKind.NOOP:
planned = PlannedPolicy(actions=(action,), probabilities=(1.0,),
selected_action=action, diagnostics={})
decision = risk_gate.validate(state, planned, params)
if not decision.approved:
action = FulfilmentAction(ActionKind.NOOP, None, None, 0, 0.0, 0)
fq_total_count += 1
prev_eq = state.account.equity
state = cwm.transition(state, (action, *cp_actions))
# Fill quality accumulation
if state.fill_quality:
fq = state.fill_quality
fq_slippage_sum += fq.slippage_bps
fq_expected_slippage_sum += fq.expected_slippage_bps
fq_price_improve_sum += fq.price_improvement_bps
fq_adverse_sum += fq.post_fill_adverse_bps
fq_value_sum += fq.fill_value_score
if fq.filled:
fq_filled_count += 1
eq = state.account.equity
peak_eq = max(peak_eq, eq)
dd = (peak_eq - eq) / max(peak_eq, 1e-12) * 10_000
max_dd = max(max_dd, dd)
ot = action.order_type.value if action.order_type else "NONE"
ot_counts[ot] += 1
tif_counts[getattr(action, 'time_in_force', 'GTC')] += 1
if action.kind == ActionKind.NOOP: noops += 1
elif action.kind in (ActionKind.CANCEL, ActionKind.CANCEL_REPLACE): cancels += 1
elif action.kind == ActionKind.CROSS_SPREAD: aggressive += 1
else: passive += 1
if action.post_only: post_onlys += 1
if action.reduce_only: reduce_onlys += 1
if eq != prev_eq and action.kind != ActionKind.NOOP: fills += 1
fq_n = max(fq_total_count, 1)
pnl = (state.account.equity - 10000.0) / 10000.0 * 10_000
pos = state.account.positions.get(state.venue.symbol, PositionState("", 0, 0, 0, 0, None, 0, None))
return {
"pnl_bps": pnl, "max_dd_bps": max_dd,
"fills": fills, "noops": noops, "cancels": cancels,
"aggressive": aggressive, "passive": passive,
"post_onlys": post_onlys, "reduce_onlys": reduce_onlys,
"order_types": dict(ot_counts), "tifs": dict(tif_counts),
"final_pos": pos.qty, "steps": steps,
# Fill quality metrics
"avg_slippage_bps": fq_slippage_sum / fq_n,
"avg_expected_slippage_bps": fq_expected_slippage_sum / fq_n,
"slippage_surprise": (fq_slippage_sum - fq_expected_slippage_sum) / fq_n,
"avg_price_improvement_bps": fq_price_improve_sum / fq_n,
"avg_adverse_bps": fq_adverse_sum / fq_n,
"avg_fill_value_score": fq_value_sum / fq_n,
"fill_rate": fq_filled_count / max(fq_total_count, 1),
}
def _baseline():
return FulfilmentPolicyParams(
version="long_e2e_35h", ucb_c=1.414, max_sims=64, max_depth=2,
rollout_depth=2, root_temperature=0.5, min_root_entropy=0.25,
quote_offsets_ticks=(0, 1, 2), quote_size_fractions=(0.10, 0.25, 0.50),
passive_ttl_ms=200, aggressive_ttl_ms=50,
maker_edge_min_bps=0.5, cross_spread_edge_min_bps=5.0,
adverse_toxicity_cancel_threshold=0.5, queue_churn_cancel_threshold=0.5,
mae_tail_cut_bps=50.0, mfe_giveback_cut_fraction=0.5,
max_time_in_loss_s=300.0, failed_recovery_cut_count=3,
recovery_velocity_min_bps_per_s=0.0,
max_symbol_notional_fraction=0.20, max_single_order_notional_fraction=0.05,
reduce_when_global_up_fraction=0.30, session_profit_lock_fraction=0.02,
w_expected_pnl=1.0, w_fill_probability=0.5, w_adverse_selection=2.0,
w_queue_priority=0.5, w_inventory_risk=1.5, w_tail_loss=5.0,
w_fee_quality=0.5, w_time_decay=0.3, w_policy_entropy=0.5,
robust_tail_weight=2.0, toxic_counterparty_weight=3.0,
low_liquidity_weight=2.0, latency_stress_weight=1.0,
)
def main():
t_start = time.time()
t_end = t_start + DURATION_S
print("=" * 80)
print("MALKHUT 3.5-HOUR INSTRUMENTED E2E")
print(f" Duration: 3.5h ({DURATION_S}s)")
print(f" CWM: HftBacktestCWM (calibrated slippage)")
print(f" Swarm: {len(SWARM)} opponents (1K)")
print(f" Assets: {', '.join(ASSETS)}")
print(f" Steps/episode: {STEPS_PER_EPISODE}")
print(f" Tracking: fill quality, slippage, improvement trends")
print("=" * 80)
print(flush=True)
cwm_factory = lambda: HftBacktestCWM(use_queue_model=True)
risk_gate = RiskGate()
params = _baseline()
factory = ScenarioFactory(exchange_id="bingx")
rng = random.Random(SEED)
# Build all scenarios
print("Building scenarios...", flush=True)
all_scenarios = []
for sym in ASSETS:
scenarios = factory.build_suite(symbols=[sym], steps_per_scenario=STEPS_PER_EPISODE, seed=SEED)
all_scenarios.extend(scenarios)
print(f" Total scenarios: {len(all_scenarios)} ({len(ASSETS)} assets)", flush=True)
# Rolling stats
recent_pnls = []
recent_fills = 0; recent_noops = 0; recent_cancels = 0
recent_aggressive = 0; recent_passive = 0
recent_post_onlys = 0; recent_reduce_onlys = 0
recent_ot_counts = defaultdict(int); recent_tif_counts = defaultdict(int)
recent_actions_total = 0; recent_episodes = 0
all_pnls = []
peak_pnl = -float("inf"); worst_pnl = float("inf")
# Fill quality rolling stats
fq_slippage_sum = 0.0; fq_expected_sum = 0.0; fq_surprise_sum = 0.0
fq_improve_sum = 0.0; fq_adverse_sum = 0.0; fq_value_sum = 0.0
fq_filled_count = 0; fq_total_count = 0
fq_n_reports = 0
# Historical fill quality per report window (for trend analysis)
fq_history = [] # list of (timestamp, avg_slippage, avg_expected, avg_surprise, avg_value, fill_rate)
# CMA-ES
cma_bests = []
codec = CMAParameterCodec()
pool = SelfPlayPool(max_size=20)
print(f"\nStarting 3.5-hour run...", flush=True)
while time.time() < t_end:
cycle = int((time.time() - t_start) / 0.1) + 1 # estimate
remaining = t_end - time.time()
if remaining < 60:
break
try:
n_episodes = min(len(all_scenarios), 20)
selected = rng.sample(all_scenarios, n_episodes)
for i, scenario in enumerate(selected):
if time.time() > t_end - 30:
break
ep_rng = random.Random(SEED + int(time.time() * 1000) + i)
ep = run_episode(
cwm=cwm_factory(), scenario=scenario, params=params,
steps=STEPS_PER_EPISODE, seed=SEED + int(time.time() * 1000) + i,
rng=ep_rng, risk_gate=risk_gate,
)
# Accumulate stats
recent_pnls.append(ep["pnl_bps"])
all_pnls.append(ep["pnl_bps"])
peak_pnl = max(peak_pnl, ep["pnl_bps"])
worst_pnl = min(worst_pnl, ep["pnl_bps"])
recent_fills += ep["fills"]
recent_noops += ep["noops"]
recent_cancels += ep["cancels"]
recent_aggressive += ep["aggressive"]
recent_passive += ep["passive"]
recent_post_onlys += ep["post_onlys"]
recent_reduce_onlys += ep["reduce_onlys"]
recent_episodes += 1
recent_actions_total += ep["steps"]
for ot, c in ep["order_types"].items():
recent_ot_counts[ot] += c
for t, c in ep["tifs"].items():
recent_tif_counts[t] += c
if len(recent_pnls) > 200:
recent_pnls = recent_pnls[-200:]
# Fill quality accumulation
fq_slippage_sum += ep["avg_slippage_bps"]
fq_expected_sum += ep["avg_expected_slippage_bps"]
fq_surprise_sum += ep["slippage_surprise"]
fq_improve_sum += ep["avg_price_improvement_bps"]
fq_adverse_sum += ep["avg_adverse_bps"]
fq_value_sum += ep["avg_fill_value_score"]
fq_filled_count += ep["fills"]
fq_total_count += ep["steps"]
fq_n_reports += 1
except Exception as e:
print(f" Error: {e}", flush=True)
# CMA-ES every 20 reports
if fq_n_reports % 20 == 0 and remaining > 600:
cma_scenarios = rng.sample(all_scenarios, min(3, len(all_scenarios)))
try:
evaluator = PolicyEvaluator(cwm_factory=cwm_factory, scoring_mode="fast")
trainer = CMAESTrainer(codec=codec, evaluator=evaluator, pool=pool, workers=0)
best = trainer.train(incumbent=params, scenarios=cma_scenarios,
budget_evals=2, seed=SEED + fq_n_reports)
cma_bests.append({"cycle": fq_n_reports, "score": best.score,
"fill_value": ep.get("avg_fill_value_score", 0)})
if best.score > params.w_expected_pnl * 10:
params = best.params
except Exception as e:
print(f" CMA-ES error: {e}", flush=True)
# Periodic report with improvement tracking
elapsed = time.time() - t_start
if fq_n_reports > 0 and fq_n_reports % 10 == 0:
n = fq_n_reports
h = elapsed / 3600; m = (elapsed % 3600) / 60
avg_slip = fq_slippage_sum / n
avg_expected = fq_expected_sum / n
avg_surprise = fq_surprise_sum / n
avg_improve = fq_improve_sum / n
avg_adverse = fq_adverse_sum / n
avg_value = fq_value_sum / n
avg_fill_rate = fq_filled_count / max(fq_total_count, 1)
# Trend: compare first half to second half
fq_history.append({
"t": elapsed, "slippage": avg_slip, "expected": avg_expected,
"surprise": avg_surprise, "improvement": avg_improve,
"adverse": avg_adverse, "value": avg_value, "fill_rate": avg_fill_rate,
})
trend_slip = "IMPROVING" if len(fq_history) >= 4 and fq_history[-1]["slippage"] < fq_history[-4]["slippage"] else "STABLE"
trend_value = "IMPROVING" if len(fq_history) >= 4 and fq_history[-1]["value"] > fq_history[-4]["value"] else "STABLE"
print(f"\n [{h:.1f}h{m:.0f}m] Cycle ~{n} | "
f"slip={avg_slip:.2f}bpx({trend_slip}) | "
f"expected={avg_expected:.2f} | "
f"surprise={avg_surprise:+.2f} | "
f"improve={avg_improve:.2f} | "
f"adverse={avg_adverse:.2f} | "
f"fill_value={avg_value:.2f}({trend_value}) | "
f"fill_rate={avg_fill_rate:.1%} | "
f"agg/pass={recent_aggressive/max(recent_passive,1):.2f}", flush=True)
# Reset window
fq_slippage_sum = 0.0; fq_expected_sum = 0.0; fq_surprise_sum = 0.0
fq_improve_sum = 0.0; fq_adverse_sum = 0.0; fq_value_sum = 0.0
fq_filled_count = 0; fq_total_count = 0; fq_n_reports = 0
recent_pnls.clear()
recent_fills = recent_noops = recent_cancels = 0
recent_aggressive = recent_passive = 0
recent_post_onlys = recent_reduce_onlys = 0
recent_ot_counts.clear(); recent_tif_counts.clear()
recent_actions_total = 0; recent_episodes = 0
# Final report
elapsed = time.time() - t_start
n = len(all_pnls)
print(f"\n{'='*80}")
print(f" FINAL REPORT — 3.5H INSTRUMENTED E2E")
print(f"{'='*80}")
if n > 0:
total_actions = sum(recent_ot_counts.values()) + recent_actions_total
print(f"\n Duration: {elapsed/3600:.1f}h ({elapsed:.0f}s)")
print(f" Total episodes: {n}")
print(f"\n PERFORMANCE")
print(f" Avg PnL: {sum(all_pnls)/n:+.1f} bps")
print(f" Median PnL: {sorted(all_pnls)[n//2]:+.1f} bps")
print(f" Best: {max(all_pnls):+.1f} bps")
print(f" Worst: {min(all_pnls):+.1f} bps")
print(f" Win rate: {sum(1 for p in all_pnls if p > 0)/n*100:.1f}%")
print(f"\n FILL QUALITY (CORE)")
if fq_history:
first = fq_history[0]; last = fq_history[-1]
print(f" Avg fill value score: {last['value']:.2f}")
print(f" Fill value trend: {first['value']:.2f} -> {last['value']:.2f} ({(last['value']-first['value'])/max(abs(first['value']),0.01)*100:+.1f}%)")
print(f" Avg fill rate: {last['fill_rate']:.1%}")
print(f" Fill rate trend: {first['fill_rate']:.1%} -> {last['fill_rate']:.1%}")
print(f"\n SLIPPAGE (CORE)")
if fq_history:
print(f" Avg actual slippage: {last['slippage']:.2f} bps")
print(f" Avg expected slip: {last['expected']:.2f} bps")
print(f" Avg slippage surprise:{last['surprise']:+.2f} bps (neg=good: actual < expected)")
print(f" Slippage trend: {first['slippage']:.2f} -> {last['slippage']:.2f} ({(last['slippage']-first['slippage'])/max(abs(first['slippage']),0.01)*100:+.1f}%)")
print(f" Avg price improvement:{last['improvement']:.2f} bps")
print(f" Avg post-fill adverse:{last['adverse']:.2f} bps")
print(f"\n 1K OPPONENT SWARM")
print(f" Aggressive: {sum(r['aggressive'] for r in fq_history):.0f}")
print(f" Passive: {sum(r['passive'] for r in fq_history):.0f}")
print(f" Post-only: {sum(r['post_onlys'] for r in fq_history):.0f}")
print(f" CMA-ES: {len(cma_bests)} cycles")
print(f"{'='*80}", flush=True)
os.makedirs("malkhut/results", exist_ok=True)
report = {
"duration_s": round(elapsed, 1), "n_episodes": n,
"n_scenarios": len(all_scenarios), "n_opponents": len(SWARM),
"avg_pnl_bps": round(sum(all_pnls)/n, 1) if n else 0,
"win_rate_pct": round(sum(1 for p in all_pnls if p > 0)/n*100, 1) if n else 0,
"fq_history": fq_history, "cma_bests": cma_bests,
}
path = f"malkhut/results/e2e_35h_{int(time.time())}.json"
with open(path, "w") as f:
json.dump(report, f, indent=2)
print(f"\nReport: {path}", flush=True)
if __name__ == "__main__":
try:
main()
except Exception as e:
import traceback
print(f"\nFATAL: {e}", flush=True)
traceback.print_exc()
sys.exit(1)