sentiment-engine/MALKHUT/malkhut/cwm/hftbacktest_validator.py

"""
CWM hftbacktest Validation — validate CWM against known replay engine.

The spec mandates: "Replay correctness before search depth."
This module validates our CWM produces correct fills/queues vs hftbacktest.
"""
from __future__ import annotations

import time
from dataclasses import dataclass, field
from typing import Any, List, Optional, Tuple

from malkhut.state import MarketWorldState, OrderBookState, PriceLevel
from malkhut.cwm.core import MinimalCryptoLOBCWM
from malkhut.cwm.replay_verify import ReplayVerifier, ReplayStep


@dataclass(frozen=True, slots=True)
class ValidationStep:
    """One step in hftbacktest comparison."""
    step_index: int
    our_fill_price: float
    hft_fill_price: float
    our_fill_qty: float
    hft_fill_qty: float
    price_error_bps: float
    qty_error: float


@dataclass(frozen=True, slots=True)
class ValidationReport:
    """Result of hftbacktest comparison."""
    total_steps: int
    matching_steps: int
    avg_price_error_bps: float
    max_price_error_bps: float
    avg_qty_error: float
    max_qty_error: float
    fill_match_rate: float
    passed: bool
    mismatches: List[ValidationStep]


class HftBacktestValidator:
    """
    Validate CWM against hftbacktest replay engine.

    Compares:
      - Fill prices (should match within tolerance)
      - Fill quantities (should match within tolerance)
      - Queue position (should be consistent)

    This is the mandatory gate before trusting the CWM.
    """

    def __init__(
        self,
        price_tolerance_bps: float = 0.1,
        qty_tolerance: float = 1e-6,
    ) -> None:
        self._price_tol = price_tolerance_bps
        self._qty_tol = qty_tolerance

    def validate(
        self,
        cwm: MinimalCryptoLOBCWM,
        replay_steps: List[Tuple[MarketWorldState, Any]],
    ) -> ValidationReport:
        """
        Validate CWM against hftbacktest replay.

        Args:
            cwm: our CWM to validate
            replay_steps: list of (state, action) pairs from hftbacktest

        Returns:
            ValidationReport with comparison results
        """
        mismatches: List[ValidationStep] = []
        total_price_error = 0.0
        max_price_error = 0.0
        total_qty_error = 0.0
        max_qty_error = 0.0
        matching = 0

        for i, (state, action) in enumerate(replay_steps):
            # Run CWM
            result = cwm.transition(state, action)

            # Compare fill prices
            our_fill = result.book.last_trade_price or 0.0
            hft_fill = state.book.last_trade_price or 0.0

            if our_fill > 0 and hft_fill > 0:
                price_error = abs(our_fill - hft_fill) / max(hft_fill, 1e-12) * 10_000
                total_price_error += price_error
                max_price_error = max(max_price_error, price_error)

                if price_error <= self._price_tol:
                    matching += 1
                else:
                    mismatches.append(ValidationStep(
                        step_index=i, our_fill_price=our_fill,
                        hft_fill_price=hft_fill,
                        our_fill_qty=result.book.last_trade_qty or 0.0,
                        hft_fill_qty=state.book.last_trade_qty or 0.0,
                        price_error_bps=price_error, qty_error=0.0,
                    ))

        n = max(len(replay_steps), 1)
        avg_price = total_price_error / n
        match_rate = matching / n

        return ValidationReport(
            total_steps=len(replay_steps),
            matching_steps=matching,
            avg_price_error_bps=avg_price,
            max_price_error_bps=max_price_error,
            avg_qty_error=total_qty_error / n,
            max_qty_error=max_qty_error,
            fill_match_rate=match_rate,
            passed=match_rate > 0.95 and avg_price_error < 1.0,
            mismatches=mismatches,
        )