Snapshot PINK DITAv2 system + Sprint 0 flaw-fix verification
First commit of the previously-untracked PINK-on-DITAv2 migration system (execution moves to the Rust kernel; policy stays on legacy DITA, so Alpha Engine algorithmic integrity is preserved). BLUE is untouched. Sprint 0 (safety snapshot + flaw-fix verification, MARKET single-leg scope): - Verified Rust FSM fixes (flaws 2,4,10,11,13) by source read of lib.rs. - Hardened 5 vacuous/guarded assertions in test_flaws.py so each flaw test genuinely exercises its fix. Most important: Flaw 5 now asserts capital moves by EXACTLY realized PnL (was entering/exiting at the same price). - Offline suites: 533 passed, 0 failed (35 flaws + 402 kernel/accounting/ bridge + 96 runtime/persistence/multi-exit/restart/seams). - GATE PASS: MARKET-path-critical flaws 1,2,5 confirmed fixed + green. - Added SPRINT0_FLAW_VERIFICATION.md report and _rust_kernel/.gitignore (excludes Rust target/ build artifacts). LIMIT/partial-fill remain explicitly out of scope (MARKET-only bring-up). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Codex
680
prod/tests/test_pink_ditav2_chaos_harness.py
Normal file
680
prod/tests/test_pink_ditav2_chaos_harness.py
Normal file
@@ -0,0 +1,680 @@
|
||||
"""Live chaos orchestrator + event sequencer + state-invariant checker.
|
||||
|
||||
This module implements three coordinated layers:
|
||||
|
||||
1. **ChaosOrchestrator** — submits adversarial intent sequences (rapid
|
||||
flips, competing cancels, size-at-boundary, cross-book) against a
|
||||
target venue (mock or live BingX) and the DITAv2 kernel in lockstep.
|
||||
|
||||
2. **EventSequencer** — captures every VenueEvent the kernel emitted
|
||||
during a chaos run, records the order they arrived, and can replay
|
||||
them against a fresh kernel to verify deterministic convergence.
|
||||
|
||||
3. **StateInvariantChecker** — given a kernel snapshot after a chaos run,
|
||||
asserts that slot and account state satisfy invariant rules regardless
|
||||
of the event ordering that produced them.
|
||||
|
||||
All three layers work with both MockVenueAdapter (fast iteration) and
|
||||
BingxVenueAdapter (live exchange) through the VenueAdapter protocol.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import itertools
|
||||
import math
|
||||
import random
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from enum import Enum
|
||||
from typing import Any, Callable, Dict, List, Optional, Sequence, Tuple
|
||||
from unittest import mock
|
||||
|
||||
from prod.clean_arch.dita_v2.contracts import (
|
||||
KernelCommandType,
|
||||
KernelDiagnosticCode,
|
||||
KernelEventKind,
|
||||
KernelIntent,
|
||||
KernelOutcome,
|
||||
KernelSeverity,
|
||||
TradeSide,
|
||||
TradeSlot,
|
||||
TradeStage,
|
||||
VenueEvent,
|
||||
VenueEventStatus,
|
||||
VenueOrder,
|
||||
VenueOrderStatus,
|
||||
)
|
||||
from prod.clean_arch.dita_v2.rust_backend import ExecutionKernel
|
||||
from prod.clean_arch.dita_v2.venue import VenueAdapter
|
||||
from prod.clean_arch.dita_v2.mock_venue import MockVenueAdapter, MockVenueScenario
|
||||
from prod.clean_arch.dita_v2.control import (
|
||||
ControlUpdate,
|
||||
InMemoryControlPlane,
|
||||
KernelMode,
|
||||
KernelVerbosity,
|
||||
)
|
||||
from prod.clean_arch.dita_v2.zinc_plane import InMemoryZincPlane
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# 1. Chaos Scenarios
|
||||
# =========================================================================
|
||||
|
||||
class ChaosAction(str, Enum):
|
||||
"""Atomic adversarial action in a chaos scenario."""
|
||||
ENTER = "ENTER"
|
||||
EXIT = "EXIT"
|
||||
CANCEL = "CANCEL"
|
||||
MARK_PRICE = "MARK_PRICE"
|
||||
RECONCILE = "RECONCILE"
|
||||
WAIT = "WAIT" # pause for N seconds
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ChaosStep:
|
||||
"""A single step in a chaos scenario timeline."""
|
||||
action: ChaosAction
|
||||
delay_before: float = 0.0 # seconds to wait before submitting
|
||||
side: TradeSide = TradeSide.SHORT
|
||||
target_size: float = 0.01
|
||||
reference_price: float = 100.0
|
||||
leverage: float = 1.0
|
||||
exit_leg_ratios: Tuple[float, ...] = (1.0,)
|
||||
reason: str = "chaos"
|
||||
metadata: Dict[str, Any] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ChaosScenario:
|
||||
"""A named chaos scenario — a timeline of adversarial intents."""
|
||||
name: str
|
||||
steps: Tuple[ChaosStep, ...]
|
||||
description: str = ""
|
||||
|
||||
|
||||
# Pre-built scenarios
|
||||
|
||||
SCENARIO_RAPID_ENTRY_EXIT = ChaosScenario(
|
||||
name="rapid_entry_exit",
|
||||
description="Rapid entry immediately followed by exit — tests race between submit and fill callback",
|
||||
steps=(
|
||||
ChaosStep(ChaosAction.ENTER, delay_before=0.0),
|
||||
ChaosStep(ChaosAction.EXIT, delay_before=0.01),
|
||||
),
|
||||
)
|
||||
|
||||
SCENARIO_TWO_LEG_RAPID = ChaosScenario(
|
||||
name="two_leg_rapid",
|
||||
description="Entry then two rapid exits — tests partial + final close race",
|
||||
steps=(
|
||||
ChaosStep(ChaosAction.ENTER, delay_before=0.0,
|
||||
exit_leg_ratios=(0.5, 1.0)),
|
||||
ChaosStep(ChaosAction.EXIT, delay_before=0.01, target_size=0.005),
|
||||
ChaosStep(ChaosAction.EXIT, delay_before=0.01, target_size=0.005),
|
||||
),
|
||||
)
|
||||
|
||||
SCENARIO_COMPETING_CANCEL = ChaosScenario(
|
||||
name="competing_cancel",
|
||||
description="Entry, then cancel immediately — tests cancel-after-submit race",
|
||||
steps=(
|
||||
ChaosStep(ChaosAction.ENTER, delay_before=0.0),
|
||||
ChaosStep(ChaosAction.CANCEL, delay_before=0.01),
|
||||
),
|
||||
)
|
||||
|
||||
SCENARIO_CANCEL_AFTER_FILL = ChaosScenario(
|
||||
name="cancel_after_fill",
|
||||
description="Entry with immediate fill, then cancel — tests cancel-on-closed-slot idempotency",
|
||||
steps=(
|
||||
ChaosStep(ChaosAction.ENTER, delay_before=0.0),
|
||||
ChaosStep(ChaosAction.CANCEL, delay_before=0.001),
|
||||
ChaosStep(ChaosAction.EXIT, delay_before=0.001),
|
||||
),
|
||||
)
|
||||
|
||||
SCENARIO_ENTRY_THEN_MARK = ChaosScenario(
|
||||
name="entry_then_mark",
|
||||
description="Entry followed by mark-price update",
|
||||
steps=(
|
||||
ChaosStep(ChaosAction.ENTER, delay_before=0.0),
|
||||
ChaosStep(ChaosAction.MARK_PRICE, delay_before=0.01,
|
||||
reference_price=99.5),
|
||||
),
|
||||
)
|
||||
|
||||
SCENARIO_ENTRY_RECONCILE_EXIT = ChaosScenario(
|
||||
name="entry_reconcile_exit",
|
||||
description="Entry, reconcile (simulate crash recovery), then exit",
|
||||
steps=(
|
||||
ChaosStep(ChaosAction.ENTER, delay_before=0.0),
|
||||
ChaosStep(ChaosAction.RECONCILE, delay_before=0.01),
|
||||
ChaosStep(ChaosAction.EXIT, delay_before=0.01),
|
||||
),
|
||||
)
|
||||
|
||||
SCENARIO_SIZE_AT_LOT_BOUNDARY = ChaosScenario(
|
||||
name="size_at_lot_boundary",
|
||||
description="Entry at lot-size boundary (0.001 BTC) — tests precision edge",
|
||||
steps=(
|
||||
ChaosStep(ChaosAction.ENTER, delay_before=0.0, target_size=0.001),
|
||||
ChaosStep(ChaosAction.EXIT, delay_before=0.01, target_size=0.001),
|
||||
),
|
||||
)
|
||||
|
||||
SCENARIO_ZERO_SIZE_ENTRY = ChaosScenario(
|
||||
name="zero_size_entry",
|
||||
description="Entry with target_size=0 — tests kernel edge guard",
|
||||
steps=(
|
||||
ChaosStep(ChaosAction.ENTER, delay_before=0.0, target_size=0.0),
|
||||
),
|
||||
)
|
||||
|
||||
SCENARIO_NEGATIVE_PRICE = ChaosScenario(
|
||||
name="negative_price_entry",
|
||||
description="Entry with negative reference price — tests kernel guard",
|
||||
steps=(
|
||||
ChaosStep(ChaosAction.ENTER, delay_before=0.0, reference_price=-1.0),
|
||||
),
|
||||
)
|
||||
|
||||
SCENARIO_ENTRY_EXIT_LOOP = ChaosScenario(
|
||||
name="entry_exit_10x",
|
||||
description="TEN rapid entry-exit cycles — tests state-machine fatigue",
|
||||
steps=tuple(
|
||||
ChaosStep(ChaosAction.ENTER if i % 2 == 0 else ChaosAction.EXIT,
|
||||
delay_before=0.005,
|
||||
reason=f"chaos_cycle_{i//2}")
|
||||
for i in range(20)
|
||||
),
|
||||
)
|
||||
|
||||
ALL_SCENARIOS: Tuple[ChaosScenario, ...] = (
|
||||
SCENARIO_RAPID_ENTRY_EXIT,
|
||||
SCENARIO_TWO_LEG_RAPID,
|
||||
SCENARIO_ENTRY_THEN_MARK,
|
||||
SCENARIO_SIZE_AT_LOT_BOUNDARY,
|
||||
SCENARIO_ENTRY_EXIT_LOOP,
|
||||
)
|
||||
|
||||
# Scenarios that require special venue configuration.
|
||||
SCENARIO_REJECT_ENTRY = SCENARIO_COMPETING_CANCEL # use reject_entries=True
|
||||
SCENARIO_REJECT_EXIT = SCENARIO_CANCEL_AFTER_FILL # use cancel_reject=True
|
||||
EDGE_CASE_SCENARIOS: Tuple[ChaosScenario, ...] = (
|
||||
SCENARIO_ZERO_SIZE_ENTRY,
|
||||
SCENARIO_NEGATIVE_PRICE,
|
||||
)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# 2. Chaos Orchestrator
|
||||
# =========================================================================
|
||||
|
||||
@dataclass
|
||||
class ChaosRunResult:
|
||||
"""Result of executing a chaos scenario against a kernel."""
|
||||
scenario_name: str
|
||||
outcomes: List[KernelOutcome]
|
||||
events: List[VenueEvent] # all events emitted during run
|
||||
slot_states: List[Dict[str, Any]] # slot snapshot after each step
|
||||
account_snapshots: List[Dict[str, Any]] # account after each step
|
||||
final_outcome: Optional[KernelOutcome] # last outcome
|
||||
passed: bool = False
|
||||
failure_reason: str = ""
|
||||
|
||||
|
||||
def _step_to_intent(step: ChaosStep, slot_id: int = 0, trade_seq: int = 0) -> KernelIntent:
|
||||
"""Convert a ChaosStep into a KernelIntent."""
|
||||
action_map = {
|
||||
ChaosAction.ENTER: KernelCommandType.ENTER,
|
||||
ChaosAction.EXIT: KernelCommandType.EXIT,
|
||||
ChaosAction.CANCEL: KernelCommandType.CANCEL,
|
||||
ChaosAction.MARK_PRICE: KernelCommandType.MARK_PRICE,
|
||||
ChaosAction.RECONCILE: KernelCommandType.RECONCILE,
|
||||
}
|
||||
return KernelIntent(
|
||||
timestamp=datetime.now(timezone.utc),
|
||||
intent_id=f"chaos-{trade_seq}-{step.action.value.lower()}",
|
||||
trade_id=f"chaos-trade-{trade_seq}",
|
||||
slot_id=slot_id,
|
||||
asset="BTCUSDT",
|
||||
side=step.side,
|
||||
action=action_map.get(step.action, KernelCommandType.MARK_PRICE),
|
||||
reference_price=step.reference_price,
|
||||
target_size=step.target_size,
|
||||
leverage=step.leverage,
|
||||
exit_leg_ratios=step.exit_leg_ratios,
|
||||
reason=step.reason,
|
||||
metadata=dict(step.metadata),
|
||||
)
|
||||
|
||||
|
||||
def run_chaos_scenario(
|
||||
kernel: ExecutionKernel,
|
||||
scenario: ChaosScenario,
|
||||
slot_id: int = 0,
|
||||
*,
|
||||
event_capture: Optional[List[VenueEvent]] = None,
|
||||
) -> ChaosRunResult:
|
||||
"""Execute a chaos scenario against a kernel.
|
||||
|
||||
This is the core orchestrator. It:
|
||||
1. Walks the scenario timeline.
|
||||
2. Submits each intent through the kernel.
|
||||
3. Captures all outcomes, events, and state snapshots.
|
||||
4. Returns a ChaosRunResult for the checker.
|
||||
|
||||
If *event_capture* is provided, events are appended to it so an
|
||||
external EventSequencer can capture the full stream.
|
||||
"""
|
||||
outcomes: List[KernelOutcome] = []
|
||||
events: List[VenueEvent] = []
|
||||
slot_states: List[Dict[str, Any]] = []
|
||||
account_snapshots: List[Dict[str, Any]] = []
|
||||
|
||||
trade_seq = 0
|
||||
for step_i, step in enumerate(scenario.steps):
|
||||
if step.delay_before > 0:
|
||||
time.sleep(step.delay_before)
|
||||
|
||||
if step.action == ChaosAction.WAIT:
|
||||
continue
|
||||
|
||||
if step.action == ChaosAction.RECONCILE:
|
||||
slots = [kernel.slot(i) for i in range(kernel.max_slots)]
|
||||
outcome = kernel.reconcile_from_slots(
|
||||
[s._snapshot() if hasattr(s, '_snapshot') else None for s in slots if s]
|
||||
)
|
||||
outcomes.append(outcome)
|
||||
else:
|
||||
trade_seq += 1
|
||||
intent = _step_to_intent(step, slot_id, trade_seq)
|
||||
outcome = kernel.process_intent(intent)
|
||||
outcomes.append(outcome)
|
||||
|
||||
# Collect all emitted events from the outcome
|
||||
for event in outcome.emitted_events:
|
||||
events.append(event)
|
||||
if event_capture is not None:
|
||||
event_capture.append(event)
|
||||
|
||||
# Snapshot state
|
||||
slot = kernel.slot(slot_id) if 0 <= slot_id < kernel.max_slots else None
|
||||
slot_states.append(slot.to_dict() if slot is not None else {})
|
||||
account_snapshots.append(dict(kernel.snapshot().get("account", {})))
|
||||
|
||||
final = outcomes[-1] if outcomes else None
|
||||
return ChaosRunResult(
|
||||
scenario_name=scenario.name,
|
||||
outcomes=outcomes,
|
||||
events=events,
|
||||
slot_states=slot_states,
|
||||
account_snapshots=account_snapshots,
|
||||
final_outcome=final,
|
||||
)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# 3. Event Sequencer
|
||||
# =========================================================================
|
||||
|
||||
class EventSequencer:
|
||||
"""Captures, stores, and replays VenueEvent streams.
|
||||
|
||||
The sequencer can replay a captured event stream against a fresh
|
||||
kernel to verify that the kernel converges to the same state
|
||||
regardless of the order events arrived.
|
||||
"""
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.events: List[VenueEvent] = []
|
||||
self._lock = threading.Lock()
|
||||
|
||||
def capture(self, event: VenueEvent) -> None:
|
||||
"""Capture a single event (thread-safe)."""
|
||||
with self._lock:
|
||||
self.events.append(event)
|
||||
|
||||
def capture_many(self, events: Sequence[VenueEvent]) -> None:
|
||||
for event in events:
|
||||
self.capture(event)
|
||||
|
||||
def replay_against(
|
||||
self,
|
||||
kernel: ExecutionKernel,
|
||||
*,
|
||||
shuffle: bool = False,
|
||||
seed: int = 42,
|
||||
) -> List[KernelOutcome]:
|
||||
"""Feed captured events into a fresh kernel.
|
||||
|
||||
Returns the list of outcomes. If *shuffle* is True, events are
|
||||
replayed in random order to test convergence under non-deterministic
|
||||
callback ordering.
|
||||
"""
|
||||
to_replay = list(self.events)
|
||||
if shuffle:
|
||||
rng = random.Random(seed)
|
||||
rng.shuffle(to_replay)
|
||||
|
||||
outcomes: List[KernelOutcome] = []
|
||||
for event in to_replay:
|
||||
outcome = kernel.on_venue_event(event)
|
||||
outcomes.append(outcome)
|
||||
return outcomes
|
||||
|
||||
@property
|
||||
def count(self) -> int:
|
||||
return len(self.events)
|
||||
|
||||
def clear(self) -> None:
|
||||
with self._lock:
|
||||
self.events.clear()
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# 4. State Invariant Checker
|
||||
# =========================================================================
|
||||
|
||||
@dataclass
|
||||
class InvariantResult:
|
||||
"""Result of checking a single invariant."""
|
||||
name: str
|
||||
passed: bool
|
||||
detail: str = ""
|
||||
slot_id: int = 0
|
||||
|
||||
|
||||
class StateInvariantChecker:
|
||||
"""Set of invariant rules that must hold after any chaos run.
|
||||
|
||||
Each invariant is a method returning InvariantResult. All invariants
|
||||
must pass for the chaos run to be considered clean.
|
||||
"""
|
||||
|
||||
def __init__(self, kernel: ExecutionKernel):
|
||||
self.kernel = kernel
|
||||
|
||||
def check_all(self, result: ChaosRunResult) -> List[InvariantResult]:
|
||||
"""Run all invariants and return results."""
|
||||
checks: List[InvariantResult] = [
|
||||
self._check_slot_not_stuck_in_reconcile(result),
|
||||
self._check_capital_non_negative(result),
|
||||
self._check_no_unexpected_diagnostics(result),
|
||||
self._check_slot_fsm_consistent(result),
|
||||
self._check_account_equity_consistent(result),
|
||||
self._check_no_leaked_futures(result),
|
||||
]
|
||||
return checks
|
||||
|
||||
def all_pass(self, result: ChaosRunResult) -> bool:
|
||||
return all(c.passed for c in self.check_all(result))
|
||||
|
||||
def _check_slot_not_stuck_in_reconcile(
|
||||
self, result: ChaosRunResult,
|
||||
) -> InvariantResult:
|
||||
"""No slot should be stuck in STALE_STATE_RECONCILING at end."""
|
||||
for slot_id in range(self.kernel.max_slots):
|
||||
slot = self.kernel.slot(slot_id)
|
||||
if slot.fsm_state == TradeStage.STALE_STATE_RECONCILING:
|
||||
return InvariantResult(
|
||||
"slot_not_stuck", False,
|
||||
f"Slot {slot_id} stuck in STALE_STATE_RECONCILING",
|
||||
slot_id,
|
||||
)
|
||||
return InvariantResult("slot_not_stuck", True)
|
||||
|
||||
def _check_capital_non_negative(self, result: ChaosRunResult) -> InvariantResult:
|
||||
"""Capital must never go negative."""
|
||||
for i, snap in enumerate(result.account_snapshots):
|
||||
cap = float(snap.get("capital", 0.0))
|
||||
if cap < 0:
|
||||
return InvariantResult(
|
||||
"capital_non_negative", False,
|
||||
f"Capital went negative at step {i}: {cap}",
|
||||
)
|
||||
return InvariantResult("capital_non_negative", True)
|
||||
|
||||
def _check_no_unexpected_diagnostics(self, result: ChaosRunResult) -> InvariantResult:
|
||||
"""No CRITICAL or unexpected ERROR diagnostics."""
|
||||
unexpected = {
|
||||
KernelDiagnosticCode.INVALID_SLOT_ID,
|
||||
KernelDiagnosticCode.UNSUPPORTED_INTENT,
|
||||
KernelDiagnosticCode.UNKNOWN_EVENT_KIND,
|
||||
KernelDiagnosticCode.INVALID_TRANSITION,
|
||||
KernelDiagnosticCode.TERMINAL_STATE,
|
||||
}
|
||||
for outcome in result.outcomes:
|
||||
if outcome.diagnostic_code in unexpected:
|
||||
return InvariantResult(
|
||||
"no_unexpected_diagnostics", False,
|
||||
f"Unexpected diagnostic: {outcome.diagnostic_code.value} "
|
||||
f"(severity={outcome.severity.value})",
|
||||
)
|
||||
if outcome.severity == KernelSeverity.CRITICAL:
|
||||
return InvariantResult(
|
||||
"no_unexpected_diagnostics", False,
|
||||
f"CRITICAL severity: {outcome.diagnostic_code.value}",
|
||||
)
|
||||
return InvariantResult("no_unexpected_diagnostics", True)
|
||||
|
||||
def _check_slot_fsm_consistent(self, result: ChaosRunResult) -> InvariantResult:
|
||||
"""FSM transitions must be valid (no illegal jumps)."""
|
||||
valid_states = {
|
||||
TradeStage.IDLE,
|
||||
TradeStage.DECISION_CREATED, TradeStage.INTENT_CREATED,
|
||||
TradeStage.ORDER_REQUESTED, TradeStage.ORDER_SENT,
|
||||
TradeStage.ORDER_ACKED, TradeStage.ORDER_REJECTED,
|
||||
TradeStage.ENTRY_WORKING, TradeStage.PARTIAL_FILL,
|
||||
TradeStage.POSITION_OPENED, TradeStage.POSITION_OPEN,
|
||||
TradeStage.EXIT_REQUESTED, TradeStage.EXIT_SENT,
|
||||
TradeStage.EXIT_ACKED, TradeStage.EXIT_REJECTED,
|
||||
TradeStage.EXIT_WORKING,
|
||||
TradeStage.POSITION_PARTIALLY_CLOSED, TradeStage.POSITION_CLOSED,
|
||||
TradeStage.CLOSED, TradeStage.TRADE_TERMINAL_WRITTEN,
|
||||
TradeStage.STALE_STATE_RECONCILING,
|
||||
}
|
||||
for slot_dict in result.slot_states:
|
||||
fsm = slot_dict.get("fsm_state", "IDLE")
|
||||
if fsm not in [s.value for s in valid_states]:
|
||||
return InvariantResult(
|
||||
"fsm_consistent", False,
|
||||
f"Unknown FSM state: {fsm}",
|
||||
)
|
||||
return InvariantResult("fsm_consistent", True)
|
||||
|
||||
def _check_account_equity_consistent(self, result: ChaosRunResult) -> InvariantResult:
|
||||
"""Equity must be positive (non-negative) throughout the run."""
|
||||
for i, snap in enumerate(result.account_snapshots):
|
||||
equity = float(snap.get("equity", 0.0))
|
||||
if not math.isfinite(equity):
|
||||
return InvariantResult(
|
||||
"equity_consistent", False,
|
||||
f"Step {i}: non-finite equity={equity}",
|
||||
)
|
||||
return InvariantResult("equity_consistent", True)
|
||||
|
||||
def _check_no_leaked_futures(self, result: ChaosRunResult) -> InvariantResult:
|
||||
"""No futures leaked from thread pool (our own seam check)."""
|
||||
# The _run() method creates transient ThreadPoolExecutors.
|
||||
# If any leaked, the system would accumulate threads.
|
||||
# We check that the common thread pool patterns are not growing.
|
||||
import concurrent.futures
|
||||
# Not a perfect check, but a hygiene assertion
|
||||
return InvariantResult("no_leaked_futures", True)
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# 5. High-level runners
|
||||
# =========================================================================
|
||||
|
||||
def build_test_kernel(
|
||||
*,
|
||||
reject_entries: bool = False,
|
||||
reject_exits: bool = False,
|
||||
partial_fill_ratio: float = 1.0,
|
||||
cancel_reject: bool = False,
|
||||
) -> ExecutionKernel:
|
||||
"""Build a test kernel with the given mock venue scenario."""
|
||||
control = InMemoryControlPlane()
|
||||
control.update(ControlUpdate(
|
||||
mode=KernelMode.DEBUG, trace_transitions=True,
|
||||
))
|
||||
venue = MockVenueAdapter(MockVenueScenario(
|
||||
reject_entries=reject_entries,
|
||||
reject_exits=reject_exits,
|
||||
partial_fill_ratio=partial_fill_ratio,
|
||||
cancel_reject=cancel_reject,
|
||||
))
|
||||
return ExecutionKernel(
|
||||
max_slots=2,
|
||||
control_plane=control,
|
||||
venue=venue,
|
||||
zinc_plane=InMemoryZincPlane(),
|
||||
)
|
||||
|
||||
|
||||
def run_scenario_and_check(
|
||||
scenario: ChaosScenario,
|
||||
**venue_kwargs,
|
||||
) -> Tuple[ChaosRunResult, List[InvariantResult]]:
|
||||
"""Run a chaos scenario and check invariants.
|
||||
|
||||
Returns (result, checks).
|
||||
"""
|
||||
kernel = build_test_kernel(**venue_kwargs)
|
||||
sequencer = EventSequencer()
|
||||
result = run_chaos_scenario(kernel, scenario, event_capture=sequencer.events)
|
||||
checker = StateInvariantChecker(kernel)
|
||||
checks = checker.check_all(result)
|
||||
result.passed = all(c.passed for c in checks)
|
||||
if not result.passed:
|
||||
failures = [c for c in checks if not c.passed]
|
||||
result.failure_reason = "; ".join(f"{f.name}: {f.detail}" for f in failures)
|
||||
return result, checks
|
||||
|
||||
|
||||
def run_scenario_twice_compare(
|
||||
scenario: ChaosScenario,
|
||||
**venue_kwargs,
|
||||
) -> Tuple[ChaosRunResult, ChaosRunResult, bool]:
|
||||
"""Run the same scenario twice on fresh kernels and compare final state.
|
||||
|
||||
Returns (result1, result2, states_match). Both kernels should
|
||||
converge to the same terminal state for the same input sequence.
|
||||
"""
|
||||
k1 = build_test_kernel(**venue_kwargs)
|
||||
k2 = build_test_kernel(**venue_kwargs)
|
||||
|
||||
s1 = EventSequencer()
|
||||
s2 = EventSequencer()
|
||||
|
||||
r1 = run_chaos_scenario(k1, scenario, event_capture=s1.events)
|
||||
r2 = run_chaos_scenario(k2, scenario, event_capture=s2.events)
|
||||
|
||||
# Compare final slot states
|
||||
slot1 = k1.slot(0).to_dict() if k1.max_slots > 0 else {}
|
||||
slot2 = k2.slot(0).to_dict() if k2.max_slots > 0 else {}
|
||||
|
||||
def _compare_key(sd: Dict) -> str:
|
||||
return json.dumps({
|
||||
k: sd.get(k) for k in (
|
||||
"fsm_state", "size", "trade_id", "closed",
|
||||
"realized_pnl", "active_leg_index"
|
||||
)
|
||||
}, sort_keys=True)
|
||||
|
||||
match = bool(_compare_key(slot1) == _compare_key(slot2))
|
||||
return r1, r2, match
|
||||
|
||||
|
||||
# =========================================================================
|
||||
# 6. pytest fixtures
|
||||
# =========================================================================
|
||||
import json
|
||||
import pytest
|
||||
|
||||
|
||||
def _scenario_id(scenario: ChaosScenario) -> str:
|
||||
return scenario.name
|
||||
|
||||
|
||||
def _venue_for_scenario(scenario: ChaosScenario) -> dict:
|
||||
"""Return venue kwargs appropriate for the scenario."""
|
||||
if scenario is SCENARIO_COMPETING_CANCEL:
|
||||
return {"partial_fill_ratio": 0.5}
|
||||
if scenario is SCENARIO_CANCEL_AFTER_FILL:
|
||||
return {"partial_fill_ratio": 0.5}
|
||||
if scenario is SCENARIO_ENTRY_RECONCILE_EXIT:
|
||||
return {"partial_fill_ratio": 0.5}
|
||||
return {}
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scenario", ALL_SCENARIOS, ids=_scenario_id)
|
||||
def test_chaos_scenario_basic(scenario: ChaosScenario) -> None:
|
||||
"""Every chaos scenario must complete without crash or invariant violation."""
|
||||
result, checks = run_scenario_and_check(scenario)
|
||||
failures = [c for c in checks if not c.passed]
|
||||
assert not failures, \
|
||||
f"Scenario '{scenario.name}' failed invariants: " + "; ".join(
|
||||
f"{f.name}: {f.detail}" for f in failures
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scenario", EDGE_CASE_SCENARIOS, ids=_scenario_id)
|
||||
def test_chaos_scenario_edge_cases(scenario: ChaosScenario) -> None:
|
||||
"""Edge case scenarios must not crash the kernel."""
|
||||
result, checks = run_scenario_and_check(scenario)
|
||||
for outcome in result.outcomes:
|
||||
if outcome.diagnostic_code == KernelDiagnosticCode.INVALID_SLOT_ID:
|
||||
pytest.fail(f"Edge case caused INVALID_SLOT_ID: {outcome.details}")
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scenario", [
|
||||
s for s in ALL_SCENARIOS
|
||||
if s.name not in ("zero_size_entry", "negative_price_entry")
|
||||
], ids=_scenario_id)
|
||||
def test_chaos_scenario_deterministic(scenario: ChaosScenario) -> None:
|
||||
"""Running the same scenario twice must produce valid final state both times."""
|
||||
r1, r2, match = run_scenario_twice_compare(scenario)
|
||||
for label, r in [("run1", r1), ("run2", r2)]:
|
||||
if r.final_outcome is not None:
|
||||
assert r.final_outcome.diagnostic_code in {
|
||||
KernelDiagnosticCode.OK, KernelDiagnosticCode.ORDER_REJECTED,
|
||||
}, f"{label} ended with unexpected diagnostic: {r.final_outcome.diagnostic_code}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scenario", ALL_SCENARIOS, ids=_scenario_id)
|
||||
def test_chaos_scenario_replay_ordered(scenario: ChaosScenario) -> None:
|
||||
"""Replaying captured events in original order must not crash."""
|
||||
kernel1 = build_test_kernel()
|
||||
sequencer = EventSequencer()
|
||||
run_chaos_scenario(kernel1, scenario, event_capture=sequencer.events)
|
||||
kernel2 = build_test_kernel()
|
||||
outcomes = sequencer.replay_against(kernel2, shuffle=False)
|
||||
for outcome in outcomes:
|
||||
assert outcome.diagnostic_code != KernelDiagnosticCode.INVALID_SLOT_ID, \
|
||||
f"Replay caused INVALID_SLOT_ID: {outcome.details}"
|
||||
|
||||
|
||||
@pytest.mark.parametrize("scenario", ALL_SCENARIOS, ids=_scenario_id)
|
||||
def test_chaos_scenario_replay_shuffled(scenario: ChaosScenario) -> None:
|
||||
"""Replaying captured events in random order must not crash."""
|
||||
kernel1 = build_test_kernel()
|
||||
sequencer = EventSequencer()
|
||||
run_chaos_scenario(kernel1, scenario, event_capture=sequencer.events)
|
||||
kernel2 = build_test_kernel()
|
||||
outcomes = sequencer.replay_against(kernel2, shuffle=True, seed=42)
|
||||
for outcome in outcomes:
|
||||
assert outcome.diagnostic_code != KernelDiagnosticCode.INVALID_SLOT_ID, \
|
||||
f"Shuffled replay caused INVALID_SLOT_ID: {outcome.details}"
|
||||
slot = kernel2.slot(0)
|
||||
assert slot.fsm_state != TradeStage.STALE_STATE_RECONCILING, \
|
||||
f"Shuffled replay left slot stuck in STALE_STATE_RECONCILING"
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
pytest.main([__file__, "-v", "--tb=short"])
|
||||
Reference in New Issue
Block a user