4b52fff8dd
clock.py: mono_ns single timebase; LatencyHistogram (raw reservoir, exact nearest-rank percentiles); PlaneClock per-plane seq clocks with strict staleness budgets (age==budget not stale — MHS FIX-9 lesson); DeadlineScheduler — single-driver timer heap with EARLY-WAKE on earlier-than-head insert (the jitter-budget mechanism), isolated callbacks. harness.py: seeded deterministic event storms (sequence-hash asserted) driving the REAL Rust ExecutionKernel via the MOCK bundle; reaction latency measured producer-stamp→post-fold across the queue hop exactly as the production account stream consumes; ACCOUNT_UPDATE wallet sentinel tracks kernel k_capital so synthetic storms never trip capital_frozen; sustained throughput reported alongside the gate. V0 GATE (prod host, 50k events, 5k concurrent deadlines, burst 8/12ms ≈ 667 ev/s offered): venue_event_reaction p99 7.19ms (<10ms budget), deadline_jitter p99 4.86ms (<25ms), zero early fires. Capacity artifacts: the 32/1ms and 16/12ms storms (archived reports) show intra-burst queueing dominating beyond ~1.3k ev/s offered at ~0.33ms/fold. 17 unit tests. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
113 lines
3.7 KiB
Python
113 lines
3.7 KiB
Python
"""V0: DeadlineScheduler — ordering, cancellation, early-wake, jitter."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import asyncio
|
|
import random
|
|
import sys
|
|
|
|
sys.path.insert(0, "/mnt/dolphinng5_predict")
|
|
|
|
import pytest
|
|
|
|
from prod.clean_arch.violet.clock import DeadlineScheduler, LatencyHistogram, mono_ns
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_deadlines_fire_in_due_order_after_shuffled_insert():
|
|
fired: list[int] = []
|
|
sched = DeadlineScheduler()
|
|
sched.start()
|
|
base = mono_ns()
|
|
order = list(range(1_000))
|
|
random.Random(7).shuffle(order)
|
|
for k in order:
|
|
due = base + (k + 1) * 1_000_000 # 1ms apart, shuffled insertion
|
|
sched.schedule_at(due, (lambda kk=k: fired.append(kk)))
|
|
await asyncio.sleep(1.3)
|
|
await sched.stop()
|
|
assert len(fired) == 1_000
|
|
assert fired == sorted(fired)
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_cancellation_prevents_fire():
|
|
fired: list[str] = []
|
|
sched = DeadlineScheduler()
|
|
sched.start()
|
|
keep = sched.schedule_in(30, lambda: fired.append("keep"))
|
|
kill = sched.schedule_in(30, lambda: fired.append("kill"))
|
|
kill.cancel()
|
|
await asyncio.sleep(0.12)
|
|
await sched.stop()
|
|
assert fired == ["keep"]
|
|
assert kill.cancelled and not kill.fired
|
|
assert keep.fired
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_early_wake_on_earlier_insert():
|
|
"""A deadline inserted BELOW the current head must fire on time even when
|
|
the driver was sleeping toward a much later head — the early-wake path."""
|
|
sched = DeadlineScheduler(max_sleep_ms=5_000) # tick alone would be too slow
|
|
jit = LatencyHistogram("j")
|
|
sched.jitter_hist = jit
|
|
sched.start()
|
|
fired_at: list[int] = []
|
|
sched.schedule_in(4_000, lambda: None) # far head; driver sleeps
|
|
await asyncio.sleep(0.05) # driver now parked
|
|
due = mono_ns() + 20_000_000 # 20ms from now
|
|
sched.schedule_at(due, lambda: fired_at.append(mono_ns()))
|
|
await asyncio.sleep(0.15)
|
|
await sched.stop()
|
|
assert fired_at, "early deadline never fired — early-wake broken"
|
|
lateness_ms = (fired_at[0] - due) / 1e6
|
|
assert lateness_ms < 25.0, f"early-wake too late: {lateness_ms:.1f}ms"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_no_early_fires_and_jitter_budget_under_cpu_noise():
|
|
jit = LatencyHistogram("jitter")
|
|
sched = DeadlineScheduler(jitter_hist=jit)
|
|
sched.start()
|
|
early = [0]
|
|
rng = random.Random(11)
|
|
dues = []
|
|
for _ in range(400):
|
|
due = mono_ns() + int(rng.uniform(10, 800) * 1e6)
|
|
dues.append(due)
|
|
sched.schedule_at(due, (lambda d=due: early.__setitem__(0, early[0] + 1) if mono_ns() < d else None))
|
|
|
|
async def cpu_noise():
|
|
end = mono_ns() + int(0.9e9)
|
|
while mono_ns() < end:
|
|
sum(i * i for i in range(2_000))
|
|
await asyncio.sleep(0)
|
|
|
|
noise = asyncio.create_task(cpu_noise())
|
|
await asyncio.sleep(1.0)
|
|
noise.cancel()
|
|
await sched.stop()
|
|
assert early[0] == 0, "deadline fired EARLY"
|
|
assert sched.fired_count == 400
|
|
p99_ms = jit.percentile_ns(0.99) / 1e6
|
|
assert p99_ms < 25.0, f"jitter p99 {p99_ms:.2f}ms exceeds 25ms budget"
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_callback_exception_does_not_kill_driver():
|
|
errors: list[str] = []
|
|
sched = DeadlineScheduler(on_error=lambda dl, e: errors.append(str(e)))
|
|
sched.start()
|
|
fired: list[str] = []
|
|
sched.schedule_in(10, lambda: (_ for _ in ()).throw(RuntimeError("boom")))
|
|
sched.schedule_in(40, lambda: fired.append("after"))
|
|
await asyncio.sleep(0.12)
|
|
await sched.stop()
|
|
assert errors and "boom" in errors[0]
|
|
assert fired == ["after"], "driver died after callback exception"
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(pytest.main([__file__, "-v"]))
|