113 lines
3.7 KiB
Python
113 lines
3.7 KiB
Python
|
|
"""V0: DeadlineScheduler — ordering, cancellation, early-wake, jitter."""
|
||
|
|
|
||
|
|
from __future__ import annotations
|
||
|
|
|
||
|
|
import asyncio
|
||
|
|
import random
|
||
|
|
import sys
|
||
|
|
|
||
|
|
sys.path.insert(0, "/mnt/dolphinng5_predict")
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
|
||
|
|
from prod.clean_arch.violet.clock import DeadlineScheduler, LatencyHistogram, mono_ns
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_deadlines_fire_in_due_order_after_shuffled_insert():
|
||
|
|
fired: list[int] = []
|
||
|
|
sched = DeadlineScheduler()
|
||
|
|
sched.start()
|
||
|
|
base = mono_ns()
|
||
|
|
order = list(range(1_000))
|
||
|
|
random.Random(7).shuffle(order)
|
||
|
|
for k in order:
|
||
|
|
due = base + (k + 1) * 1_000_000 # 1ms apart, shuffled insertion
|
||
|
|
sched.schedule_at(due, (lambda kk=k: fired.append(kk)))
|
||
|
|
await asyncio.sleep(1.3)
|
||
|
|
await sched.stop()
|
||
|
|
assert len(fired) == 1_000
|
||
|
|
assert fired == sorted(fired)
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_cancellation_prevents_fire():
|
||
|
|
fired: list[str] = []
|
||
|
|
sched = DeadlineScheduler()
|
||
|
|
sched.start()
|
||
|
|
keep = sched.schedule_in(30, lambda: fired.append("keep"))
|
||
|
|
kill = sched.schedule_in(30, lambda: fired.append("kill"))
|
||
|
|
kill.cancel()
|
||
|
|
await asyncio.sleep(0.12)
|
||
|
|
await sched.stop()
|
||
|
|
assert fired == ["keep"]
|
||
|
|
assert kill.cancelled and not kill.fired
|
||
|
|
assert keep.fired
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_early_wake_on_earlier_insert():
|
||
|
|
"""A deadline inserted BELOW the current head must fire on time even when
|
||
|
|
the driver was sleeping toward a much later head — the early-wake path."""
|
||
|
|
sched = DeadlineScheduler(max_sleep_ms=5_000) # tick alone would be too slow
|
||
|
|
jit = LatencyHistogram("j")
|
||
|
|
sched.jitter_hist = jit
|
||
|
|
sched.start()
|
||
|
|
fired_at: list[int] = []
|
||
|
|
sched.schedule_in(4_000, lambda: None) # far head; driver sleeps
|
||
|
|
await asyncio.sleep(0.05) # driver now parked
|
||
|
|
due = mono_ns() + 20_000_000 # 20ms from now
|
||
|
|
sched.schedule_at(due, lambda: fired_at.append(mono_ns()))
|
||
|
|
await asyncio.sleep(0.15)
|
||
|
|
await sched.stop()
|
||
|
|
assert fired_at, "early deadline never fired — early-wake broken"
|
||
|
|
lateness_ms = (fired_at[0] - due) / 1e6
|
||
|
|
assert lateness_ms < 25.0, f"early-wake too late: {lateness_ms:.1f}ms"
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_no_early_fires_and_jitter_budget_under_cpu_noise():
|
||
|
|
jit = LatencyHistogram("jitter")
|
||
|
|
sched = DeadlineScheduler(jitter_hist=jit)
|
||
|
|
sched.start()
|
||
|
|
early = [0]
|
||
|
|
rng = random.Random(11)
|
||
|
|
dues = []
|
||
|
|
for _ in range(400):
|
||
|
|
due = mono_ns() + int(rng.uniform(10, 800) * 1e6)
|
||
|
|
dues.append(due)
|
||
|
|
sched.schedule_at(due, (lambda d=due: early.__setitem__(0, early[0] + 1) if mono_ns() < d else None))
|
||
|
|
|
||
|
|
async def cpu_noise():
|
||
|
|
end = mono_ns() + int(0.9e9)
|
||
|
|
while mono_ns() < end:
|
||
|
|
sum(i * i for i in range(2_000))
|
||
|
|
await asyncio.sleep(0)
|
||
|
|
|
||
|
|
noise = asyncio.create_task(cpu_noise())
|
||
|
|
await asyncio.sleep(1.0)
|
||
|
|
noise.cancel()
|
||
|
|
await sched.stop()
|
||
|
|
assert early[0] == 0, "deadline fired EARLY"
|
||
|
|
assert sched.fired_count == 400
|
||
|
|
p99_ms = jit.percentile_ns(0.99) / 1e6
|
||
|
|
assert p99_ms < 25.0, f"jitter p99 {p99_ms:.2f}ms exceeds 25ms budget"
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_callback_exception_does_not_kill_driver():
|
||
|
|
errors: list[str] = []
|
||
|
|
sched = DeadlineScheduler(on_error=lambda dl, e: errors.append(str(e)))
|
||
|
|
sched.start()
|
||
|
|
fired: list[str] = []
|
||
|
|
sched.schedule_in(10, lambda: (_ for _ in ()).throw(RuntimeError("boom")))
|
||
|
|
sched.schedule_in(40, lambda: fired.append("after"))
|
||
|
|
await asyncio.sleep(0.12)
|
||
|
|
await sched.stop()
|
||
|
|
assert errors and "boom" in errors[0]
|
||
|
|
assert fired == ["after"], "driver died after callback exception"
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
raise SystemExit(pytest.main([__file__, "-v"]))
|