dita_v2(bingx_venue): telemetry OFF the exec path — bounded non-blocking lane

Standing steer (HJ, months): non-blocking behaviour + explicit execution priority.
Telemetry is not on the ladder at all, yet it sat ON the order path, inline and able to
raise — which is how it orphaned 6 live positions.

Exec path now does exactly one thing: pack primitives, append to a bounded ring, return.
No plane write, no snapshot construction, no I/O, no lock, no raise. Snapshot build +
plane publish move to a daemon drain lane. Ring is deque(maxlen=4096): drops OLDEST on
full and counts the drops — telemetry loss is always preferable to exec backpressure,
but it stays observable.

Measured: 1000 exec-path calls against a 2s-BLOCKING plane = 3.65 ms total (3.65 us/call).
Inline, that was 2000 s of stall. Wedged lane + 50k pushes -> ring pinned at 4096,
45903 counted drops, exec path never backpressured. Healthy plane still gets 5/5.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Codex
2026-07-13 08:58:52 +02:00
parent 832c85b233
commit 40e85ab8d1

View File

@@ -13,6 +13,7 @@ import inspect
import itertools import itertools
import re import re
import threading import threading
from collections import deque
from datetime import datetime, timezone from datetime import datetime, timezone
from dataclasses import replace from dataclasses import replace
from typing import Any, Iterable, List, Optional from typing import Any, Iterable, List, Optional
@@ -282,8 +283,64 @@ class BingxVenueAdapter(VenueAdapter):
except Exception: except Exception:
pass pass
# ── telemetry lane ───────────────────────────────────────────────────────
# Telemetry is NOT on the execution priority ladder (P0 kill/emergency,
# P1 SL/ADVSL, P2 exits, P3 ENTER, P4 maintenance) — it ranks below all of
# it. It therefore must never sit ON the exec path, never block it, and
# never raise into it.
#
# The exec path does exactly one thing: append a tuple of primitives to a
# bounded ring and return. deque(maxlen=N).append is O(1), never blocks, and
# drops the OLDEST record when full — telemetry loss is always preferable to
# exec-path backpressure. Snapshot construction and the plane write happen on
# the drain lane, off the critical section entirely.
_TEL_RING_MAX = 4096
def _start_telemetry_lane(self) -> None:
if getattr(self, "_tel_thread", None) is not None:
return
self._tel_ring: Any = deque(maxlen=self._TEL_RING_MAX)
self._tel_wake = threading.Event()
self._tel_dropped = 0
thread = threading.Thread(
target=self._drain_telemetry, name="bingx-telemetry-lane", daemon=True
)
self._tel_thread = thread
thread.start()
def _drain_telemetry(self) -> None:
"""Low-priority lane: build snapshots + publish. Never touches the exec path."""
while True:
self._tel_wake.wait(timeout=0.5)
self._tel_wake.clear()
ring = getattr(self, "_tel_ring", None)
if ring is None:
continue
while True:
try:
fields = ring.popleft()
except IndexError:
break
try:
plane = self._telemetry_plane
publish = (
getattr(plane, "publish_venue", None) if plane is not None else None
)
if publish is None:
continue
publish(VenueTelemetrySnapshot(**fields))
except Exception as exc: # lane-local: never escapes to exec
import logging as _log
_log.getLogger(__name__).warning(
"FIX(bingx_venue): telemetry lane publish failed (phase=%s): %s",
fields.get("phase", "?"), exc,
)
def set_telemetry_plane(self, zinc_plane: Any | None) -> None: def set_telemetry_plane(self, zinc_plane: Any | None) -> None:
self._telemetry_plane = zinc_plane self._telemetry_plane = zinc_plane
if zinc_plane is not None:
self._start_telemetry_lane()
def _publish_telemetry( def _publish_telemetry(
self, self,
@@ -302,20 +359,19 @@ class BingxVenueAdapter(VenueAdapter):
client_order_id: str = "", client_order_id: str = "",
details: dict[str, Any] | None = None, details: dict[str, Any] | None = None,
) -> None: ) -> None:
# TOTAL: this method cannot raise. It runs INLINE on the order path — # EXEC PATH. Non-blocking and total: pack primitives, append to the ring,
# submit()/submit_async() call it immediately after the venue has already # return. No plane write, no snapshot construction, no I/O, no lock, no
# accepted the order. An exception escaping here reaches the caller's # raise. Telemetry ranks below every rung of the execution ladder and gets
# submit guard, which synthesises a REJECTED event and rolls the FSM back # dropped (oldest-first) before it is ever allowed to cost the order path
# while the venue keeps the position (orphan; 6 of them on 2026-07-13). # a microsecond.
# #
# The guard used to cover only publish(), leaving the attribute/coercion # Everything below used to run INLINE, immediately after the venue had
# block below it unprotected — which is how a signature skew orphaned live # already accepted the order — so a raise here reached the caller's submit
# positions past a defence that existed precisely to prevent it. Everything # guard, which synthesised a REJECTED and rolled the FSM back while the
# that can raise now lives inside the try. Observability never vetoes a fill. # venue kept the position (6 orphans, 2026-07-13).
try: try:
plane = self._telemetry_plane ring = getattr(self, "_tel_ring", None)
publish = getattr(plane, "publish_venue", None) if plane is not None else None if ring is None or self._telemetry_plane is None:
if publish is None:
return return
slot_id = 0 slot_id = 0
trade_id = "" trade_id = ""
@@ -335,36 +391,42 @@ class BingxVenueAdapter(VenueAdapter):
trade_id = str(order.internal_trade_id or trade_id) trade_id = str(order.internal_trade_id or trade_id)
asset = str(order.metadata.get("asset") or asset) asset = str(order.metadata.get("asset") or asset)
side = order.side or side side = order.side or side
publish( if len(ring) == ring.maxlen:
VenueTelemetrySnapshot( # Full: deque drops the oldest on append. Count it — silent
phase=phase, # telemetry loss must still be observable.
status=status, self._tel_dropped = getattr(self, "_tel_dropped", 0) + 1
venue="bingx", # Timestamp is stamped HERE (event time), not on the drain lane.
endpoint=endpoint, ring.append(
method=method, {
intent_id=intent_id, "phase": phase,
trade_id=trade_id, "status": status,
slot_id=slot_id, "venue": "bingx",
asset=asset, "endpoint": endpoint,
side=side, "method": method,
action=action, "intent_id": intent_id,
order_id=str(order_id or getattr(order, "venue_order_id", "") or ""), "trade_id": trade_id,
client_order_id=str( "slot_id": slot_id,
"asset": asset,
"side": side,
"action": action,
"order_id": str(order_id or getattr(order, "venue_order_id", "") or ""),
"client_order_id": str(
client_order_id or getattr(order, "venue_client_id", "") or "" client_order_id or getattr(order, "venue_client_id", "") or ""
), ),
venue_order_status=venue_order_status, "venue_order_status": venue_order_status,
venue_event_kind=venue_event_kind, "venue_event_kind": venue_event_kind,
message=message, "message": message,
retry_after_ms=int(retry_after_ms or 0), "retry_after_ms": int(retry_after_ms or 0),
timestamp=datetime.now(timezone.utc), "timestamp": datetime.now(timezone.utc),
details=dict(details or {}), "details": dict(details or {}),
) }
) )
self._tel_wake.set()
except Exception as exc: except Exception as exc:
import logging as _log import logging as _log
_log.getLogger(__name__).warning( _log.getLogger(__name__).warning(
"FIX(bingx_venue): telemetry publish failed (phase=%s) — swallowed, " "FIX(bingx_venue): telemetry enqueue failed (phase=%s) — swallowed, "
"order path unaffected: %s", "order path unaffected: %s",
phase, exc, phase, exc,
) )