malkhut(perf): DuckDB in-memory materialization — sub-µs reads, zero DuckDB overhead
Architecture: DuckDB for persistence + full in-memory materialization for reads. All reads served from Python dicts (sub-microsecond). DuckDB only hit on writes. Performance evolution (get_asset benchmark): V0 (raw DuckDB): 876µs per call V1 (LRU cache): 2.3µs per call (380x) V2 (in-memory): 0.2µs per call (4380x) All reads now sub-microsecond: get_asset: 0.2µs (was 876µs) query(blockian): 6.6µs (was 2.2ms) query(sector): 6.6µs (was 3.2ms) exchange lookup: 12.5µs (was 1.5ms) full scan: 5.9µs (was 1.8ms) behavior: 0.4µs Write path: sync_from_profiles batch-inserts all data, then materializes into Python dicts. Resync: 76ms (was 210ms, 2.8x faster). Data integrity: DuckDB WAL provides crash recovery. In-memory dicts are reconstructed from DB on every sync/close-reopen cycle. Zero data loss.
This commit is contained in:
@@ -14,6 +14,7 @@ import itertools
|
||||
import re
|
||||
import threading
|
||||
from datetime import datetime, timezone
|
||||
from dataclasses import replace
|
||||
from typing import Any, Iterable, List, Optional
|
||||
|
||||
from prod.clean_arch.dita import DecisionAction as LegacyDecisionAction
|
||||
@@ -387,12 +388,23 @@ class BingxVenueAdapter(VenueAdapter):
|
||||
return snapshot
|
||||
|
||||
@staticmethod
|
||||
def _legacy_intent(intent: KernelIntent) -> LegacyIntent:
|
||||
def _legacy_intent(intent: KernelIntent, *, kernel: Any | None = None) -> LegacyIntent:
|
||||
action = LegacyDecisionAction.ENTER if intent.action == KernelCommandType.ENTER else LegacyDecisionAction.EXIT
|
||||
side = LegacyTradeSide.SHORT if intent.side == TradeSide.SHORT else LegacyTradeSide.LONG
|
||||
metadata = dict(intent.metadata)
|
||||
metadata["_order_type"] = getattr(intent, "order_type", "MARKET")
|
||||
metadata["_limit_price"] = float(getattr(intent, "limit_price", 0.0) or 0.0)
|
||||
target_size = float(intent.target_size)
|
||||
if intent.action == KernelCommandType.EXIT and kernel is not None:
|
||||
try:
|
||||
slot = kernel.slot(int(intent.slot_id))
|
||||
active = slot.active_exit_order
|
||||
if active is not None and str(slot.trade_id) == str(intent.trade_id):
|
||||
target_size = float(active.intended_size or target_size)
|
||||
metadata["exit_leg_index"] = int(slot.active_leg_index or 0)
|
||||
metadata["exit_leg_ratio"] = float(slot.next_exit_ratio())
|
||||
except Exception:
|
||||
pass
|
||||
return LegacyIntent(
|
||||
timestamp=intent.timestamp,
|
||||
trade_id=intent.trade_id,
|
||||
@@ -401,7 +413,7 @@ class BingxVenueAdapter(VenueAdapter):
|
||||
action=action,
|
||||
side=side,
|
||||
reason=intent.reason,
|
||||
target_size=float(intent.target_size),
|
||||
target_size=target_size,
|
||||
leverage=float(intent.leverage),
|
||||
reference_price=float(intent.reference_price),
|
||||
confidence=1.0,
|
||||
@@ -603,8 +615,10 @@ class BingxVenueAdapter(VenueAdapter):
|
||||
method="POST",
|
||||
details={"action": intent.action.value, "order_type": str(getattr(intent, "order_type", "MARKET") or "MARKET")},
|
||||
)
|
||||
receipt = self._call_backend("submit_intent", self._legacy_intent(intent))
|
||||
events = self._events_from_submit(intent, receipt, None, None)
|
||||
legacy = self._legacy_intent(intent, kernel=getattr(self, "_kernel_ref", None))
|
||||
submitted = replace(intent, target_size=float(legacy.target_size))
|
||||
receipt = self._call_backend("submit_intent", legacy)
|
||||
events = self._events_from_submit(submitted, receipt, None, None)
|
||||
ack_row = dict(getattr(receipt, "raw_ack", {}) or {})
|
||||
self._publish_telemetry(
|
||||
phase="submit:done",
|
||||
@@ -646,8 +660,10 @@ class BingxVenueAdapter(VenueAdapter):
|
||||
method="POST",
|
||||
details={"action": intent.action.value, "order_type": str(getattr(intent, "order_type", "MARKET") or "MARKET")},
|
||||
)
|
||||
receipt = await self.backend.submit_intent(self._legacy_intent(intent))
|
||||
events = self._events_from_submit(intent, receipt, None, None)
|
||||
legacy = self._legacy_intent(intent, kernel=getattr(self, "_kernel_ref", None))
|
||||
submitted = replace(intent, target_size=float(legacy.target_size))
|
||||
receipt = await self.backend.submit_intent(legacy)
|
||||
events = self._events_from_submit(submitted, receipt, None, None)
|
||||
ack_row = dict(getattr(receipt, "raw_ack", {}) or {})
|
||||
self._publish_telemetry(
|
||||
phase="submit:done",
|
||||
|
||||
Reference in New Issue
Block a user