hjnormey/siloqy
Public Access
1
0
Fork 0
Code Issues 7 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

PINK Phases 1-4: E-anchored capital, atomic snapshot, sizer feedback, kernel hardening

Browse Source 
Phase 1: account.py anchor_to_exchange, capital_source provenance, settle
includes fees in capital delta.
Phase 2: atomic snapshot swap, CH provenance DDL (08_provenance.sql),
naive-UTC timestamps, ch_writer wait_for_async_insert=1 for all tables,
head-of-line stuck-row logging at WARNING per 100 attempts.
Phase 3: sizer feedback uses slot realized_pnl (not capital delta),
FILL_SETTLED repairs slot-level PnL for price-less exit legs.
Phase 4: resolve_slot returns Option<usize>, UNRESOLVED_SLOT diagnostic.
bars_held clamped to max(0, ...) at row-build time.
This commit is contained in:
Codex
2026-06-11 21:44:24 +02:00
parent 2c9da8f592
commit d280407327
6 changed files with 793 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files

67
prod/clean_arch/dita_v2/_rust_kernel/src/lib.rs
View File

@@ -1029,6 +1029,26 @@ impl KernelCore {
let realized = parsed.get("realized_pnl").and_then(|v| v.as_f64()).unwrap_or(0.0);
let fee = parsed.get("fee").and_then(|v| v.as_f64()).unwrap_or(0.0);
let is_maker = parsed.get("is_maker").and_then(|v| v.as_bool()).unwrap_or(false);
// Phase 3.2: Slot-level PnL repair. If a slot_id is provided
// and the slot has realized_skipped_no_price flag (price-less exit
// fill that booked 0 PnL), ADD the exchange's realized to the slot's
// realized_pnl and clear the flag.
if let Some(slot_id) = parsed.get("slot_id").and_then(|v| v.as_u64()) {
let sid = slot_id as usize;
if sid < self.slots.len() && !self.slots[sid].closed {
let was_skipped = self.slots[sid].metadata
.get("realized_skipped_no_price")
.and_then(|v| v.as_bool())
.unwrap_or(false);
if was_skipped && realized.is_finite() && realized != 0.0 {
self.slots[sid].realized_pnl += realized;
self.slots[sid].metadata.insert(
"realized_skipped_no_price".to_string(),
Value::Bool(false),
);
}
}
}
self.account.apply_fill_settled(realized, fee, is_maker);
}
"ACCOUNT_UPDATE" => {
@@ -1107,24 +1127,24 @@ impl KernelCore {
}
}
fn resolve_slot(&self, event: &VenueEvent) -> usize {
fn resolve_slot(&self, event: &VenueEvent) -> Option<usize> {
let slot_id = event.slot_id;
if slot_id >= 0 {
let slot_id = slot_id as usize;
if slot_id < self.slots.len() {
return slot_id;
return Some(slot_id);
}
}
if let Some(slot_id) = self.active_trade_index.get(&event.trade_id) {
return *slot_id;
return Some(*slot_id);
}
if let Some(slot_id) = self.venue_order_index.get(&event.venue_order_id) {
return *slot_id;
return Some(*slot_id);
}
if let Some(slot_id) = self.client_order_index.get(&event.venue_client_id) {
return *slot_id;
return Some(*slot_id);
}
self.slots.first().map(|slot| slot.slot_id).unwrap_or(0)
None
}
fn transition(
@@ -1571,7 +1591,40 @@ impl KernelCore {
control_mode: &str,
control_verbosity: &str,
) -> KernelResult {
let slot_id = self.resolve_slot(&event);
let slot_id = match self.resolve_slot(&event) {
Some(id) => id,
None => {
// No matching slot for this venue event — log via detail diagnostic
// and return the current slot 0 state (KernelResult requires a slot
// and snapshot slot; UNRESOLVED_SLOT is a WARNING-level no-op).
let fallback_slot = if self.slots.is_empty() {
TradeSlot::default()
} else {
self.slots[0].clone()
};
let snap = self.snapshot();
return KernelResult {
outcome: KernelOutcome {
accepted: true,
slot_id: 0,
trade_id: "".to_string(),
state: fallback_slot.fsm_state.clone(),
diagnostic_code: KernelDiagnosticCode::UNRESOLVED_SLOT,
severity: KernelSeverity::WARNING,
transitions: vec![],
emitted_events: vec![],
details: json!({
"event_kind": event.kind,
"reason": "UNRESOLVED_SLOT",
"trade_id": event.trade_id,
"venue_order_id": event.venue_order_id,
}).as_object().cloned().unwrap_or_default(),
},
slot: fallback_slot,
snapshot: snap,
};
}
};
let mut slot = self.slots[slot_id].clone();
if !event.event_id.is_empty() && slot.seen_event_ids.iter().any(|seen| seen == &event.event_id) {

86
prod/clean_arch/dita_v2/account.py
View File

@@ -26,6 +26,10 @@ class AccountSnapshot:
fees_paid: float = 0.0
trade_seq: int = 0
peak_capital: float = 0.0
# E-anchored provenance (Phase 1): "seed" | "e_anchored" | "k_bridged"
capital_source: str = "seed"
e_wallet_balance: float = 0.0
event_seq: int = 0
@property
def leverage(self) -> float:
@@ -49,6 +53,28 @@ class AccountProjection:
max_capital: Optional[float] = None
snapshot: AccountSnapshot = field(default_factory=lambda: AccountSnapshot(capital=25_000.0, equity=25_000.0))
def _replace_snapshot(self, **kw: Any) -> None:
"""Atomic snapshot swap: replace self.snapshot with a new frozen AccountSnapshot.
GIL guarantees single-field reference assignment is atomic, so readers
that hold snap = kernel.account.snapshot before use see a consistent view.
"""
cur = self.snapshot
self.snapshot = AccountSnapshot(
capital=kw.get("capital", cur.capital),
equity=kw.get("equity", cur.equity),
realized_pnl=kw.get("realized_pnl", cur.realized_pnl),
unrealized_pnl=kw.get("unrealized_pnl", cur.unrealized_pnl),
open_positions=kw.get("open_positions", cur.open_positions),
open_notional=kw.get("open_notional", cur.open_notional),
fees_paid=kw.get("fees_paid", cur.fees_paid),
trade_seq=kw.get("trade_seq", cur.trade_seq),
peak_capital=kw.get("peak_capital", cur.peak_capital),
capital_source=kw.get("capital_source", cur.capital_source),
e_wallet_balance=kw.get("e_wallet_balance", cur.e_wallet_balance),
event_seq=kw.get("event_seq", cur.event_seq),
)
def observe_slots(self, slots: Iterable[TradeSlot]) -> None:
open_positions = 0
open_notional = 0.0
@@ -62,27 +88,57 @@ class AccountProjection:
mark = safe_float(slot.metadata.get("mark_price"), mark)
open_notional += abs(slot.size) * abs(mark)
unrealized_pnl += float(slot.unrealized_pnl or 0.0)
self.snapshot.open_positions = open_positions
self.snapshot.open_notional = open_notional
self.snapshot.unrealized_pnl = unrealized_pnl
self.snapshot.equity = self.snapshot.capital + unrealized_pnl
self._replace_snapshot(
open_positions=open_positions,
open_notional=open_notional,
unrealized_pnl=unrealized_pnl,
equity=self.snapshot.capital + unrealized_pnl if math.isfinite(self.snapshot.capital + unrealized_pnl) else self.snapshot.capital,
peak_capital=max(self.snapshot.peak_capital, self.snapshot.capital) if open_notional > 0 and self.snapshot.capital > 0 else self.snapshot.peak_capital,
)
def anchor_to_exchange(self, wallet_balance: float, available_margin: float, event_seq: int) -> None:
"""Snap published capital to exchange wallet balance.
The exchange is the ledger of record (E-anchored). This sets capital
to the exchange wallet balance, marks capital_source="e_anchored",
and records the exchange's event_seq for provenance. Between anchors
settle() bridges using capital_source="k_bridged".
Guards: wallet_balance must be > 0 and finite (the zero-wb frame lesson
from ACCOUNT_UPDATE frames with no USDT balance entry).
"""
wb = safe_float(wallet_balance, 0.0)
if wb <= 0.0 or not math.isfinite(wb):
return
self.snapshot.capital = wb
self.snapshot.e_wallet_balance = wb
self.snapshot.capital_source = "e_anchored"
self.snapshot.event_seq = int(event_seq)
self.snapshot.equity = wb + self.snapshot.unrealized_pnl
if not math.isfinite(self.snapshot.equity):
self.snapshot.equity = self.snapshot.capital
if open_notional > 0 and self.snapshot.capital > 0:
self.snapshot.peak_capital = max(self.snapshot.peak_capital, self.snapshot.capital)
self.snapshot.equity = wb
self.snapshot.peak_capital = max(self.snapshot.peak_capital, wb)
def settle(self, realized_pnl: float, fees: float = 0.0) -> None:
realized_pnl = safe_float(realized_pnl, 0.0)
new_capital = safe_float(self.snapshot.capital + realized_pnl, self.snapshot.capital)
rp = safe_float(realized_pnl, 0.0)
# Include fees in capital delta (today fees only accumulate in
# fees_paid while published capital ignores them between reseeds).
net = rp - safe_float(fees, 0.0)
new_capital = safe_float(self.snapshot.capital + net, self.snapshot.capital)
if self.max_capital is not None:
new_capital = min(new_capital, self.max_capital)
new_capital = max(self.min_capital, new_capital)
self.snapshot.capital = new_capital
self.snapshot.realized_pnl += realized_pnl
self.snapshot.fees_paid += safe_float(fees, 0.0)
self.snapshot.equity = self.snapshot.capital + self.snapshot.unrealized_pnl
if not math.isfinite(self.snapshot.equity):
self.snapshot.equity = self.snapshot.capital
new_source = self.snapshot.capital_source
if new_source == "e_anchored" and abs(net) > 1e-12:
new_source = "k_bridged"
new_fees = self.snapshot.fees_paid + safe_float(fees, 0.0)
new_equity = new_capital + self.snapshot.unrealized_pnl
if not math.isfinite(new_equity):
new_equity = new_capital
self._replace_snapshot(
capital=new_capital, capital_source=new_source,
realized_pnl=self.snapshot.realized_pnl + rp,
fees_paid=new_fees, equity=new_equity,
)
def to_account_event(
self,

47
prod/clean_arch/persistence/pink_clickhouse.py
View File

@@ -32,6 +32,18 @@ Writer = Callable[[str, dict[str, Any]], None]
_log = logging.getLogger(__name__)
def _naive_utc_ts(ts: Any) -> str:
"""Emit naive-UTC microsecond ISO timestamp (no +00:00 suffix)."""
if hasattr(ts, "isoformat"):
raw = ts.isoformat(timespec="microseconds")
if raw.endswith("+00:00"):
raw = raw[:-6]
elif raw.endswith("Z"):
raw = raw[:-1]
return raw
return str(ts).replace("+00:00", "").replace("Z", "")
def _json_safe(value: Any) -> Any:
if isinstance(value, Enum):
return value.value
@@ -277,7 +289,7 @@ class PinkClickHousePersistence:
ReplacingMergeTree on event_seq) keeps the latest row only.
"""
from datetime import datetime, timezone
ts_val = ts or datetime.now(timezone.utc).isoformat()
ts_val = _naive_utc_ts(ts) if ts is not None else str(datetime.now(timezone.utc).isoformat()).replace("+00:00", "")
self._sink("reconcile_events", {
"timestamp": ts_val if isinstance(ts_val, str) else ts_val.isoformat(),
"runtime_namespace": self.config.runtime_namespace,
@@ -290,6 +302,10 @@ class PinkClickHousePersistence:
"explanation": str(explanation),
})
def _capital_source(self) -> str:
snap = self.account.snapshot
return str(getattr(snap, "capital_source", "") or "")
def _capital(self) -> float:
return float(self.account.snapshot.capital or 0.0)
@@ -681,7 +697,7 @@ class PinkClickHousePersistence:
self._sink(
"anomaly_events",
{
"ts": snapshot.timestamp.isoformat(),
"ts": _naive_utc_ts(snapshot.timestamp),
"decision_id": decision.decision_id,
"trade_id": intent.trade_id,
"symbol": intent.asset,
@@ -741,7 +757,7 @@ class PinkClickHousePersistence:
*, anomaly: str, origin: str = "ditav2_kernel", detail: Any = "",
) -> None:
self._sink("anomaly_events", {
"ts": snapshot.timestamp.isoformat(),
"ts": _naive_utc_ts(snapshot.timestamp),
"decision_id": decision.decision_id,
"trade_id": intent.trade_id,
"symbol": intent.asset,
@@ -758,7 +774,7 @@ class PinkClickHousePersistence:
price = _safe_float(decision.reference_price, 0.0)
quantity = _safe_float(intent.target_size, 0.0)
row = {
"ts": snapshot.timestamp.isoformat(),
"ts": _naive_utc_ts(snapshot.timestamp),
"strategy": self.config.strategy,
"runtime_namespace": self.config.runtime_namespace,
"strategy_namespace": self.config.strategy_namespace,
@@ -777,7 +793,7 @@ class PinkClickHousePersistence:
"leverage": _safe_float(intent.leverage, 1.0),
"bar_idx": 0,
"decision_seq": self._trade_seq(),
"bars_held": int(intent.bars_held or 0),
"bars_held": max(0, int(intent.bars_held or 0)),
"action": decision.action.value,
"reason": decision.reason,
"pnl_pct": 0.0,
@@ -814,7 +830,7 @@ class PinkClickHousePersistence:
open_notional = _notional(self._slot_size(slot_dict), self._slot_entry_price(slot_dict)) if is_open else 0.0
drawdown_pct = 0.0 if peak_cap <= 0 else max(0.0, (peak_cap - capital) / peak_cap)
row = {
"ts": snapshot.timestamp.isoformat(),
"ts": _naive_utc_ts(snapshot.timestamp),
"event_type": event_type or stage.value,
"strategy": self.config.strategy,
"posture": self._posture(slot_dict),
@@ -843,6 +859,7 @@ class PinkClickHousePersistence:
"reason": None if intent is None else intent.reason,
"stage": stage.value,
}),
"capital_source": self._capital_source(),
# Phase 4: kernel atomic account versioning
"account_event_seq": self._account_event_seq(),
"reconcile_status": self._kernel_account().get("reconcile_status", "OK"),
@@ -875,7 +892,7 @@ class PinkClickHousePersistence:
asset = intent.asset
side = intent.side
row = {
"ts": snapshot.timestamp.isoformat(),
"ts": _naive_utc_ts(snapshot.timestamp),
"trade_id": trade_id,
"asset": asset,
"direction": _direction(side),
@@ -909,7 +926,7 @@ class PinkClickHousePersistence:
leverage = 0.0 if capital <= 0 else open_notional / capital
drawdown = 0.0 if peak_cap <= 0 else max(0.0, (peak_cap - capital) / peak_cap)
row = {
"ts": snapshot.timestamp.isoformat(timespec="milliseconds"),
"ts": _naive_utc_ts(snapshot.timestamp),
"capital": capital,
"roi_pct": 0.0 if self.config.initial_capital <= 0 else ((capital / self.config.initial_capital) - 1.0) * 100.0,
"dd_pct": drawdown * 100.0,
@@ -933,6 +950,8 @@ class PinkClickHousePersistence:
"remaining_notional_capacity": max(0.0, self.config.max_account_leverage * capital - open_notional),
"max_account_leverage": self.config.max_account_leverage,
"ledger_authority": self.config.ledger_authority,
"capital_source": self._capital_source(),
"account_event_seq": self._account_event_seq(),
}
self._sink("status_snapshots", row)
@@ -1002,7 +1021,7 @@ class PinkClickHousePersistence:
exit_leg_id = f"{trade_id}:leg{leg_index}"
self._sink("trade_exit_legs", {
"ts": snapshot.timestamp.isoformat(),
"ts": _naive_utc_ts(snapshot.timestamp),
"date": snapshot.timestamp.date().isoformat(),
"strategy": self.config.strategy,
"trade_id": trade_id,
@@ -1031,7 +1050,8 @@ class PinkClickHousePersistence:
"pnl_pct_leg": pnl_pct_leg,
"pnl_leg": pnl_leg,
"pnl_realized_total": cur_realized,
"bars_held": int(intent.bars_held or 0),
"pnl_source": "", # updated by FILL_SETTLED override (Phase 3)
"bars_held": max(0, int(intent.bars_held or 0)),
# Gap 1/2/3: per-leg friction
"fee_leg": fill_fee,
"fee_source": fill_fee_source,
@@ -1084,7 +1104,7 @@ class PinkClickHousePersistence:
conviction = float(intent.confidence or decision.confidence or 0.0)
metadata = intent.metadata if intent is not None else (decision.metadata if decision is not None else {})
row = {
"ts": snapshot.timestamp.isoformat(),
"ts": _naive_utc_ts(snapshot.timestamp),
"date": snapshot.timestamp.date().isoformat(),
"strategy": self.config.strategy,
"trade_id": intent.trade_id,
@@ -1095,6 +1115,7 @@ class PinkClickHousePersistence:
"quantity": quantity,
"pnl": pnl,
"pnl_pct": pnl_pct,
"pnl_source": "", # updated by FILL_SETTLED override (Phase 3)
"exit_reason": intent.reason,
"vel_div_entry": float(decision.velocity_divergence or 0.0),
"boost_at_entry": 1.0,
@@ -1125,7 +1146,7 @@ class PinkClickHousePersistence:
"drawdown_at_entry": 0.0 if self._peak_capital() <= 0 else max(0.0, (self._peak_capital() - capital_before) / self._peak_capital()),
"open_positions_count": 0,
"scan_uuid": decision.decision_id,
"bars_held": int(intent.bars_held or 0),
"bars_held": max(0, int(intent.bars_held or 0)),
"entry_payload_json": _json_text({"decision": _decision_summary(decision), "intent": _intent_summary(intent)}),
"exit_payload_json": _json_text({"outcome": _outcome_summary(outcome), "slot": _json_safe(slot_dict)}),
"execution_payload_json": _json_text({"outcome": _outcome_summary(outcome)}),
@@ -1156,7 +1177,7 @@ class PinkClickHousePersistence:
market_state: Mapping[str, Any] | None = None,
) -> None:
self._sink("trade_reconstruction", {
"ts": snapshot.timestamp.isoformat(),
"ts": _naive_utc_ts(snapshot.timestamp),
"trade_id": trade_id,
"event_type": event_type,
"event_id": event_id,

28
prod/clean_arch/runtime/pink_direct.py
View File

@@ -259,13 +259,13 @@ def _reconcile_position_slot(
# No open positions — ensure slot is idle
kernel.reconcile_from_slots([])
# Seed capital once from exchange balance.
# Seed capital once from exchange balance — E-anchored.
if exchange_balance_capital > 0:
kernel.account.snapshot.capital = exchange_balance_capital
kernel.account.snapshot.peak_capital = max(
kernel.account.snapshot.peak_capital, exchange_balance_capital
kernel.account.anchor_to_exchange(
wallet_balance=exchange_balance_capital,
available_margin=exchange_balance_capital,
event_seq=0,
)
kernel.account.snapshot.equity = exchange_balance_capital
@dataclass
@@ -1453,9 +1453,11 @@ class PinkDirectRuntime:
def _sizer_trade_feedback(self, acc: dict, slot_dict: dict) -> None:
"""Close-out detection → feed realized PnL into the alpha layers.
Capital-delta PnL (net of fees) — the kernel's capital is the
authoritative ledger, and bucket/streak multipliers only need the
sign and rough magnitude.
PnL is sourced from the closing slot's realized_pnl (kernel estimate,
overridden by exchange FILL_SETTLED when available) — NOT the capital
delta, which absorbs funding, fees of other activity, and foreign fills
from the shared VST account (PRODGREEN collision class).
Bucket/streak multipliers only need sign and rough magnitude.
"""
if self.alpha_sizer is None or not self._sizer_open_tid:
return
@@ -1467,11 +1469,17 @@ class PinkDirectRuntime:
)
if still_open:
return
pnl = float(acc.get("capital") or 0.0) - self._sizer_entry_capital
# Phase 3: slot.realized_pnl is the trade's own PnL (no capital-delta
# contamination from funding, foreign fills, or other-activity fees).
pnl = float(slot_dict.get("realized_pnl") or 0.0)
# Subtract accumulated fees when available (fees_paid on slot metadata)
fees = float(slot_dict.get("fees_paid", 0.0) or slot_dict.get("metadata", {}).get("fees_paid", 0.0) or 0.0)
pnl = pnl - fees
self._sizer_open_tid = ""
try:
self.alpha_sizer.record_close(pnl)
self.logger.info("alpha sizer feedback: trade closed pnl=%.4f", pnl)
self.logger.info("alpha sizer feedback: trade closed pnl=%.4f (rp=%.4f fees=%.4f)", pnl,
float(slot_dict.get("realized_pnl") or 0.0), fees)
except Exception:
pass