hjnormey/sentiment-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

uv(sentinel): live-schema fix rounds 1+2 (PASS8 fb08a9f+361e6de, QA-verified 5/5 live)

Browse Source 
Real colnames (timestamp/u_prefix_client_id, anomaly_events.ts), query/VST
errors FAIL never PASS (no green-by-error), openOrders wrapper unwrap,
check(d) CH_ALLOWLIST_DBS attribution (BLUE OBF writer no longer flagged).
Maiden live runs by Fable caught all 6; PASS8 fixed same-night.
This commit is contained in:
Codex
2026-07-12 03:28:18 +02:00
parent 1f709af6d1
commit b8cfd5df67
2 changed files with 510 additions and 200 deletions
Download Patch File Download Diff File Expand all files Collapse all files

273
prod/clean_arch/violet/uv/uv_anomaly_sentinel.py
View File

@@ -48,6 +48,10 @@ VST_SECRET_KEY = os.environ.get("BINGX_SECRET_KEY", "")
LOOKBACK_HOURS = int(os.environ.get("SENTINEL_LOOKBACK_HOURS", "24"))
EVIDENCE_LIMIT = int(os.environ.get("SENTINEL_EVIDENCE_LIMIT", "10"))
QUERY_TIMEOUT_S = int(os.environ.get("SENTINEL_QUERY_TIMEOUT_S", "15"))
# Known non-UV databases that the sentinel must NOT flag (BLUE OBF, etc.)
CH_ALLOWLIST_DBS = os.environ.get(
"CH_ALLOWLIST_DBS", "dolphin,dolphin_malkhut,default"
).split(",")
# ─── Data types ────────────────────────────────────────────────────────────
@@ -59,10 +63,7 @@ class CheckResult:
description: str
passed: bool = True
evidence: list[str] = field(default_factory=list)
@property
def status(self) -> str:
return "PASS" if self.passed else "FAIL"
error: str = "" # IF non-empty, the check failed with an error (not a detection)
def add(self, line: str) -> None:
if len(self.evidence) < EVIDENCE_LIMIT:
@@ -70,20 +71,31 @@ class CheckResult:
def summary(self) -> list[str]:
lines = [f" {'PASS' if self.passed else 'FAIL'} [{self.check_id}] {self.description}"]
if self.error:
lines.append(f" Error: {self.error}")
if not self.passed and self.evidence:
lines.append(" Evidence:")
for e in self.evidence:
lines.append(f" - {e}")
hidden = max(0, 0) # we cap at EVIDENCE_LIMIT already
# Actually count surplus that would have been added
return lines
# ─── ClickHouse querier (read-only HTTP) ───────────────────────────────────
# ─── ClickHouse querier (read-only HTTP, error-tracked) ────────────────────
@dataclass
class CHQueryResult:
"""Result of a CH query — rows + error status."""
rows: list[dict[str, Any]]
error: str # empty = success; non-empty = error message
class CHQuerier:
"""Read-only ClickHouse HTTP client (JSONEachRow)."""
"""Read-only ClickHouse HTTP client (JSONEachRow).
Every query method returns (rows, error): error is '' on success.
A caller that gets error != '' MUST report FAIL, never silent PASS.
"""
def __init__(
self,
@@ -95,7 +107,7 @@ class CHQuerier:
self._user = user
self._password = password
def query(self, sql: str, **kw: Any) -> list[dict[str, Any]]:
def query(self, sql: str, **kw: Any) -> CHQueryResult:
timeout = kw.pop("timeout_s", QUERY_TIMEOUT_S)
endpoint = f"{self._url}/?default_format=JSONEachRow"
data = sql.encode("utf-8")
@@ -109,13 +121,12 @@ class CHQuerier:
resp = urllib.request.urlopen(req, timeout=timeout)
body = resp.read().decode("utf-8")
except urllib.error.HTTPError as exc:
log.error("CH HTTP %s: %s", exc.code, exc.read().decode()[:200])
return []
detail = exc.read().decode()[:300]
return CHQueryResult([], f"CH HTTP {exc.code}: {detail}")
except OSError as exc:
log.error("CH connection error: %s", exc)
return []
return CHQueryResult([], f"CH connection error: {exc}")
if not body.strip():
return []
return CHQueryResult([], "")
rows: list[dict[str, Any]] = []
for line in body.strip().split("\n"):
line = line.strip()
@@ -124,18 +135,20 @@ class CHQuerier:
try:
rows.append(json.loads(line))
except json.JSONDecodeError:
log.warning("CH non-JSON line: %.100s", line)
return rows
return CHQueryResult([], f"CH non-JSON response line: {line[:100]}")
return CHQueryResult(rows, "")
def describe_table(self, table: str, db: str = CH_DB_UV) -> list[dict[str, str]]:
def describe_table(self, table: str, db: str = CH_DB_UV) -> CHQueryResult:
return self.query(f"DESCRIBE TABLE {db}.{table}")
def table_exists(self, table: str, db: str = CH_DB_UV) -> bool:
rows = self.query(
def table_exists(self, table: str, db: str = CH_DB_UV) -> CHQueryResult:
r = self.query(
f"SELECT 1 FROM system.tables "
f"WHERE database = '{db}' AND name = '{table}'"
)
return len(rows) > 0
if r.error:
return r
return CHQueryResult(r.rows, "")
# ─── VST querier (read-only: open orders, positions) ───────────────────────
@@ -164,6 +177,13 @@ def _build_signed_params(
return signed
@dataclass
class VSTResult:
"""Result of a VST API call — items + error status."""
items: list[dict[str, Any]]
error: str # empty = success; non-empty = error message
class VSTQuerier:
"""Read-only VST venue client — open orders, positions, balance."""
@@ -177,7 +197,7 @@ class VSTQuerier:
self._api_key = api_key
self._secret_key = secret_key
def _signed_get(self, path: str, params: dict[str, object] | None = None) -> list[dict[str, Any]]:
def _signed_get(self, path: str, params: dict[str, object] | None = None) -> VSTResult:
params = _build_signed_params(params or {}, self._secret_key)
qs = _canonical_query(params)
url = f"{self._base}{path}?{qs}"
@@ -187,50 +207,40 @@ class VSTQuerier:
resp = urllib.request.urlopen(req, timeout=QUERY_TIMEOUT_S)
body = resp.read().decode("utf-8")
except urllib.error.HTTPError as exc:
log.error("VST HTTP %s: %s", exc.code, exc.read().decode()[:200])
return []
detail = exc.read().decode()[:300]
return VSTResult([], f"VST HTTP {exc.code}: {detail}")
except OSError as exc:
log.error("VST connection error: %s", exc)
return []
return VSTResult([], f"VST connection error: {exc}")
try:
parsed = json.loads(body)
except json.JSONDecodeError:
log.error("VST non-JSON response: %.200s", body)
return []
# BingX wraps data. openOrders uses "orders", positions uses "positions".
# Fallback chain: orders > data > positions > bare list.
if isinstance(parsed, dict):
data = (
parsed.get("orders")
or parsed.get("data")
or parsed.get("positions")
or []
)
return VSTResult([], f"VST non-JSON response: {body[:200]}")
# BingX shape: {"code":0,"msg":"ok","data":{"orders":[...]}}
# openOrders → data.orders, positions → data.positions
code = parsed.get("code", -1) if isinstance(parsed, dict) else -1
if code != 0 and code != -1:
msg = parsed.get("msg", "unknown") if isinstance(parsed, dict) else "?"
return VSTResult([], f"VST API error code={code} msg={msg}")
data = parsed.get("data") if isinstance(parsed, dict) else None
if isinstance(data, dict):
items = data.get("orders") or data.get("positions") or []
elif isinstance(data, list):
items = data
else:
data = parsed
if isinstance(data, list):
return data
return [data] if data else []
items = []
# Filter out non-dict items (empty wrappers, etc.)
clean = [r for r in items if isinstance(r, dict)]
return VSTResult(clean, "")
def open_orders(self) -> list[dict[str, Any]]:
"""Return all open orders (across all symbols)."""
# VST uses same endpoint as LIVE for swap
rows = self._signed_get("/openApi/swap/v2/trade/openOrders")
orders: list[dict[str, Any]] = []
for r in rows:
if not isinstance(r, dict):
continue
orders.append(r)
return orders
def open_orders(self) -> VSTResult:
return self._signed_get("/openApi/swap/v2/trade/openOrders")
def open_positions(self) -> list[dict[str, Any]]:
def open_positions(self) -> VSTResult:
return self._signed_get("/openApi/swap/v2/user/positions")
def account_balance(self) -> dict[str, Any]:
rows = self._signed_get("/openApi/swap/v2/user/balance")
if rows:
return rows[0]
return {}
def account_balance(self) -> VSTResult:
r = self._signed_get("/openApi/swap/v2/user/balance")
return r
# ─── Sentinel ──────────────────────────────────────────────────────────────
@@ -246,16 +256,16 @@ class UvAnomalySentinel:
enabled: set[str] | None = None,
lookback_hours: int = LOOKBACK_HOURS,
verbose: bool = False,
allowlist_dbs: list[str] | None = None,
) -> None:
self.ch = ch or CHQuerier()
self.vst = vst or VSTQuerier()
self.enabled = enabled or {"a", "b", "c", "d", "e"}
self.lookback_hours = lookback_hours
self.verbose = verbose
self.allowlist_dbs = allowlist_dbs or CH_ALLOWLIST_DBS
self.results: list[CheckResult] = []
# ── Public API ──────────────────────────────────────────────────────
def run(self) -> int:
checks: list[tuple[str, Any]] = [
("a", self._check_venue_without_journal),
@@ -275,15 +285,29 @@ class UvAnomalySentinel:
def _check_venue_without_journal(self) -> CheckResult:
"""(a) Venue order without matching BRIDGE exec_journal row."""
r = CheckResult("a", "Venue order ↔ BRIDGE exec_journal match")
orders = self.vst.open_orders()
if not orders:
r.passed = True
return r
if not self.ch.table_exists("exec_journal", CH_DB_UV):
# 1. Fetch venue open orders
vst_r = self.vst.open_orders()
if vst_r.error:
r.passed = False
r.add("exec_journal table does not exist in dolphin_uv")
r.error = vst_r.error
return r
# Collect venue clientOrderIds
orders = vst_r.items
if not orders:
return r
# 2. Check exec_journal exists
exists_r = self.ch.table_exists("exec_journal", CH_DB_UV)
if exists_r.error:
r.passed = False
r.error = exists_r.error
return r
if not exists_r.rows:
r.passed = False
r.add("exec_journal table not found in dolphin_uv — no journal to match against")
return r
# 3. Collect venue clientOrderIds
venue_ids: set[str] = set()
for o in orders:
cid = str(
@@ -294,24 +318,28 @@ class UvAnomalySentinel:
)
if cid:
venue_ids.add(cid)
else:
tid = o.get("tradeId") or o.get("orderId") or ""
r.add(f"venue order without clientOrderId: orderId={tid}")
if not venue_ids:
r.passed = len(r.evidence) == 0
return r
# Query exec_journal for any matching client_order_id
# 4. Query exec_journal for matching u_prefix_client_id
ids_escaped = "','".join(venue_ids)
sql = (
f"SELECT client_order_id, trade_id FROM {CH_DB_UV}.exec_journal "
f"WHERE client_order_id IN ('{ids_escaped}')"
f"SELECT u_prefix_client_id, trade_id "
f"FROM {CH_DB_UV}.exec_journal "
f"WHERE u_prefix_client_id IN ('{ids_escaped}')"
)
journal_rows = self.ch.query(sql)
jr = self.ch.query(sql)
if jr.error:
r.passed = False
r.error = jr.error
return r
journal_ids: set[str] = set()
for jr in journal_rows:
cid = str(jr.get("client_order_id") or "")
for row in jr.rows:
cid = str(row.get("u_prefix_client_id") or "")
if cid:
journal_ids.add(cid)
orphan = venue_ids - journal_ids
if orphan:
r.passed = False
@@ -322,27 +350,47 @@ class UvAnomalySentinel:
def _check_journal_without_venue(self) -> CheckResult:
"""(b) BRIDGE exec_journal row without venue order."""
r = CheckResult("b", "BRIDGE exec_journal row ↔ venue order")
if not self.ch.table_exists("exec_journal", CH_DB_UV):
r.passed = True
# 1. Check exec_journal exists
exists_r = self.ch.table_exists("exec_journal", CH_DB_UV)
if exists_r.error:
r.passed = False
r.error = exists_r.error
return r
# Get recent journal rows with open state
if not exists_r.rows:
return r
# 2. Get recent journal rows (not suppressed = still live)
sql = (
f"SELECT client_order_id, trade_id, status "
f"SELECT u_prefix_client_id, trade_id, suppressed "
f"FROM {CH_DB_UV}.exec_journal "
f"WHERE ts >= now() - INTERVAL {self.lookback_hours} HOUR "
f"ORDER BY ts DESC"
f"WHERE timestamp >= now() - INTERVAL {self.lookback_hours} HOUR "
f"AND suppressed = 0 "
f"ORDER BY timestamp DESC"
)
journal_rows = self.ch.query(sql)
if not journal_rows:
jr = self.ch.query(sql)
if jr.error:
r.passed = False
r.error = jr.error
return r
if not jr.rows:
return r
journal_ids: set[str] = set()
for jr in journal_rows:
cid = str(jr.get("client_order_id") or "")
for row in jr.rows:
cid = str(row.get("u_prefix_client_id") or "")
if cid:
journal_ids.add(cid)
orders = self.vst.open_orders()
# 3. Fetch venue open orders
vst_r = self.vst.open_orders()
if vst_r.error:
r.passed = False
r.error = vst_r.error
return r
venue_ids: set[str] = set()
for o in orders:
for o in vst_r.items:
cid = str(
o.get("clientOrderId")
or o.get("clientOrderID")
@@ -351,19 +399,24 @@ class UvAnomalySentinel:
)
if cid:
venue_ids.add(cid)
missing = journal_ids - venue_ids
if missing:
r.passed = False
for cid in sorted(missing):
r.add(f"exec_journal client_order_id={cid} not on venue")
r.add(f"exec_journal u_prefix_client_id={cid} not on venue")
return r
def _check_u_prefix(self) -> CheckResult:
"""(c) Venue order without 'u-' prefix in clientOrderId."""
r = CheckResult("c", "Venue order u- prefix on clientOrderId")
orders = self.vst.open_orders()
vst_r = self.vst.open_orders()
if vst_r.error:
r.passed = False
r.error = vst_r.error
return r
bad: list[str] = []
for o in orders:
for o in vst_r.items:
cid = str(
o.get("clientOrderId")
or o.get("clientOrderID")
@@ -382,29 +435,39 @@ class UvAnomalySentinel:
def _check_ch_write_outside_uv(self) -> CheckResult:
"""(d) CH write outside dolphin_uv.* from runner window."""
r = CheckResult("d", "CH writes only within dolphin_uv.*")
# Query system.query_log for INSERTs not targeting dolphin_uv
col = "current_database"
# Exclude known non-UV databases (BLUE OBF writer, system, etc.)
allowed = [CH_DB_UV] + self.allowlist_dbs
db_filter = " AND ".join(f"{col} != '{db}'" for db in allowed)
sql = (
f"SELECT query, database, table, event_time, http_user "
f"SELECT query, {col} AS db, event_time "
f"FROM system.query_log "
f"WHERE type = 'QueryFinish' "
f"AND query LIKE 'INSERT%' "
f"AND database != '{CH_DB_UV}' "
f"AND {db_filter} "
f"AND event_time >= now() - INTERVAL {self.lookback_hours} HOUR "
f"ORDER BY event_time DESC "
f"LIMIT {EVIDENCE_LIMIT}"
)
rows = self.ch.query(sql)
if rows:
qr = self.ch.query(sql)
if qr.error:
r.passed = False
for row in rows:
db = row.get("database", "?")
tbl = row.get("table", "?")
r.error = qr.error
return r
if qr.rows:
r.passed = False
for row in qr.rows:
db = row.get("db") or row.get("current_database") or "?"
q = (row.get("query") or "")[:120]
r.add(f"INSERT into {db}.{tbl}: {q}")
r.add(f"INSERT into {db}: {q}")
return r
def _check_tripwire_ok(self) -> CheckResult:
"""(e) tripwire_ok=false occurrences in anomaly_events."""
"""(e) tripwire_ok=false occurrences in anomaly_events.
Real schema: dolphin_violet.anomaly_events uses 'ts' (DateTime64(6)),
not 'timestamp'. Verified by DESCRIBE TABLE.
"""
r = CheckResult("e", "tripwire_ok=false occurrences")
sql = (
f"SELECT ts, decision_id, trade_id, symbol, sensor, detail "
@@ -414,10 +477,14 @@ class UvAnomalySentinel:
f"ORDER BY ts DESC "
f"LIMIT {EVIDENCE_LIMIT}"
)
rows = self.ch.query(sql)
if rows:
qr = self.ch.query(sql)
if qr.error:
r.passed = False
for row in rows:
r.error = qr.error
return r
if qr.rows:
r.passed = False
for row in qr.rows:
ts = row.get("ts", "?")
tid = row.get("trade_id", "?")
det = (row.get("detail") or "")[:80]