hjnormey/siloqy
Public Access
1
0
Fork 0
Code Issues 7 Pull Requests 2 Actions Packages Projects Releases Wiki Activity

VIOLET V1a: dolphin_violet DDL set + statement-wise applier (applied + verified)

Browse Source 
14 tables consolidated from the LIVE post-ALTER dolphin_pink schema
(maras_tp + provenance folded into base CREATEs — a fresh DB never replays
ALTER chains) + violet_feed_divergence (scan-vs-venue divergence metric,
session_id + plane seqs + mono_ns). apply_violet_ddl.py posts ONE statement
per HTTP request (multi-statement posts fail — proven on pink 08), is
idempotent (all IF NOT EXISTS, double-apply tested live), and verifies the
expected table set. Applied to live CH: verify all 14 present.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Codex
2026-06-12 15:40:05 +02:00
parent 4b52fff8dd
commit 99e529c32a
19 changed files with 753 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
prod/clickhouse/__init__.py Normal file
View File

5
prod/clickhouse/violet/00_create_database.sql Normal file
View File

@@ -0,0 +1,5 @@
-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema
-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12.
-- Apply via apply_violet_ddl.py (one statement per HTTP POST).
CREATE DATABASE IF NOT EXISTS dolphin_violet;

55
prod/clickhouse/violet/01_policy_events.sql Normal file
View File

@@ -0,0 +1,55 @@
-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema
-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12.
-- Apply via apply_violet_ddl.py (one statement per HTTP POST).
CREATE TABLE IF NOT EXISTS dolphin_violet.policy_events
(
`ts` DateTime64(6, 'UTC'),
`ts_day` Date MATERIALIZED toDate(ts),
`strategy` LowCardinality(String),
`runtime_namespace` LowCardinality(String) DEFAULT '',
`strategy_namespace` LowCardinality(String) DEFAULT '',
`event_namespace` LowCardinality(String) DEFAULT '',
`actor_name` LowCardinality(String) DEFAULT '',
`exec_venue` LowCardinality(String) DEFAULT '',
`data_venue` LowCardinality(String) DEFAULT '',
`source` LowCardinality(String),
`trade_id` String,
`asset` LowCardinality(String),
`side` LowCardinality(String),
`entry_price` Float64,
`current_price` Float64,
`quantity` Float64,
`notional` Float64,
`leverage` Float32,
`bar_idx` UInt32,
`decision_seq` UInt32,
`bars_held` UInt16,
`action` LowCardinality(String),
`reason` LowCardinality(String),
`pnl_pct` Float32,
`mfe` Float32,
`mae` Float32,
`mfe_risk` Float32,
`mae_risk` Float32,
`exit_pressure` Float32,
`rv_comp` Float32,
`mae_thresh1` Float32,
`bounce_score` Float32,
`bounce_risk` Float32,
`ob_imbalance` Float32,
`vel_div_entry` Float32,
`vel_div_now` Float32,
`v50_vel` Float32,
`v750_vel` Float32,
`exf_funding` Float32,
`exf_dvol` Float32,
`exf_fear_greed` Float32,
`exf_taker` Float32,
`posture` LowCardinality(String)
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(ts)
ORDER BY (ts_day, trade_id, decision_seq, ts)
TTL ts_day + toIntervalDay(180)
SETTINGS index_granularity = 8192;

22
prod/clickhouse/violet/02_trade_reconstruction.sql Normal file
View File

@@ -0,0 +1,22 @@
-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema
-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12.
-- Apply via apply_violet_ddl.py (one statement per HTTP POST).
CREATE TABLE IF NOT EXISTS dolphin_violet.trade_reconstruction
(
`ts` DateTime64(6, 'UTC'),
`ts_day` Date MATERIALIZED toDate(ts),
`trade_id` String,
`event_type` LowCardinality(String),
`event_id` String,
`payload_json` String,
`market_state_bundle_json` String DEFAULT '',
`tp_base_pct` Float32 DEFAULT 0,
`tp_effective_pct` Float32 DEFAULT 0,
`our_leverage` Float32 DEFAULT 0
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(ts)
ORDER BY (ts_day, trade_id, event_type, event_id)
TTL ts_day + toIntervalDay(180)
SETTINGS index_granularity = 8192;

45
prod/clickhouse/violet/03_trade_exit_legs.sql Normal file
View File

@@ -0,0 +1,45 @@
-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema
-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12.
-- Apply via apply_violet_ddl.py (one statement per HTTP POST).
CREATE TABLE IF NOT EXISTS dolphin_violet.trade_exit_legs
(
`ts` DateTime64(6, 'UTC'),
`ts_day` Date MATERIALIZED toDate(ts),
`date` Date,
`strategy` LowCardinality(String),
`trade_id` String,
`chain_root_trade_id` String DEFAULT '',
`chain_head_leg_id` String DEFAULT '',
`chain_prev_leg_id` String DEFAULT '',
`chain_seq` UInt32 DEFAULT 0,
`chain_token` String DEFAULT '',
`chain_mode` LowCardinality(String) DEFAULT '',
`exit_leg_id` String,
`exit_seq` UInt32,
`command_id` String DEFAULT '',
`source` LowCardinality(String),
`reason` LowCardinality(String),
`asset` LowCardinality(String),
`side` LowCardinality(String),
`entry_price` Float64,
`exit_price` Float64,
`fraction` Float32,
`exit_notional` Float64,
`remaining_notional` Float64,
`remaining_qty` Float64,
`pnl_pct_leg` Float32,
`pnl_leg` Float64,
`pnl_realized_total` Float64,
`bars_held` UInt16,
`fee_leg` Float64 DEFAULT 0,
`fee_source` LowCardinality(String) DEFAULT '',
`is_maker` UInt8 DEFAULT 0,
`slippage_bps` Float32 DEFAULT 0,
`pnl_source` LowCardinality(String) DEFAULT ''
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(ts)
ORDER BY (ts_day, trade_id, exit_seq, ts)
TTL ts_day + toIntervalDay(180)
SETTINGS index_granularity = 8192;

30
prod/clickhouse/violet/04_position_state.sql Normal file
View File

@@ -0,0 +1,30 @@
-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema
-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12.
-- Apply via apply_violet_ddl.py (one statement per HTTP POST).
CREATE TABLE IF NOT EXISTS dolphin_violet.position_state
(
`ts` DateTime64(6, 'UTC'),
`trade_id` String,
`asset` LowCardinality(String),
`direction` Int8,
`entry_price` Float64,
`quantity` Float64,
`notional` Float64,
`leverage` Float32,
`bucket_id` Int32 DEFAULT -1,
`entry_bar` Int32 DEFAULT 0,
`status` LowCardinality(String),
`exit_reason` LowCardinality(String) DEFAULT '',
`pnl` Float64 DEFAULT 0,
`bars_held` UInt16 DEFAULT 0,
`market_state_bundle_json` String DEFAULT '',
`tp_base_pct` Float32 DEFAULT 0,
`tp_effective_pct` Float32 DEFAULT 0,
`our_leverage` Float32 DEFAULT 0
)
ENGINE = ReplacingMergeTree(ts)
PARTITION BY toYYYYMM(ts)
ORDER BY (trade_id, ts)
TTL toDateTime(ts) + toIntervalDay(180)
SETTINGS index_granularity = 8192;

22
prod/clickhouse/violet/05_anomaly_events.sql Normal file
View File

@@ -0,0 +1,22 @@
-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema
-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12.
-- Apply via apply_violet_ddl.py (one statement per HTTP POST).
CREATE TABLE IF NOT EXISTS dolphin_violet.anomaly_events
(
`ts` DateTime64(6, 'UTC'),
`ts_day` Date MATERIALIZED toDate(ts),
`decision_id` String,
`trade_id` String,
`symbol` LowCardinality(String),
`anomaly` LowCardinality(String),
`origin` LowCardinality(String),
`sensor` LowCardinality(String) DEFAULT '',
`detail` String DEFAULT '',
`rm_meta` Float32 DEFAULT 0
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(ts)
ORDER BY (ts_day, trade_id, anomaly, ts)
TTL ts_day + toIntervalDay(180)
SETTINGS index_granularity = 8192;

36
prod/clickhouse/violet/06_account_events.sql Normal file
View File

@@ -0,0 +1,36 @@
-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema
-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12.
-- Apply via apply_violet_ddl.py (one statement per HTTP POST).
CREATE TABLE IF NOT EXISTS dolphin_violet.account_events
(
`ts` DateTime64(6, 'UTC'),
`event_type` LowCardinality(String),
`strategy` LowCardinality(String),
`posture` LowCardinality(String),
`capital` Float64,
`peak_capital` Float64,
`drawdown_pct` Float32,
`pnl_today` Float64 DEFAULT 0,
`trades_today` UInt16 DEFAULT 0,
`open_positions` UInt8 DEFAULT 0,
`boost` Float32 DEFAULT 1.,
`beta` Float32 DEFAULT 0.,
`current_open_notional` Float64 DEFAULT 0,
`current_account_leverage` Float32 DEFAULT 0,
`exchange_leverage` UInt8 DEFAULT 0,
`exchange_leverage_mode` LowCardinality(String) DEFAULT '',
`leverage_mapping_rule` LowCardinality(String) DEFAULT '',
`notes` String DEFAULT '',
`runtime_namespace` LowCardinality(String) DEFAULT '',
`strategy_namespace` LowCardinality(String) DEFAULT '',
`event_namespace` LowCardinality(String) DEFAULT '',
`actor_name` LowCardinality(String) DEFAULT '',
`exec_venue` LowCardinality(String) DEFAULT '',
`data_venue` LowCardinality(String) DEFAULT '',
`capital_source` LowCardinality(String) DEFAULT '',
`account_event_seq` UInt64 DEFAULT 0
)
ENGINE = MergeTree
ORDER BY (ts, event_type)
SETTINGS index_granularity = 8192;

37
prod/clickhouse/violet/07_status_snapshots.sql Normal file
View File

@@ -0,0 +1,37 @@
-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema
-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12.
-- Apply via apply_violet_ddl.py (one statement per HTTP POST).
CREATE TABLE IF NOT EXISTS dolphin_violet.status_snapshots
(
`ts` DateTime64(3, 'UTC'),
`capital` Float64,
`roi_pct` Float32,
`dd_pct` Float32,
`trades_executed` UInt16,
`posture` LowCardinality(String),
`rm` Float32,
`vel_div` Float32,
`vol_ok` UInt8,
`phase` LowCardinality(String),
`mhs_status` LowCardinality(String),
`boost` Float32,
`cat5` Float32,
`conviction_multiplier` Float32 DEFAULT 0,
`exchange_leverage` UInt8 DEFAULT 0,
`exchange_leverage_mode` LowCardinality(String) DEFAULT '',
`leverage_mapping_rule` LowCardinality(String) DEFAULT '',
`account_capital` Float64 DEFAULT 0,
`portfolio_capital` Float64 DEFAULT 0,
`current_open_notional` Float64 DEFAULT 0,
`current_account_leverage` Float32 DEFAULT 0,
`remaining_notional_capacity` Float64 DEFAULT 0,
`max_account_leverage` Float32 DEFAULT 0,
`ledger_authority` LowCardinality(String) DEFAULT '',
`capital_source` LowCardinality(String) DEFAULT '',
`account_event_seq` UInt64 DEFAULT 0
)
ENGINE = MergeTree
ORDER BY ts
TTL toDateTime(ts) + toIntervalDay(180)
SETTINGS index_granularity = 8192;

69
prod/clickhouse/violet/08_trade_events.sql Normal file
View File

@@ -0,0 +1,69 @@
-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema
-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12.
-- Apply via apply_violet_ddl.py (one statement per HTTP POST).
CREATE TABLE IF NOT EXISTS dolphin_violet.trade_events
(
`ts` DateTime64(6, 'UTC'),
`trade_id` String DEFAULT '',
`date` Date,
`strategy` LowCardinality(String),
`asset` LowCardinality(String),
`side` LowCardinality(String),
`entry_price` Float64,
`exit_price` Float64 DEFAULT 0,
`quantity` Float64,
`pnl` Float64 DEFAULT 0,
`pnl_pct` Float32 DEFAULT 0,
`exit_reason` LowCardinality(String) DEFAULT '',
`vel_div_entry` Float32,
`boost_at_entry` Float32,
`beta_at_entry` Float32,
`posture` LowCardinality(String),
`leverage` Float32,
`conviction_multiplier` Float32 DEFAULT 0,
`exchange_leverage` UInt8 DEFAULT 0,
`exchange_leverage_mode` LowCardinality(String) DEFAULT '',
`leverage_mapping_rule` LowCardinality(String) DEFAULT '',
`account_capital` Float64 DEFAULT 0,
`portfolio_capital` Float64 DEFAULT 0,
`current_open_notional` Float64 DEFAULT 0,
`remaining_notional_capacity` Float64 DEFAULT 0,
`max_account_leverage` Float32 DEFAULT 0,
`margin_required` Float64 DEFAULT 0,
`ledger_authority` LowCardinality(String) DEFAULT '',
`regime_signal` Int8 DEFAULT 0,
`capital_before` Float64 DEFAULT 0,
`capital_after` Float64 DEFAULT 0,
`peak_capital` Float64 DEFAULT 0,
`drawdown_at_entry` Float32 DEFAULT 0,
`open_positions_count` UInt8 DEFAULT 0,
`scan_uuid` String DEFAULT '',
`bars_held` UInt16 DEFAULT 0,
`entry_payload_json` String DEFAULT '',
`exit_payload_json` String DEFAULT '',
`execution_payload_json` String DEFAULT '',
`friction_payload_json` String DEFAULT '',
`event_payload_json` String DEFAULT '',
`runtime_namespace` LowCardinality(String) DEFAULT '',
`strategy_namespace` LowCardinality(String) DEFAULT '',
`event_namespace` LowCardinality(String) DEFAULT '',
`actor_name` LowCardinality(String) DEFAULT '',
`exec_venue` LowCardinality(String) DEFAULT '',
`data_venue` LowCardinality(String) DEFAULT '',
`market_state_bundle_json` String DEFAULT '',
`tp_base_pct` Float32 DEFAULT 0,
`tp_effective_pct` Float32 DEFAULT 0,
`our_leverage` Float32 DEFAULT 0,
`fee` Float64 DEFAULT 0,
`fee_source` LowCardinality(String) DEFAULT '',
`is_maker` UInt8 DEFAULT 0,
`slippage_bps` Float32 DEFAULT 0,
`mark_at_submit` Float64 DEFAULT 0,
`exchange_ts` Int64 DEFAULT 0,
`pnl_source` LowCardinality(String) DEFAULT ''
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(ts)
ORDER BY (ts, asset)
SETTINGS index_granularity = 8192;

55
prod/clickhouse/violet/09_v7_decision_events.sql Normal file
View File

@@ -0,0 +1,55 @@
-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema
-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12.
-- Apply via apply_violet_ddl.py (one statement per HTTP POST).
CREATE TABLE IF NOT EXISTS dolphin_violet.v7_decision_events
(
`ts` DateTime64(6, 'UTC'),
`ts_day` Date MATERIALIZED toDate(ts),
`strategy` LowCardinality(String),
`source` LowCardinality(String),
`trade_id` String,
`asset` LowCardinality(String),
`side` LowCardinality(String),
`entry_price` Float64,
`current_price` Float64,
`quantity` Float64,
`notional` Float64,
`leverage` Float32,
`bar_idx` UInt32,
`decision_seq` UInt32,
`bars_held` UInt16,
`action` LowCardinality(String),
`reason` LowCardinality(String),
`pnl_pct` Float32,
`mfe` Float32,
`mae` Float32,
`mfe_risk` Float32,
`mae_risk` Float32,
`exit_pressure` Float32,
`rv_comp` Float32,
`mae_thresh1` Float32,
`bounce_score` Float32,
`bounce_risk` Float32,
`ob_imbalance` Float32,
`vel_div_entry` Float32,
`vel_div_now` Float32,
`v50_vel` Float32,
`v750_vel` Float32,
`exf_funding` Float32,
`exf_dvol` Float32,
`exf_fear_greed` Float32,
`exf_taker` Float32,
`posture` LowCardinality(String),
`runtime_namespace` LowCardinality(String) DEFAULT '',
`strategy_namespace` LowCardinality(String) DEFAULT '',
`event_namespace` LowCardinality(String) DEFAULT '',
`actor_name` LowCardinality(String) DEFAULT '',
`exec_venue` LowCardinality(String) DEFAULT '',
`data_venue` LowCardinality(String) DEFAULT ''
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(ts)
ORDER BY (ts_day, trade_id, decision_seq, ts)
TTL ts_day + toIntervalDay(180)
SETTINGS index_granularity = 8192;

33
prod/clickhouse/violet/10_adaptive_exit_shadow.sql Normal file
View File

@@ -0,0 +1,33 @@
-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema
-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12.
-- Apply via apply_violet_ddl.py (one statement per HTTP POST).
CREATE TABLE IF NOT EXISTS dolphin_violet.adaptive_exit_shadow
(
`ts` DateTime64(6, 'UTC'),
`ts_day` Date MATERIALIZED toDate(ts),
`trade_id` String,
`asset` LowCardinality(String),
`bucket_id` UInt8,
`bars_held` UInt16,
`mae_norm` Float32,
`mfe_norm` Float32,
`tau_norm` Float32,
`p_cont` Float32,
`vel_div_entry` Float32,
`vel_div_now` Float32,
`action` LowCardinality(String),
`exit_reason` LowCardinality(String),
`actual_exit` LowCardinality(String),
`pnl_pct` Float32,
`runtime_namespace` LowCardinality(String) DEFAULT '',
`strategy_namespace` LowCardinality(String) DEFAULT '',
`event_namespace` LowCardinality(String) DEFAULT '',
`actor_name` LowCardinality(String) DEFAULT '',
`exec_venue` LowCardinality(String) DEFAULT '',
`data_venue` LowCardinality(String) DEFAULT ''
)
ENGINE = MergeTree
ORDER BY (ts_day, asset, ts)
TTL ts_day + toIntervalDay(90)
SETTINGS index_granularity = 8192;

21
prod/clickhouse/violet/11_fee_settled_events.sql Normal file
View File

@@ -0,0 +1,21 @@
-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema
-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12.
-- Apply via apply_violet_ddl.py (one statement per HTTP POST).
CREATE TABLE IF NOT EXISTS dolphin_violet.fee_settled_events
(
`ts` DateTime64(6, 'UTC'),
`trade_id` String DEFAULT '',
`fee` Float64 DEFAULT 0,
`fee_asset` LowCardinality(String) DEFAULT 'USDT',
`fee_source` LowCardinality(String) DEFAULT 'WS_SETTLED',
`is_maker` UInt8 DEFAULT 0,
`exchange_ts` Int64 DEFAULT 0,
`realized_pnl_delta` Float64 DEFAULT 0,
`runtime_namespace` LowCardinality(String) DEFAULT '',
`strategy` LowCardinality(String) DEFAULT ''
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(ts)
ORDER BY (trade_id, ts)
SETTINGS index_granularity = 8192;

49
prod/clickhouse/violet/12_sc_bucket_gauge_shadow.sql Normal file
View File

@@ -0,0 +1,49 @@
-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema
-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12.
-- Apply via apply_violet_ddl.py (one statement per HTTP POST).
CREATE TABLE IF NOT EXISTS dolphin_violet.sc_bucket_gauge_shadow
(
`ts` DateTime64(6, 'UTC'),
`ts_day` Date MATERIALIZED toDate(ts),
`strategy` LowCardinality(String),
`trade_id` String,
`asset` LowCardinality(String),
`bucket_id` UInt8,
`scan_number` UInt32,
`bar_idx` UInt32,
`sc` Float32,
`current_mult` Float32,
`recommended_size_mult` Float32,
`recommended_tp_mult` Float32,
`recommended_hold_mult` Float32,
`recommended_sc_cut` Float32,
`current_label` LowCardinality(String),
`recommended_label` LowCardinality(String),
`confidence` Float32,
`model_score` Float32,
`current_score` Float32,
`vel_div` Float32,
`exf_signal` Float32,
`exf_funding` Float32,
`exf_dvol` Float32,
`exf_fear_greed` Float32,
`exf_taker` Float32,
`obf_spread_bps` Float32,
`obf_depth_1pct_usd` Float32,
`obf_depth_quality` Float32,
`obf_fill_probability` Float32,
`obf_imbalance` Float32,
`obf_agreement_pct` Float32,
`obf_regime_signal` Float32,
`recent_trade_count` UInt16,
`recent_win_rate` Float32,
`recent_avg_pnl_pct` Float32,
`recent_loss_streak` UInt16,
`context_json` String,
`outcome_json` String
)
ENGINE = MergeTree
ORDER BY (ts_day, strategy, asset, ts)
TTL ts_day + toIntervalDay(120)
SETTINGS index_granularity = 8192;

44
prod/clickhouse/violet/13_sc_threshold_advisor_shadow.sql Normal file
View File

@@ -0,0 +1,44 @@
-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema
-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12.
-- Apply via apply_violet_ddl.py (one statement per HTTP POST).
CREATE TABLE IF NOT EXISTS dolphin_violet.sc_threshold_advisor_shadow
(
`ts` DateTime64(6, 'UTC'),
`ts_day` Date MATERIALIZED toDate(ts),
`strategy` LowCardinality(String),
`trade_id` String,
`asset` LowCardinality(String),
`scan_number` UInt32,
`bar_idx` UInt32,
`sc` Float32,
`current_mult` Float32,
`recommended_mult` Float32,
`recommended_sc_cut` Float32,
`current_label` LowCardinality(String),
`recommended_label` LowCardinality(String),
`confidence` Float32,
`model_score` Float32,
`current_score` Float32,
`vel_div` Float32,
`exf_funding` Float32,
`exf_dvol` Float32,
`exf_fear_greed` Float32,
`exf_taker` Float32,
`exf_signal` Float32,
`recent_trade_count` UInt16,
`recent_win_rate` Float32,
`recent_avg_pnl_pct` Float32,
`recent_loss_streak` UInt16,
`session` LowCardinality(String),
`dow` UInt8,
`slot_15m` LowCardinality(String),
`decision_source` LowCardinality(String),
`model_version` LowCardinality(String),
`context_json` String,
`outcome_json` String
)
ENGINE = MergeTree
ORDER BY (ts_day, strategy, asset, ts)
TTL ts_day + toIntervalDay(120)
SETTINGS index_granularity = 8192;

20
prod/clickhouse/violet/20_violet_feed_divergence.sql Normal file
View File

@@ -0,0 +1,20 @@
-- VIOLET V1: scan-vs-venue price divergence metric (the FET 0.2176-vs-0.1878
-- detector). One row per (scan, asset-with-fresh-venue-mid). session_id is a
-- boot-time UUID — mono_ns is meaningless across restarts without it.
-- Apply via apply_violet_ddl.py (one statement per HTTP POST).
CREATE TABLE IF NOT EXISTS dolphin_violet.violet_feed_divergence
(
`ts` DateTime64(3, 'UTC'),
`session_id` String,
`asset` LowCardinality(String),
`scan_price` Float64,
`venue_mid` Float64,
`divergence_bps` Float64,
`scan_seq` UInt64,
`venue_seq` UInt64,
`mono_ns` UInt64
)
ENGINE = MergeTree
ORDER BY (asset, ts)
TTL toDate(ts) + toIntervalDay(180);

0
prod/clickhouse/violet/__init__.py Normal file
View File

117
prod/clickhouse/violet/apply_violet_ddl.py Normal file
View File

@@ -0,0 +1,117 @@
#!/usr/bin/env python3
"""Apply the dolphin_violet DDL set — ONE statement per HTTP POST.
ClickHouse's HTTP interface rejects multi-statement posts (proven on
2026-06-12 when prod/clickhouse/pink/08_provenance.sql failed as one body),
so this applier splits each .sql file into single statements and posts them
individually. Idempotent by construction: every statement in the set is
CREATE ... IF NOT EXISTS.
Usage:
python3 prod/clickhouse/violet/apply_violet_ddl.py # apply + verify
python3 prod/clickhouse/violet/apply_violet_ddl.py --dry-run # print statements
python3 prod/clickhouse/violet/apply_violet_ddl.py --verify # verify only
Exit codes: 0 = applied + verified; 1 = a statement failed or verify found
missing tables.
"""
from __future__ import annotations
import argparse
import os
import sys
import urllib.parse
import urllib.request
from pathlib import Path
from typing import Iterator, List, Tuple
SQL_DIR = Path(__file__).resolve().parent
EXPECTED_TABLES = {
"policy_events", "trade_reconstruction", "trade_exit_legs",
"position_state", "anomaly_events", "account_events",
"status_snapshots", "trade_events", "v7_decision_events",
"adaptive_exit_shadow", "fee_settled_events",
"sc_bucket_gauge_shadow", "sc_threshold_advisor_shadow",
"violet_feed_divergence",
}
def _strip_comments(sql: str) -> str:
lines = []
for line in sql.splitlines():
if line.strip().startswith("--"):
continue
lines.append(line)
return "\n".join(lines)
def iter_statements(sql_dir: Path = SQL_DIR) -> Iterator[Tuple[str, str]]:
"""Yield (filename, statement) pairs in sorted file order."""
for path in sorted(sql_dir.glob("*.sql")):
body = _strip_comments(path.read_text())
for stmt in body.split(";"):
stmt = stmt.strip()
if stmt:
yield (path.name, stmt)
def apply_statement(stmt: str, *, url: str, user: str, password: str) -> None:
req = urllib.request.Request(url + "/", data=stmt.encode(), method="POST")
req.add_header("X-ClickHouse-User", user)
req.add_header("X-ClickHouse-Key", password)
with urllib.request.urlopen(req, timeout=30) as resp:
resp.read()
def verify(*, url: str, user: str, password: str) -> List[str]:
"""Return the list of expected tables missing from dolphin_violet."""
q = urllib.parse.quote_plus("SHOW TABLES FROM dolphin_violet")
req = urllib.request.Request(f"{url}/?query={q}", method="POST")
req.add_header("X-ClickHouse-User", user)
req.add_header("X-ClickHouse-Key", password)
try:
with urllib.request.urlopen(req, timeout=15) as resp:
present = set(resp.read().decode().split())
except Exception as exc: # noqa: BLE001 — db may not exist yet
print(f"verify: SHOW TABLES failed ({exc})")
return sorted(EXPECTED_TABLES)
return sorted(EXPECTED_TABLES - present)
def main(argv: List[str] | None = None) -> int:
ap = argparse.ArgumentParser(description="Apply dolphin_violet DDLs")
ap.add_argument("--ch-url", default=os.environ.get("CH_URL", "http://localhost:8123"))
ap.add_argument("--user", default=os.environ.get("CH_USER", "dolphin"))
ap.add_argument("--password", default=os.environ.get("CH_PASS", "dolphin_ch_2026"))
ap.add_argument("--dry-run", action="store_true")
ap.add_argument("--verify", action="store_true", help="verify only, no apply")
args = ap.parse_args(argv)
if not args.verify:
for fname, stmt in iter_statements():
head = stmt.splitlines()[0][:90]
if args.dry_run:
print(f"[dry-run] {fname}: {head}")
continue
try:
apply_statement(stmt, url=args.ch_url, user=args.user,
password=args.password)
print(f"OK {fname}: {head}")
except Exception as exc: # noqa: BLE001
print(f"FAIL {fname}: {head}\n {exc}")
return 1
if args.dry_run:
return 0
missing = verify(url=args.ch_url, user=args.user, password=args.password)
if missing:
print(f"verify: MISSING tables in dolphin_violet: {missing}")
return 1
print(f"verify: all {len(EXPECTED_TABLES)} expected tables present")
return 0
if __name__ == "__main__":
raise SystemExit(main())

93
prod/clickhouse/violet/test_apply_violet_ddl.py Normal file
View File

@@ -0,0 +1,93 @@
"""V1: apply_violet_ddl — splitting, one-POST-per-statement, idempotency."""
from __future__ import annotations
import sys
from pathlib import Path
sys.path.insert(0, "/mnt/dolphinng5_predict")
import pytest
from prod.clickhouse.violet.apply_violet_ddl import (
EXPECTED_TABLES,
SQL_DIR,
_strip_comments,
iter_statements,
main,
)
def test_statement_splitting_strips_comments_and_trailing_semis(tmp_path):
f = tmp_path / "01_x.sql"
f.write_text("-- a comment\nCREATE TABLE IF NOT EXISTS d.t (x Int8)\nENGINE=Memory;\n\n-- tail\n")
stmts = list(iter_statements(tmp_path))
assert len(stmts) == 1
fname, stmt = stmts[0]
assert fname == "01_x.sql"
assert "-- " not in stmt
assert not stmt.endswith(";")
def test_real_sql_dir_yields_one_create_per_file():
"""Every shipped .sql file must contain exactly ONE statement (the
one-POST-per-statement contract is enforceable only if files stay
single-statement)."""
by_file: dict[str, int] = {}
for fname, stmt in iter_statements(SQL_DIR):
by_file[fname] = by_file.get(fname, 0) + 1
assert stmt.upper().startswith(("CREATE TABLE IF NOT EXISTS", "CREATE DATABASE IF NOT EXISTS")), (
f"{fname}: non-idempotent or non-CREATE statement"
)
assert "dolphin_pink" not in stmt, f"{fname}: leftover dolphin_pink reference"
assert all(n == 1 for n in by_file.values()), f"multi-statement files: {by_file}"
# every expected table has a CREATE
created = {
stmt.split("dolphin_violet.")[1].split("\n")[0].split(" ")[0].split("(")[0].strip("`")
for _, stmt in iter_statements(SQL_DIR)
if "dolphin_violet." in stmt
}
assert EXPECTED_TABLES <= created, f"missing CREATEs: {EXPECTED_TABLES - created}"
def test_one_post_per_statement_and_double_apply_idempotent(monkeypatch):
posts: list[str] = []
def fake_apply(stmt, *, url, user, password):
posts.append(stmt)
import prod.clickhouse.violet.apply_violet_ddl as mod
monkeypatch.setattr(mod, "apply_statement", fake_apply)
monkeypatch.setattr(mod, "verify", lambda **kw: [])
rc1 = main(["--ch-url", "http://fake:0"])
n_first = len(posts)
rc2 = main(["--ch-url", "http://fake:0"])
assert rc1 == 0 and rc2 == 0
assert len(posts) == 2 * n_first # same statements re-posted
assert n_first == len(list(iter_statements(SQL_DIR)))
assert all(";" not in p for p in posts) # never multi-statement bodies
def test_verify_failure_path(monkeypatch):
import prod.clickhouse.violet.apply_violet_ddl as mod
monkeypatch.setattr(mod, "apply_statement", lambda *a, **k: None)
monkeypatch.setattr(mod, "verify", lambda **kw: ["violet_feed_divergence"])
assert main(["--ch-url", "http://fake:0"]) == 1
def test_dry_run_posts_nothing(monkeypatch, capsys):
import prod.clickhouse.violet.apply_violet_ddl as mod
def boom(*a, **k):
raise AssertionError("dry-run must not POST")
monkeypatch.setattr(mod, "apply_statement", boom)
assert main(["--dry-run", "--ch-url", "http://fake:0"]) == 0
out = capsys.readouterr().out
assert "[dry-run]" in out
if __name__ == "__main__":
raise SystemExit(pytest.main([__file__, "-v"]))