From 99e529c32ab353027fd8e53e5219d5c949ec06ce Mon Sep 17 00:00:00 2001 From: Codex Date: Fri, 12 Jun 2026 15:40:05 +0200 Subject: [PATCH] VIOLET V1a: dolphin_violet DDL set + statement-wise applier (applied + verified) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 14 tables consolidated from the LIVE post-ALTER dolphin_pink schema (maras_tp + provenance folded into base CREATEs — a fresh DB never replays ALTER chains) + violet_feed_divergence (scan-vs-venue divergence metric, session_id + plane seqs + mono_ns). apply_violet_ddl.py posts ONE statement per HTTP request (multi-statement posts fail — proven on pink 08), is idempotent (all IF NOT EXISTS, double-apply tested live), and verifies the expected table set. Applied to live CH: verify all 14 present. Co-Authored-By: Claude Fable 5 --- prod/clickhouse/__init__.py | 0 prod/clickhouse/violet/00_create_database.sql | 5 + prod/clickhouse/violet/01_policy_events.sql | 55 ++++++++ .../violet/02_trade_reconstruction.sql | 22 ++++ prod/clickhouse/violet/03_trade_exit_legs.sql | 45 +++++++ prod/clickhouse/violet/04_position_state.sql | 30 +++++ prod/clickhouse/violet/05_anomaly_events.sql | 22 ++++ prod/clickhouse/violet/06_account_events.sql | 36 ++++++ .../clickhouse/violet/07_status_snapshots.sql | 37 ++++++ prod/clickhouse/violet/08_trade_events.sql | 69 +++++++++++ .../violet/09_v7_decision_events.sql | 55 ++++++++ .../violet/10_adaptive_exit_shadow.sql | 33 +++++ .../violet/11_fee_settled_events.sql | 21 ++++ .../violet/12_sc_bucket_gauge_shadow.sql | 49 ++++++++ .../violet/13_sc_threshold_advisor_shadow.sql | 44 +++++++ .../violet/20_violet_feed_divergence.sql | 20 +++ prod/clickhouse/violet/__init__.py | 0 prod/clickhouse/violet/apply_violet_ddl.py | 117 ++++++++++++++++++ .../violet/test_apply_violet_ddl.py | 93 ++++++++++++++ 19 files changed, 753 insertions(+) create mode 100644 prod/clickhouse/__init__.py create mode 100644 prod/clickhouse/violet/00_create_database.sql create mode 100644 prod/clickhouse/violet/01_policy_events.sql create mode 100644 prod/clickhouse/violet/02_trade_reconstruction.sql create mode 100644 prod/clickhouse/violet/03_trade_exit_legs.sql create mode 100644 prod/clickhouse/violet/04_position_state.sql create mode 100644 prod/clickhouse/violet/05_anomaly_events.sql create mode 100644 prod/clickhouse/violet/06_account_events.sql create mode 100644 prod/clickhouse/violet/07_status_snapshots.sql create mode 100644 prod/clickhouse/violet/08_trade_events.sql create mode 100644 prod/clickhouse/violet/09_v7_decision_events.sql create mode 100644 prod/clickhouse/violet/10_adaptive_exit_shadow.sql create mode 100644 prod/clickhouse/violet/11_fee_settled_events.sql create mode 100644 prod/clickhouse/violet/12_sc_bucket_gauge_shadow.sql create mode 100644 prod/clickhouse/violet/13_sc_threshold_advisor_shadow.sql create mode 100644 prod/clickhouse/violet/20_violet_feed_divergence.sql create mode 100644 prod/clickhouse/violet/__init__.py create mode 100644 prod/clickhouse/violet/apply_violet_ddl.py create mode 100644 prod/clickhouse/violet/test_apply_violet_ddl.py diff --git a/prod/clickhouse/__init__.py b/prod/clickhouse/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/prod/clickhouse/violet/00_create_database.sql b/prod/clickhouse/violet/00_create_database.sql new file mode 100644 index 0000000..47f5ac6 --- /dev/null +++ b/prod/clickhouse/violet/00_create_database.sql @@ -0,0 +1,5 @@ +-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema +-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12. +-- Apply via apply_violet_ddl.py (one statement per HTTP POST). + +CREATE DATABASE IF NOT EXISTS dolphin_violet; diff --git a/prod/clickhouse/violet/01_policy_events.sql b/prod/clickhouse/violet/01_policy_events.sql new file mode 100644 index 0000000..8df4227 --- /dev/null +++ b/prod/clickhouse/violet/01_policy_events.sql @@ -0,0 +1,55 @@ +-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema +-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12. +-- Apply via apply_violet_ddl.py (one statement per HTTP POST). + +CREATE TABLE IF NOT EXISTS dolphin_violet.policy_events +( + `ts` DateTime64(6, 'UTC'), + `ts_day` Date MATERIALIZED toDate(ts), + `strategy` LowCardinality(String), + `runtime_namespace` LowCardinality(String) DEFAULT '', + `strategy_namespace` LowCardinality(String) DEFAULT '', + `event_namespace` LowCardinality(String) DEFAULT '', + `actor_name` LowCardinality(String) DEFAULT '', + `exec_venue` LowCardinality(String) DEFAULT '', + `data_venue` LowCardinality(String) DEFAULT '', + `source` LowCardinality(String), + `trade_id` String, + `asset` LowCardinality(String), + `side` LowCardinality(String), + `entry_price` Float64, + `current_price` Float64, + `quantity` Float64, + `notional` Float64, + `leverage` Float32, + `bar_idx` UInt32, + `decision_seq` UInt32, + `bars_held` UInt16, + `action` LowCardinality(String), + `reason` LowCardinality(String), + `pnl_pct` Float32, + `mfe` Float32, + `mae` Float32, + `mfe_risk` Float32, + `mae_risk` Float32, + `exit_pressure` Float32, + `rv_comp` Float32, + `mae_thresh1` Float32, + `bounce_score` Float32, + `bounce_risk` Float32, + `ob_imbalance` Float32, + `vel_div_entry` Float32, + `vel_div_now` Float32, + `v50_vel` Float32, + `v750_vel` Float32, + `exf_funding` Float32, + `exf_dvol` Float32, + `exf_fear_greed` Float32, + `exf_taker` Float32, + `posture` LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(ts) +ORDER BY (ts_day, trade_id, decision_seq, ts) +TTL ts_day + toIntervalDay(180) +SETTINGS index_granularity = 8192; diff --git a/prod/clickhouse/violet/02_trade_reconstruction.sql b/prod/clickhouse/violet/02_trade_reconstruction.sql new file mode 100644 index 0000000..e0fa87e --- /dev/null +++ b/prod/clickhouse/violet/02_trade_reconstruction.sql @@ -0,0 +1,22 @@ +-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema +-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12. +-- Apply via apply_violet_ddl.py (one statement per HTTP POST). + +CREATE TABLE IF NOT EXISTS dolphin_violet.trade_reconstruction +( + `ts` DateTime64(6, 'UTC'), + `ts_day` Date MATERIALIZED toDate(ts), + `trade_id` String, + `event_type` LowCardinality(String), + `event_id` String, + `payload_json` String, + `market_state_bundle_json` String DEFAULT '', + `tp_base_pct` Float32 DEFAULT 0, + `tp_effective_pct` Float32 DEFAULT 0, + `our_leverage` Float32 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(ts) +ORDER BY (ts_day, trade_id, event_type, event_id) +TTL ts_day + toIntervalDay(180) +SETTINGS index_granularity = 8192; diff --git a/prod/clickhouse/violet/03_trade_exit_legs.sql b/prod/clickhouse/violet/03_trade_exit_legs.sql new file mode 100644 index 0000000..91e8032 --- /dev/null +++ b/prod/clickhouse/violet/03_trade_exit_legs.sql @@ -0,0 +1,45 @@ +-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema +-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12. +-- Apply via apply_violet_ddl.py (one statement per HTTP POST). + +CREATE TABLE IF NOT EXISTS dolphin_violet.trade_exit_legs +( + `ts` DateTime64(6, 'UTC'), + `ts_day` Date MATERIALIZED toDate(ts), + `date` Date, + `strategy` LowCardinality(String), + `trade_id` String, + `chain_root_trade_id` String DEFAULT '', + `chain_head_leg_id` String DEFAULT '', + `chain_prev_leg_id` String DEFAULT '', + `chain_seq` UInt32 DEFAULT 0, + `chain_token` String DEFAULT '', + `chain_mode` LowCardinality(String) DEFAULT '', + `exit_leg_id` String, + `exit_seq` UInt32, + `command_id` String DEFAULT '', + `source` LowCardinality(String), + `reason` LowCardinality(String), + `asset` LowCardinality(String), + `side` LowCardinality(String), + `entry_price` Float64, + `exit_price` Float64, + `fraction` Float32, + `exit_notional` Float64, + `remaining_notional` Float64, + `remaining_qty` Float64, + `pnl_pct_leg` Float32, + `pnl_leg` Float64, + `pnl_realized_total` Float64, + `bars_held` UInt16, + `fee_leg` Float64 DEFAULT 0, + `fee_source` LowCardinality(String) DEFAULT '', + `is_maker` UInt8 DEFAULT 0, + `slippage_bps` Float32 DEFAULT 0, + `pnl_source` LowCardinality(String) DEFAULT '' +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(ts) +ORDER BY (ts_day, trade_id, exit_seq, ts) +TTL ts_day + toIntervalDay(180) +SETTINGS index_granularity = 8192; diff --git a/prod/clickhouse/violet/04_position_state.sql b/prod/clickhouse/violet/04_position_state.sql new file mode 100644 index 0000000..460f9ab --- /dev/null +++ b/prod/clickhouse/violet/04_position_state.sql @@ -0,0 +1,30 @@ +-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema +-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12. +-- Apply via apply_violet_ddl.py (one statement per HTTP POST). + +CREATE TABLE IF NOT EXISTS dolphin_violet.position_state +( + `ts` DateTime64(6, 'UTC'), + `trade_id` String, + `asset` LowCardinality(String), + `direction` Int8, + `entry_price` Float64, + `quantity` Float64, + `notional` Float64, + `leverage` Float32, + `bucket_id` Int32 DEFAULT -1, + `entry_bar` Int32 DEFAULT 0, + `status` LowCardinality(String), + `exit_reason` LowCardinality(String) DEFAULT '', + `pnl` Float64 DEFAULT 0, + `bars_held` UInt16 DEFAULT 0, + `market_state_bundle_json` String DEFAULT '', + `tp_base_pct` Float32 DEFAULT 0, + `tp_effective_pct` Float32 DEFAULT 0, + `our_leverage` Float32 DEFAULT 0 +) +ENGINE = ReplacingMergeTree(ts) +PARTITION BY toYYYYMM(ts) +ORDER BY (trade_id, ts) +TTL toDateTime(ts) + toIntervalDay(180) +SETTINGS index_granularity = 8192; diff --git a/prod/clickhouse/violet/05_anomaly_events.sql b/prod/clickhouse/violet/05_anomaly_events.sql new file mode 100644 index 0000000..d372980 --- /dev/null +++ b/prod/clickhouse/violet/05_anomaly_events.sql @@ -0,0 +1,22 @@ +-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema +-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12. +-- Apply via apply_violet_ddl.py (one statement per HTTP POST). + +CREATE TABLE IF NOT EXISTS dolphin_violet.anomaly_events +( + `ts` DateTime64(6, 'UTC'), + `ts_day` Date MATERIALIZED toDate(ts), + `decision_id` String, + `trade_id` String, + `symbol` LowCardinality(String), + `anomaly` LowCardinality(String), + `origin` LowCardinality(String), + `sensor` LowCardinality(String) DEFAULT '', + `detail` String DEFAULT '', + `rm_meta` Float32 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(ts) +ORDER BY (ts_day, trade_id, anomaly, ts) +TTL ts_day + toIntervalDay(180) +SETTINGS index_granularity = 8192; diff --git a/prod/clickhouse/violet/06_account_events.sql b/prod/clickhouse/violet/06_account_events.sql new file mode 100644 index 0000000..eb78cfb --- /dev/null +++ b/prod/clickhouse/violet/06_account_events.sql @@ -0,0 +1,36 @@ +-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema +-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12. +-- Apply via apply_violet_ddl.py (one statement per HTTP POST). + +CREATE TABLE IF NOT EXISTS dolphin_violet.account_events +( + `ts` DateTime64(6, 'UTC'), + `event_type` LowCardinality(String), + `strategy` LowCardinality(String), + `posture` LowCardinality(String), + `capital` Float64, + `peak_capital` Float64, + `drawdown_pct` Float32, + `pnl_today` Float64 DEFAULT 0, + `trades_today` UInt16 DEFAULT 0, + `open_positions` UInt8 DEFAULT 0, + `boost` Float32 DEFAULT 1., + `beta` Float32 DEFAULT 0., + `current_open_notional` Float64 DEFAULT 0, + `current_account_leverage` Float32 DEFAULT 0, + `exchange_leverage` UInt8 DEFAULT 0, + `exchange_leverage_mode` LowCardinality(String) DEFAULT '', + `leverage_mapping_rule` LowCardinality(String) DEFAULT '', + `notes` String DEFAULT '', + `runtime_namespace` LowCardinality(String) DEFAULT '', + `strategy_namespace` LowCardinality(String) DEFAULT '', + `event_namespace` LowCardinality(String) DEFAULT '', + `actor_name` LowCardinality(String) DEFAULT '', + `exec_venue` LowCardinality(String) DEFAULT '', + `data_venue` LowCardinality(String) DEFAULT '', + `capital_source` LowCardinality(String) DEFAULT '', + `account_event_seq` UInt64 DEFAULT 0 +) +ENGINE = MergeTree +ORDER BY (ts, event_type) +SETTINGS index_granularity = 8192; diff --git a/prod/clickhouse/violet/07_status_snapshots.sql b/prod/clickhouse/violet/07_status_snapshots.sql new file mode 100644 index 0000000..b766685 --- /dev/null +++ b/prod/clickhouse/violet/07_status_snapshots.sql @@ -0,0 +1,37 @@ +-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema +-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12. +-- Apply via apply_violet_ddl.py (one statement per HTTP POST). + +CREATE TABLE IF NOT EXISTS dolphin_violet.status_snapshots +( + `ts` DateTime64(3, 'UTC'), + `capital` Float64, + `roi_pct` Float32, + `dd_pct` Float32, + `trades_executed` UInt16, + `posture` LowCardinality(String), + `rm` Float32, + `vel_div` Float32, + `vol_ok` UInt8, + `phase` LowCardinality(String), + `mhs_status` LowCardinality(String), + `boost` Float32, + `cat5` Float32, + `conviction_multiplier` Float32 DEFAULT 0, + `exchange_leverage` UInt8 DEFAULT 0, + `exchange_leverage_mode` LowCardinality(String) DEFAULT '', + `leverage_mapping_rule` LowCardinality(String) DEFAULT '', + `account_capital` Float64 DEFAULT 0, + `portfolio_capital` Float64 DEFAULT 0, + `current_open_notional` Float64 DEFAULT 0, + `current_account_leverage` Float32 DEFAULT 0, + `remaining_notional_capacity` Float64 DEFAULT 0, + `max_account_leverage` Float32 DEFAULT 0, + `ledger_authority` LowCardinality(String) DEFAULT '', + `capital_source` LowCardinality(String) DEFAULT '', + `account_event_seq` UInt64 DEFAULT 0 +) +ENGINE = MergeTree +ORDER BY ts +TTL toDateTime(ts) + toIntervalDay(180) +SETTINGS index_granularity = 8192; diff --git a/prod/clickhouse/violet/08_trade_events.sql b/prod/clickhouse/violet/08_trade_events.sql new file mode 100644 index 0000000..633d185 --- /dev/null +++ b/prod/clickhouse/violet/08_trade_events.sql @@ -0,0 +1,69 @@ +-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema +-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12. +-- Apply via apply_violet_ddl.py (one statement per HTTP POST). + +CREATE TABLE IF NOT EXISTS dolphin_violet.trade_events +( + `ts` DateTime64(6, 'UTC'), + `trade_id` String DEFAULT '', + `date` Date, + `strategy` LowCardinality(String), + `asset` LowCardinality(String), + `side` LowCardinality(String), + `entry_price` Float64, + `exit_price` Float64 DEFAULT 0, + `quantity` Float64, + `pnl` Float64 DEFAULT 0, + `pnl_pct` Float32 DEFAULT 0, + `exit_reason` LowCardinality(String) DEFAULT '', + `vel_div_entry` Float32, + `boost_at_entry` Float32, + `beta_at_entry` Float32, + `posture` LowCardinality(String), + `leverage` Float32, + `conviction_multiplier` Float32 DEFAULT 0, + `exchange_leverage` UInt8 DEFAULT 0, + `exchange_leverage_mode` LowCardinality(String) DEFAULT '', + `leverage_mapping_rule` LowCardinality(String) DEFAULT '', + `account_capital` Float64 DEFAULT 0, + `portfolio_capital` Float64 DEFAULT 0, + `current_open_notional` Float64 DEFAULT 0, + `remaining_notional_capacity` Float64 DEFAULT 0, + `max_account_leverage` Float32 DEFAULT 0, + `margin_required` Float64 DEFAULT 0, + `ledger_authority` LowCardinality(String) DEFAULT '', + `regime_signal` Int8 DEFAULT 0, + `capital_before` Float64 DEFAULT 0, + `capital_after` Float64 DEFAULT 0, + `peak_capital` Float64 DEFAULT 0, + `drawdown_at_entry` Float32 DEFAULT 0, + `open_positions_count` UInt8 DEFAULT 0, + `scan_uuid` String DEFAULT '', + `bars_held` UInt16 DEFAULT 0, + `entry_payload_json` String DEFAULT '', + `exit_payload_json` String DEFAULT '', + `execution_payload_json` String DEFAULT '', + `friction_payload_json` String DEFAULT '', + `event_payload_json` String DEFAULT '', + `runtime_namespace` LowCardinality(String) DEFAULT '', + `strategy_namespace` LowCardinality(String) DEFAULT '', + `event_namespace` LowCardinality(String) DEFAULT '', + `actor_name` LowCardinality(String) DEFAULT '', + `exec_venue` LowCardinality(String) DEFAULT '', + `data_venue` LowCardinality(String) DEFAULT '', + `market_state_bundle_json` String DEFAULT '', + `tp_base_pct` Float32 DEFAULT 0, + `tp_effective_pct` Float32 DEFAULT 0, + `our_leverage` Float32 DEFAULT 0, + `fee` Float64 DEFAULT 0, + `fee_source` LowCardinality(String) DEFAULT '', + `is_maker` UInt8 DEFAULT 0, + `slippage_bps` Float32 DEFAULT 0, + `mark_at_submit` Float64 DEFAULT 0, + `exchange_ts` Int64 DEFAULT 0, + `pnl_source` LowCardinality(String) DEFAULT '' +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(ts) +ORDER BY (ts, asset) +SETTINGS index_granularity = 8192; diff --git a/prod/clickhouse/violet/09_v7_decision_events.sql b/prod/clickhouse/violet/09_v7_decision_events.sql new file mode 100644 index 0000000..16995c3 --- /dev/null +++ b/prod/clickhouse/violet/09_v7_decision_events.sql @@ -0,0 +1,55 @@ +-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema +-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12. +-- Apply via apply_violet_ddl.py (one statement per HTTP POST). + +CREATE TABLE IF NOT EXISTS dolphin_violet.v7_decision_events +( + `ts` DateTime64(6, 'UTC'), + `ts_day` Date MATERIALIZED toDate(ts), + `strategy` LowCardinality(String), + `source` LowCardinality(String), + `trade_id` String, + `asset` LowCardinality(String), + `side` LowCardinality(String), + `entry_price` Float64, + `current_price` Float64, + `quantity` Float64, + `notional` Float64, + `leverage` Float32, + `bar_idx` UInt32, + `decision_seq` UInt32, + `bars_held` UInt16, + `action` LowCardinality(String), + `reason` LowCardinality(String), + `pnl_pct` Float32, + `mfe` Float32, + `mae` Float32, + `mfe_risk` Float32, + `mae_risk` Float32, + `exit_pressure` Float32, + `rv_comp` Float32, + `mae_thresh1` Float32, + `bounce_score` Float32, + `bounce_risk` Float32, + `ob_imbalance` Float32, + `vel_div_entry` Float32, + `vel_div_now` Float32, + `v50_vel` Float32, + `v750_vel` Float32, + `exf_funding` Float32, + `exf_dvol` Float32, + `exf_fear_greed` Float32, + `exf_taker` Float32, + `posture` LowCardinality(String), + `runtime_namespace` LowCardinality(String) DEFAULT '', + `strategy_namespace` LowCardinality(String) DEFAULT '', + `event_namespace` LowCardinality(String) DEFAULT '', + `actor_name` LowCardinality(String) DEFAULT '', + `exec_venue` LowCardinality(String) DEFAULT '', + `data_venue` LowCardinality(String) DEFAULT '' +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(ts) +ORDER BY (ts_day, trade_id, decision_seq, ts) +TTL ts_day + toIntervalDay(180) +SETTINGS index_granularity = 8192; diff --git a/prod/clickhouse/violet/10_adaptive_exit_shadow.sql b/prod/clickhouse/violet/10_adaptive_exit_shadow.sql new file mode 100644 index 0000000..af78df0 --- /dev/null +++ b/prod/clickhouse/violet/10_adaptive_exit_shadow.sql @@ -0,0 +1,33 @@ +-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema +-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12. +-- Apply via apply_violet_ddl.py (one statement per HTTP POST). + +CREATE TABLE IF NOT EXISTS dolphin_violet.adaptive_exit_shadow +( + `ts` DateTime64(6, 'UTC'), + `ts_day` Date MATERIALIZED toDate(ts), + `trade_id` String, + `asset` LowCardinality(String), + `bucket_id` UInt8, + `bars_held` UInt16, + `mae_norm` Float32, + `mfe_norm` Float32, + `tau_norm` Float32, + `p_cont` Float32, + `vel_div_entry` Float32, + `vel_div_now` Float32, + `action` LowCardinality(String), + `exit_reason` LowCardinality(String), + `actual_exit` LowCardinality(String), + `pnl_pct` Float32, + `runtime_namespace` LowCardinality(String) DEFAULT '', + `strategy_namespace` LowCardinality(String) DEFAULT '', + `event_namespace` LowCardinality(String) DEFAULT '', + `actor_name` LowCardinality(String) DEFAULT '', + `exec_venue` LowCardinality(String) DEFAULT '', + `data_venue` LowCardinality(String) DEFAULT '' +) +ENGINE = MergeTree +ORDER BY (ts_day, asset, ts) +TTL ts_day + toIntervalDay(90) +SETTINGS index_granularity = 8192; diff --git a/prod/clickhouse/violet/11_fee_settled_events.sql b/prod/clickhouse/violet/11_fee_settled_events.sql new file mode 100644 index 0000000..36ac26c --- /dev/null +++ b/prod/clickhouse/violet/11_fee_settled_events.sql @@ -0,0 +1,21 @@ +-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema +-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12. +-- Apply via apply_violet_ddl.py (one statement per HTTP POST). + +CREATE TABLE IF NOT EXISTS dolphin_violet.fee_settled_events +( + `ts` DateTime64(6, 'UTC'), + `trade_id` String DEFAULT '', + `fee` Float64 DEFAULT 0, + `fee_asset` LowCardinality(String) DEFAULT 'USDT', + `fee_source` LowCardinality(String) DEFAULT 'WS_SETTLED', + `is_maker` UInt8 DEFAULT 0, + `exchange_ts` Int64 DEFAULT 0, + `realized_pnl_delta` Float64 DEFAULT 0, + `runtime_namespace` LowCardinality(String) DEFAULT '', + `strategy` LowCardinality(String) DEFAULT '' +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(ts) +ORDER BY (trade_id, ts) +SETTINGS index_granularity = 8192; diff --git a/prod/clickhouse/violet/12_sc_bucket_gauge_shadow.sql b/prod/clickhouse/violet/12_sc_bucket_gauge_shadow.sql new file mode 100644 index 0000000..423c8ba --- /dev/null +++ b/prod/clickhouse/violet/12_sc_bucket_gauge_shadow.sql @@ -0,0 +1,49 @@ +-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema +-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12. +-- Apply via apply_violet_ddl.py (one statement per HTTP POST). + +CREATE TABLE IF NOT EXISTS dolphin_violet.sc_bucket_gauge_shadow +( + `ts` DateTime64(6, 'UTC'), + `ts_day` Date MATERIALIZED toDate(ts), + `strategy` LowCardinality(String), + `trade_id` String, + `asset` LowCardinality(String), + `bucket_id` UInt8, + `scan_number` UInt32, + `bar_idx` UInt32, + `sc` Float32, + `current_mult` Float32, + `recommended_size_mult` Float32, + `recommended_tp_mult` Float32, + `recommended_hold_mult` Float32, + `recommended_sc_cut` Float32, + `current_label` LowCardinality(String), + `recommended_label` LowCardinality(String), + `confidence` Float32, + `model_score` Float32, + `current_score` Float32, + `vel_div` Float32, + `exf_signal` Float32, + `exf_funding` Float32, + `exf_dvol` Float32, + `exf_fear_greed` Float32, + `exf_taker` Float32, + `obf_spread_bps` Float32, + `obf_depth_1pct_usd` Float32, + `obf_depth_quality` Float32, + `obf_fill_probability` Float32, + `obf_imbalance` Float32, + `obf_agreement_pct` Float32, + `obf_regime_signal` Float32, + `recent_trade_count` UInt16, + `recent_win_rate` Float32, + `recent_avg_pnl_pct` Float32, + `recent_loss_streak` UInt16, + `context_json` String, + `outcome_json` String +) +ENGINE = MergeTree +ORDER BY (ts_day, strategy, asset, ts) +TTL ts_day + toIntervalDay(120) +SETTINGS index_granularity = 8192; diff --git a/prod/clickhouse/violet/13_sc_threshold_advisor_shadow.sql b/prod/clickhouse/violet/13_sc_threshold_advisor_shadow.sql new file mode 100644 index 0000000..a921822 --- /dev/null +++ b/prod/clickhouse/violet/13_sc_threshold_advisor_shadow.sql @@ -0,0 +1,44 @@ +-- dolphin_violet DDL — consolidated from the LIVE dolphin_pink schema +-- (post-ALTER: includes maras_tp + provenance columns). Generated 2026-06-12. +-- Apply via apply_violet_ddl.py (one statement per HTTP POST). + +CREATE TABLE IF NOT EXISTS dolphin_violet.sc_threshold_advisor_shadow +( + `ts` DateTime64(6, 'UTC'), + `ts_day` Date MATERIALIZED toDate(ts), + `strategy` LowCardinality(String), + `trade_id` String, + `asset` LowCardinality(String), + `scan_number` UInt32, + `bar_idx` UInt32, + `sc` Float32, + `current_mult` Float32, + `recommended_mult` Float32, + `recommended_sc_cut` Float32, + `current_label` LowCardinality(String), + `recommended_label` LowCardinality(String), + `confidence` Float32, + `model_score` Float32, + `current_score` Float32, + `vel_div` Float32, + `exf_funding` Float32, + `exf_dvol` Float32, + `exf_fear_greed` Float32, + `exf_taker` Float32, + `exf_signal` Float32, + `recent_trade_count` UInt16, + `recent_win_rate` Float32, + `recent_avg_pnl_pct` Float32, + `recent_loss_streak` UInt16, + `session` LowCardinality(String), + `dow` UInt8, + `slot_15m` LowCardinality(String), + `decision_source` LowCardinality(String), + `model_version` LowCardinality(String), + `context_json` String, + `outcome_json` String +) +ENGINE = MergeTree +ORDER BY (ts_day, strategy, asset, ts) +TTL ts_day + toIntervalDay(120) +SETTINGS index_granularity = 8192; diff --git a/prod/clickhouse/violet/20_violet_feed_divergence.sql b/prod/clickhouse/violet/20_violet_feed_divergence.sql new file mode 100644 index 0000000..4581d70 --- /dev/null +++ b/prod/clickhouse/violet/20_violet_feed_divergence.sql @@ -0,0 +1,20 @@ +-- VIOLET V1: scan-vs-venue price divergence metric (the FET 0.2176-vs-0.1878 +-- detector). One row per (scan, asset-with-fresh-venue-mid). session_id is a +-- boot-time UUID — mono_ns is meaningless across restarts without it. +-- Apply via apply_violet_ddl.py (one statement per HTTP POST). + +CREATE TABLE IF NOT EXISTS dolphin_violet.violet_feed_divergence +( + `ts` DateTime64(3, 'UTC'), + `session_id` String, + `asset` LowCardinality(String), + `scan_price` Float64, + `venue_mid` Float64, + `divergence_bps` Float64, + `scan_seq` UInt64, + `venue_seq` UInt64, + `mono_ns` UInt64 +) +ENGINE = MergeTree +ORDER BY (asset, ts) +TTL toDate(ts) + toIntervalDay(180); diff --git a/prod/clickhouse/violet/__init__.py b/prod/clickhouse/violet/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/prod/clickhouse/violet/apply_violet_ddl.py b/prod/clickhouse/violet/apply_violet_ddl.py new file mode 100644 index 0000000..0e24c31 --- /dev/null +++ b/prod/clickhouse/violet/apply_violet_ddl.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +"""Apply the dolphin_violet DDL set — ONE statement per HTTP POST. + +ClickHouse's HTTP interface rejects multi-statement posts (proven on +2026-06-12 when prod/clickhouse/pink/08_provenance.sql failed as one body), +so this applier splits each .sql file into single statements and posts them +individually. Idempotent by construction: every statement in the set is +CREATE ... IF NOT EXISTS. + +Usage: + python3 prod/clickhouse/violet/apply_violet_ddl.py # apply + verify + python3 prod/clickhouse/violet/apply_violet_ddl.py --dry-run # print statements + python3 prod/clickhouse/violet/apply_violet_ddl.py --verify # verify only + +Exit codes: 0 = applied + verified; 1 = a statement failed or verify found +missing tables. +""" + +from __future__ import annotations + +import argparse +import os +import sys +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Iterator, List, Tuple + +SQL_DIR = Path(__file__).resolve().parent + +EXPECTED_TABLES = { + "policy_events", "trade_reconstruction", "trade_exit_legs", + "position_state", "anomaly_events", "account_events", + "status_snapshots", "trade_events", "v7_decision_events", + "adaptive_exit_shadow", "fee_settled_events", + "sc_bucket_gauge_shadow", "sc_threshold_advisor_shadow", + "violet_feed_divergence", +} + + +def _strip_comments(sql: str) -> str: + lines = [] + for line in sql.splitlines(): + if line.strip().startswith("--"): + continue + lines.append(line) + return "\n".join(lines) + + +def iter_statements(sql_dir: Path = SQL_DIR) -> Iterator[Tuple[str, str]]: + """Yield (filename, statement) pairs in sorted file order.""" + for path in sorted(sql_dir.glob("*.sql")): + body = _strip_comments(path.read_text()) + for stmt in body.split(";"): + stmt = stmt.strip() + if stmt: + yield (path.name, stmt) + + +def apply_statement(stmt: str, *, url: str, user: str, password: str) -> None: + req = urllib.request.Request(url + "/", data=stmt.encode(), method="POST") + req.add_header("X-ClickHouse-User", user) + req.add_header("X-ClickHouse-Key", password) + with urllib.request.urlopen(req, timeout=30) as resp: + resp.read() + + +def verify(*, url: str, user: str, password: str) -> List[str]: + """Return the list of expected tables missing from dolphin_violet.""" + q = urllib.parse.quote_plus("SHOW TABLES FROM dolphin_violet") + req = urllib.request.Request(f"{url}/?query={q}", method="POST") + req.add_header("X-ClickHouse-User", user) + req.add_header("X-ClickHouse-Key", password) + try: + with urllib.request.urlopen(req, timeout=15) as resp: + present = set(resp.read().decode().split()) + except Exception as exc: # noqa: BLE001 — db may not exist yet + print(f"verify: SHOW TABLES failed ({exc})") + return sorted(EXPECTED_TABLES) + return sorted(EXPECTED_TABLES - present) + + +def main(argv: List[str] | None = None) -> int: + ap = argparse.ArgumentParser(description="Apply dolphin_violet DDLs") + ap.add_argument("--ch-url", default=os.environ.get("CH_URL", "http://localhost:8123")) + ap.add_argument("--user", default=os.environ.get("CH_USER", "dolphin")) + ap.add_argument("--password", default=os.environ.get("CH_PASS", "dolphin_ch_2026")) + ap.add_argument("--dry-run", action="store_true") + ap.add_argument("--verify", action="store_true", help="verify only, no apply") + args = ap.parse_args(argv) + + if not args.verify: + for fname, stmt in iter_statements(): + head = stmt.splitlines()[0][:90] + if args.dry_run: + print(f"[dry-run] {fname}: {head}") + continue + try: + apply_statement(stmt, url=args.ch_url, user=args.user, + password=args.password) + print(f"OK {fname}: {head}") + except Exception as exc: # noqa: BLE001 + print(f"FAIL {fname}: {head}\n {exc}") + return 1 + if args.dry_run: + return 0 + + missing = verify(url=args.ch_url, user=args.user, password=args.password) + if missing: + print(f"verify: MISSING tables in dolphin_violet: {missing}") + return 1 + print(f"verify: all {len(EXPECTED_TABLES)} expected tables present") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/prod/clickhouse/violet/test_apply_violet_ddl.py b/prod/clickhouse/violet/test_apply_violet_ddl.py new file mode 100644 index 0000000..35a033b --- /dev/null +++ b/prod/clickhouse/violet/test_apply_violet_ddl.py @@ -0,0 +1,93 @@ +"""V1: apply_violet_ddl — splitting, one-POST-per-statement, idempotency.""" + +from __future__ import annotations + +import sys +from pathlib import Path + +sys.path.insert(0, "/mnt/dolphinng5_predict") + +import pytest + +from prod.clickhouse.violet.apply_violet_ddl import ( + EXPECTED_TABLES, + SQL_DIR, + _strip_comments, + iter_statements, + main, +) + + +def test_statement_splitting_strips_comments_and_trailing_semis(tmp_path): + f = tmp_path / "01_x.sql" + f.write_text("-- a comment\nCREATE TABLE IF NOT EXISTS d.t (x Int8)\nENGINE=Memory;\n\n-- tail\n") + stmts = list(iter_statements(tmp_path)) + assert len(stmts) == 1 + fname, stmt = stmts[0] + assert fname == "01_x.sql" + assert "-- " not in stmt + assert not stmt.endswith(";") + + +def test_real_sql_dir_yields_one_create_per_file(): + """Every shipped .sql file must contain exactly ONE statement (the + one-POST-per-statement contract is enforceable only if files stay + single-statement).""" + by_file: dict[str, int] = {} + for fname, stmt in iter_statements(SQL_DIR): + by_file[fname] = by_file.get(fname, 0) + 1 + assert stmt.upper().startswith(("CREATE TABLE IF NOT EXISTS", "CREATE DATABASE IF NOT EXISTS")), ( + f"{fname}: non-idempotent or non-CREATE statement" + ) + assert "dolphin_pink" not in stmt, f"{fname}: leftover dolphin_pink reference" + assert all(n == 1 for n in by_file.values()), f"multi-statement files: {by_file}" + # every expected table has a CREATE + created = { + stmt.split("dolphin_violet.")[1].split("\n")[0].split(" ")[0].split("(")[0].strip("`") + for _, stmt in iter_statements(SQL_DIR) + if "dolphin_violet." in stmt + } + assert EXPECTED_TABLES <= created, f"missing CREATEs: {EXPECTED_TABLES - created}" + + +def test_one_post_per_statement_and_double_apply_idempotent(monkeypatch): + posts: list[str] = [] + + def fake_apply(stmt, *, url, user, password): + posts.append(stmt) + + import prod.clickhouse.violet.apply_violet_ddl as mod + + monkeypatch.setattr(mod, "apply_statement", fake_apply) + monkeypatch.setattr(mod, "verify", lambda **kw: []) + rc1 = main(["--ch-url", "http://fake:0"]) + n_first = len(posts) + rc2 = main(["--ch-url", "http://fake:0"]) + assert rc1 == 0 and rc2 == 0 + assert len(posts) == 2 * n_first # same statements re-posted + assert n_first == len(list(iter_statements(SQL_DIR))) + assert all(";" not in p for p in posts) # never multi-statement bodies + + +def test_verify_failure_path(monkeypatch): + import prod.clickhouse.violet.apply_violet_ddl as mod + + monkeypatch.setattr(mod, "apply_statement", lambda *a, **k: None) + monkeypatch.setattr(mod, "verify", lambda **kw: ["violet_feed_divergence"]) + assert main(["--ch-url", "http://fake:0"]) == 1 + + +def test_dry_run_posts_nothing(monkeypatch, capsys): + import prod.clickhouse.violet.apply_violet_ddl as mod + + def boom(*a, **k): + raise AssertionError("dry-run must not POST") + + monkeypatch.setattr(mod, "apply_statement", boom) + assert main(["--dry-run", "--ch-url", "http://fake:0"]) == 0 + out = capsys.readouterr().out + assert "[dry-run]" in out + + +if __name__ == "__main__": + raise SystemExit(pytest.main([__file__, "-v"]))