From dc90f6d77e03a550f628e22027b170bc628e89e2 Mon Sep 17 00:00:00 2001 From: Codex Date: Sun, 12 Jul 2026 19:13:25 +0200 Subject: [PATCH] clickhouse: commit exit-leg/reconstruction/maras DDLs + UV/violet DDLs (M1 vendor reconcile, upstream-first) --- .../01_alter_trade_events_trade_id.sql | 2 + .../02_create_trade_reconstruction.sql | 18 +++ prod/clickhouse/03_create_trade_exit_legs.sql | 36 +++++ .../04_alter_trade_events_maras_tp.sql | 5 + .../05_alter_position_state_maras_tp.sql | 5 + .../06_alter_trade_events_event_types.sql | 18 +++ ...06_alter_trade_reconstruction_maras_tp.sql | 5 + prod/clickhouse/config.xml | 53 ++++++++ prod/clickhouse/pink/00_create_database.sql | 1 + .../pink/01_create_policy_events.sql | 51 +++++++ .../pink/02_create_trade_reconstruction.sql | 18 +++ .../pink/03_create_trade_exit_legs.sql | 36 +++++ .../pink/04_create_position_state.sql | 26 ++++ .../pink/05_alter_trade_events_maras_tp.sql | 5 + .../pink/05_create_anomaly_events.sql | 18 +++ .../pink/06_alter_position_state_maras_tp.sql | 5 + ...07_alter_trade_reconstruction_maras_tp.sql | 5 + prod/clickhouse/pink/account_events.sql | 30 +++++ prod/clickhouse/pink/adaptive_exit_shadow.sql | 29 ++++ prod/clickhouse/pink/status_snapshots.sql | 31 +++++ prod/clickhouse/pink/trade_events.sql | 58 ++++++++ prod/clickhouse/pink/v7_decision_events.sql | 51 +++++++ .../prodgreen/00_create_database.sql | 1 + .../01_alter_account_events_leverage.sql | 6 + .../01_alter_status_snapshots_leverage.sql | 12 ++ .../01_alter_trade_events_full_payload.sql | 18 +++ .../02_create_trade_reconstruction.sql | 14 ++ .../prodgreen/03_create_trade_exit_legs.sql | 36 +++++ prod/clickhouse/prodgreen/account_events.sql | 24 ++++ .../prodgreen/adaptive_exit_shadow.sql | 23 ++++ .../clickhouse/prodgreen/status_snapshots.sql | 31 +++++ prod/clickhouse/prodgreen/trade_events.sql | 48 +++++++ .../prodgreen/v7_decision_events.sql | 45 +++++++ prod/clickhouse/users.xml | 21 +++ prod/clickhouse/uv/00_create_database.sql | 5 + prod/clickhouse/uv/01_prime_decisions.sql | 27 ++++ prod/clickhouse/uv/02_prime_scans.sql | 20 +++ prod/clickhouse/uv/03_exec_journal.sql | 30 +++++ prod/clickhouse/uv/04_max_hold_ingress.sql | 26 ++++ prod/clickhouse/uv/04_stop_watch_events.sql | 25 ++++ .../uv/05_alter_prime_decisions_corrupt.sql | 8 ++ prod/clickhouse/uv/05_tp_exit_ingress.sql | 32 +++++ prod/clickhouse/uv/apply_uv_ddl.py | 125 ++++++++++++++++++ prod/clickhouse/uv/prime_decisions.sql | 42 ++++++ .../clickhouse/violet/30_violet_economics.sql | 23 ++++ .../violet/31_violet_decision_divergence.sql | 19 +++ 46 files changed, 1167 insertions(+) create mode 100644 prod/clickhouse/01_alter_trade_events_trade_id.sql create mode 100644 prod/clickhouse/02_create_trade_reconstruction.sql create mode 100644 prod/clickhouse/03_create_trade_exit_legs.sql create mode 100644 prod/clickhouse/04_alter_trade_events_maras_tp.sql create mode 100644 prod/clickhouse/05_alter_position_state_maras_tp.sql create mode 100644 prod/clickhouse/06_alter_trade_events_event_types.sql create mode 100644 prod/clickhouse/06_alter_trade_reconstruction_maras_tp.sql create mode 100644 prod/clickhouse/config.xml create mode 100644 prod/clickhouse/pink/00_create_database.sql create mode 100644 prod/clickhouse/pink/01_create_policy_events.sql create mode 100644 prod/clickhouse/pink/02_create_trade_reconstruction.sql create mode 100644 prod/clickhouse/pink/03_create_trade_exit_legs.sql create mode 100644 prod/clickhouse/pink/04_create_position_state.sql create mode 100644 prod/clickhouse/pink/05_alter_trade_events_maras_tp.sql create mode 100644 prod/clickhouse/pink/05_create_anomaly_events.sql create mode 100644 prod/clickhouse/pink/06_alter_position_state_maras_tp.sql create mode 100644 prod/clickhouse/pink/07_alter_trade_reconstruction_maras_tp.sql create mode 100644 prod/clickhouse/pink/account_events.sql create mode 100644 prod/clickhouse/pink/adaptive_exit_shadow.sql create mode 100644 prod/clickhouse/pink/status_snapshots.sql create mode 100644 prod/clickhouse/pink/trade_events.sql create mode 100644 prod/clickhouse/pink/v7_decision_events.sql create mode 100644 prod/clickhouse/prodgreen/00_create_database.sql create mode 100644 prod/clickhouse/prodgreen/01_alter_account_events_leverage.sql create mode 100644 prod/clickhouse/prodgreen/01_alter_status_snapshots_leverage.sql create mode 100644 prod/clickhouse/prodgreen/01_alter_trade_events_full_payload.sql create mode 100644 prod/clickhouse/prodgreen/02_create_trade_reconstruction.sql create mode 100644 prod/clickhouse/prodgreen/03_create_trade_exit_legs.sql create mode 100644 prod/clickhouse/prodgreen/account_events.sql create mode 100644 prod/clickhouse/prodgreen/adaptive_exit_shadow.sql create mode 100644 prod/clickhouse/prodgreen/status_snapshots.sql create mode 100644 prod/clickhouse/prodgreen/trade_events.sql create mode 100644 prod/clickhouse/prodgreen/v7_decision_events.sql create mode 100644 prod/clickhouse/users.xml create mode 100644 prod/clickhouse/uv/00_create_database.sql create mode 100644 prod/clickhouse/uv/01_prime_decisions.sql create mode 100644 prod/clickhouse/uv/02_prime_scans.sql create mode 100644 prod/clickhouse/uv/03_exec_journal.sql create mode 100644 prod/clickhouse/uv/04_max_hold_ingress.sql create mode 100644 prod/clickhouse/uv/04_stop_watch_events.sql create mode 100644 prod/clickhouse/uv/05_alter_prime_decisions_corrupt.sql create mode 100644 prod/clickhouse/uv/05_tp_exit_ingress.sql create mode 100644 prod/clickhouse/uv/apply_uv_ddl.py create mode 100644 prod/clickhouse/uv/prime_decisions.sql create mode 100644 prod/clickhouse/violet/30_violet_economics.sql create mode 100644 prod/clickhouse/violet/31_violet_decision_divergence.sql diff --git a/prod/clickhouse/01_alter_trade_events_trade_id.sql b/prod/clickhouse/01_alter_trade_events_trade_id.sql new file mode 100644 index 0000000..64aabaf --- /dev/null +++ b/prod/clickhouse/01_alter_trade_events_trade_id.sql @@ -0,0 +1,2 @@ +ALTER TABLE dolphin.trade_events + ADD COLUMN IF NOT EXISTS trade_id String DEFAULT '' AFTER ts; diff --git a/prod/clickhouse/02_create_trade_reconstruction.sql b/prod/clickhouse/02_create_trade_reconstruction.sql new file mode 100644 index 0000000..5a547d0 --- /dev/null +++ b/prod/clickhouse/02_create_trade_reconstruction.sql @@ -0,0 +1,18 @@ +CREATE TABLE IF NOT EXISTS dolphin.trade_reconstruction +( + `ts` DateTime64(6, 'UTC'), + `ts_day` Date MATERIALIZED toDate(ts), + `trade_id` String, + `event_type` LowCardinality(String), + `event_id` String, + `payload_json` String, + `market_state_bundle_json` String DEFAULT '', + `tp_base_pct` Float32 DEFAULT 0, + `tp_effective_pct` Float32 DEFAULT 0, + `our_leverage` Float32 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(ts) +ORDER BY (ts_day, trade_id, event_type, event_id) +TTL ts_day + toIntervalDay(180) +SETTINGS index_granularity = 8192 diff --git a/prod/clickhouse/03_create_trade_exit_legs.sql b/prod/clickhouse/03_create_trade_exit_legs.sql new file mode 100644 index 0000000..d7a6d26 --- /dev/null +++ b/prod/clickhouse/03_create_trade_exit_legs.sql @@ -0,0 +1,36 @@ +CREATE TABLE IF NOT EXISTS dolphin.trade_exit_legs +( + `ts` DateTime64(6, 'UTC'), + `ts_day` Date MATERIALIZED toDate(ts), + `date` Date, + `strategy` LowCardinality(String), + `trade_id` String, + `chain_root_trade_id` String DEFAULT '', + `chain_head_leg_id` String DEFAULT '', + `chain_prev_leg_id` String DEFAULT '', + `chain_seq` UInt32 DEFAULT 0, + `chain_token` String DEFAULT '', + `chain_mode` LowCardinality(String) DEFAULT '', + `exit_leg_id` String, + `exit_seq` UInt32, + `command_id` String DEFAULT '', + `source` LowCardinality(String), + `reason` LowCardinality(String), + `asset` LowCardinality(String), + `side` LowCardinality(String), + `entry_price` Float64, + `exit_price` Float64, + `fraction` Float32, + `exit_notional` Float64, + `remaining_notional` Float64, + `remaining_qty` Float64, + `pnl_pct_leg` Float32, + `pnl_leg` Float64, + `pnl_realized_total` Float64, + `bars_held` UInt16 +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(ts) +ORDER BY (ts_day, trade_id, exit_seq, ts) +TTL ts_day + toIntervalDay(180) +SETTINGS index_granularity = 8192 diff --git a/prod/clickhouse/04_alter_trade_events_maras_tp.sql b/prod/clickhouse/04_alter_trade_events_maras_tp.sql new file mode 100644 index 0000000..db97824 --- /dev/null +++ b/prod/clickhouse/04_alter_trade_events_maras_tp.sql @@ -0,0 +1,5 @@ +ALTER TABLE dolphin.trade_events + ADD COLUMN IF NOT EXISTS market_state_bundle_json String DEFAULT '', + ADD COLUMN IF NOT EXISTS tp_base_pct Float32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS tp_effective_pct Float32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS our_leverage Float32 DEFAULT 0; diff --git a/prod/clickhouse/05_alter_position_state_maras_tp.sql b/prod/clickhouse/05_alter_position_state_maras_tp.sql new file mode 100644 index 0000000..fd1d602 --- /dev/null +++ b/prod/clickhouse/05_alter_position_state_maras_tp.sql @@ -0,0 +1,5 @@ +ALTER TABLE dolphin.position_state + ADD COLUMN IF NOT EXISTS market_state_bundle_json String DEFAULT '', + ADD COLUMN IF NOT EXISTS tp_base_pct Float32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS tp_effective_pct Float32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS our_leverage Float32 DEFAULT 0; diff --git a/prod/clickhouse/06_alter_trade_events_event_types.sql b/prod/clickhouse/06_alter_trade_events_event_types.sql new file mode 100644 index 0000000..772d77a --- /dev/null +++ b/prod/clickhouse/06_alter_trade_events_event_types.sql @@ -0,0 +1,18 @@ +ALTER TABLE dolphin.trade_events + ADD COLUMN IF NOT EXISTS event_type LowCardinality(String) DEFAULT 'CLOSE', + ADD COLUMN IF NOT EXISTS event_id String DEFAULT '', + ADD COLUMN IF NOT EXISTS chain_root_trade_id String DEFAULT '', + ADD COLUMN IF NOT EXISTS chain_head_leg_id String DEFAULT '', + ADD COLUMN IF NOT EXISTS chain_prev_leg_id String DEFAULT '', + ADD COLUMN IF NOT EXISTS chain_seq UInt16 DEFAULT 0, + ADD COLUMN IF NOT EXISTS chain_token String DEFAULT '', + ADD COLUMN IF NOT EXISTS chain_mode LowCardinality(String) DEFAULT '', + ADD COLUMN IF NOT EXISTS exit_leg_id String DEFAULT '', + ADD COLUMN IF NOT EXISTS exit_seq UInt16 DEFAULT 0, + ADD COLUMN IF NOT EXISTS retraction_legs UInt16 DEFAULT 0, + ADD COLUMN IF NOT EXISTS retraction_realized_total Float64 DEFAULT 0, + ADD COLUMN IF NOT EXISTS pnl_leg Float64 DEFAULT 0, + ADD COLUMN IF NOT EXISTS pnl_realized_total Float64 DEFAULT 0, + ADD COLUMN IF NOT EXISTS exit_notional Float64 DEFAULT 0, + ADD COLUMN IF NOT EXISTS remaining_notional Float64 DEFAULT 0, + ADD COLUMN IF NOT EXISTS remaining_qty Float64 DEFAULT 0; diff --git a/prod/clickhouse/06_alter_trade_reconstruction_maras_tp.sql b/prod/clickhouse/06_alter_trade_reconstruction_maras_tp.sql new file mode 100644 index 0000000..66404f8 --- /dev/null +++ b/prod/clickhouse/06_alter_trade_reconstruction_maras_tp.sql @@ -0,0 +1,5 @@ +ALTER TABLE dolphin.trade_reconstruction + ADD COLUMN IF NOT EXISTS market_state_bundle_json String DEFAULT '', + ADD COLUMN IF NOT EXISTS tp_base_pct Float32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS tp_effective_pct Float32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS our_leverage Float32 DEFAULT 0; diff --git a/prod/clickhouse/config.xml b/prod/clickhouse/config.xml new file mode 100644 index 0000000..11446fe --- /dev/null +++ b/prod/clickhouse/config.xml @@ -0,0 +1,53 @@ + + + warning + /var/log/clickhouse-server/clickhouse-server.log + /var/log/clickhouse-server/clickhouse-server.err.log + 500M + 3 + + 0.0.0.0 + 8123 + 9000 + 0.4 + 4 + 5368709120 + + + + system + query_log
+ event_date + INTERVAL 3 DAY + 7500 + + + system + query_thread_log
+ event_date + INTERVAL 3 DAY + 7500 + + + system + metric_log
+ event_date + INTERVAL 3 DAY + 1000 + + + system + asynchronous_metric_log
+ event_date + INTERVAL 3 DAY + + + system + trace_log
+ event_date + INTERVAL 3 DAY + 7500 + + + system + text_log
+ event_date + INTERVAL 3 DAY + warning + + diff --git a/prod/clickhouse/pink/00_create_database.sql b/prod/clickhouse/pink/00_create_database.sql new file mode 100644 index 0000000..e0d4f5a --- /dev/null +++ b/prod/clickhouse/pink/00_create_database.sql @@ -0,0 +1 @@ +CREATE DATABASE IF NOT EXISTS dolphin_pink; diff --git a/prod/clickhouse/pink/01_create_policy_events.sql b/prod/clickhouse/pink/01_create_policy_events.sql new file mode 100644 index 0000000..e530420 --- /dev/null +++ b/prod/clickhouse/pink/01_create_policy_events.sql @@ -0,0 +1,51 @@ +CREATE TABLE IF NOT EXISTS dolphin_pink.policy_events +( + `ts` DateTime64(6, 'UTC'), + `ts_day` Date MATERIALIZED toDate(ts), + `strategy` LowCardinality(String), + `runtime_namespace` LowCardinality(String) DEFAULT '', + `strategy_namespace` LowCardinality(String) DEFAULT '', + `event_namespace` LowCardinality(String) DEFAULT '', + `actor_name` LowCardinality(String) DEFAULT '', + `exec_venue` LowCardinality(String) DEFAULT '', + `data_venue` LowCardinality(String) DEFAULT '', + `source` LowCardinality(String), + `trade_id` String, + `asset` LowCardinality(String), + `side` LowCardinality(String), + `entry_price` Float64, + `current_price` Float64, + `quantity` Float64, + `notional` Float64, + `leverage` Float32, + `bar_idx` UInt32, + `decision_seq` UInt32, + `bars_held` UInt16, + `action` LowCardinality(String), + `reason` LowCardinality(String), + `pnl_pct` Float32, + `mfe` Float32, + `mae` Float32, + `mfe_risk` Float32, + `mae_risk` Float32, + `exit_pressure` Float32, + `rv_comp` Float32, + `mae_thresh1` Float32, + `bounce_score` Float32, + `bounce_risk` Float32, + `ob_imbalance` Float32, + `vel_div_entry` Float32, + `vel_div_now` Float32, + `v50_vel` Float32, + `v750_vel` Float32, + `exf_funding` Float32, + `exf_dvol` Float32, + `exf_fear_greed` Float32, + `exf_taker` Float32, + `posture` LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(ts) +ORDER BY (ts_day, trade_id, decision_seq, ts) +TTL ts_day + toIntervalDay(180) +SETTINGS index_granularity = 8192 diff --git a/prod/clickhouse/pink/02_create_trade_reconstruction.sql b/prod/clickhouse/pink/02_create_trade_reconstruction.sql new file mode 100644 index 0000000..6b4ad0d --- /dev/null +++ b/prod/clickhouse/pink/02_create_trade_reconstruction.sql @@ -0,0 +1,18 @@ +CREATE TABLE IF NOT EXISTS dolphin_pink.trade_reconstruction +( + `ts` DateTime64(6, 'UTC'), + `ts_day` Date MATERIALIZED toDate(ts), + `trade_id` String, + `event_type` LowCardinality(String), + `event_id` String, + `payload_json` String, + `market_state_bundle_json` String DEFAULT '', + `tp_base_pct` Float32 DEFAULT 0, + `tp_effective_pct` Float32 DEFAULT 0, + `our_leverage` Float32 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(ts) +ORDER BY (ts_day, trade_id, event_type, event_id) +TTL ts_day + toIntervalDay(180) +SETTINGS index_granularity = 8192 diff --git a/prod/clickhouse/pink/03_create_trade_exit_legs.sql b/prod/clickhouse/pink/03_create_trade_exit_legs.sql new file mode 100644 index 0000000..f96b0f4 --- /dev/null +++ b/prod/clickhouse/pink/03_create_trade_exit_legs.sql @@ -0,0 +1,36 @@ +CREATE TABLE IF NOT EXISTS dolphin_pink.trade_exit_legs +( + `ts` DateTime64(6, 'UTC'), + `ts_day` Date MATERIALIZED toDate(ts), + `date` Date, + `strategy` LowCardinality(String), + `trade_id` String, + `chain_root_trade_id` String DEFAULT '', + `chain_head_leg_id` String DEFAULT '', + `chain_prev_leg_id` String DEFAULT '', + `chain_seq` UInt32 DEFAULT 0, + `chain_token` String DEFAULT '', + `chain_mode` LowCardinality(String) DEFAULT '', + `exit_leg_id` String, + `exit_seq` UInt32, + `command_id` String DEFAULT '', + `source` LowCardinality(String), + `reason` LowCardinality(String), + `asset` LowCardinality(String), + `side` LowCardinality(String), + `entry_price` Float64, + `exit_price` Float64, + `fraction` Float32, + `exit_notional` Float64, + `remaining_notional` Float64, + `remaining_qty` Float64, + `pnl_pct_leg` Float32, + `pnl_leg` Float64, + `pnl_realized_total` Float64, + `bars_held` UInt16 +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(ts) +ORDER BY (ts_day, trade_id, exit_seq, ts) +TTL ts_day + toIntervalDay(180) +SETTINGS index_granularity = 8192 diff --git a/prod/clickhouse/pink/04_create_position_state.sql b/prod/clickhouse/pink/04_create_position_state.sql new file mode 100644 index 0000000..ac51f51 --- /dev/null +++ b/prod/clickhouse/pink/04_create_position_state.sql @@ -0,0 +1,26 @@ +CREATE TABLE IF NOT EXISTS dolphin_pink.position_state +( + `ts` DateTime64(6, 'UTC'), + `trade_id` String, + `asset` LowCardinality(String), + `direction` Int8, + `entry_price` Float64, + `quantity` Float64, + `notional` Float64, + `leverage` Float32, + `bucket_id` Int32 DEFAULT -1, + `entry_bar` Int32 DEFAULT 0, + `status` LowCardinality(String), + `exit_reason` LowCardinality(String) DEFAULT '', + `pnl` Float64 DEFAULT 0, + `bars_held` UInt16 DEFAULT 0, + `market_state_bundle_json` String DEFAULT '', + `tp_base_pct` Float32 DEFAULT 0, + `tp_effective_pct` Float32 DEFAULT 0, + `our_leverage` Float32 DEFAULT 0 +) +ENGINE = ReplacingMergeTree(ts) +PARTITION BY toYYYYMM(ts) +ORDER BY (trade_id, ts) +TTL toDateTime(ts) + toIntervalDay(180) +SETTINGS index_granularity = 8192 diff --git a/prod/clickhouse/pink/05_alter_trade_events_maras_tp.sql b/prod/clickhouse/pink/05_alter_trade_events_maras_tp.sql new file mode 100644 index 0000000..213aefa --- /dev/null +++ b/prod/clickhouse/pink/05_alter_trade_events_maras_tp.sql @@ -0,0 +1,5 @@ +ALTER TABLE dolphin_pink.trade_events + ADD COLUMN IF NOT EXISTS market_state_bundle_json String DEFAULT '', + ADD COLUMN IF NOT EXISTS tp_base_pct Float32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS tp_effective_pct Float32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS our_leverage Float32 DEFAULT 0; diff --git a/prod/clickhouse/pink/05_create_anomaly_events.sql b/prod/clickhouse/pink/05_create_anomaly_events.sql new file mode 100644 index 0000000..8237879 --- /dev/null +++ b/prod/clickhouse/pink/05_create_anomaly_events.sql @@ -0,0 +1,18 @@ +CREATE TABLE IF NOT EXISTS dolphin_pink.anomaly_events +( + `ts` DateTime64(6, 'UTC'), + `ts_day` Date MATERIALIZED toDate(ts), + `decision_id` String, + `trade_id` String, + `symbol` LowCardinality(String), + `anomaly` LowCardinality(String), + `origin` LowCardinality(String), + `sensor` LowCardinality(String) DEFAULT '', + `detail` String DEFAULT '', + `rm_meta` Float32 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(ts) +ORDER BY (ts_day, trade_id, anomaly, ts) +TTL ts_day + toIntervalDay(180) +SETTINGS index_granularity = 8192 diff --git a/prod/clickhouse/pink/06_alter_position_state_maras_tp.sql b/prod/clickhouse/pink/06_alter_position_state_maras_tp.sql new file mode 100644 index 0000000..6853977 --- /dev/null +++ b/prod/clickhouse/pink/06_alter_position_state_maras_tp.sql @@ -0,0 +1,5 @@ +ALTER TABLE dolphin_pink.position_state + ADD COLUMN IF NOT EXISTS market_state_bundle_json String DEFAULT '', + ADD COLUMN IF NOT EXISTS tp_base_pct Float32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS tp_effective_pct Float32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS our_leverage Float32 DEFAULT 0; diff --git a/prod/clickhouse/pink/07_alter_trade_reconstruction_maras_tp.sql b/prod/clickhouse/pink/07_alter_trade_reconstruction_maras_tp.sql new file mode 100644 index 0000000..38e860f --- /dev/null +++ b/prod/clickhouse/pink/07_alter_trade_reconstruction_maras_tp.sql @@ -0,0 +1,5 @@ +ALTER TABLE dolphin_pink.trade_reconstruction + ADD COLUMN IF NOT EXISTS market_state_bundle_json String DEFAULT '', + ADD COLUMN IF NOT EXISTS tp_base_pct Float32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS tp_effective_pct Float32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS our_leverage Float32 DEFAULT 0; diff --git a/prod/clickhouse/pink/account_events.sql b/prod/clickhouse/pink/account_events.sql new file mode 100644 index 0000000..e8a728c --- /dev/null +++ b/prod/clickhouse/pink/account_events.sql @@ -0,0 +1,30 @@ +CREATE TABLE dolphin_pink.account_events +( + `ts` DateTime64(6, 'UTC'), + `event_type` LowCardinality(String), + `strategy` LowCardinality(String), + `posture` LowCardinality(String), + `capital` Float64, + `peak_capital` Float64, + `drawdown_pct` Float32, + `pnl_today` Float64 DEFAULT 0, + `trades_today` UInt16 DEFAULT 0, + `open_positions` UInt8 DEFAULT 0, + `boost` Float32 DEFAULT 1., + `beta` Float32 DEFAULT 0., + `current_open_notional` Float64 DEFAULT 0, + `current_account_leverage` Float32 DEFAULT 0, + `exchange_leverage` UInt8 DEFAULT 0, + `exchange_leverage_mode` LowCardinality(String) DEFAULT '', + `leverage_mapping_rule` LowCardinality(String) DEFAULT '', + `runtime_namespace` LowCardinality(String) DEFAULT '', + `strategy_namespace` LowCardinality(String) DEFAULT '', + `event_namespace` LowCardinality(String) DEFAULT '', + `actor_name` LowCardinality(String) DEFAULT '', + `exec_venue` LowCardinality(String) DEFAULT '', + `data_venue` LowCardinality(String) DEFAULT '', + `notes` String DEFAULT '' +) +ENGINE = MergeTree +ORDER BY (ts, event_type) +SETTINGS index_granularity = 8192 diff --git a/prod/clickhouse/pink/adaptive_exit_shadow.sql b/prod/clickhouse/pink/adaptive_exit_shadow.sql new file mode 100644 index 0000000..31c9889 --- /dev/null +++ b/prod/clickhouse/pink/adaptive_exit_shadow.sql @@ -0,0 +1,29 @@ +CREATE TABLE dolphin_pink.adaptive_exit_shadow +( + `ts` DateTime64(6, 'UTC'), + `ts_day` Date MATERIALIZED toDate(ts), + `trade_id` String, + `asset` LowCardinality(String), + `runtime_namespace` LowCardinality(String) DEFAULT '', + `strategy_namespace` LowCardinality(String) DEFAULT '', + `event_namespace` LowCardinality(String) DEFAULT '', + `actor_name` LowCardinality(String) DEFAULT '', + `exec_venue` LowCardinality(String) DEFAULT '', + `data_venue` LowCardinality(String) DEFAULT '', + `bucket_id` UInt8, + `bars_held` UInt16, + `mae_norm` Float32, + `mfe_norm` Float32, + `tau_norm` Float32, + `p_cont` Float32, + `vel_div_entry` Float32, + `vel_div_now` Float32, + `action` LowCardinality(String), + `exit_reason` LowCardinality(String), + `actual_exit` LowCardinality(String), + `pnl_pct` Float32 +) +ENGINE = MergeTree +ORDER BY (ts_day, asset, ts) +TTL ts_day + toIntervalDay(90) +SETTINGS index_granularity = 8192 diff --git a/prod/clickhouse/pink/status_snapshots.sql b/prod/clickhouse/pink/status_snapshots.sql new file mode 100644 index 0000000..df074e3 --- /dev/null +++ b/prod/clickhouse/pink/status_snapshots.sql @@ -0,0 +1,31 @@ +CREATE TABLE dolphin_pink.status_snapshots +( + `ts` DateTime64(3, 'UTC'), + `capital` Float64, + `roi_pct` Float32, + `dd_pct` Float32, + `trades_executed` UInt16, + `posture` LowCardinality(String), + `rm` Float32, + `vel_div` Float32, + `vol_ok` UInt8, + `phase` LowCardinality(String), + `mhs_status` LowCardinality(String), + `boost` Float32, + `cat5` Float32, + `conviction_multiplier` Float32 DEFAULT 0, + `exchange_leverage` UInt8 DEFAULT 0, + `exchange_leverage_mode` LowCardinality(String) DEFAULT '', + `leverage_mapping_rule` LowCardinality(String) DEFAULT '', + `account_capital` Float64 DEFAULT 0, + `portfolio_capital` Float64 DEFAULT 0, + `current_open_notional` Float64 DEFAULT 0, + `current_account_leverage` Float32 DEFAULT 0, + `remaining_notional_capacity` Float64 DEFAULT 0, + `max_account_leverage` Float32 DEFAULT 0, + `ledger_authority` LowCardinality(String) DEFAULT '' +) +ENGINE = MergeTree +ORDER BY ts +TTL toDateTime(ts) + toIntervalDay(180) +SETTINGS index_granularity = 8192 diff --git a/prod/clickhouse/pink/trade_events.sql b/prod/clickhouse/pink/trade_events.sql new file mode 100644 index 0000000..ffc6ba9 --- /dev/null +++ b/prod/clickhouse/pink/trade_events.sql @@ -0,0 +1,58 @@ +CREATE TABLE dolphin_pink.trade_events +( + `ts` DateTime64(6, 'UTC'), + `trade_id` String DEFAULT '', + `date` Date, + `strategy` LowCardinality(String), + `asset` LowCardinality(String), + `side` LowCardinality(String), + `entry_price` Float64, + `exit_price` Float64 DEFAULT 0, + `quantity` Float64, + `pnl` Float64 DEFAULT 0, + `pnl_pct` Float32 DEFAULT 0, + `exit_reason` LowCardinality(String) DEFAULT '', + `vel_div_entry` Float32, + `boost_at_entry` Float32, + `beta_at_entry` Float32, + `posture` LowCardinality(String), + `leverage` Float32, + `conviction_multiplier` Float32 DEFAULT 0, + `exchange_leverage` UInt8 DEFAULT 0, + `exchange_leverage_mode` LowCardinality(String) DEFAULT '', + `leverage_mapping_rule` LowCardinality(String) DEFAULT '', + `runtime_namespace` LowCardinality(String) DEFAULT '', + `strategy_namespace` LowCardinality(String) DEFAULT '', + `event_namespace` LowCardinality(String) DEFAULT '', + `actor_name` LowCardinality(String) DEFAULT '', + `exec_venue` LowCardinality(String) DEFAULT '', + `data_venue` LowCardinality(String) DEFAULT '', + `account_capital` Float64 DEFAULT 0, + `portfolio_capital` Float64 DEFAULT 0, + `current_open_notional` Float64 DEFAULT 0, + `remaining_notional_capacity` Float64 DEFAULT 0, + `max_account_leverage` Float32 DEFAULT 0, + `margin_required` Float64 DEFAULT 0, + `ledger_authority` LowCardinality(String) DEFAULT '', + `regime_signal` Int8 DEFAULT 0, + `capital_before` Float64 DEFAULT 0, + `capital_after` Float64 DEFAULT 0, + `peak_capital` Float64 DEFAULT 0, + `drawdown_at_entry` Float32 DEFAULT 0, + `open_positions_count` UInt8 DEFAULT 0, + `scan_uuid` String DEFAULT '', + `bars_held` UInt16 DEFAULT 0, + `entry_payload_json` String DEFAULT '', + `exit_payload_json` String DEFAULT '', + `execution_payload_json` String DEFAULT '', + `friction_payload_json` String DEFAULT '', + `event_payload_json` String DEFAULT '', + `market_state_bundle_json` String DEFAULT '', + `tp_base_pct` Float32 DEFAULT 0, + `tp_effective_pct` Float32 DEFAULT 0, + `our_leverage` Float32 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(ts) +ORDER BY (ts, asset) +SETTINGS index_granularity = 8192 diff --git a/prod/clickhouse/pink/v7_decision_events.sql b/prod/clickhouse/pink/v7_decision_events.sql new file mode 100644 index 0000000..1262fbb --- /dev/null +++ b/prod/clickhouse/pink/v7_decision_events.sql @@ -0,0 +1,51 @@ +CREATE TABLE IF NOT EXISTS dolphin_pink.v7_decision_events +( + `ts` DateTime64(6, 'UTC'), + `ts_day` Date MATERIALIZED toDate(ts), + `strategy` LowCardinality(String), + `runtime_namespace` LowCardinality(String) DEFAULT '', + `strategy_namespace` LowCardinality(String) DEFAULT '', + `event_namespace` LowCardinality(String) DEFAULT '', + `actor_name` LowCardinality(String) DEFAULT '', + `exec_venue` LowCardinality(String) DEFAULT '', + `data_venue` LowCardinality(String) DEFAULT '', + `source` LowCardinality(String), + `trade_id` String, + `asset` LowCardinality(String), + `side` LowCardinality(String), + `entry_price` Float64, + `current_price` Float64, + `quantity` Float64, + `notional` Float64, + `leverage` Float32, + `bar_idx` UInt32, + `decision_seq` UInt32, + `bars_held` UInt16, + `action` LowCardinality(String), + `reason` LowCardinality(String), + `pnl_pct` Float32, + `mfe` Float32, + `mae` Float32, + `mfe_risk` Float32, + `mae_risk` Float32, + `exit_pressure` Float32, + `rv_comp` Float32, + `mae_thresh1` Float32, + `bounce_score` Float32, + `bounce_risk` Float32, + `ob_imbalance` Float32, + `vel_div_entry` Float32, + `vel_div_now` Float32, + `v50_vel` Float32, + `v750_vel` Float32, + `exf_funding` Float32, + `exf_dvol` Float32, + `exf_fear_greed` Float32, + `exf_taker` Float32, + `posture` LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(ts) +ORDER BY (ts_day, trade_id, decision_seq, ts) +TTL ts_day + toIntervalDay(180) +SETTINGS index_granularity = 8192 diff --git a/prod/clickhouse/prodgreen/00_create_database.sql b/prod/clickhouse/prodgreen/00_create_database.sql new file mode 100644 index 0000000..a937495 --- /dev/null +++ b/prod/clickhouse/prodgreen/00_create_database.sql @@ -0,0 +1 @@ +CREATE DATABASE IF NOT EXISTS dolphin_prodgreen; diff --git a/prod/clickhouse/prodgreen/01_alter_account_events_leverage.sql b/prod/clickhouse/prodgreen/01_alter_account_events_leverage.sql new file mode 100644 index 0000000..c58ed99 --- /dev/null +++ b/prod/clickhouse/prodgreen/01_alter_account_events_leverage.sql @@ -0,0 +1,6 @@ +ALTER TABLE dolphin_prodgreen.account_events + ADD COLUMN IF NOT EXISTS current_open_notional Float64 DEFAULT 0, + ADD COLUMN IF NOT EXISTS current_account_leverage Float32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS exchange_leverage UInt8 DEFAULT 0, + ADD COLUMN IF NOT EXISTS exchange_leverage_mode LowCardinality(String) DEFAULT '', + ADD COLUMN IF NOT EXISTS leverage_mapping_rule LowCardinality(String) DEFAULT ''; diff --git a/prod/clickhouse/prodgreen/01_alter_status_snapshots_leverage.sql b/prod/clickhouse/prodgreen/01_alter_status_snapshots_leverage.sql new file mode 100644 index 0000000..801d664 --- /dev/null +++ b/prod/clickhouse/prodgreen/01_alter_status_snapshots_leverage.sql @@ -0,0 +1,12 @@ +ALTER TABLE dolphin_prodgreen.status_snapshots + ADD COLUMN IF NOT EXISTS conviction_multiplier Float32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS exchange_leverage UInt8 DEFAULT 0, + ADD COLUMN IF NOT EXISTS exchange_leverage_mode LowCardinality(String) DEFAULT '', + ADD COLUMN IF NOT EXISTS leverage_mapping_rule LowCardinality(String) DEFAULT '', + ADD COLUMN IF NOT EXISTS account_capital Float64 DEFAULT 0, + ADD COLUMN IF NOT EXISTS portfolio_capital Float64 DEFAULT 0, + ADD COLUMN IF NOT EXISTS current_open_notional Float64 DEFAULT 0, + ADD COLUMN IF NOT EXISTS current_account_leverage Float32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS remaining_notional_capacity Float64 DEFAULT 0, + ADD COLUMN IF NOT EXISTS max_account_leverage Float32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS ledger_authority LowCardinality(String) DEFAULT ''; diff --git a/prod/clickhouse/prodgreen/01_alter_trade_events_full_payload.sql b/prod/clickhouse/prodgreen/01_alter_trade_events_full_payload.sql new file mode 100644 index 0000000..49e3516 --- /dev/null +++ b/prod/clickhouse/prodgreen/01_alter_trade_events_full_payload.sql @@ -0,0 +1,18 @@ +ALTER TABLE dolphin_prodgreen.trade_events + ADD COLUMN IF NOT EXISTS trade_id String DEFAULT '' AFTER ts, + ADD COLUMN IF NOT EXISTS conviction_multiplier Float32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS exchange_leverage UInt8 DEFAULT 0, + ADD COLUMN IF NOT EXISTS exchange_leverage_mode LowCardinality(String) DEFAULT '', + ADD COLUMN IF NOT EXISTS leverage_mapping_rule LowCardinality(String) DEFAULT '', + ADD COLUMN IF NOT EXISTS account_capital Float64 DEFAULT 0, + ADD COLUMN IF NOT EXISTS portfolio_capital Float64 DEFAULT 0, + ADD COLUMN IF NOT EXISTS current_open_notional Float64 DEFAULT 0, + ADD COLUMN IF NOT EXISTS remaining_notional_capacity Float64 DEFAULT 0, + ADD COLUMN IF NOT EXISTS max_account_leverage Float32 DEFAULT 0, + ADD COLUMN IF NOT EXISTS margin_required Float64 DEFAULT 0, + ADD COLUMN IF NOT EXISTS ledger_authority LowCardinality(String) DEFAULT '', + ADD COLUMN IF NOT EXISTS entry_payload_json String DEFAULT '', + ADD COLUMN IF NOT EXISTS exit_payload_json String DEFAULT '', + ADD COLUMN IF NOT EXISTS execution_payload_json String DEFAULT '', + ADD COLUMN IF NOT EXISTS friction_payload_json String DEFAULT '', + ADD COLUMN IF NOT EXISTS event_payload_json String DEFAULT ''; diff --git a/prod/clickhouse/prodgreen/02_create_trade_reconstruction.sql b/prod/clickhouse/prodgreen/02_create_trade_reconstruction.sql new file mode 100644 index 0000000..5e93d4c --- /dev/null +++ b/prod/clickhouse/prodgreen/02_create_trade_reconstruction.sql @@ -0,0 +1,14 @@ +CREATE TABLE IF NOT EXISTS dolphin_prodgreen.trade_reconstruction +( + `ts` DateTime64(6, 'UTC'), + `ts_day` Date MATERIALIZED toDate(ts), + `trade_id` String, + `event_type` LowCardinality(String), + `event_id` String, + `payload_json` String +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(ts) +ORDER BY (ts_day, trade_id, event_type, event_id) +TTL ts_day + toIntervalDay(180) +SETTINGS index_granularity = 8192 diff --git a/prod/clickhouse/prodgreen/03_create_trade_exit_legs.sql b/prod/clickhouse/prodgreen/03_create_trade_exit_legs.sql new file mode 100644 index 0000000..0bcf8ac --- /dev/null +++ b/prod/clickhouse/prodgreen/03_create_trade_exit_legs.sql @@ -0,0 +1,36 @@ +CREATE TABLE IF NOT EXISTS dolphin_prodgreen.trade_exit_legs +( + `ts` DateTime64(6, 'UTC'), + `ts_day` Date MATERIALIZED toDate(ts), + `date` Date, + `strategy` LowCardinality(String), + `trade_id` String, + `chain_root_trade_id` String DEFAULT '', + `chain_head_leg_id` String DEFAULT '', + `chain_prev_leg_id` String DEFAULT '', + `chain_seq` UInt32 DEFAULT 0, + `chain_token` String DEFAULT '', + `chain_mode` LowCardinality(String) DEFAULT '', + `exit_leg_id` String, + `exit_seq` UInt32, + `command_id` String DEFAULT '', + `source` LowCardinality(String), + `reason` LowCardinality(String), + `asset` LowCardinality(String), + `side` LowCardinality(String), + `entry_price` Float64, + `exit_price` Float64, + `fraction` Float32, + `exit_notional` Float64, + `remaining_notional` Float64, + `remaining_qty` Float64, + `pnl_pct_leg` Float32, + `pnl_leg` Float64, + `pnl_realized_total` Float64, + `bars_held` UInt16 +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(ts) +ORDER BY (ts_day, trade_id, exit_seq, ts) +TTL ts_day + toIntervalDay(180) +SETTINGS index_granularity = 8192 diff --git a/prod/clickhouse/prodgreen/account_events.sql b/prod/clickhouse/prodgreen/account_events.sql new file mode 100644 index 0000000..894b211 --- /dev/null +++ b/prod/clickhouse/prodgreen/account_events.sql @@ -0,0 +1,24 @@ +CREATE TABLE dolphin_prodgreen.account_events +( + `ts` DateTime64(6, 'UTC'), + `event_type` LowCardinality(String), + `strategy` LowCardinality(String), + `posture` LowCardinality(String), + `capital` Float64, + `peak_capital` Float64, + `drawdown_pct` Float32, + `pnl_today` Float64 DEFAULT 0, + `trades_today` UInt16 DEFAULT 0, + `open_positions` UInt8 DEFAULT 0, + `boost` Float32 DEFAULT 1., + `beta` Float32 DEFAULT 0., + `current_open_notional` Float64 DEFAULT 0, + `current_account_leverage` Float32 DEFAULT 0, + `exchange_leverage` UInt8 DEFAULT 0, + `exchange_leverage_mode` LowCardinality(String) DEFAULT '', + `leverage_mapping_rule` LowCardinality(String) DEFAULT '', + `notes` String DEFAULT '' +) +ENGINE = MergeTree +ORDER BY (ts, event_type) +SETTINGS index_granularity = 8192 diff --git a/prod/clickhouse/prodgreen/adaptive_exit_shadow.sql b/prod/clickhouse/prodgreen/adaptive_exit_shadow.sql new file mode 100644 index 0000000..0af9eff --- /dev/null +++ b/prod/clickhouse/prodgreen/adaptive_exit_shadow.sql @@ -0,0 +1,23 @@ +CREATE TABLE dolphin_prodgreen.adaptive_exit_shadow +( + `ts` DateTime64(6, 'UTC'), + `ts_day` Date MATERIALIZED toDate(ts), + `trade_id` String, + `asset` LowCardinality(String), + `bucket_id` UInt8, + `bars_held` UInt16, + `mae_norm` Float32, + `mfe_norm` Float32, + `tau_norm` Float32, + `p_cont` Float32, + `vel_div_entry` Float32, + `vel_div_now` Float32, + `action` LowCardinality(String), + `exit_reason` LowCardinality(String), + `actual_exit` LowCardinality(String), + `pnl_pct` Float32 +) +ENGINE = MergeTree +ORDER BY (ts_day, asset, ts) +TTL ts_day + toIntervalDay(90) +SETTINGS index_granularity = 8192 diff --git a/prod/clickhouse/prodgreen/status_snapshots.sql b/prod/clickhouse/prodgreen/status_snapshots.sql new file mode 100644 index 0000000..59df77f --- /dev/null +++ b/prod/clickhouse/prodgreen/status_snapshots.sql @@ -0,0 +1,31 @@ +CREATE TABLE dolphin_prodgreen.status_snapshots +( + `ts` DateTime64(3, 'UTC'), + `capital` Float64, + `roi_pct` Float32, + `dd_pct` Float32, + `trades_executed` UInt16, + `posture` LowCardinality(String), + `rm` Float32, + `vel_div` Float32, + `vol_ok` UInt8, + `phase` LowCardinality(String), + `mhs_status` LowCardinality(String), + `boost` Float32, + `cat5` Float32, + `conviction_multiplier` Float32 DEFAULT 0, + `exchange_leverage` UInt8 DEFAULT 0, + `exchange_leverage_mode` LowCardinality(String) DEFAULT '', + `leverage_mapping_rule` LowCardinality(String) DEFAULT '', + `account_capital` Float64 DEFAULT 0, + `portfolio_capital` Float64 DEFAULT 0, + `current_open_notional` Float64 DEFAULT 0, + `current_account_leverage` Float32 DEFAULT 0, + `remaining_notional_capacity` Float64 DEFAULT 0, + `max_account_leverage` Float32 DEFAULT 0, + `ledger_authority` LowCardinality(String) DEFAULT '' +) +ENGINE = MergeTree +ORDER BY ts +TTL toDateTime(ts) + toIntervalDay(180) +SETTINGS index_granularity = 8192 diff --git a/prod/clickhouse/prodgreen/trade_events.sql b/prod/clickhouse/prodgreen/trade_events.sql new file mode 100644 index 0000000..4962fca --- /dev/null +++ b/prod/clickhouse/prodgreen/trade_events.sql @@ -0,0 +1,48 @@ +CREATE TABLE dolphin_prodgreen.trade_events +( + `ts` DateTime64(6, 'UTC'), + `trade_id` String DEFAULT '', + `date` Date, + `strategy` LowCardinality(String), + `asset` LowCardinality(String), + `side` LowCardinality(String), + `entry_price` Float64, + `exit_price` Float64 DEFAULT 0, + `quantity` Float64, + `pnl` Float64 DEFAULT 0, + `pnl_pct` Float32 DEFAULT 0, + `exit_reason` LowCardinality(String) DEFAULT '', + `vel_div_entry` Float32, + `boost_at_entry` Float32, + `beta_at_entry` Float32, + `posture` LowCardinality(String), + `leverage` Float32, + `conviction_multiplier` Float32 DEFAULT 0, + `exchange_leverage` UInt8 DEFAULT 0, + `exchange_leverage_mode` LowCardinality(String) DEFAULT '', + `leverage_mapping_rule` LowCardinality(String) DEFAULT '', + `account_capital` Float64 DEFAULT 0, + `portfolio_capital` Float64 DEFAULT 0, + `current_open_notional` Float64 DEFAULT 0, + `remaining_notional_capacity` Float64 DEFAULT 0, + `max_account_leverage` Float32 DEFAULT 0, + `margin_required` Float64 DEFAULT 0, + `ledger_authority` LowCardinality(String) DEFAULT '', + `regime_signal` Int8 DEFAULT 0, + `capital_before` Float64 DEFAULT 0, + `capital_after` Float64 DEFAULT 0, + `peak_capital` Float64 DEFAULT 0, + `drawdown_at_entry` Float32 DEFAULT 0, + `open_positions_count` UInt8 DEFAULT 0, + `scan_uuid` String DEFAULT '', + `bars_held` UInt16 DEFAULT 0, + `entry_payload_json` String DEFAULT '', + `exit_payload_json` String DEFAULT '', + `execution_payload_json` String DEFAULT '', + `friction_payload_json` String DEFAULT '', + `event_payload_json` String DEFAULT '' +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(ts) +ORDER BY (ts, asset) +SETTINGS index_granularity = 8192 diff --git a/prod/clickhouse/prodgreen/v7_decision_events.sql b/prod/clickhouse/prodgreen/v7_decision_events.sql new file mode 100644 index 0000000..e7f839f --- /dev/null +++ b/prod/clickhouse/prodgreen/v7_decision_events.sql @@ -0,0 +1,45 @@ +CREATE TABLE dolphin_prodgreen.v7_decision_events +( + `ts` DateTime64(6, 'UTC'), + `ts_day` Date MATERIALIZED toDate(ts), + `strategy` LowCardinality(String), + `source` LowCardinality(String), + `trade_id` String, + `asset` LowCardinality(String), + `side` LowCardinality(String), + `entry_price` Float64, + `current_price` Float64, + `quantity` Float64, + `notional` Float64, + `leverage` Float32, + `bar_idx` UInt32, + `decision_seq` UInt32, + `bars_held` UInt16, + `action` LowCardinality(String), + `reason` LowCardinality(String), + `pnl_pct` Float32, + `mfe` Float32, + `mae` Float32, + `mfe_risk` Float32, + `mae_risk` Float32, + `exit_pressure` Float32, + `rv_comp` Float32, + `mae_thresh1` Float32, + `bounce_score` Float32, + `bounce_risk` Float32, + `ob_imbalance` Float32, + `vel_div_entry` Float32, + `vel_div_now` Float32, + `v50_vel` Float32, + `v750_vel` Float32, + `exf_funding` Float32, + `exf_dvol` Float32, + `exf_fear_greed` Float32, + `exf_taker` Float32, + `posture` LowCardinality(String) +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(ts) +ORDER BY (ts_day, trade_id, decision_seq, ts) +TTL ts_day + toIntervalDay(180) +SETTINGS index_granularity = 8192 diff --git a/prod/clickhouse/users.xml b/prod/clickhouse/users.xml new file mode 100644 index 0000000..ad7646c --- /dev/null +++ b/prod/clickhouse/users.xml @@ -0,0 +1,21 @@ + + + + dolphin_ch_2026 + + ::/0 + + default + default + + 1 + 0 + 4 + + best_effort + + + + diff --git a/prod/clickhouse/uv/00_create_database.sql b/prod/clickhouse/uv/00_create_database.sql new file mode 100644 index 0000000..11f4460 --- /dev/null +++ b/prod/clickhouse/uv/00_create_database.sql @@ -0,0 +1,5 @@ +-- dolphin_uv database — created if not exists +-- This database holds PRIME's parity-review data only. +-- NEVER touches dolphin.* (BLUE's live namespace). + +CREATE DATABASE IF NOT EXISTS dolphin_uv diff --git a/prod/clickhouse/uv/01_prime_decisions.sql b/prod/clickhouse/uv/01_prime_decisions.sql new file mode 100644 index 0000000..e19f5bb --- /dev/null +++ b/prod/clickhouse/uv/01_prime_decisions.sql @@ -0,0 +1,27 @@ +-- dolphin_uv.prime_decisions — PRIME proceedings journal (parity-review sink) +-- Created by T2 Part 1 journal hardening +-- Schema matches build_journal_frame() in blue_prime/journal.py + +CREATE TABLE IF NOT EXISTS dolphin_uv.prime_decisions +( + ts DateTime64(3), -- UTC, 'YYYY-MM-DD HH:MM:SS.fff' + scan_number UInt64, + mono_ns UInt64, + vel_div Float64, + posture String, + esof_json String, + vol_ok UInt8, + trade_direction Int8, + efsm_pending_slots UInt8, + hooks_json String, + has_entry UInt8, + asset String, + side String, + leverage Float64, + bar_idx UInt32, + entry_json String, + deps_json String, +) +ENGINE = MergeTree() +ORDER BY (scan_number, ts) +SETTINGS index_granularity = 8192 diff --git a/prod/clickhouse/uv/02_prime_scans.sql b/prod/clickhouse/uv/02_prime_scans.sql new file mode 100644 index 0000000..df12b29 --- /dev/null +++ b/prod/clickhouse/uv/02_prime_scans.sql @@ -0,0 +1,20 @@ +-- dolphin_uv.prime_scans — raw scan surface per NG7 scan +-- Created by T2 Part 1 journal hardening (scope addition) +-- Captures the raw scan surface that BLUE receives from Hazelcast +-- This is the input surface that currently exists nowhere durable. +-- Closes the input-recording gap forever. + +CREATE TABLE IF NOT EXISTS dolphin_uv.prime_scans +( + ts DateTime64(3), -- UTC, 'YYYY-MM-DD HH:MM:SS.fff' + scan_number UInt64, + assets String, -- JSON array of asset symbols + asset_prices String, -- JSON array of prices + vel_div Float64, + w50_velocity Float64, + w750_velocity Float64, + instability_50 Float64, +) +ENGINE = MergeTree() +ORDER BY (scan_number, ts) +SETTINGS index_granularity = 8192 diff --git a/prod/clickhouse/uv/03_exec_journal.sql b/prod/clickhouse/uv/03_exec_journal.sql new file mode 100644 index 0000000..fc244d5 --- /dev/null +++ b/prod/clickhouse/uv/03_exec_journal.sql @@ -0,0 +1,30 @@ +-- dolphin_uv.exec_journal — Promotion bridge journal +-- Records every bridged and SUPPRESSED intent from the PRIME promotion bridge. +-- T12: promotion bridge (PRIME decision → KernelIntent → T9 seam) + +CREATE TABLE IF NOT EXISTS dolphin_uv.exec_journal +( + kind String, -- 'BRIDGE' + suppressed UInt8, -- 0=bridged, 1=suppressed (shadow mode) + timestamp DateTime64(3), -- UTC 'YYYY-MM-DD HH:MM:SS.fff' + scan_number UInt64, + intent_id String, + trade_id String, + slot_id UInt32, + asset String, + side String, + action String, -- ENTER/EXIT + reference_price Float64, + target_size Float64, + leverage Float64, + has_uv_prefix UInt8, -- always 1 (u- prefix) + u_prefix_client_id String, -- u- prefixed clientOrderId + suppress_reason String, -- reason if suppressed + promo_metadata String -- JSON metadata from promotion bridge +) +ENGINE = MergeTree() +ORDER BY (scan_number, timestamp) +-- NO TTL: exec_journal is the audit trail matching venue orders to journaled +-- intents (arming checklist §3.6). Retention doctrine: audit surfaces never +-- auto-expire (same ruling as prime_decisions / prime_scans at T2P1 merge). +SETTINGS index_granularity = 8192 diff --git a/prod/clickhouse/uv/04_max_hold_ingress.sql b/prod/clickhouse/uv/04_max_hold_ingress.sql new file mode 100644 index 0000000..11bc916 --- /dev/null +++ b/prod/clickhouse/uv/04_max_hold_ingress.sql @@ -0,0 +1,26 @@ +-- Full per-evaluation ingress for BLUE-faithful UV MAX_HOLD modulation. +-- UV namespace only. Apply during the normal certify-then-restart window. + +CREATE TABLE IF NOT EXISTS dolphin_uv.max_hold_ingress +( + timestamp DateTime64(3, 'UTC'), + trade_id String, + asset LowCardinality(String), + side LowCardinality(String), + evaluation_bar UInt64, + bars_held UInt32, + cascade_count UInt16, + regime_signal Int8, + imbalance_ma5 Float64, + effective_imbalance Float64, + withdrawal_velocity Float64, + base_max_hold_bars UInt32, + max_hold_bars UInt32, + branch LowCardinality(String), + source LowCardinality(String), + fallback UInt8, + ingress_json String +) +ENGINE = MergeTree() +ORDER BY (trade_id, evaluation_bar, timestamp) +SETTINGS index_granularity = 8192; diff --git a/prod/clickhouse/uv/04_stop_watch_events.sql b/prod/clickhouse/uv/04_stop_watch_events.sql new file mode 100644 index 0000000..adcfe53 --- /dev/null +++ b/prod/clickhouse/uv/04_stop_watch_events.sql @@ -0,0 +1,25 @@ +-- UV shadow stop-watcher events — C11 overshoot ledger +-- READ-ONLY wrt BLUE sources. INSERT only via StopWatcher. +-- Namespace: dolphin_uv ONLY. + +CREATE TABLE IF NOT EXISTS dolphin_uv.stop_watch_events +( + `ts` DateTime64(3, 'UTC'), + `trade_id` String, + `asset` LowCardinality(String), + `side` LowCardinality(String), + `entry_price` Float64, + `breach_price` Float64, + `adverse_pct` Float32, + `spread_bps` Float32, + `depth_1pct_usd` Float64, + `obf_lag_ms` Float32, + `kind` LowCardinality(String), + `escalation_step` UInt16 DEFAULT 0, + `resolved_exit_adverse_pct` Float32 DEFAULT 0, + `resolved_excess_vs_contract` Float32 DEFAULT 0 +) +ENGINE = MergeTree +PARTITION BY toYYYYMM(ts) +ORDER BY (ts, trade_id) +SETTINGS index_granularity = 8192; diff --git a/prod/clickhouse/uv/05_alter_prime_decisions_corrupt.sql b/prod/clickhouse/uv/05_alter_prime_decisions_corrupt.sql new file mode 100644 index 0000000..7bfa96c --- /dev/null +++ b/prod/clickhouse/uv/05_alter_prime_decisions_corrupt.sql @@ -0,0 +1,8 @@ +-- T13: Add corrupt-scan columns to existing dolphin_uv.prime_decisions +-- Applied at merge; idempotent via IF NOT EXISTS. + +ALTER TABLE dolphin_uv.prime_decisions + ADD COLUMN IF NOT EXISTS corrupt_scan UInt8 DEFAULT 0; + +ALTER TABLE dolphin_uv.prime_decisions + ADD COLUMN IF NOT EXISTS corrupt_meta_json String DEFAULT ''; diff --git a/prod/clickhouse/uv/05_tp_exit_ingress.sql b/prod/clickhouse/uv/05_tp_exit_ingress.sql new file mode 100644 index 0000000..ccfa86c --- /dev/null +++ b/prod/clickhouse/uv/05_tp_exit_ingress.sql @@ -0,0 +1,32 @@ +-- Full per-evaluation ingress and TP diagnostics for BLUE-faithful UV exits. +-- UV namespace only. Apply during the normal certify-then-restart window. + +CREATE TABLE IF NOT EXISTS dolphin_uv.tp_exit_ingress +( + timestamp DateTime64(3, 'UTC'), + trade_id String, + asset LowCardinality(String), + side LowCardinality(String), + evaluation_bar UInt64, + pnl_pct Float64, + max_favorable_pct Float64, + our_leverage Float64, + base_tp_pct Float64, + tp_pre_floor_pct Float64, + tp_effective_pct Float64, + tp_mod_factor Float64, + tp_floor_pct Float64, + tp_floor_enabled UInt8, + tp_floor_armed UInt8, + cascade_count UInt16, + regime_signal Int8, + imbalance_ma5 Float64, + withdrawal_velocity Float64, + branch LowCardinality(String), + source LowCardinality(String), + fallback UInt8, + ingress_json String +) +ENGINE = MergeTree() +ORDER BY (trade_id, evaluation_bar, timestamp) +SETTINGS index_granularity = 8192; diff --git a/prod/clickhouse/uv/apply_uv_ddl.py b/prod/clickhouse/uv/apply_uv_ddl.py new file mode 100644 index 0000000..29fc755 --- /dev/null +++ b/prod/clickhouse/uv/apply_uv_ddl.py @@ -0,0 +1,125 @@ +#!/usr/bin/env python3 +"""Apply the dolphin_uv DDL set — ONE statement per HTTP POST. + +ClickHouse's HTTP interface rejects multi-statement posts (proven on +2026-06-12 when prod/clickhouse/pink/08_provenance.sql failed as one body), +so this applier splits each .sql file into single statements and posts them +individually. Idempotent by construction: every statement in the set is +CREATE ... IF NOT EXISTS. + +Usage: + python3 prod/clickhouse/uv/apply_uv_ddl.py # apply + verify + python3 prod/clickhouse/uv/apply_uv_ddl.py --dry-run # print statements + python3 prod/clickhouse/uv/apply_uv_ddl.py --verify # verify only + +Exit codes: 0 = applied + verified; 1 = a statement failed or verify found +missing tables. +""" + +from __future__ import annotations + +import argparse +import os +import sys +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Iterator, List, Tuple + +SQL_DIR = Path(__file__).resolve().parent + +EXPECTED_TABLES = { + "prime_decisions", + "prime_scans", + "exec_journal", + "stop_watch_events", +} + + +def _strip_comments(sql: str) -> str: + lines = [] + for line in sql.splitlines(): + if line.strip().startswith("--"): + continue + lines.append(line) + return "\n".join(lines) + + +def iter_statements(sql_dir: Path = SQL_DIR) -> Iterator[Tuple[str, str]]: + """Yield (filename, statement) pairs in sorted file order.""" + for path in sorted(sql_dir.glob("[0-9]*.sql")): # only numbered files, skip alter scripts + body = _strip_comments(path.read_text()) + for stmt in body.split(";"): + stmt = stmt.strip() + if stmt: + yield (path.name, stmt) + + +def apply_statement(stmt: str, *, url: str, user: str, password: str) -> None: + import base64 + token = f"{user}:{password}".encode() + auth_header = f"Basic {base64.b64encode(token).decode()}" + req = urllib.request.Request(url + "/", data=stmt.encode(), method="POST") + req.add_header("Authorization", auth_header) + with urllib.request.urlopen(req, timeout=30) as resp: + resp.read() + + +def verify(*, url: str, user: str, password: str) -> List[str]: + """Return the list of expected tables missing from dolphin_uv.""" + import base64 + token = f"{user}:{password}".encode() + auth_header = f"Basic {base64.b64encode(token).decode()}" + q = urllib.parse.quote_plus("SHOW TABLES FROM dolphin_uv") + req = urllib.request.Request(f"{url}/?query={q}", method="POST") + req.add_header("Authorization", auth_header) + try: + with urllib.request.urlopen(req, timeout=15) as resp: + present = set(resp.read().decode().split()) + except Exception as exc: # noqa: BLE001 — db may not exist yet + print(f"verify: SHOW TABLES failed ({exc})") + return sorted(EXPECTED_TABLES) + return sorted(EXPECTED_TABLES - present) + + +def main(argv: List[str] | None = None) -> int: + ap = argparse.ArgumentParser(description="Apply dolphin_uv DDLs") + ap.add_argument("--ch-url", default=os.environ.get("CH_URL", "http://localhost:8123")) + ap.add_argument("--user", default=os.environ.get("CH_USER", "dolphin")) + ap.add_argument("--password", default=os.environ.get("CH_PASS", "dolphin_ch_2026")) + ap.add_argument("--dry-run", action="store_true") + ap.add_argument("--verify", action="store_true", help="verify only, no apply") + args = ap.parse_args(argv) + + if not args.verify: + for fname, stmt in iter_statements(): + head = stmt.splitlines()[0][:90] + if args.dry_run: + print(f"[dry-run] {fname}: {head}") + continue + try: + apply_statement(stmt, url=args.ch_url, user=args.user, + password=args.password) + print(f"OK {fname}: {head}") + except Exception as exc: + if "alter" in fname.lower(): + print(f"NOTE {fname}: {head}\n {exc} (expected if table already has columns)") + else: + print(f"FAIL {fname}: {head}\n {exc}") + return 1 + except Exception as exc: # noqa: BLE001 + print(f"FAIL {fname}: {head}\n {exc}") + return 1 + if args.dry_run: + return 0 + + missing = verify(url=args.ch_url, user=args.user, password=args.password) + if missing: + print(f"verify: MISSING tables in dolphin_uv: {missing}") + return 1 + print(f"verify: all {len(EXPECTED_TABLES)} expected tables present") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/prod/clickhouse/uv/prime_decisions.sql b/prod/clickhouse/uv/prime_decisions.sql new file mode 100644 index 0000000..e2f4686 --- /dev/null +++ b/prod/clickhouse/uv/prime_decisions.sql @@ -0,0 +1,42 @@ +-- PASS4_UV_WIRE.5 — PRIME proceedings journal DDL +-- ============================================================================ +-- Namespace: dolphin_uv (NOT dolphin.* — that's BLUE's live namespace, +-- polluting it is FORBIDDEN). Create the table, then journal.py writes into +-- it via HTTP INSERT JSONEachRow. MergeTree ordered by (scan_number, ts) +-- so parity diff against dolphin.trade_events is a fast scan-order join. +-- ============================================================================ + +CREATE DATABASE IF NOT EXISTS dolphin_uv; + +CREATE TABLE IF NOT EXISTS dolphin_uv.prime_decisions +( + -- identity + ts DateTime64(3, 'UTC') COMMENT 'wall-clock timestamp of the scan', + scan_number UInt32 COMMENT 'scan number (joins to trade_events)', + mono_ns Int64 COMMENT 'monotonic ns since epoch for ordering', + + -- inputs (from ScanContext) + vel_div Float64 COMMENT 'velocity divergence (non-finite -> 0)', + posture LowCardinality(String) COMMENT 'APEX | TURTLE | HIBERNATE | STALKER | ...', + esof_json String COMMENT 'esof_payload as JSON (might be None)', + vol_ok UInt8 COMMENT 'vol_ok flag (0/1)', + trade_direction Int8 COMMENT '-1 SHORT / 0 flat / 1 LONG', + efsm_pending_slots UInt8 COMMENT 'EFSM pending slot count (>=0)', + + -- 11 hook effects (from HookRunner frame) + hooks_json String COMMENT 'HookEffect list: name, ok, note, mutations, elapsed_us', + + -- decision (from step_bar) + has_entry UInt8 COMMENT '1 = entry generated', + asset LowCardinality(String) COMMENT 'traded asset symbol', + side LowCardinality(String) COMMENT 'BUY | SELL | ...', + leverage Float64 COMMENT 'applied leverage', + bar_idx Int32 COMMENT 'bar index when entry fired', + entry_json String COMMENT 'full entry dict as JSON', + + -- deps status + deps_json String COMMENT 'dep status map as JSON' +) +ENGINE = MergeTree() +ORDER BY (scan_number, ts) +COMMENT 'PRIME proceedings -- parity-diff sink against dolphin.trade_events'; diff --git a/prod/clickhouse/violet/30_violet_economics.sql b/prod/clickhouse/violet/30_violet_economics.sql new file mode 100644 index 0000000..02bb7a2 --- /dev/null +++ b/prod/clickhouse/violet/30_violet_economics.sql @@ -0,0 +1,23 @@ +-- VIOLET PASS9: economics/observability — exactly-one-row-per-event sink. +-- Columns match EconomicsRow field set (asserted by test_violet_economics_sink). +-- ReplacingMergeTree keyed on (session_id, event_seq) so a re-emit collapses to +-- ONE row — the PINK duplicate-emission debt, paid down. +-- wait_for_async_insert=1 on the economics write path (set in the sink). +-- Apply via apply_violet_ddl.py (one statement per HTTP POST). + +CREATE TABLE IF NOT EXISTS dolphin_violet.violet_economics +( + `session_id` String, + `event_seq` UInt64, + `asset` LowCardinality(String), + `realized_pnl` Float64, + `fee` Float64, + `capital_after` Float64, + `pnl_source` LowCardinality(String), + `capital_source` LowCardinality(String), + `ts` DateTime64(3, 'UTC'), + `mono_ns` UInt64 +) +ENGINE = ReplacingMergeTree +ORDER BY (session_id, event_seq, ts) +TTL toDate(ts) + toIntervalDay(180); diff --git a/prod/clickhouse/violet/31_violet_decision_divergence.sql b/prod/clickhouse/violet/31_violet_decision_divergence.sql new file mode 100644 index 0000000..172d473 --- /dev/null +++ b/prod/clickhouse/violet/31_violet_decision_divergence.sql @@ -0,0 +1,19 @@ +-- VIOLET PASS9: per-factor VIOLET-vs-BLUE decision divergence monitor (v2). +-- One row per (asset, factor, event_seq). Extends the V1 feed-divergence +-- idea to the factor plane (boost/beta/mc_scale/esof/ob/dc/posture etc.). +-- Columns match DecisionDivergenceRow field set (asserted by test). +-- Apply via apply_violet_ddl.py (one statement per HTTP POST). + +CREATE TABLE IF NOT EXISTS dolphin_violet.violet_decision_divergence +( + `ts` DateTime64(3, 'UTC'), + `asset` LowCardinality(String), + `factor` LowCardinality(String), + `violet_value` Float64, + `blue_value` Float64, + `abs_err` Float64, + `event_seq` UInt64 +) +ENGINE = MergeTree +ORDER BY (asset, factor, ts) +TTL toDate(ts) + toIntervalDay(180);