hjnormey/sentiment-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

clickhouse: commit exit-leg/reconstruction/maras DDLs + UV/violet DDLs (M1 vendor reconcile, upstream-first)

Browse Source
This commit is contained in:
Codex
2026-07-12 19:13:25 +02:00
parent d763f77516
commit dc90f6d77e
46 changed files with 1167 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
prod/clickhouse/uv/00_create_database.sql Normal file
View File

@@ -0,0 +1,5 @@
-- dolphin_uv database — created if not exists
-- This database holds PRIME's parity-review data only.
-- NEVER touches dolphin.* (BLUE's live namespace).
CREATE DATABASE IF NOT EXISTS dolphin_uv

27
prod/clickhouse/uv/01_prime_decisions.sql Normal file
View File

@@ -0,0 +1,27 @@
-- dolphin_uv.prime_decisions — PRIME proceedings journal (parity-review sink)
-- Created by T2 Part 1 journal hardening
-- Schema matches build_journal_frame() in blue_prime/journal.py
CREATE TABLE IF NOT EXISTS dolphin_uv.prime_decisions
(
ts DateTime64(3), -- UTC, 'YYYY-MM-DD HH:MM:SS.fff'
scan_number UInt64,
mono_ns UInt64,
vel_div Float64,
posture String,
esof_json String,
vol_ok UInt8,
trade_direction Int8,
efsm_pending_slots UInt8,
hooks_json String,
has_entry UInt8,
asset String,
side String,
leverage Float64,
bar_idx UInt32,
entry_json String,
deps_json String,
)
ENGINE = MergeTree()
ORDER BY (scan_number, ts)
SETTINGS index_granularity = 8192

20
prod/clickhouse/uv/02_prime_scans.sql Normal file
View File

@@ -0,0 +1,20 @@
-- dolphin_uv.prime_scans — raw scan surface per NG7 scan
-- Created by T2 Part 1 journal hardening (scope addition)
-- Captures the raw scan surface that BLUE receives from Hazelcast
-- This is the input surface that currently exists nowhere durable.
-- Closes the input-recording gap forever.
CREATE TABLE IF NOT EXISTS dolphin_uv.prime_scans
(
ts DateTime64(3), -- UTC, 'YYYY-MM-DD HH:MM:SS.fff'
scan_number UInt64,
assets String, -- JSON array of asset symbols
asset_prices String, -- JSON array of prices
vel_div Float64,
w50_velocity Float64,
w750_velocity Float64,
instability_50 Float64,
)
ENGINE = MergeTree()
ORDER BY (scan_number, ts)
SETTINGS index_granularity = 8192

30
prod/clickhouse/uv/03_exec_journal.sql Normal file
View File

@@ -0,0 +1,30 @@
-- dolphin_uv.exec_journal — Promotion bridge journal
-- Records every bridged and SUPPRESSED intent from the PRIME promotion bridge.
-- T12: promotion bridge (PRIME decision → KernelIntent → T9 seam)
CREATE TABLE IF NOT EXISTS dolphin_uv.exec_journal
(
kind String, -- 'BRIDGE'
suppressed UInt8, -- 0=bridged, 1=suppressed (shadow mode)
timestamp DateTime64(3), -- UTC 'YYYY-MM-DD HH:MM:SS.fff'
scan_number UInt64,
intent_id String,
trade_id String,
slot_id UInt32,
asset String,
side String,
action String, -- ENTER/EXIT
reference_price Float64,
target_size Float64,
leverage Float64,
has_uv_prefix UInt8, -- always 1 (u- prefix)
u_prefix_client_id String, -- u- prefixed clientOrderId
suppress_reason String, -- reason if suppressed
promo_metadata String -- JSON metadata from promotion bridge
)
ENGINE = MergeTree()
ORDER BY (scan_number, timestamp)
-- NO TTL: exec_journal is the audit trail matching venue orders to journaled
-- intents (arming checklist §3.6). Retention doctrine: audit surfaces never
-- auto-expire (same ruling as prime_decisions / prime_scans at T2P1 merge).
SETTINGS index_granularity = 8192

26
prod/clickhouse/uv/04_max_hold_ingress.sql Normal file
View File

@@ -0,0 +1,26 @@
-- Full per-evaluation ingress for BLUE-faithful UV MAX_HOLD modulation.
-- UV namespace only. Apply during the normal certify-then-restart window.
CREATE TABLE IF NOT EXISTS dolphin_uv.max_hold_ingress
(
timestamp DateTime64(3, 'UTC'),
trade_id String,
asset LowCardinality(String),
side LowCardinality(String),
evaluation_bar UInt64,
bars_held UInt32,
cascade_count UInt16,
regime_signal Int8,
imbalance_ma5 Float64,
effective_imbalance Float64,
withdrawal_velocity Float64,
base_max_hold_bars UInt32,
max_hold_bars UInt32,
branch LowCardinality(String),
source LowCardinality(String),
fallback UInt8,
ingress_json String
)
ENGINE = MergeTree()
ORDER BY (trade_id, evaluation_bar, timestamp)
SETTINGS index_granularity = 8192;

25
prod/clickhouse/uv/04_stop_watch_events.sql Normal file
View File

@@ -0,0 +1,25 @@
-- UV shadow stop-watcher events — C11 overshoot ledger
-- READ-ONLY wrt BLUE sources. INSERT only via StopWatcher.
-- Namespace: dolphin_uv ONLY.
CREATE TABLE IF NOT EXISTS dolphin_uv.stop_watch_events
(
`ts` DateTime64(3, 'UTC'),
`trade_id` String,
`asset` LowCardinality(String),
`side` LowCardinality(String),
`entry_price` Float64,
`breach_price` Float64,
`adverse_pct` Float32,
`spread_bps` Float32,
`depth_1pct_usd` Float64,
`obf_lag_ms` Float32,
`kind` LowCardinality(String),
`escalation_step` UInt16 DEFAULT 0,
`resolved_exit_adverse_pct` Float32 DEFAULT 0,
`resolved_excess_vs_contract` Float32 DEFAULT 0
)
ENGINE = MergeTree
PARTITION BY toYYYYMM(ts)
ORDER BY (ts, trade_id)
SETTINGS index_granularity = 8192;

8
prod/clickhouse/uv/05_alter_prime_decisions_corrupt.sql Normal file
View File

@@ -0,0 +1,8 @@
-- T13: Add corrupt-scan columns to existing dolphin_uv.prime_decisions
-- Applied at merge; idempotent via IF NOT EXISTS.
ALTER TABLE dolphin_uv.prime_decisions
ADD COLUMN IF NOT EXISTS corrupt_scan UInt8 DEFAULT 0;
ALTER TABLE dolphin_uv.prime_decisions
ADD COLUMN IF NOT EXISTS corrupt_meta_json String DEFAULT '';

32
prod/clickhouse/uv/05_tp_exit_ingress.sql Normal file
View File

@@ -0,0 +1,32 @@
-- Full per-evaluation ingress and TP diagnostics for BLUE-faithful UV exits.
-- UV namespace only. Apply during the normal certify-then-restart window.
CREATE TABLE IF NOT EXISTS dolphin_uv.tp_exit_ingress
(
timestamp DateTime64(3, 'UTC'),
trade_id String,
asset LowCardinality(String),
side LowCardinality(String),
evaluation_bar UInt64,
pnl_pct Float64,
max_favorable_pct Float64,
our_leverage Float64,
base_tp_pct Float64,
tp_pre_floor_pct Float64,
tp_effective_pct Float64,
tp_mod_factor Float64,
tp_floor_pct Float64,
tp_floor_enabled UInt8,
tp_floor_armed UInt8,
cascade_count UInt16,
regime_signal Int8,
imbalance_ma5 Float64,
withdrawal_velocity Float64,
branch LowCardinality(String),
source LowCardinality(String),
fallback UInt8,
ingress_json String
)
ENGINE = MergeTree()
ORDER BY (trade_id, evaluation_bar, timestamp)
SETTINGS index_granularity = 8192;

125
prod/clickhouse/uv/apply_uv_ddl.py Normal file
View File

@@ -0,0 +1,125 @@
#!/usr/bin/env python3
"""Apply the dolphin_uv DDL set — ONE statement per HTTP POST.
ClickHouse's HTTP interface rejects multi-statement posts (proven on
2026-06-12 when prod/clickhouse/pink/08_provenance.sql failed as one body),
so this applier splits each .sql file into single statements and posts them
individually. Idempotent by construction: every statement in the set is
CREATE ... IF NOT EXISTS.
Usage:
python3 prod/clickhouse/uv/apply_uv_ddl.py # apply + verify
python3 prod/clickhouse/uv/apply_uv_ddl.py --dry-run # print statements
python3 prod/clickhouse/uv/apply_uv_ddl.py --verify # verify only
Exit codes: 0 = applied + verified; 1 = a statement failed or verify found
missing tables.
"""
from __future__ import annotations
import argparse
import os
import sys
import urllib.parse
import urllib.request
from pathlib import Path
from typing import Iterator, List, Tuple
SQL_DIR = Path(__file__).resolve().parent
EXPECTED_TABLES = {
"prime_decisions",
"prime_scans",
"exec_journal",
"stop_watch_events",
}
def _strip_comments(sql: str) -> str:
lines = []
for line in sql.splitlines():
if line.strip().startswith("--"):
continue
lines.append(line)
return "\n".join(lines)
def iter_statements(sql_dir: Path = SQL_DIR) -> Iterator[Tuple[str, str]]:
"""Yield (filename, statement) pairs in sorted file order."""
for path in sorted(sql_dir.glob("[0-9]*.sql")): # only numbered files, skip alter scripts
body = _strip_comments(path.read_text())
for stmt in body.split(";"):
stmt = stmt.strip()
if stmt:
yield (path.name, stmt)
def apply_statement(stmt: str, *, url: str, user: str, password: str) -> None:
import base64
token = f"{user}:{password}".encode()
auth_header = f"Basic {base64.b64encode(token).decode()}"
req = urllib.request.Request(url + "/", data=stmt.encode(), method="POST")
req.add_header("Authorization", auth_header)
with urllib.request.urlopen(req, timeout=30) as resp:
resp.read()
def verify(*, url: str, user: str, password: str) -> List[str]:
"""Return the list of expected tables missing from dolphin_uv."""
import base64
token = f"{user}:{password}".encode()
auth_header = f"Basic {base64.b64encode(token).decode()}"
q = urllib.parse.quote_plus("SHOW TABLES FROM dolphin_uv")
req = urllib.request.Request(f"{url}/?query={q}", method="POST")
req.add_header("Authorization", auth_header)
try:
with urllib.request.urlopen(req, timeout=15) as resp:
present = set(resp.read().decode().split())
except Exception as exc: # noqa: BLE001 — db may not exist yet
print(f"verify: SHOW TABLES failed ({exc})")
return sorted(EXPECTED_TABLES)
return sorted(EXPECTED_TABLES - present)
def main(argv: List[str] | None = None) -> int:
ap = argparse.ArgumentParser(description="Apply dolphin_uv DDLs")
ap.add_argument("--ch-url", default=os.environ.get("CH_URL", "http://localhost:8123"))
ap.add_argument("--user", default=os.environ.get("CH_USER", "dolphin"))
ap.add_argument("--password", default=os.environ.get("CH_PASS", "dolphin_ch_2026"))
ap.add_argument("--dry-run", action="store_true")
ap.add_argument("--verify", action="store_true", help="verify only, no apply")
args = ap.parse_args(argv)
if not args.verify:
for fname, stmt in iter_statements():
head = stmt.splitlines()[0][:90]
if args.dry_run:
print(f"[dry-run] {fname}: {head}")
continue
try:
apply_statement(stmt, url=args.ch_url, user=args.user,
password=args.password)
print(f"OK {fname}: {head}")
except Exception as exc:
if "alter" in fname.lower():
print(f"NOTE {fname}: {head}\n {exc} (expected if table already has columns)")
else:
print(f"FAIL {fname}: {head}\n {exc}")
return 1
except Exception as exc: # noqa: BLE001
print(f"FAIL {fname}: {head}\n {exc}")
return 1
if args.dry_run:
return 0
missing = verify(url=args.ch_url, user=args.user, password=args.password)
if missing:
print(f"verify: MISSING tables in dolphin_uv: {missing}")
return 1
print(f"verify: all {len(EXPECTED_TABLES)} expected tables present")
return 0
if __name__ == "__main__":
raise SystemExit(main())

42
prod/clickhouse/uv/prime_decisions.sql Normal file
View File

@@ -0,0 +1,42 @@
-- PASS4_UV_WIRE.5 — PRIME proceedings journal DDL
-- ============================================================================
-- Namespace: dolphin_uv (NOT dolphin.* — that's BLUE's live namespace,
-- polluting it is FORBIDDEN). Create the table, then journal.py writes into
-- it via HTTP INSERT JSONEachRow. MergeTree ordered by (scan_number, ts)
-- so parity diff against dolphin.trade_events is a fast scan-order join.
-- ============================================================================
CREATE DATABASE IF NOT EXISTS dolphin_uv;
CREATE TABLE IF NOT EXISTS dolphin_uv.prime_decisions
(
-- identity
ts DateTime64(3, 'UTC') COMMENT 'wall-clock timestamp of the scan',
scan_number UInt32 COMMENT 'scan number (joins to trade_events)',
mono_ns Int64 COMMENT 'monotonic ns since epoch for ordering',
-- inputs (from ScanContext)
vel_div Float64 COMMENT 'velocity divergence (non-finite -> 0)',
posture LowCardinality(String) COMMENT 'APEX | TURTLE | HIBERNATE | STALKER | ...',
esof_json String COMMENT 'esof_payload as JSON (might be None)',
vol_ok UInt8 COMMENT 'vol_ok flag (0/1)',
trade_direction Int8 COMMENT '-1 SHORT / 0 flat / 1 LONG',
efsm_pending_slots UInt8 COMMENT 'EFSM pending slot count (>=0)',
-- 11 hook effects (from HookRunner frame)
hooks_json String COMMENT 'HookEffect list: name, ok, note, mutations, elapsed_us',
-- decision (from step_bar)
has_entry UInt8 COMMENT '1 = entry generated',
asset LowCardinality(String) COMMENT 'traded asset symbol',
side LowCardinality(String) COMMENT 'BUY | SELL | ...',
leverage Float64 COMMENT 'applied leverage',
bar_idx Int32 COMMENT 'bar index when entry fired',
entry_json String COMMENT 'full entry dict as JSON',
-- deps status
deps_json String COMMENT 'dep status map as JSON'
)
ENGINE = MergeTree()
ORDER BY (scan_number, ts)
COMMENT 'PRIME proceedings -- parity-diff sink against dolphin.trade_events';