99e529c32a
14 tables consolidated from the LIVE post-ALTER dolphin_pink schema (maras_tp + provenance folded into base CREATEs — a fresh DB never replays ALTER chains) + violet_feed_divergence (scan-vs-venue divergence metric, session_id + plane seqs + mono_ns). apply_violet_ddl.py posts ONE statement per HTTP request (multi-statement posts fail — proven on pink 08), is idempotent (all IF NOT EXISTS, double-apply tested live), and verifies the expected table set. Applied to live CH: verify all 14 present. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
118 lines
4.3 KiB
Python
118 lines
4.3 KiB
Python
#!/usr/bin/env python3
|
|
"""Apply the dolphin_violet DDL set — ONE statement per HTTP POST.
|
|
|
|
ClickHouse's HTTP interface rejects multi-statement posts (proven on
|
|
2026-06-12 when prod/clickhouse/pink/08_provenance.sql failed as one body),
|
|
so this applier splits each .sql file into single statements and posts them
|
|
individually. Idempotent by construction: every statement in the set is
|
|
CREATE ... IF NOT EXISTS.
|
|
|
|
Usage:
|
|
python3 prod/clickhouse/violet/apply_violet_ddl.py # apply + verify
|
|
python3 prod/clickhouse/violet/apply_violet_ddl.py --dry-run # print statements
|
|
python3 prod/clickhouse/violet/apply_violet_ddl.py --verify # verify only
|
|
|
|
Exit codes: 0 = applied + verified; 1 = a statement failed or verify found
|
|
missing tables.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import os
|
|
import sys
|
|
import urllib.parse
|
|
import urllib.request
|
|
from pathlib import Path
|
|
from typing import Iterator, List, Tuple
|
|
|
|
SQL_DIR = Path(__file__).resolve().parent
|
|
|
|
EXPECTED_TABLES = {
|
|
"policy_events", "trade_reconstruction", "trade_exit_legs",
|
|
"position_state", "anomaly_events", "account_events",
|
|
"status_snapshots", "trade_events", "v7_decision_events",
|
|
"adaptive_exit_shadow", "fee_settled_events",
|
|
"sc_bucket_gauge_shadow", "sc_threshold_advisor_shadow",
|
|
"violet_feed_divergence",
|
|
}
|
|
|
|
|
|
def _strip_comments(sql: str) -> str:
|
|
lines = []
|
|
for line in sql.splitlines():
|
|
if line.strip().startswith("--"):
|
|
continue
|
|
lines.append(line)
|
|
return "\n".join(lines)
|
|
|
|
|
|
def iter_statements(sql_dir: Path = SQL_DIR) -> Iterator[Tuple[str, str]]:
|
|
"""Yield (filename, statement) pairs in sorted file order."""
|
|
for path in sorted(sql_dir.glob("*.sql")):
|
|
body = _strip_comments(path.read_text())
|
|
for stmt in body.split(";"):
|
|
stmt = stmt.strip()
|
|
if stmt:
|
|
yield (path.name, stmt)
|
|
|
|
|
|
def apply_statement(stmt: str, *, url: str, user: str, password: str) -> None:
|
|
req = urllib.request.Request(url + "/", data=stmt.encode(), method="POST")
|
|
req.add_header("X-ClickHouse-User", user)
|
|
req.add_header("X-ClickHouse-Key", password)
|
|
with urllib.request.urlopen(req, timeout=30) as resp:
|
|
resp.read()
|
|
|
|
|
|
def verify(*, url: str, user: str, password: str) -> List[str]:
|
|
"""Return the list of expected tables missing from dolphin_violet."""
|
|
q = urllib.parse.quote_plus("SHOW TABLES FROM dolphin_violet")
|
|
req = urllib.request.Request(f"{url}/?query={q}", method="POST")
|
|
req.add_header("X-ClickHouse-User", user)
|
|
req.add_header("X-ClickHouse-Key", password)
|
|
try:
|
|
with urllib.request.urlopen(req, timeout=15) as resp:
|
|
present = set(resp.read().decode().split())
|
|
except Exception as exc: # noqa: BLE001 — db may not exist yet
|
|
print(f"verify: SHOW TABLES failed ({exc})")
|
|
return sorted(EXPECTED_TABLES)
|
|
return sorted(EXPECTED_TABLES - present)
|
|
|
|
|
|
def main(argv: List[str] | None = None) -> int:
|
|
ap = argparse.ArgumentParser(description="Apply dolphin_violet DDLs")
|
|
ap.add_argument("--ch-url", default=os.environ.get("CH_URL", "http://localhost:8123"))
|
|
ap.add_argument("--user", default=os.environ.get("CH_USER", "dolphin"))
|
|
ap.add_argument("--password", default=os.environ.get("CH_PASS", "dolphin_ch_2026"))
|
|
ap.add_argument("--dry-run", action="store_true")
|
|
ap.add_argument("--verify", action="store_true", help="verify only, no apply")
|
|
args = ap.parse_args(argv)
|
|
|
|
if not args.verify:
|
|
for fname, stmt in iter_statements():
|
|
head = stmt.splitlines()[0][:90]
|
|
if args.dry_run:
|
|
print(f"[dry-run] {fname}: {head}")
|
|
continue
|
|
try:
|
|
apply_statement(stmt, url=args.ch_url, user=args.user,
|
|
password=args.password)
|
|
print(f"OK {fname}: {head}")
|
|
except Exception as exc: # noqa: BLE001
|
|
print(f"FAIL {fname}: {head}\n {exc}")
|
|
return 1
|
|
if args.dry_run:
|
|
return 0
|
|
|
|
missing = verify(url=args.ch_url, user=args.user, password=args.password)
|
|
if missing:
|
|
print(f"verify: MISSING tables in dolphin_violet: {missing}")
|
|
return 1
|
|
print(f"verify: all {len(EXPECTED_TABLES)} expected tables present")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|