initial: import DOLPHIN baseline 2026-04-21 from dolphinng5_predict working tree
Includes core prod + GREEN/BLUE subsystems: - prod/ (BLUE harness, configs, scripts, docs) - nautilus_dolphin/ (GREEN Nautilus-native impl + dvae/ preserved) - adaptive_exit/ (AEM engine + models/bucket_assignments.pkl) - Observability/ (EsoF advisor, TUI, dashboards) - external_factors/ (EsoF producer) - mc_forewarning_qlabs_fork/ (MC regime/envelope) Excludes runtime caches, logs, backups, and reproducible artifacts per .gitignore.
This commit is contained in:
113
prod/exf_prefect_flow.py
Executable file
113
prod/exf_prefect_flow.py
Executable file
@@ -0,0 +1,113 @@
|
||||
from prefect import flow, task, get_run_logger
|
||||
from prefect.task_runners import ConcurrentTaskRunner
|
||||
from datetime import datetime, timezone
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent))
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / "external_factors"))
|
||||
|
||||
HZ_KEY = "exf_latest"
|
||||
HZ_MAP = "DOLPHIN_FEATURES"
|
||||
ACB_KEYS = ["funding_btc","funding_eth","dvol_btc","dvol_eth","fng","vix","ls_btc","taker","oi_btc"]
|
||||
|
||||
@task(name="hz_push", retries=3, retry_delay_seconds=1)
|
||||
def hz_push_task(client, payload: dict):
|
||||
import json
|
||||
try:
|
||||
payload = dict(payload)
|
||||
payload["_pushed_at"] = datetime.now(timezone.utc).isoformat()
|
||||
client.get_map(HZ_MAP).blocking().put(HZ_KEY, json.dumps(payload))
|
||||
return True
|
||||
except Exception as e:
|
||||
return False
|
||||
|
||||
@flow(name="exf-live", task_runner=ConcurrentTaskRunner(), log_prints=True)
|
||||
def exf_live_flow(warmup_s: int = 20):
|
||||
from realtime_exf_service import RealTimeExFService
|
||||
from exf_persistence import ExFPersistenceService
|
||||
from _hz_push import make_hz_client
|
||||
import time
|
||||
|
||||
log = get_run_logger()
|
||||
|
||||
# Start services
|
||||
svc = RealTimeExFService()
|
||||
svc.start()
|
||||
log.info(f"RealTimeExFService started — warmup {warmup_s}s")
|
||||
time.sleep(warmup_s)
|
||||
|
||||
persist = ExFPersistenceService(flush_interval_s=300)
|
||||
persist.start()
|
||||
log.info("ExFPersistenceService started")
|
||||
|
||||
client = make_hz_client()
|
||||
log.info("Hazelcast connected")
|
||||
|
||||
pushes = 0
|
||||
last_log = 0
|
||||
|
||||
log.info(f"Main loop starting — pushing every 0.5s")
|
||||
|
||||
try:
|
||||
while True:
|
||||
t0 = time.monotonic()
|
||||
|
||||
# Get indicators
|
||||
indicators = svc.get_indicators(dual_sample=True)
|
||||
staleness = indicators.pop("_staleness", {})
|
||||
|
||||
# Build payload
|
||||
payload = {k: v for k, v in indicators.items()
|
||||
if isinstance(v, (int, float, str, bool))}
|
||||
payload["_staleness_s"] = {k: round(v, 1)
|
||||
for k, v in staleness.items()
|
||||
if isinstance(v, (int, float))}
|
||||
|
||||
# Check ACB
|
||||
acb_present = [k for k in ACB_KEYS
|
||||
if payload.get(k) is not None
|
||||
and isinstance(payload[k], (int, float))
|
||||
and payload[k] == payload[k]]
|
||||
|
||||
payload["_acb_ready"] = len(acb_present) == len(ACB_KEYS)
|
||||
payload["_acb_present"] = f"{len(acb_present)}/{len(ACB_KEYS)}"
|
||||
payload["_acb_missing"] = [k for k in ACB_KEYS if k not in acb_present]
|
||||
payload["_ok_count"] = len([k for k in payload.keys() if not k.startswith('_')])
|
||||
payload["_timestamp"] = datetime.now(timezone.utc).isoformat()
|
||||
payload["_push_seq"] = pushes
|
||||
|
||||
# Push to HZ
|
||||
try:
|
||||
hz_push_task.submit(client, payload).result(timeout=2.0)
|
||||
pushes += 1
|
||||
except Exception as e:
|
||||
log.warning(f"HZ push failed: {e}")
|
||||
|
||||
# Persist
|
||||
persist.update_snapshot(payload)
|
||||
|
||||
# Log status every 60s
|
||||
if time.time() - last_log > 60:
|
||||
stats = persist.get_stats()
|
||||
log.info(
|
||||
f"Status | pushes={pushes} | "
|
||||
f"acb={payload['_acb_present']} ready={payload['_acb_ready']} | "
|
||||
f"files={stats.get('files_written', 0)}"
|
||||
)
|
||||
last_log = time.time()
|
||||
|
||||
# Maintain interval
|
||||
elapsed = time.monotonic() - t0
|
||||
time.sleep(max(0.0, 0.5 - elapsed))
|
||||
|
||||
except KeyboardInterrupt:
|
||||
log.info("Interrupted")
|
||||
finally:
|
||||
svc.stop()
|
||||
persist.stop()
|
||||
client.shutdown()
|
||||
log.info(f"Stopped. Total pushes: {pushes}")
|
||||
|
||||
if __name__ == "__main__":
|
||||
exf_live_flow()
|
||||
Reference in New Issue
Block a user