114 lines
4.0 KiB
Python
114 lines
4.0 KiB
Python
|
from prefect import flow, task, get_run_logger
|
||
|
|
from prefect.task_runners import ConcurrentTaskRunner
|
||
|
|
from datetime import datetime, timezone
|
||
|
|
import sys
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
sys.path.insert(0, str(Path(__file__).parent))
|
||
|
|
sys.path.insert(0, str(Path(__file__).parent.parent / "external_factors"))
|
||
|
|
|
||
|
|
HZ_KEY = "exf_latest"
|
||
|
|
HZ_MAP = "DOLPHIN_FEATURES"
|
||
|
|
ACB_KEYS = ["funding_btc","funding_eth","dvol_btc","dvol_eth","fng","vix","ls_btc","taker","oi_btc"]
|
||
|
|
|
||
|
|
@task(name="hz_push", retries=3, retry_delay_seconds=1)
|
||
|
|
def hz_push_task(client, payload: dict):
|
||
|
|
import json
|
||
|
|
try:
|
||
|
|
payload = dict(payload)
|
||
|
|
payload["_pushed_at"] = datetime.now(timezone.utc).isoformat()
|
||
|
|
client.get_map(HZ_MAP).blocking().put(HZ_KEY, json.dumps(payload))
|
||
|
|
return True
|
||
|
|
except Exception as e:
|
||
|
|
return False
|
||
|
|
|
||
|
|
@flow(name="exf-live", task_runner=ConcurrentTaskRunner(), log_prints=True)
|
||
|
|
def exf_live_flow(warmup_s: int = 20):
|
||
|
|
from realtime_exf_service import RealTimeExFService
|
||
|
|
from exf_persistence import ExFPersistenceService
|
||
|
|
from _hz_push import make_hz_client
|
||
|
|
import time
|
||
|
|
|
||
|
|
log = get_run_logger()
|
||
|
|
|
||
|
|
# Start services
|
||
|
|
svc = RealTimeExFService()
|
||
|
|
svc.start()
|
||
|
|
log.info(f"RealTimeExFService started — warmup {warmup_s}s")
|
||
|
|
time.sleep(warmup_s)
|
||
|
|
|
||
|
|
persist = ExFPersistenceService(flush_interval_s=300)
|
||
|
|
persist.start()
|
||
|
|
log.info("ExFPersistenceService started")
|
||
|
|
|
||
|
|
client = make_hz_client()
|
||
|
|
log.info("Hazelcast connected")
|
||
|
|
|
||
|
|
pushes = 0
|
||
|
|
last_log = 0
|
||
|
|
|
||
|
|
log.info(f"Main loop starting — pushing every 0.5s")
|
||
|
|
|
||
|
|
try:
|
||
|
|
while True:
|
||
|
|
t0 = time.monotonic()
|
||
|
|
|
||
|
|
# Get indicators
|
||
|
|
indicators = svc.get_indicators(dual_sample=True)
|
||
|
|
staleness = indicators.pop("_staleness", {})
|
||
|
|
|
||
|
|
# Build payload
|
||
|
|
payload = {k: v for k, v in indicators.items()
|
||
|
|
if isinstance(v, (int, float, str, bool))}
|
||
|
|
payload["_staleness_s"] = {k: round(v, 1)
|
||
|
|
for k, v in staleness.items()
|
||
|
|
if isinstance(v, (int, float))}
|
||
|
|
|
||
|
|
# Check ACB
|
||
|
|
acb_present = [k for k in ACB_KEYS
|
||
|
|
if payload.get(k) is not None
|
||
|
|
and isinstance(payload[k], (int, float))
|
||
|
|
and payload[k] == payload[k]]
|
||
|
|
|
||
|
|
payload["_acb_ready"] = len(acb_present) == len(ACB_KEYS)
|
||
|
|
payload["_acb_present"] = f"{len(acb_present)}/{len(ACB_KEYS)}"
|
||
|
|
payload["_acb_missing"] = [k for k in ACB_KEYS if k not in acb_present]
|
||
|
|
payload["_ok_count"] = len([k for k in payload.keys() if not k.startswith('_')])
|
||
|
|
payload["_timestamp"] = datetime.now(timezone.utc).isoformat()
|
||
|
|
payload["_push_seq"] = pushes
|
||
|
|
|
||
|
|
# Push to HZ
|
||
|
|
try:
|
||
|
|
hz_push_task.submit(client, payload).result(timeout=2.0)
|
||
|
|
pushes += 1
|
||
|
|
except Exception as e:
|
||
|
|
log.warning(f"HZ push failed: {e}")
|
||
|
|
|
||
|
|
# Persist
|
||
|
|
persist.update_snapshot(payload)
|
||
|
|
|
||
|
|
# Log status every 60s
|
||
|
|
if time.time() - last_log > 60:
|
||
|
|
stats = persist.get_stats()
|
||
|
|
log.info(
|
||
|
|
f"Status | pushes={pushes} | "
|
||
|
|
f"acb={payload['_acb_present']} ready={payload['_acb_ready']} | "
|
||
|
|
f"files={stats.get('files_written', 0)}"
|
||
|
|
)
|
||
|
|
last_log = time.time()
|
||
|
|
|
||
|
|
# Maintain interval
|
||
|
|
elapsed = time.monotonic() - t0
|
||
|
|
time.sleep(max(0.0, 0.5 - elapsed))
|
||
|
|
|
||
|
|
except KeyboardInterrupt:
|
||
|
|
log.info("Interrupted")
|
||
|
|
finally:
|
||
|
|
svc.stop()
|
||
|
|
persist.stop()
|
||
|
|
client.shutdown()
|
||
|
|
log.info(f"Stopped. Total pushes: {pushes}")
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
exf_live_flow()
|