from prefect import flow, task, get_run_logger from prefect.task_runners import ConcurrentTaskRunner from datetime import datetime, timezone import sys from pathlib import Path sys.path.insert(0, str(Path(__file__).parent)) sys.path.insert(0, str(Path(__file__).parent.parent / "external_factors")) HZ_KEY = "exf_latest" HZ_MAP = "DOLPHIN_FEATURES" ACB_KEYS = ["funding_btc","funding_eth","dvol_btc","dvol_eth","fng","vix","ls_btc","taker","oi_btc"] @task(name="hz_push", retries=3, retry_delay_seconds=1) def hz_push_task(client, payload: dict): import json try: payload = dict(payload) payload["_pushed_at"] = datetime.now(timezone.utc).isoformat() client.get_map(HZ_MAP).blocking().put(HZ_KEY, json.dumps(payload)) return True except Exception as e: return False @flow(name="exf-live", task_runner=ConcurrentTaskRunner(), log_prints=True) def exf_live_flow(warmup_s: int = 20): from realtime_exf_service import RealTimeExFService from exf_persistence import ExFPersistenceService from _hz_push import make_hz_client import time log = get_run_logger() # Start services svc = RealTimeExFService() svc.start() log.info(f"RealTimeExFService started — warmup {warmup_s}s") time.sleep(warmup_s) persist = ExFPersistenceService(flush_interval_s=300) persist.start() log.info("ExFPersistenceService started") client = make_hz_client() log.info("Hazelcast connected") pushes = 0 last_log = 0 log.info(f"Main loop starting — pushing every 0.5s") try: while True: t0 = time.monotonic() # Get indicators indicators = svc.get_indicators(dual_sample=True) staleness = indicators.pop("_staleness", {}) # Build payload payload = {k: v for k, v in indicators.items() if isinstance(v, (int, float, str, bool))} payload["_staleness_s"] = {k: round(v, 1) for k, v in staleness.items() if isinstance(v, (int, float))} # Check ACB acb_present = [k for k in ACB_KEYS if payload.get(k) is not None and isinstance(payload[k], (int, float)) and payload[k] == payload[k]] payload["_acb_ready"] = len(acb_present) == len(ACB_KEYS) payload["_acb_present"] = f"{len(acb_present)}/{len(ACB_KEYS)}" payload["_acb_missing"] = [k for k in ACB_KEYS if k not in acb_present] payload["_ok_count"] = len([k for k in payload.keys() if not k.startswith('_')]) payload["_timestamp"] = datetime.now(timezone.utc).isoformat() payload["_push_seq"] = pushes # Push to HZ try: hz_push_task.submit(client, payload).result(timeout=2.0) pushes += 1 except Exception as e: log.warning(f"HZ push failed: {e}") # Persist persist.update_snapshot(payload) # Log status every 60s if time.time() - last_log > 60: stats = persist.get_stats() log.info( f"Status | pushes={pushes} | " f"acb={payload['_acb_present']} ready={payload['_acb_ready']} | " f"files={stats.get('files_written', 0)}" ) last_log = time.time() # Maintain interval elapsed = time.monotonic() - t0 time.sleep(max(0.0, 0.5 - elapsed)) except KeyboardInterrupt: log.info("Interrupted") finally: svc.stop() persist.stop() client.shutdown() log.info(f"Stopped. Total pushes: {pushes}") if __name__ == "__main__": exf_live_flow()