#!/usr/bin/env python3 """ EXTERNAL FACTORS MATRIX v5.0 - DOLPHIN Compatible with BACKFILL ================================================================ 85 indicators with HISTORICAL query support where available. BACKFILL CAPABILITY: FULL HISTORY (51): CoinMetrics, FRED, DeFi Llama TVL/stables, F&G, Binance funding/OI PARTIAL (12): Deribit DVOL, CoinGecko prices, DEX volume CURRENT ONLY (22): Mempool, order books, spreads, dominance Author: HJ / Claude | Version: 5.0.0 """ import asyncio import aiohttp import numpy as np from dataclasses import dataclass from typing import Dict, List, Optional, Any, Tuple from datetime import datetime, timezone from collections import deque from enum import Enum import json class Category(Enum): DERIVATIVES = "derivatives" ONCHAIN = "onchain" DEFI = "defi" MACRO = "macro" SENTIMENT = "sentiment" MICROSTRUCTURE = "microstructure" class Stationarity(Enum): STATIONARY = "stationary" TREND_UP = "trend_up" EPISODIC = "episodic" class HistoricalSupport(Enum): FULL = "full" # Any historical date PARTIAL = "partial" # Limited history CURRENT = "current" # Real-time only @dataclass class Indicator: id: int name: str category: Category source: str url: str parser: str stationarity: Stationarity historical: HistoricalSupport hist_url: str = "" hist_resolution: str = "" description: str = "" @dataclass class Config: timeout: int = 15 max_concurrent: int = 15 cache_ttl: int = 30 fred_api_key: str = "" # fmt: off INDICATORS: List[Indicator] = [ # DERIVATIVES - Binance (1-10) - Most have FULL history Indicator(1, "funding_btc", Category.DERIVATIVES, "binance", "https://fapi.binance.com/fapi/v1/fundingRate?symbol=BTCUSDT&limit=1", "parse_binance_funding", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://fapi.binance.com/fapi/v1/fundingRate?symbol=BTCUSDT&startTime={start_ms}&endTime={end_ms}&limit=1", "8h", "BTC funding - FULL via startTime/endTime"), Indicator(2, "funding_eth", Category.DERIVATIVES, "binance", "https://fapi.binance.com/fapi/v1/fundingRate?symbol=ETHUSDT&limit=1", "parse_binance_funding", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://fapi.binance.com/fapi/v1/fundingRate?symbol=ETHUSDT&startTime={start_ms}&endTime={end_ms}&limit=1", "8h", "ETH funding"), Indicator(3, "oi_btc", Category.DERIVATIVES, "binance", "https://fapi.binance.com/fapi/v1/openInterest?symbol=BTCUSDT", "parse_binance_oi", Stationarity.TREND_UP, HistoricalSupport.FULL, "https://fapi.binance.com/futures/data/openInterestHist?symbol=BTCUSDT&period=1h&startTime={start_ms}&endTime={end_ms}&limit=1", "1h", "BTC OI - FULL via openInterestHist"), Indicator(4, "oi_eth", Category.DERIVATIVES, "binance", "https://fapi.binance.com/fapi/v1/openInterest?symbol=ETHUSDT", "parse_binance_oi", Stationarity.TREND_UP, HistoricalSupport.FULL, "https://fapi.binance.com/futures/data/openInterestHist?symbol=ETHUSDT&period=1h&startTime={start_ms}&endTime={end_ms}&limit=1", "1h", "ETH OI"), Indicator(5, "ls_btc", Category.DERIVATIVES, "binance", "https://fapi.binance.com/futures/data/globalLongShortAccountRatio?symbol=BTCUSDT&period=1h&limit=1", "parse_binance_ls", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://fapi.binance.com/futures/data/globalLongShortAccountRatio?symbol=BTCUSDT&period=1h&startTime={start_ms}&endTime={end_ms}&limit=1", "1h", "L/S ratio - FULL"), Indicator(6, "ls_eth", Category.DERIVATIVES, "binance", "https://fapi.binance.com/futures/data/globalLongShortAccountRatio?symbol=ETHUSDT&period=1h&limit=1", "parse_binance_ls", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://fapi.binance.com/futures/data/globalLongShortAccountRatio?symbol=ETHUSDT&period=1h&startTime={start_ms}&endTime={end_ms}&limit=1", "1h", "ETH L/S"), Indicator(7, "ls_top", Category.DERIVATIVES, "binance", "https://fapi.binance.com/futures/data/topLongShortAccountRatio?symbol=BTCUSDT&period=1h&limit=1", "parse_binance_ls", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://fapi.binance.com/futures/data/topLongShortAccountRatio?symbol=BTCUSDT&period=1h&startTime={start_ms}&endTime={end_ms}&limit=1", "1h", "Top trader L/S"), Indicator(8, "taker", Category.DERIVATIVES, "binance", "https://fapi.binance.com/futures/data/takerlongshortRatio?symbol=BTCUSDT&period=1h&limit=1", "parse_binance_taker", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://fapi.binance.com/futures/data/takerlongshortRatio?symbol=BTCUSDT&period=1h&startTime={start_ms}&endTime={end_ms}&limit=1", "1h", "Taker ratio"), Indicator(9, "basis", Category.DERIVATIVES, "binance", "https://fapi.binance.com/fapi/v1/premiumIndex?symbol=BTCUSDT", "parse_binance_basis", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "Basis - CURRENT"), Indicator(10, "liq_proxy", Category.DERIVATIVES, "binance", "https://fapi.binance.com/fapi/v1/ticker/24hr?symbol=BTCUSDT", "parse_liq_proxy", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "Liq proxy - CURRENT"), # DERIVATIVES - Deribit (11-18) Indicator(11, "dvol_btc", Category.DERIVATIVES, "deribit", "https://www.deribit.com/api/v2/public/get_volatility_index_data?currency=BTC&resolution=3600&count=1", "parse_deribit_dvol", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://www.deribit.com/api/v2/public/get_volatility_index_data?currency=BTC&resolution=3600&start_timestamp={start_ms}&end_timestamp={end_ms}", "1h", "DVOL - FULL"), Indicator(12, "dvol_eth", Category.DERIVATIVES, "deribit", "https://www.deribit.com/api/v2/public/get_volatility_index_data?currency=ETH&resolution=3600&count=1", "parse_deribit_dvol", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://www.deribit.com/api/v2/public/get_volatility_index_data?currency=ETH&resolution=3600&start_timestamp={start_ms}&end_timestamp={end_ms}", "1h", "ETH DVOL"), Indicator(13, "pcr_vol", Category.DERIVATIVES, "deribit", "https://www.deribit.com/api/v2/public/get_book_summary_by_currency?currency=BTC&kind=option", "parse_deribit_pcr", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "PCR - CURRENT"), Indicator(14, "pcr_oi", Category.DERIVATIVES, "deribit", "https://www.deribit.com/api/v2/public/get_book_summary_by_currency?currency=BTC&kind=option", "parse_deribit_pcr_oi", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "PCR OI - CURRENT"), Indicator(15, "pcr_eth", Category.DERIVATIVES, "deribit", "https://www.deribit.com/api/v2/public/get_book_summary_by_currency?currency=ETH&kind=option", "parse_deribit_pcr", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "ETH PCR - CURRENT"), Indicator(16, "opt_oi", Category.DERIVATIVES, "deribit", "https://www.deribit.com/api/v2/public/get_book_summary_by_currency?currency=BTC&kind=option", "parse_deribit_oi", Stationarity.TREND_UP, HistoricalSupport.CURRENT, "", "", "Options OI - CURRENT"), Indicator(17, "fund_dbt_btc", Category.DERIVATIVES, "deribit", "https://www.deribit.com/api/v2/public/get_funding_rate_value?instrument_name=BTC-PERPETUAL", "parse_deribit_fund", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://www.deribit.com/api/v2/public/get_funding_rate_history?instrument_name=BTC-PERPETUAL&start_timestamp={start_ms}&end_timestamp={end_ms}", "8h", "Deribit fund - FULL"), Indicator(18, "fund_dbt_eth", Category.DERIVATIVES, "deribit", "https://www.deribit.com/api/v2/public/get_funding_rate_value?instrument_name=ETH-PERPETUAL", "parse_deribit_fund", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://www.deribit.com/api/v2/public/get_funding_rate_history?instrument_name=ETH-PERPETUAL&start_timestamp={start_ms}&end_timestamp={end_ms}", "8h", "Deribit ETH fund"), # ONCHAIN - CoinMetrics (19-30) - ALL FULL HISTORY Indicator(19, "rcap_btc", Category.ONCHAIN, "coinmetrics", "https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=CapRealUSD&frequency=1d&page_size=1", "parse_cm", Stationarity.TREND_UP, HistoricalSupport.FULL, "https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=CapRealUSD&frequency=1d&start_time={date}T00:00:00Z&end_time={date}T23:59:59Z", "1d", "Realized cap - FULL"), Indicator(20, "mvrv", Category.ONCHAIN, "coinmetrics", "https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=CapMrktCurUSD,CapRealUSD&frequency=1d&page_size=1", "parse_cm_mvrv", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=CapMrktCurUSD,CapRealUSD&frequency=1d&start_time={date}T00:00:00Z&end_time={date}T23:59:59Z", "1d", "MVRV - FULL"), Indicator(21, "nupl", Category.ONCHAIN, "coinmetrics", "https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=CapMrktCurUSD,CapRealUSD&frequency=1d&page_size=1", "parse_cm_nupl", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=CapMrktCurUSD,CapRealUSD&frequency=1d&start_time={date}T00:00:00Z&end_time={date}T23:59:59Z", "1d", "NUPL - FULL"), Indicator(22, "addr_btc", Category.ONCHAIN, "coinmetrics", "https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=AdrActCnt&frequency=1d&page_size=1", "parse_cm", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=AdrActCnt&frequency=1d&start_time={date}T00:00:00Z&end_time={date}T23:59:59Z", "1d", "Active addr - FULL"), Indicator(23, "addr_eth", Category.ONCHAIN, "coinmetrics", "https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=eth&metrics=AdrActCnt&frequency=1d&page_size=1", "parse_cm", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=eth&metrics=AdrActCnt&frequency=1d&start_time={date}T00:00:00Z&end_time={date}T23:59:59Z", "1d", "ETH addr - FULL"), Indicator(24, "txcnt", Category.ONCHAIN, "coinmetrics", "https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=TxCnt&frequency=1d&page_size=1", "parse_cm", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=TxCnt&frequency=1d&start_time={date}T00:00:00Z&end_time={date}T23:59:59Z", "1d", "TX count - FULL"), Indicator(25, "fees_btc", Category.ONCHAIN, "coinmetrics", "https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=FeeTotUSD&frequency=1d&page_size=1", "parse_cm", Stationarity.EPISODIC, HistoricalSupport.FULL, "https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=FeeTotUSD&frequency=1d&start_time={date}T00:00:00Z&end_time={date}T23:59:59Z", "1d", "BTC fees - FULL"), Indicator(26, "fees_eth", Category.ONCHAIN, "coinmetrics", "https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=eth&metrics=FeeTotUSD&frequency=1d&page_size=1", "parse_cm", Stationarity.EPISODIC, HistoricalSupport.FULL, "https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=eth&metrics=FeeTotUSD&frequency=1d&start_time={date}T00:00:00Z&end_time={date}T23:59:59Z", "1d", "ETH fees - FULL"), Indicator(27, "nvt", Category.ONCHAIN, "coinmetrics", "https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=NVTAdj&frequency=1d&page_size=1", "parse_cm", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=NVTAdj&frequency=1d&start_time={date}T00:00:00Z&end_time={date}T23:59:59Z", "1d", "NVT - FULL"), Indicator(28, "velocity", Category.ONCHAIN, "coinmetrics", "https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=VelCur1yr&frequency=1d&page_size=1", "parse_cm", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=VelCur1yr&frequency=1d&start_time={date}T00:00:00Z&end_time={date}T23:59:59Z", "1d", "Velocity - FULL"), Indicator(29, "sply_act", Category.ONCHAIN, "coinmetrics", "https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=SplyAct1yr&frequency=1d&page_size=1", "parse_cm", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=SplyAct1yr&frequency=1d&start_time={date}T00:00:00Z&end_time={date}T23:59:59Z", "1d", "Active supply - FULL"), Indicator(30, "rcap_eth", Category.ONCHAIN, "coinmetrics", "https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=eth&metrics=CapRealUSD&frequency=1d&page_size=1", "parse_cm", Stationarity.TREND_UP, HistoricalSupport.FULL, "https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=eth&metrics=CapRealUSD&frequency=1d&start_time={date}T00:00:00Z&end_time={date}T23:59:59Z", "1d", "ETH rcap - FULL"), # ONCHAIN - Blockchain.info (31-37) Indicator(31, "hashrate", Category.ONCHAIN, "blockchain", "https://blockchain.info/q/hashrate", "parse_bc", Stationarity.TREND_UP, HistoricalSupport.FULL, "https://api.blockchain.info/charts/hash-rate?timespan=1days&start={date}&format=json", "1d", "Hashrate - FULL"), Indicator(32, "difficulty", Category.ONCHAIN, "blockchain", "https://blockchain.info/q/getdifficulty", "parse_bc", Stationarity.TREND_UP, HistoricalSupport.FULL, "https://api.blockchain.info/charts/difficulty?timespan=1days&start={date}&format=json", "1d", "Difficulty - FULL"), Indicator(33, "blk_int", Category.ONCHAIN, "blockchain", "https://blockchain.info/q/interval", "parse_bc_int", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "Block int - CURRENT"), Indicator(34, "unconf", Category.ONCHAIN, "blockchain", "https://blockchain.info/q/unconfirmedcount", "parse_bc", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "Unconf - CURRENT"), Indicator(35, "tx_blk", Category.ONCHAIN, "blockchain", "https://blockchain.info/q/nperblock", "parse_bc", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://api.blockchain.info/charts/n-transactions-per-block?timespan=1days&start={date}&format=json", "1d", "TX/blk - FULL"), Indicator(36, "total_btc", Category.ONCHAIN, "blockchain", "https://blockchain.info/q/totalbc", "parse_bc_btc", Stationarity.TREND_UP, HistoricalSupport.FULL, "https://api.blockchain.info/charts/total-bitcoins?timespan=1days&start={date}&format=json", "1d", "Total BTC - FULL"), Indicator(37, "mcap_bc", Category.ONCHAIN, "blockchain", "https://blockchain.info/q/marketcap", "parse_bc", Stationarity.TREND_UP, HistoricalSupport.FULL, "https://api.blockchain.info/charts/market-cap?timespan=1days&start={date}&format=json", "1d", "Mcap - FULL"), # ONCHAIN - Mempool (38-42) - ALL CURRENT Indicator(38, "mp_cnt", Category.ONCHAIN, "mempool", "https://mempool.space/api/mempool", "parse_mp_cnt", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "Mempool - CURRENT"), Indicator(39, "mp_mb", Category.ONCHAIN, "mempool", "https://mempool.space/api/mempool", "parse_mp_mb", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "Mempool MB - CURRENT"), Indicator(40, "fee_fast", Category.ONCHAIN, "mempool", "https://mempool.space/api/v1/fees/recommended", "parse_fee_fast", Stationarity.EPISODIC, HistoricalSupport.CURRENT, "", "", "Fast fee - CURRENT"), Indicator(41, "fee_med", Category.ONCHAIN, "mempool", "https://mempool.space/api/v1/fees/recommended", "parse_fee_med", Stationarity.EPISODIC, HistoricalSupport.CURRENT, "", "", "Med fee - CURRENT"), Indicator(42, "fee_slow", Category.ONCHAIN, "mempool", "https://mempool.space/api/v1/fees/recommended", "parse_fee_slow", Stationarity.EPISODIC, HistoricalSupport.CURRENT, "", "", "Slow fee - CURRENT"), # DEFI - DeFi Llama (43-51) Indicator(43, "tvl", Category.DEFI, "defillama", "https://api.llama.fi/v2/historicalChainTvl", "parse_dl_tvl", Stationarity.TREND_UP, HistoricalSupport.FULL, "https://api.llama.fi/v2/historicalChainTvl", "1d", "TVL - FULL (filter client-side)"), Indicator(44, "tvl_eth", Category.DEFI, "defillama", "https://api.llama.fi/v2/historicalChainTvl/Ethereum", "parse_dl_tvl", Stationarity.TREND_UP, HistoricalSupport.FULL, "https://api.llama.fi/v2/historicalChainTvl/Ethereum", "1d", "ETH TVL - FULL"), Indicator(45, "stables", Category.DEFI, "defillama", "https://stablecoins.llama.fi/stablecoins?includePrices=false", "parse_dl_stables", Stationarity.TREND_UP, HistoricalSupport.FULL, "https://stablecoins.llama.fi/stablecoincharts/all?stablecoin=1", "1d", "Stables - FULL"), Indicator(46, "usdt", Category.DEFI, "defillama", "https://stablecoins.llama.fi/stablecoin/tether", "parse_dl_single", Stationarity.TREND_UP, HistoricalSupport.FULL, "https://stablecoins.llama.fi/stablecoincharts/all?stablecoin=1", "1d", "USDT - FULL"), Indicator(47, "usdc", Category.DEFI, "defillama", "https://stablecoins.llama.fi/stablecoin/usd-coin", "parse_dl_single", Stationarity.TREND_UP, HistoricalSupport.FULL, "https://stablecoins.llama.fi/stablecoincharts/all?stablecoin=2", "1d", "USDC - FULL"), Indicator(48, "dex_vol", Category.DEFI, "defillama", "https://api.llama.fi/overview/dexs?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=true", "parse_dl_dex", Stationarity.EPISODIC, HistoricalSupport.PARTIAL, "", "1d", "DEX vol - PARTIAL"), Indicator(49, "bridge", Category.DEFI, "defillama", "https://bridges.llama.fi/bridges?includeChains=false", "parse_dl_bridge", Stationarity.EPISODIC, HistoricalSupport.PARTIAL, "", "1d", "Bridge - PARTIAL"), Indicator(50, "yields", Category.DEFI, "defillama", "https://yields.llama.fi/pools", "parse_dl_yields", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "Yields - CURRENT"), Indicator(51, "fees", Category.DEFI, "defillama", "https://api.llama.fi/overview/fees?excludeTotalDataChart=true", "parse_dl_fees", Stationarity.EPISODIC, HistoricalSupport.PARTIAL, "", "1d", "Fees - PARTIAL"), # MACRO - FRED (52-65) - ALL FULL HISTORY (decades) Indicator(52, "dxy", Category.MACRO, "fred", "DTWEXBGS", "parse_fred", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://api.stlouisfed.org/fred/series/observations?series_id=DTWEXBGS&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1d", "DXY - FULL"), Indicator(53, "us10y", Category.MACRO, "fred", "DGS10", "parse_fred", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://api.stlouisfed.org/fred/series/observations?series_id=DGS10&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1d", "10Y - FULL"), Indicator(54, "us2y", Category.MACRO, "fred", "DGS2", "parse_fred", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://api.stlouisfed.org/fred/series/observations?series_id=DGS2&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1d", "2Y - FULL"), Indicator(55, "ycurve", Category.MACRO, "fred", "T10Y2Y", "parse_fred", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://api.stlouisfed.org/fred/series/observations?series_id=T10Y2Y&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1d", "Yield curve - FULL"), Indicator(56, "vix", Category.MACRO, "fred", "VIXCLS", "parse_fred", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://api.stlouisfed.org/fred/series/observations?series_id=VIXCLS&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1d", "VIX - FULL"), Indicator(57, "fedfunds", Category.MACRO, "fred", "DFF", "parse_fred", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://api.stlouisfed.org/fred/series/observations?series_id=DFF&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1d", "Fed funds - FULL"), Indicator(58, "m2", Category.MACRO, "fred", "WM2NS", "parse_fred", Stationarity.TREND_UP, HistoricalSupport.FULL, "https://api.stlouisfed.org/fred/series/observations?series_id=WM2NS&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1w", "M2 - FULL"), Indicator(59, "cpi", Category.MACRO, "fred", "CPIAUCSL", "parse_fred", Stationarity.TREND_UP, HistoricalSupport.FULL, "https://api.stlouisfed.org/fred/series/observations?series_id=CPIAUCSL&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1m", "CPI - FULL"), Indicator(60, "sp500", Category.MACRO, "fred", "SP500", "parse_fred", Stationarity.TREND_UP, HistoricalSupport.FULL, "https://api.stlouisfed.org/fred/series/observations?series_id=SP500&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1d", "S&P - FULL"), Indicator(61, "gold", Category.MACRO, "fred", "GOLDAMGBD228NLBM", "parse_fred", Stationarity.TREND_UP, HistoricalSupport.FULL, "https://api.stlouisfed.org/fred/series/observations?series_id=GOLDAMGBD228NLBM&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1d", "Gold - FULL"), Indicator(62, "hy_spread", Category.MACRO, "fred", "BAMLH0A0HYM2", "parse_fred", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://api.stlouisfed.org/fred/series/observations?series_id=BAMLH0A0HYM2&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1d", "HY spread - FULL"), Indicator(63, "be5y", Category.MACRO, "fred", "T5YIE", "parse_fred", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://api.stlouisfed.org/fred/series/observations?series_id=T5YIE&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1d", "Breakeven - FULL"), Indicator(64, "nfci", Category.MACRO, "fred", "NFCI", "parse_fred", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://api.stlouisfed.org/fred/series/observations?series_id=NFCI&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1w", "NFCI - FULL"), Indicator(65, "claims", Category.MACRO, "fred", "ICSA", "parse_fred", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://api.stlouisfed.org/fred/series/observations?series_id=ICSA&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1w", "Claims - FULL"), # SENTIMENT (66-72) - F&G has FULL history Indicator(66, "fng", Category.SENTIMENT, "alternative", "https://api.alternative.me/fng/?limit=1", "parse_fng", Stationarity.STATIONARY, HistoricalSupport.FULL, "https://api.alternative.me/fng/?limit=1000&date_format=us", "1d", "F&G - FULL (returns history, filter)"), Indicator(67, "fng_prev", Category.SENTIMENT, "alternative", "https://api.alternative.me/fng/?limit=2", "parse_fng_prev", Stationarity.STATIONARY, HistoricalSupport.FULL, "", "1d", "Prev F&G"), Indicator(68, "fng_week", Category.SENTIMENT, "alternative", "https://api.alternative.me/fng/?limit=7", "parse_fng_week", Stationarity.STATIONARY, HistoricalSupport.FULL, "", "1d", "Week F&G"), Indicator(69, "fng_vol", Category.SENTIMENT, "alternative", "https://api.alternative.me/fng/?limit=1", "parse_fng", Stationarity.STATIONARY, HistoricalSupport.FULL, "", "1d", "Vol proxy"), Indicator(70, "fng_mom", Category.SENTIMENT, "alternative", "https://api.alternative.me/fng/?limit=1", "parse_fng", Stationarity.STATIONARY, HistoricalSupport.FULL, "", "1d", "Mom proxy"), Indicator(71, "fng_soc", Category.SENTIMENT, "alternative", "https://api.alternative.me/fng/?limit=1", "parse_fng", Stationarity.STATIONARY, HistoricalSupport.FULL, "", "1d", "Social proxy"), Indicator(72, "fng_dom", Category.SENTIMENT, "alternative", "https://api.alternative.me/fng/?limit=1", "parse_fng", Stationarity.STATIONARY, HistoricalSupport.FULL, "", "1d", "Dom proxy"), # MICROSTRUCTURE (73-80) - Most CURRENT Indicator(73, "imbal_btc", Category.MICROSTRUCTURE, "binance", "https://api.binance.com/api/v3/depth?symbol=BTCUSDT&limit=100", "parse_imbal", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "Imbalance - CURRENT"), Indicator(74, "imbal_eth", Category.MICROSTRUCTURE, "binance", "https://api.binance.com/api/v3/depth?symbol=ETHUSDT&limit=100", "parse_imbal", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "ETH imbal - CURRENT"), Indicator(75, "spread", Category.MICROSTRUCTURE, "binance", "https://api.binance.com/api/v3/ticker/bookTicker?symbol=BTCUSDT", "parse_spread", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "Spread - CURRENT"), Indicator(76, "chg24_btc", Category.MICROSTRUCTURE, "binance", "https://api.binance.com/api/v3/ticker/24hr?symbol=BTCUSDT", "parse_chg", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "24h chg - CURRENT"), Indicator(77, "chg24_eth", Category.MICROSTRUCTURE, "binance", "https://api.binance.com/api/v3/ticker/24hr?symbol=ETHUSDT", "parse_chg", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "ETH 24h - CURRENT"), Indicator(78, "vol24", Category.MICROSTRUCTURE, "binance", "https://api.binance.com/api/v3/ticker/24hr?symbol=BTCUSDT", "parse_vol", Stationarity.EPISODIC, HistoricalSupport.FULL, "https://api.binance.com/api/v3/klines?symbol=BTCUSDT&interval=1d&startTime={start_ms}&endTime={end_ms}&limit=1", "1d", "Volume - FULL via klines"), Indicator(79, "dispersion", Category.MICROSTRUCTURE, "binance", "https://api.binance.com/api/v3/ticker/24hr", "parse_disp", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "Dispersion - CURRENT"), Indicator(80, "correlation", Category.MICROSTRUCTURE, "binance", "https://api.binance.com/api/v3/ticker/24hr", "parse_corr", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "Correlation - CURRENT"), # MARKET - CoinGecko (81-85) Indicator(81, "btc_price", Category.MACRO, "coingecko", "https://api.coingecko.com/api/v3/simple/price?ids=bitcoin&vs_currencies=usd", "parse_cg_btc", Stationarity.TREND_UP, HistoricalSupport.FULL, "https://api.coingecko.com/api/v3/coins/bitcoin/history?date={date_dmy}", "1d", "BTC price - FULL"), Indicator(82, "eth_price", Category.MACRO, "coingecko", "https://api.coingecko.com/api/v3/simple/price?ids=ethereum&vs_currencies=usd", "parse_cg_eth", Stationarity.TREND_UP, HistoricalSupport.FULL, "https://api.coingecko.com/api/v3/coins/ethereum/history?date={date_dmy}", "1d", "ETH price - FULL"), Indicator(83, "mcap", Category.MACRO, "coingecko", "https://api.coingecko.com/api/v3/global", "parse_cg_mcap", Stationarity.TREND_UP, HistoricalSupport.PARTIAL, "", "1d", "Mcap - PARTIAL"), Indicator(84, "btc_dom", Category.MACRO, "coingecko", "https://api.coingecko.com/api/v3/global", "parse_cg_dom_btc", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "BTC dom - CURRENT"), Indicator(85, "eth_dom", Category.MACRO, "coingecko", "https://api.coingecko.com/api/v3/global", "parse_cg_dom_eth", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "ETH dom - CURRENT"), ] # fmt: on N_INDICATORS = len(INDICATORS) class StationarityTransformer: def __init__(self, lookback: int = 10): self.history: Dict[int, deque] = {i: deque(maxlen=lookback+1) for i in range(1, N_INDICATORS+1)} def transform(self, ind_id: int, raw: float) -> float: ind = INDICATORS[ind_id - 1] hist = self.history[ind_id] hist.append(raw) if ind.stationarity == Stationarity.STATIONARY: return raw if ind.stationarity == Stationarity.TREND_UP: return (raw - hist[-2]) / abs(hist[-2]) if len(hist) >= 2 and hist[-2] != 0 else 0.0 if ind.stationarity == Stationarity.EPISODIC: if len(hist) < 3: return 0.0 m, s = np.mean(list(hist)), np.std(list(hist)) return (raw - m) / s if s > 0 else 0.0 return raw def transform_matrix(self, raw: np.ndarray) -> np.ndarray: return np.array([self.transform(i+1, raw[i]) for i in range(len(raw))]) class ExternalFactorsFetcher: def __init__(self, config: Config = None): self.config = config or Config() self.cache: Dict[str, Tuple[float, Any]] = {} import time as t; self._time = t def _build_hist_url(self, ind: Indicator, dt: datetime) -> Optional[str]: if ind.historical == HistoricalSupport.CURRENT or not ind.hist_url: return None url = ind.hist_url date_str = dt.strftime("%Y-%m-%d") date_dmy = dt.strftime("%d-%m-%Y") start_ms = int(dt.replace(hour=0, minute=0, second=0).timestamp() * 1000) end_ms = int(dt.replace(hour=23, minute=59, second=59).timestamp() * 1000) key = self.config.fred_api_key or "DEMO_KEY" return url.replace("{date}", date_str).replace("{date_dmy}", date_dmy).replace("{start_ms}", str(start_ms)).replace("{end_ms}", str(end_ms)).replace("{key}", key) async def _fetch(self, session, url: str) -> Optional[Any]: if url in self.cache: ct, cd = self.cache[url] if self._time.time() - ct < self.config.cache_ttl: return cd try: async with session.get(url, timeout=aiohttp.ClientTimeout(total=self.config.timeout), headers={"User-Agent": "Mozilla/5.0"}) as r: if r.status == 200: d = await r.json() if 'json' in r.headers.get('Content-Type', '') else await r.text() if isinstance(d, str): try: d = json.loads(d) except: pass self.cache[url] = (self._time.time(), d) return d except: pass return None def _fred_url(self, series: str) -> str: return f"https://api.stlouisfed.org/fred/series/observations?series_id={series}&api_key={self.config.fred_api_key or 'DEMO_KEY'}&file_type=json&sort_order=desc&limit=1" # Parsers def parse_binance_funding(self, d): return float(d[0]['fundingRate']) if isinstance(d, list) and d else 0.0 def parse_binance_oi(self, d): if isinstance(d, list) and d: return float(d[-1].get('sumOpenInterest', 0)) return float(d.get('openInterest', 0)) if isinstance(d, dict) else 0.0 def parse_binance_ls(self, d): return float(d[-1]['longShortRatio']) if isinstance(d, list) and d else 1.0 def parse_binance_taker(self, d): return float(d[-1]['buySellRatio']) if isinstance(d, list) and d else 1.0 def parse_binance_basis(self, d): return float(d.get('lastFundingRate', 0)) * 365 * 3 if isinstance(d, dict) else 0.0 def parse_liq_proxy(self, d): return np.tanh(float(d.get('priceChangePercent', 0)) / 10) if isinstance(d, dict) else 0.0 def parse_deribit_dvol(self, d): if isinstance(d, dict) and 'result' in d and isinstance(d['result'], dict) and 'data' in d['result'] and d['result']['data']: return float(d['result']['data'][-1][4]) if len(d['result']['data'][-1]) > 4 else 0.0 return 0.0 def parse_deribit_pcr(self, d): if isinstance(d, dict) and 'result' in d: r = d['result'] p = sum(float(o.get('volume', 0)) for o in r if '-P' in o.get('instrument_name', '')) c = sum(float(o.get('volume', 0)) for o in r if '-C' in o.get('instrument_name', '')) return p / c if c > 0 else 1.0 return 1.0 def parse_deribit_pcr_oi(self, d): if isinstance(d, dict) and 'result' in d: r = d['result'] p = sum(float(o.get('open_interest', 0)) for o in r if '-P' in o.get('instrument_name', '')) c = sum(float(o.get('open_interest', 0)) for o in r if '-C' in o.get('instrument_name', '')) return p / c if c > 0 else 1.0 return 1.0 def parse_deribit_oi(self, d): return sum(float(o.get('open_interest', 0)) for o in d['result']) if isinstance(d, dict) and 'result' in d else 0.0 def parse_deribit_fund(self, d): if isinstance(d, dict) and 'result' in d: r = d['result'] return float(r[-1].get('interest_8h', 0)) if isinstance(r, list) and r else float(r) return 0.0 def parse_cm(self, d): if isinstance(d, dict) and 'data' in d and d['data']: for k, v in d['data'][-1].items(): if k not in ['asset', 'time']: try: return float(v) except: pass return 0.0 def parse_cm_mvrv(self, d): if isinstance(d, dict) and 'data' in d and d['data']: r = d['data'][-1] m, rc = float(r.get('CapMrktCurUSD', 0)), float(r.get('CapRealUSD', 1)) return m / rc if rc > 0 else 0.0 return 0.0 def parse_cm_nupl(self, d): if isinstance(d, dict) and 'data' in d and d['data']: r = d['data'][-1] m, rc = float(r.get('CapMrktCurUSD', 0)), float(r.get('CapRealUSD', 1)) return (m - rc) / m if m > 0 else 0.0 return 0.0 def parse_bc(self, d): if isinstance(d, (int, float)): return float(d) if isinstance(d, str): try: return float(d) except: pass if isinstance(d, dict) and 'values' in d and d['values']: return float(d['values'][-1].get('y', 0)) return 0.0 def parse_bc_int(self, d): v = self.parse_bc(d); return abs(v - 600) / 600 if v > 0 else 0.0 def parse_bc_btc(self, d): v = self.parse_bc(d); return v / 1e8 if v > 0 else 0.0 def parse_mp_cnt(self, d): return float(d.get('count', 0)) if isinstance(d, dict) else 0.0 def parse_mp_mb(self, d): return float(d.get('vsize', 0)) / 1e6 if isinstance(d, dict) else 0.0 def parse_fee_fast(self, d): return float(d.get('fastestFee', 0)) if isinstance(d, dict) else 0.0 def parse_fee_med(self, d): return float(d.get('halfHourFee', 0)) if isinstance(d, dict) else 0.0 def parse_fee_slow(self, d): return float(d.get('economyFee', 0)) if isinstance(d, dict) else 0.0 def parse_dl_tvl(self, d, target_date: datetime = None): if isinstance(d, list) and d: if target_date: ts = int(target_date.timestamp()) for e in reversed(d): if e.get('date', 0) <= ts: return float(e.get('tvl', 0)) return float(d[-1].get('tvl', 0)) return 0.0 def parse_dl_stables(self, d): if isinstance(d, dict) and 'peggedAssets' in d: return sum(float(a.get('circulating', {}).get('peggedUSD', 0)) for a in d['peggedAssets']) return 0.0 def parse_dl_single(self, d): if isinstance(d, dict) and 'tokens' in d and d['tokens']: return float(d['tokens'][-1].get('circulating', {}).get('peggedUSD', 0)) return 0.0 def parse_dl_dex(self, d): return float(d.get('total24h', 0)) if isinstance(d, dict) else 0.0 def parse_dl_bridge(self, d): if isinstance(d, dict) and 'bridges' in d: return sum(float(b.get('lastDayVolume', 0)) for b in d['bridges']) return 0.0 def parse_dl_yields(self, d): if isinstance(d, dict) and 'data' in d: apys = [float(p.get('apy', 0)) for p in d['data'][:100] if p.get('apy')] return np.mean(apys) if apys else 0.0 return 0.0 def parse_dl_fees(self, d): return float(d.get('total24h', 0)) if isinstance(d, dict) else 0.0 def parse_fred(self, d): if isinstance(d, dict) and 'observations' in d and d['observations']: v = d['observations'][-1].get('value', '.') if v != '.': try: return float(v) except: pass return 0.0 def parse_fng(self, d): return float(d['data'][0]['value']) if isinstance(d, dict) and 'data' in d and d['data'] else 50.0 def parse_fng_prev(self, d): return float(d['data'][1]['value']) if isinstance(d, dict) and 'data' in d and len(d['data']) > 1 else 50.0 def parse_fng_week(self, d): return np.mean([float(x['value']) for x in d['data'][:7]]) if isinstance(d, dict) and 'data' in d and len(d['data']) >= 7 else 50.0 def parse_imbal(self, d): if isinstance(d, dict): bv = sum(float(b[1]) for b in d.get('bids', [])[:50]) av = sum(float(a[1]) for a in d.get('asks', [])[:50]) t = bv + av return (bv - av) / t if t > 0 else 0.0 return 0.0 def parse_spread(self, d): if isinstance(d, dict): b, a = float(d.get('bidPrice', 0)), float(d.get('askPrice', 0)) return (a - b) / b * 10000 if b > 0 else 0.0 return 0.0 def parse_chg(self, d): return float(d.get('priceChangePercent', 0)) if isinstance(d, dict) else 0.0 def parse_vol(self, d): if isinstance(d, dict): return float(d.get('quoteVolume', 0)) if isinstance(d, list) and d and isinstance(d[0], list): return float(d[-1][7]) return 0.0 def parse_disp(self, d): if isinstance(d, list) and len(d) > 10: chg = [float(t['priceChangePercent']) for t in d if t.get('symbol', '').endswith('USDT') and 'priceChangePercent' in t] return float(np.std(chg[:50])) if len(chg) > 5 else 0.0 return 0.0 def parse_corr(self, d): disp = self.parse_disp(d); return 1 / (1 + disp) if disp > 0 else 0.5 def parse_cg_btc(self, d): if isinstance(d, dict) and 'bitcoin' in d: return float(d['bitcoin']['usd']) if isinstance(d, dict) and 'market_data' in d: return float(d['market_data'].get('current_price', {}).get('usd', 0)) return 0.0 def parse_cg_eth(self, d): if isinstance(d, dict) and 'ethereum' in d: return float(d['ethereum']['usd']) if isinstance(d, dict) and 'market_data' in d: return float(d['market_data'].get('current_price', {}).get('usd', 0)) return 0.0 def parse_cg_mcap(self, d): return float(d['data']['total_market_cap']['usd']) if isinstance(d, dict) and 'data' in d else 0.0 def parse_cg_dom_btc(self, d): return float(d['data']['market_cap_percentage']['btc']) if isinstance(d, dict) and 'data' in d else 0.0 def parse_cg_dom_eth(self, d): return float(d['data']['market_cap_percentage']['eth']) if isinstance(d, dict) and 'data' in d else 0.0 async def fetch_indicator(self, session, ind: Indicator, target_date: datetime = None) -> Tuple[int, str, float, bool]: if target_date and ind.historical != HistoricalSupport.CURRENT: url = self._build_hist_url(ind, target_date) else: url = self._fred_url(ind.url) if ind.source == "fred" else ind.url if url is None: return (ind.id, ind.name, 0.0, False) data = await self._fetch(session, url) if data is None: return (ind.id, ind.name, 0.0, False) parser = getattr(self, ind.parser, None) if parser is None: return (ind.id, ind.name, 0.0, False) try: value = parser(data) return (ind.id, ind.name, value, value != 0.0 or 'imbal' in ind.name) except: return (ind.id, ind.name, 0.0, False) async def fetch_all(self, target_date: datetime = None) -> Dict[str, Any]: connector = aiohttp.TCPConnector(limit=self.config.max_concurrent) async with aiohttp.ClientSession(connector=connector) as session: results = await asyncio.gather(*[self.fetch_indicator(session, ind, target_date) for ind in INDICATORS]) matrix = np.zeros(N_INDICATORS) success = 0 details = {} for idx, name, value, ok in results: matrix[idx - 1] = value if ok: success += 1 details[idx] = {'name': name, 'value': value, 'success': ok} return {'matrix': matrix, 'timestamp': (target_date or datetime.now(timezone.utc)).isoformat(), 'success_count': success, 'total': N_INDICATORS, 'details': details} def fetch_sync(self, target_date: datetime = None) -> Dict[str, Any]: return asyncio.run(self.fetch_all(target_date)) class ExternalFactorsMatrix: """DOLPHIN interface with BACKFILL. Usage: efm.update() or efm.update(datetime(2024,6,15))""" def __init__(self, config: Config = None): self.config = config or Config() self.fetcher = ExternalFactorsFetcher(self.config) self.transformer = StationarityTransformer() self.raw_matrix: Optional[np.ndarray] = None self.stationary_matrix: Optional[np.ndarray] = None self.last_result: Optional[Dict] = None def update(self, target_date: datetime = None) -> np.ndarray: self.last_result = self.fetcher.fetch_sync(target_date) self.raw_matrix = self.last_result['matrix'] self.stationary_matrix = self.transformer.transform_matrix(self.raw_matrix) return self.stationary_matrix def update_raw(self, target_date: datetime = None) -> np.ndarray: self.last_result = self.fetcher.fetch_sync(target_date) self.raw_matrix = self.last_result['matrix'] return self.raw_matrix def get_indicator_names(self) -> List[str]: return [i.name for i in INDICATORS] def get_backfillable(self) -> List[Tuple[int, str, str]]: return [(i.id, i.name, i.hist_resolution) for i in INDICATORS if i.historical in [HistoricalSupport.FULL, HistoricalSupport.PARTIAL]] def get_current_only(self) -> List[Tuple[int, str]]: return [(i.id, i.name) for i in INDICATORS if i.historical == HistoricalSupport.CURRENT] def summary(self) -> str: if not self.last_result: return "No data." r = self.last_result f = sum(1 for i in INDICATORS if i.historical == HistoricalSupport.FULL) p = sum(1 for i in INDICATORS if i.historical == HistoricalSupport.PARTIAL) c = sum(1 for i in INDICATORS if i.historical == HistoricalSupport.CURRENT) return f"Success: {r['success_count']}/{r['total']} | Historical: FULL={f}, PARTIAL={p}, CURRENT={c}" if __name__ == "__main__": print(f"EXTERNAL FACTORS v5.0 - {N_INDICATORS} indicators with BACKFILL") f = [i for i in INDICATORS if i.historical == HistoricalSupport.FULL] p = [i for i in INDICATORS if i.historical == HistoricalSupport.PARTIAL] c = [i for i in INDICATORS if i.historical == HistoricalSupport.CURRENT] print(f"\nFULL: {len(f)} | PARTIAL: {len(p)} | CURRENT: {len(c)}") print("\nFULL HISTORY indicators:") for i in f: print(f" {i.id:2d}. {i.name:15s} [{i.hist_resolution:3s}] {i.source}") print("\nCURRENT ONLY:") for i in c: print(f" {i.id:2d}. {i.name:15s} - {i.description}")