#!/usr/bin/env python3 """ INDICATOR SOURCES v5.0 - API Reference with Historical Support =============================================================== Documents all 85 indicators with their backfill capability. """ SOURCES = { "binance": {"url": "fapi.binance.com / api.binance.com", "auth": "None", "limit": "1200/min", "history": "FULL (startTime/endTime)"}, "deribit": {"url": "deribit.com/api/v2/public", "auth": "None", "limit": "20/sec", "history": "FULL for DVOL/funding"}, "coinmetrics": {"url": "community-api.coinmetrics.io/v4", "auth": "None", "limit": "10/6sec", "history": "FULL (start_time/end_time)"}, "fred": {"url": "api.stlouisfed.org/fred", "auth": "Free key", "limit": "120/min", "history": "FULL (decades)"}, "defillama": {"url": "api.llama.fi", "auth": "None", "limit": "Generous", "history": "FULL for TVL/stables"}, "alternative": {"url": "api.alternative.me", "auth": "None", "limit": "Unlimited", "history": "FULL (limit=N param)"}, "blockchain": {"url": "blockchain.info", "auth": "None", "limit": "Generous", "history": "FULL via charts API"}, "mempool": {"url": "mempool.space/api", "auth": "None", "limit": "Generous", "history": "NONE (real-time only)"}, "coingecko": {"url": "api.coingecko.com/api/v3", "auth": "None (demo)", "limit": "30/min", "history": "FULL for prices"}, } # Historical URL templates for backfill HISTORICAL_ENDPOINTS = { # BINANCE - All support startTime/endTime in milliseconds "binance_funding": "https://fapi.binance.com/fapi/v1/fundingRate?symbol={SYMBOL}&startTime={start_ms}&endTime={end_ms}&limit=1000", "binance_oi_hist": "https://fapi.binance.com/futures/data/openInterestHist?symbol={SYMBOL}&period=1h&startTime={start_ms}&endTime={end_ms}&limit=500", "binance_ls_hist": "https://fapi.binance.com/futures/data/globalLongShortAccountRatio?symbol={SYMBOL}&period=1h&startTime={start_ms}&endTime={end_ms}&limit=500", "binance_taker_hist": "https://fapi.binance.com/futures/data/takerlongshortRatio?symbol={SYMBOL}&period=1h&startTime={start_ms}&endTime={end_ms}&limit=500", "binance_klines": "https://api.binance.com/api/v3/klines?symbol={SYMBOL}&interval=1d&startTime={start_ms}&endTime={end_ms}&limit=1", # DERIBIT - Uses start_timestamp/end_timestamp in milliseconds "deribit_dvol": "https://www.deribit.com/api/v2/public/get_volatility_index_data?currency={CURRENCY}&resolution=3600&start_timestamp={start_ms}&end_timestamp={end_ms}", "deribit_funding_hist": "https://www.deribit.com/api/v2/public/get_funding_rate_history?instrument_name={INSTRUMENT}&start_timestamp={start_ms}&end_timestamp={end_ms}", # COINMETRICS - Uses ISO date format "coinmetrics": "https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets={asset}&metrics={metric}&frequency=1d&start_time={date}T00:00:00Z&end_time={date}T23:59:59Z", # FRED - Uses observation_start/observation_end in YYYY-MM-DD "fred": "https://api.stlouisfed.org/fred/series/observations?series_id={series}&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", # DEFILLAMA - Returns full history, filter client-side "defillama_tvl": "https://api.llama.fi/v2/historicalChainTvl", # Filter by date client-side "defillama_tvl_chain": "https://api.llama.fi/v2/historicalChainTvl/{chain}", "defillama_stables": "https://stablecoins.llama.fi/stablecoincharts/all?stablecoin={id}", # 1=USDT, 2=USDC # BLOCKCHAIN.INFO - Uses start param in YYYY-MM-DD "blockchain_charts": "https://api.blockchain.info/charts/{chart}?timespan=1days&start={date}&format=json", # COINGECKO - Uses DD-MM-YYYY format "coingecko_history": "https://api.coingecko.com/api/v3/coins/{id}/history?date={date_dmy}", # ALTERNATIVE.ME - Returns N days of history "fng_history": "https://api.alternative.me/fng/?limit=1000&date_format=us", # Filter client-side } HISTORICAL_SUPPORT = { # FULL HISTORY (51 indicators) "full": [ # Binance derivatives (1, "funding_btc", "8h", "Funding rate history via startTime/endTime"), (2, "funding_eth", "8h", "ETH funding"), (3, "oi_btc", "1h", "Open interest history via openInterestHist endpoint"), (4, "oi_eth", "1h", "ETH OI"), (5, "ls_btc", "1h", "Long/short ratio history"), (6, "ls_eth", "1h", "ETH L/S"), (7, "ls_top", "1h", "Top trader L/S"), (8, "taker", "1h", "Taker ratio history"), # Deribit (11, "dvol_btc", "1h", "DVOL via get_volatility_index_data"), (12, "dvol_eth", "1h", "ETH DVOL"), (17, "fund_dbt_btc", "8h", "Deribit funding via get_funding_rate_history"), (18, "fund_dbt_eth", "8h", "ETH Deribit funding"), # CoinMetrics (ALL have full history) (19, "rcap_btc", "1d", "CoinMetrics: CapRealUSD"), (20, "mvrv", "1d", "CoinMetrics: derived from CapMrktCurUSD/CapRealUSD"), (21, "nupl", "1d", "CoinMetrics: derived"), (22, "addr_btc", "1d", "CoinMetrics: AdrActCnt"), (23, "addr_eth", "1d", "CoinMetrics: ETH AdrActCnt"), (24, "txcnt", "1d", "CoinMetrics: TxCnt"), (25, "fees_btc", "1d", "CoinMetrics: FeeTotUSD"), (26, "fees_eth", "1d", "CoinMetrics: ETH FeeTotUSD"), (27, "nvt", "1d", "CoinMetrics: NVTAdj"), (28, "velocity", "1d", "CoinMetrics: VelCur1yr"), (29, "sply_act", "1d", "CoinMetrics: SplyAct1yr"), (30, "rcap_eth", "1d", "CoinMetrics: ETH CapRealUSD"), # Blockchain.info charts (31, "hashrate", "1d", "Blockchain.info: hash-rate chart"), (32, "difficulty", "1d", "Blockchain.info: difficulty chart"), (35, "tx_blk", "1d", "Blockchain.info: n-transactions-per-block chart"), (36, "total_btc", "1d", "Blockchain.info: total-bitcoins chart"), (37, "mcap_bc", "1d", "Blockchain.info: market-cap chart"), # DeFi Llama (43, "tvl", "1d", "DeFi Llama: historicalChainTvl (returns all, filter client-side)"), (44, "tvl_eth", "1d", "DeFi Llama: ETH TVL"), (45, "stables", "1d", "DeFi Llama: stablecoincharts"), (46, "usdt", "1d", "DeFi Llama: stablecoin ID=1"), (47, "usdc", "1d", "DeFi Llama: stablecoin ID=2"), # FRED (ALL have decades of history) (52, "dxy", "1d", "FRED: DTWEXBGS"), (53, "us10y", "1d", "FRED: DGS10"), (54, "us2y", "1d", "FRED: DGS2"), (55, "ycurve", "1d", "FRED: T10Y2Y"), (56, "vix", "1d", "FRED: VIXCLS"), (57, "fedfunds", "1d", "FRED: DFF"), (58, "m2", "1w", "FRED: WM2NS (weekly)"), (59, "cpi", "1m", "FRED: CPIAUCSL (monthly)"), (60, "sp500", "1d", "FRED: SP500"), (61, "gold", "1d", "FRED: GOLDAMGBD228NLBM"), (62, "hy_spread", "1d", "FRED: BAMLH0A0HYM2"), (63, "be5y", "1d", "FRED: T5YIE"), (64, "nfci", "1w", "FRED: NFCI (weekly)"), (65, "claims", "1w", "FRED: ICSA (weekly)"), # Alternative.me (66, "fng", "1d", "Alternative.me: limit param returns history"), (67, "fng_prev", "1d", ""), (68, "fng_week", "1d", ""), (69, "fng_vol", "1d", ""), (70, "fng_mom", "1d", ""), (71, "fng_soc", "1d", ""), (72, "fng_dom", "1d", ""), # CoinGecko (81, "btc_price", "1d", "CoinGecko: /coins/{id}/history"), (82, "eth_price", "1d", "CoinGecko: /coins/{id}/history"), # Binance klines (78, "vol24", "1d", "Binance: klines endpoint"), ], # PARTIAL HISTORY (12 indicators) "partial": [ (48, "dex_vol", "1d", "DeFi Llama: recent history in response"), (49, "bridge", "1d", "DeFi Llama: bridgevolume endpoint"), (51, "fees", "1d", "DeFi Llama: fees overview"), (83, "mcap", "1d", "CoinGecko: market_cap_chart (limited)"), ], # CURRENT ONLY (22 indicators) "current": [ (9, "basis", "Binance premium index - real-time only"), (10, "liq_proxy", "Derived from 24hr ticker - real-time"), (13, "pcr_vol", "Deribit options summary - real-time"), (14, "pcr_oi", "Deribit options OI - real-time"), (15, "pcr_eth", "Deribit ETH options - real-time"), (16, "opt_oi", "Deribit total options OI - real-time"), (33, "blk_int", "Blockchain.info simple query - real-time"), (34, "unconf", "Blockchain.info unconfirmed - real-time"), (38, "mp_cnt", "Mempool.space - NO historical API"), (39, "mp_mb", "Mempool.space - NO historical API"), (40, "fee_fast", "Mempool.space - NO historical API"), (41, "fee_med", "Mempool.space - NO historical API"), (42, "fee_slow", "Mempool.space - NO historical API"), (50, "yields", "DeFi Llama yields - real-time"), (73, "imbal_btc", "Order book depth - real-time"), (74, "imbal_eth", "Order book depth - real-time"), (75, "spread", "Book ticker - real-time"), (76, "chg24_btc", "24hr ticker - real-time"), (77, "chg24_eth", "24hr ticker - real-time"), (79, "dispersion", "Calculated from 24hr - real-time"), (80, "correlation", "Calculated from 24hr - real-time"), (84, "btc_dom", "CoinGecko global - real-time"), (85, "eth_dom", "CoinGecko global - real-time"), ], } BACKFILL_NOTES = """ BACKFILL STRATEGY ================= 1. DAILY BACKFILL (Most indicators): - CoinMetrics, FRED, DeFi Llama TVL, Blockchain.info charts - Use: efm.update(datetime(2024, 6, 15)) 2. HOURLY BACKFILL (Binance derivatives): - OI, L/S ratio, taker ratio have 1h resolution - Funding rate has 8h resolution 3. APIS RETURNING FULL HISTORY: - DeFi Llama TVL: Returns ALL history, filter client-side by timestamp - Alternative.me F&G: Use limit=1000 to get ~3 years of history - Blockchain.info charts: Use start= param with date 4. MISSING HISTORICAL DATA: - Mempool fees: Build your own collector - Order book imbalance: Build your own collector - Spreads: Build your own collector 5. RECOMMENDED APPROACH FOR TRAINING: a) Backfill what's available (51 indicators with FULL history) b) For CURRENT-only indicators, either: - Accept NaN/0 for historical periods - Build collectors to capture going forward - Use proxy indicators (e.g., volatility proxy for mempool fees) """ if __name__ == "__main__": print("INDICATOR SOURCES v5.0") print("=" * 60) print("\nData Sources:") for src, info in SOURCES.items(): print(f" {src:12s}: {info['auth']:10s} | {info['limit']:12s} | {info['history']}") print(f"\nHistorical Support:") print(f" FULL: {len(HISTORICAL_SUPPORT['full'])} indicators") print(f" PARTIAL: {len(HISTORICAL_SUPPORT['partial'])} indicators") print(f" CURRENT: {len(HISTORICAL_SUPPORT['current'])} indicators") print(BACKFILL_NOTES)