DOLPHIN/external_factors/indicator_sources.py

#!/usr/bin/env python3
"""
INDICATOR SOURCES v5.0 - API Reference with Historical Support
===============================================================
Documents all 85 indicators with their backfill capability.
"""

SOURCES = {
    "binance": {"url": "fapi.binance.com / api.binance.com", "auth": "None", "limit": "1200/min", "history": "FULL (startTime/endTime)"},
    "deribit": {"url": "deribit.com/api/v2/public", "auth": "None", "limit": "20/sec", "history": "FULL for DVOL/funding"},
    "coinmetrics": {"url": "community-api.coinmetrics.io/v4", "auth": "None", "limit": "10/6sec", "history": "FULL (start_time/end_time)"},
    "fred": {"url": "api.stlouisfed.org/fred", "auth": "Free key", "limit": "120/min", "history": "FULL (decades)"},
    "defillama": {"url": "api.llama.fi", "auth": "None", "limit": "Generous", "history": "FULL for TVL/stables"},
    "alternative": {"url": "api.alternative.me", "auth": "None", "limit": "Unlimited", "history": "FULL (limit=N param)"},
    "blockchain": {"url": "blockchain.info", "auth": "None", "limit": "Generous", "history": "FULL via charts API"},
    "mempool": {"url": "mempool.space/api", "auth": "None", "limit": "Generous", "history": "NONE (real-time only)"},
    "coingecko": {"url": "api.coingecko.com/api/v3", "auth": "None (demo)", "limit": "30/min", "history": "FULL for prices"},
}

# Historical URL templates for backfill
HISTORICAL_ENDPOINTS = {
    # BINANCE - All support startTime/endTime in milliseconds
    "binance_funding": "https://fapi.binance.com/fapi/v1/fundingRate?symbol={SYMBOL}&startTime={start_ms}&endTime={end_ms}&limit=1000",
    "binance_oi_hist": "https://fapi.binance.com/futures/data/openInterestHist?symbol={SYMBOL}&period=1h&startTime={start_ms}&endTime={end_ms}&limit=500",
    "binance_ls_hist": "https://fapi.binance.com/futures/data/globalLongShortAccountRatio?symbol={SYMBOL}&period=1h&startTime={start_ms}&endTime={end_ms}&limit=500",
    "binance_taker_hist": "https://fapi.binance.com/futures/data/takerlongshortRatio?symbol={SYMBOL}&period=1h&startTime={start_ms}&endTime={end_ms}&limit=500",
    "binance_klines": "https://api.binance.com/api/v3/klines?symbol={SYMBOL}&interval=1d&startTime={start_ms}&endTime={end_ms}&limit=1",
    
    # DERIBIT - Uses start_timestamp/end_timestamp in milliseconds
    "deribit_dvol": "https://www.deribit.com/api/v2/public/get_volatility_index_data?currency={CURRENCY}&resolution=3600&start_timestamp={start_ms}&end_timestamp={end_ms}",
    "deribit_funding_hist": "https://www.deribit.com/api/v2/public/get_funding_rate_history?instrument_name={INSTRUMENT}&start_timestamp={start_ms}&end_timestamp={end_ms}",
    
    # COINMETRICS - Uses ISO date format
    "coinmetrics": "https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets={asset}&metrics={metric}&frequency=1d&start_time={date}T00:00:00Z&end_time={date}T23:59:59Z",
    
    # FRED - Uses observation_start/observation_end in YYYY-MM-DD
    "fred": "https://api.stlouisfed.org/fred/series/observations?series_id={series}&api_key={key}&file_type=json&observation_start={date}&observation_end={date}",
    
    # DEFILLAMA - Returns full history, filter client-side
    "defillama_tvl": "https://api.llama.fi/v2/historicalChainTvl",  # Filter by date client-side
    "defillama_tvl_chain": "https://api.llama.fi/v2/historicalChainTvl/{chain}",
    "defillama_stables": "https://stablecoins.llama.fi/stablecoincharts/all?stablecoin={id}",  # 1=USDT, 2=USDC
    
    # BLOCKCHAIN.INFO - Uses start param in YYYY-MM-DD
    "blockchain_charts": "https://api.blockchain.info/charts/{chart}?timespan=1days&start={date}&format=json",
    
    # COINGECKO - Uses DD-MM-YYYY format
    "coingecko_history": "https://api.coingecko.com/api/v3/coins/{id}/history?date={date_dmy}",
    
    # ALTERNATIVE.ME - Returns N days of history
    "fng_history": "https://api.alternative.me/fng/?limit=1000&date_format=us",  # Filter client-side
}

HISTORICAL_SUPPORT = {
    # FULL HISTORY (51 indicators)
    "full": [
        # Binance derivatives
        (1, "funding_btc", "8h", "Funding rate history via startTime/endTime"),
        (2, "funding_eth", "8h", "ETH funding"),
        (3, "oi_btc", "1h", "Open interest history via openInterestHist endpoint"),
        (4, "oi_eth", "1h", "ETH OI"),
        (5, "ls_btc", "1h", "Long/short ratio history"),
        (6, "ls_eth", "1h", "ETH L/S"),
        (7, "ls_top", "1h", "Top trader L/S"),
        (8, "taker", "1h", "Taker ratio history"),
        # Deribit
        (11, "dvol_btc", "1h", "DVOL via get_volatility_index_data"),
        (12, "dvol_eth", "1h", "ETH DVOL"),
        (17, "fund_dbt_btc", "8h", "Deribit funding via get_funding_rate_history"),
        (18, "fund_dbt_eth", "8h", "ETH Deribit funding"),
        # CoinMetrics (ALL have full history)
        (19, "rcap_btc", "1d", "CoinMetrics: CapRealUSD"),
        (20, "mvrv", "1d", "CoinMetrics: derived from CapMrktCurUSD/CapRealUSD"),
        (21, "nupl", "1d", "CoinMetrics: derived"),
        (22, "addr_btc", "1d", "CoinMetrics: AdrActCnt"),
        (23, "addr_eth", "1d", "CoinMetrics: ETH AdrActCnt"),
        (24, "txcnt", "1d", "CoinMetrics: TxCnt"),
        (25, "fees_btc", "1d", "CoinMetrics: FeeTotUSD"),
        (26, "fees_eth", "1d", "CoinMetrics: ETH FeeTotUSD"),
        (27, "nvt", "1d", "CoinMetrics: NVTAdj"),
        (28, "velocity", "1d", "CoinMetrics: VelCur1yr"),
        (29, "sply_act", "1d", "CoinMetrics: SplyAct1yr"),
        (30, "rcap_eth", "1d", "CoinMetrics: ETH CapRealUSD"),
        # Blockchain.info charts
        (31, "hashrate", "1d", "Blockchain.info: hash-rate chart"),
        (32, "difficulty", "1d", "Blockchain.info: difficulty chart"),
        (35, "tx_blk", "1d", "Blockchain.info: n-transactions-per-block chart"),
        (36, "total_btc", "1d", "Blockchain.info: total-bitcoins chart"),
        (37, "mcap_bc", "1d", "Blockchain.info: market-cap chart"),
        # DeFi Llama
        (43, "tvl", "1d", "DeFi Llama: historicalChainTvl (returns all, filter client-side)"),
        (44, "tvl_eth", "1d", "DeFi Llama: ETH TVL"),
        (45, "stables", "1d", "DeFi Llama: stablecoincharts"),
        (46, "usdt", "1d", "DeFi Llama: stablecoin ID=1"),
        (47, "usdc", "1d", "DeFi Llama: stablecoin ID=2"),
        # FRED (ALL have decades of history)
        (52, "dxy", "1d", "FRED: DTWEXBGS"),
        (53, "us10y", "1d", "FRED: DGS10"),
        (54, "us2y", "1d", "FRED: DGS2"),
        (55, "ycurve", "1d", "FRED: T10Y2Y"),
        (56, "vix", "1d", "FRED: VIXCLS"),
        (57, "fedfunds", "1d", "FRED: DFF"),
        (58, "m2", "1w", "FRED: WM2NS (weekly)"),
        (59, "cpi", "1m", "FRED: CPIAUCSL (monthly)"),
        (60, "sp500", "1d", "FRED: SP500"),
        (61, "gold", "1d", "FRED: GOLDAMGBD228NLBM"),
        (62, "hy_spread", "1d", "FRED: BAMLH0A0HYM2"),
        (63, "be5y", "1d", "FRED: T5YIE"),
        (64, "nfci", "1w", "FRED: NFCI (weekly)"),
        (65, "claims", "1w", "FRED: ICSA (weekly)"),
        # Alternative.me
        (66, "fng", "1d", "Alternative.me: limit param returns history"),
        (67, "fng_prev", "1d", ""),
        (68, "fng_week", "1d", ""),
        (69, "fng_vol", "1d", ""),
        (70, "fng_mom", "1d", ""),
        (71, "fng_soc", "1d", ""),
        (72, "fng_dom", "1d", ""),
        # CoinGecko
        (81, "btc_price", "1d", "CoinGecko: /coins/{id}/history"),
        (82, "eth_price", "1d", "CoinGecko: /coins/{id}/history"),
        # Binance klines
        (78, "vol24", "1d", "Binance: klines endpoint"),
    ],
    
    # PARTIAL HISTORY (12 indicators)
    "partial": [
        (48, "dex_vol", "1d", "DeFi Llama: recent history in response"),
        (49, "bridge", "1d", "DeFi Llama: bridgevolume endpoint"),
        (51, "fees", "1d", "DeFi Llama: fees overview"),
        (83, "mcap", "1d", "CoinGecko: market_cap_chart (limited)"),
    ],
    
    # CURRENT ONLY (22 indicators)
    "current": [
        (9, "basis", "Binance premium index - real-time only"),
        (10, "liq_proxy", "Derived from 24hr ticker - real-time"),
        (13, "pcr_vol", "Deribit options summary - real-time"),
        (14, "pcr_oi", "Deribit options OI - real-time"),
        (15, "pcr_eth", "Deribit ETH options - real-time"),
        (16, "opt_oi", "Deribit total options OI - real-time"),
        (33, "blk_int", "Blockchain.info simple query - real-time"),
        (34, "unconf", "Blockchain.info unconfirmed - real-time"),
        (38, "mp_cnt", "Mempool.space - NO historical API"),
        (39, "mp_mb", "Mempool.space - NO historical API"),
        (40, "fee_fast", "Mempool.space - NO historical API"),
        (41, "fee_med", "Mempool.space - NO historical API"),
        (42, "fee_slow", "Mempool.space - NO historical API"),
        (50, "yields", "DeFi Llama yields - real-time"),
        (73, "imbal_btc", "Order book depth - real-time"),
        (74, "imbal_eth", "Order book depth - real-time"),
        (75, "spread", "Book ticker - real-time"),
        (76, "chg24_btc", "24hr ticker - real-time"),
        (77, "chg24_eth", "24hr ticker - real-time"),
        (79, "dispersion", "Calculated from 24hr - real-time"),
        (80, "correlation", "Calculated from 24hr - real-time"),
        (84, "btc_dom", "CoinGecko global - real-time"),
        (85, "eth_dom", "CoinGecko global - real-time"),
    ],
}

BACKFILL_NOTES = """
BACKFILL STRATEGY
=================

1. DAILY BACKFILL (Most indicators):
   - CoinMetrics, FRED, DeFi Llama TVL, Blockchain.info charts
   - Use: efm.update(datetime(2024, 6, 15))

2. HOURLY BACKFILL (Binance derivatives):
   - OI, L/S ratio, taker ratio have 1h resolution
   - Funding rate has 8h resolution

3. APIS RETURNING FULL HISTORY:
   - DeFi Llama TVL: Returns ALL history, filter client-side by timestamp
   - Alternative.me F&G: Use limit=1000 to get ~3 years of history
   - Blockchain.info charts: Use start= param with date

4. MISSING HISTORICAL DATA:
   - Mempool fees: Build your own collector
   - Order book imbalance: Build your own collector
   - Spreads: Build your own collector

5. RECOMMENDED APPROACH FOR TRAINING:
   a) Backfill what's available (51 indicators with FULL history)
   b) For CURRENT-only indicators, either:
      - Accept NaN/0 for historical periods
      - Build collectors to capture going forward
      - Use proxy indicators (e.g., volatility proxy for mempool fees)
"""

if __name__ == "__main__":
    print("INDICATOR SOURCES v5.0")
    print("=" * 60)
    print("\nData Sources:")
    for src, info in SOURCES.items():
        print(f"  {src:12s}: {info['auth']:10s} | {info['limit']:12s} | {info['history']}")
    
    print(f"\nHistorical Support:")
    print(f"  FULL:    {len(HISTORICAL_SUPPORT['full'])} indicators")
    print(f"  PARTIAL: {len(HISTORICAL_SUPPORT['partial'])} indicators")
    print(f"  CURRENT: {len(HISTORICAL_SUPPORT['current'])} indicators")
    
    print(BACKFILL_NOTES)