01c19662cb
Includes core prod + GREEN/BLUE subsystems: - prod/ (BLUE harness, configs, scripts, docs) - nautilus_dolphin/ (GREEN Nautilus-native impl + dvae/ preserved) - adaptive_exit/ (AEM engine + models/bucket_assignments.pkl) - Observability/ (EsoF advisor, TUI, dashboards) - external_factors/ (EsoF producer) - mc_forewarning_qlabs_fork/ (MC regime/envelope) Excludes runtime caches, logs, backups, and reproducible artifacts per .gitignore.
613 lines
43 KiB
Python
Executable File
613 lines
43 KiB
Python
Executable File
#!/usr/bin/env python3
|
|
"""
|
|
EXTERNAL FACTORS MATRIX v5.0 - DOLPHIN Compatible with BACKFILL
|
|
================================================================
|
|
85 indicators with HISTORICAL query support where available.
|
|
|
|
BACKFILL CAPABILITY:
|
|
FULL HISTORY (51): CoinMetrics, FRED, DeFi Llama TVL/stables, F&G, Binance funding/OI
|
|
PARTIAL (12): Deribit DVOL, CoinGecko prices, DEX volume
|
|
CURRENT ONLY (22): Mempool, order books, spreads, dominance
|
|
|
|
Author: HJ / Claude | Version: 5.0.0
|
|
"""
|
|
|
|
import asyncio
|
|
import aiohttp
|
|
import numpy as np
|
|
from dataclasses import dataclass
|
|
from typing import Dict, List, Optional, Any, Tuple
|
|
from datetime import datetime, timezone
|
|
from collections import deque
|
|
from enum import Enum
|
|
import json
|
|
|
|
class Category(Enum):
|
|
DERIVATIVES = "derivatives"
|
|
ONCHAIN = "onchain"
|
|
DEFI = "defi"
|
|
MACRO = "macro"
|
|
SENTIMENT = "sentiment"
|
|
MICROSTRUCTURE = "microstructure"
|
|
|
|
class Stationarity(Enum):
|
|
STATIONARY = "stationary"
|
|
TREND_UP = "trend_up"
|
|
EPISODIC = "episodic"
|
|
|
|
class HistoricalSupport(Enum):
|
|
FULL = "full" # Any historical date
|
|
PARTIAL = "partial" # Limited history
|
|
CURRENT = "current" # Real-time only
|
|
|
|
@dataclass
|
|
class Indicator:
|
|
id: int
|
|
name: str
|
|
category: Category
|
|
source: str
|
|
url: str
|
|
parser: str
|
|
stationarity: Stationarity
|
|
historical: HistoricalSupport
|
|
hist_url: str = ""
|
|
hist_resolution: str = ""
|
|
description: str = ""
|
|
|
|
@dataclass
|
|
class Config:
|
|
timeout: int = 15
|
|
max_concurrent: int = 15
|
|
cache_ttl: int = 30
|
|
fred_api_key: str = ""
|
|
|
|
# fmt: off
|
|
INDICATORS: List[Indicator] = [
|
|
# DERIVATIVES - Binance (1-10) - Most have FULL history
|
|
Indicator(1, "funding_btc", Category.DERIVATIVES, "binance",
|
|
"https://fapi.binance.com/fapi/v1/fundingRate?symbol=BTCUSDT&limit=1",
|
|
"parse_binance_funding", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://fapi.binance.com/fapi/v1/fundingRate?symbol=BTCUSDT&startTime={start_ms}&endTime={end_ms}&limit=1",
|
|
"8h", "BTC funding - FULL via startTime/endTime"),
|
|
Indicator(2, "funding_eth", Category.DERIVATIVES, "binance",
|
|
"https://fapi.binance.com/fapi/v1/fundingRate?symbol=ETHUSDT&limit=1",
|
|
"parse_binance_funding", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://fapi.binance.com/fapi/v1/fundingRate?symbol=ETHUSDT&startTime={start_ms}&endTime={end_ms}&limit=1",
|
|
"8h", "ETH funding"),
|
|
Indicator(3, "oi_btc", Category.DERIVATIVES, "binance",
|
|
"https://fapi.binance.com/fapi/v1/openInterest?symbol=BTCUSDT",
|
|
"parse_binance_oi", Stationarity.TREND_UP, HistoricalSupport.FULL,
|
|
"https://fapi.binance.com/futures/data/openInterestHist?symbol=BTCUSDT&period=1h&startTime={start_ms}&endTime={end_ms}&limit=1",
|
|
"1h", "BTC OI - FULL via openInterestHist"),
|
|
Indicator(4, "oi_eth", Category.DERIVATIVES, "binance",
|
|
"https://fapi.binance.com/fapi/v1/openInterest?symbol=ETHUSDT",
|
|
"parse_binance_oi", Stationarity.TREND_UP, HistoricalSupport.FULL,
|
|
"https://fapi.binance.com/futures/data/openInterestHist?symbol=ETHUSDT&period=1h&startTime={start_ms}&endTime={end_ms}&limit=1",
|
|
"1h", "ETH OI"),
|
|
Indicator(5, "ls_btc", Category.DERIVATIVES, "binance",
|
|
"https://fapi.binance.com/futures/data/globalLongShortAccountRatio?symbol=BTCUSDT&period=1h&limit=1",
|
|
"parse_binance_ls", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://fapi.binance.com/futures/data/globalLongShortAccountRatio?symbol=BTCUSDT&period=1h&startTime={start_ms}&endTime={end_ms}&limit=1",
|
|
"1h", "L/S ratio - FULL"),
|
|
Indicator(6, "ls_eth", Category.DERIVATIVES, "binance",
|
|
"https://fapi.binance.com/futures/data/globalLongShortAccountRatio?symbol=ETHUSDT&period=1h&limit=1",
|
|
"parse_binance_ls", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://fapi.binance.com/futures/data/globalLongShortAccountRatio?symbol=ETHUSDT&period=1h&startTime={start_ms}&endTime={end_ms}&limit=1",
|
|
"1h", "ETH L/S"),
|
|
Indicator(7, "ls_top", Category.DERIVATIVES, "binance",
|
|
"https://fapi.binance.com/futures/data/topLongShortAccountRatio?symbol=BTCUSDT&period=1h&limit=1",
|
|
"parse_binance_ls", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://fapi.binance.com/futures/data/topLongShortAccountRatio?symbol=BTCUSDT&period=1h&startTime={start_ms}&endTime={end_ms}&limit=1",
|
|
"1h", "Top trader L/S"),
|
|
Indicator(8, "taker", Category.DERIVATIVES, "binance",
|
|
"https://fapi.binance.com/futures/data/takerlongshortRatio?symbol=BTCUSDT&period=1h&limit=1",
|
|
"parse_binance_taker", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://fapi.binance.com/futures/data/takerlongshortRatio?symbol=BTCUSDT&period=1h&startTime={start_ms}&endTime={end_ms}&limit=1",
|
|
"1h", "Taker ratio"),
|
|
Indicator(9, "basis", Category.DERIVATIVES, "binance",
|
|
"https://fapi.binance.com/fapi/v1/premiumIndex?symbol=BTCUSDT",
|
|
"parse_binance_basis", Stationarity.STATIONARY, HistoricalSupport.CURRENT,
|
|
"", "", "Basis - CURRENT"),
|
|
Indicator(10, "liq_proxy", Category.DERIVATIVES, "binance",
|
|
"https://fapi.binance.com/fapi/v1/ticker/24hr?symbol=BTCUSDT",
|
|
"parse_liq_proxy", Stationarity.STATIONARY, HistoricalSupport.CURRENT,
|
|
"", "", "Liq proxy - CURRENT"),
|
|
# DERIVATIVES - Deribit (11-18)
|
|
Indicator(11, "dvol_btc", Category.DERIVATIVES, "deribit",
|
|
"https://www.deribit.com/api/v2/public/get_volatility_index_data?currency=BTC&resolution=3600&count=1",
|
|
"parse_deribit_dvol", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://www.deribit.com/api/v2/public/get_volatility_index_data?currency=BTC&resolution=3600&start_timestamp={start_ms}&end_timestamp={end_ms}",
|
|
"1h", "DVOL - FULL"),
|
|
Indicator(12, "dvol_eth", Category.DERIVATIVES, "deribit",
|
|
"https://www.deribit.com/api/v2/public/get_volatility_index_data?currency=ETH&resolution=3600&count=1",
|
|
"parse_deribit_dvol", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://www.deribit.com/api/v2/public/get_volatility_index_data?currency=ETH&resolution=3600&start_timestamp={start_ms}&end_timestamp={end_ms}",
|
|
"1h", "ETH DVOL"),
|
|
Indicator(13, "pcr_vol", Category.DERIVATIVES, "deribit",
|
|
"https://www.deribit.com/api/v2/public/get_book_summary_by_currency?currency=BTC&kind=option",
|
|
"parse_deribit_pcr", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "PCR - CURRENT"),
|
|
Indicator(14, "pcr_oi", Category.DERIVATIVES, "deribit",
|
|
"https://www.deribit.com/api/v2/public/get_book_summary_by_currency?currency=BTC&kind=option",
|
|
"parse_deribit_pcr_oi", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "PCR OI - CURRENT"),
|
|
Indicator(15, "pcr_eth", Category.DERIVATIVES, "deribit",
|
|
"https://www.deribit.com/api/v2/public/get_book_summary_by_currency?currency=ETH&kind=option",
|
|
"parse_deribit_pcr", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "ETH PCR - CURRENT"),
|
|
Indicator(16, "opt_oi", Category.DERIVATIVES, "deribit",
|
|
"https://www.deribit.com/api/v2/public/get_book_summary_by_currency?currency=BTC&kind=option",
|
|
"parse_deribit_oi", Stationarity.TREND_UP, HistoricalSupport.CURRENT, "", "", "Options OI - CURRENT"),
|
|
Indicator(17, "fund_dbt_btc", Category.DERIVATIVES, "deribit",
|
|
"https://www.deribit.com/api/v2/public/get_funding_rate_value?instrument_name=BTC-PERPETUAL",
|
|
"parse_deribit_fund", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://www.deribit.com/api/v2/public/get_funding_rate_history?instrument_name=BTC-PERPETUAL&start_timestamp={start_ms}&end_timestamp={end_ms}",
|
|
"8h", "Deribit fund - FULL"),
|
|
Indicator(18, "fund_dbt_eth", Category.DERIVATIVES, "deribit",
|
|
"https://www.deribit.com/api/v2/public/get_funding_rate_value?instrument_name=ETH-PERPETUAL",
|
|
"parse_deribit_fund", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://www.deribit.com/api/v2/public/get_funding_rate_history?instrument_name=ETH-PERPETUAL&start_timestamp={start_ms}&end_timestamp={end_ms}",
|
|
"8h", "Deribit ETH fund"),
|
|
# ONCHAIN - CoinMetrics (19-30) - ALL FULL HISTORY
|
|
Indicator(19, "rcap_btc", Category.ONCHAIN, "coinmetrics",
|
|
"https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=CapRealUSD&frequency=1d&page_size=1",
|
|
"parse_cm", Stationarity.TREND_UP, HistoricalSupport.FULL,
|
|
"https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=CapRealUSD&frequency=1d&start_time={date}T00:00:00Z&end_time={date}T23:59:59Z",
|
|
"1d", "Realized cap - FULL"),
|
|
Indicator(20, "mvrv", Category.ONCHAIN, "coinmetrics",
|
|
"https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=CapMrktCurUSD,CapRealUSD&frequency=1d&page_size=1",
|
|
"parse_cm_mvrv", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=CapMrktCurUSD,CapRealUSD&frequency=1d&start_time={date}T00:00:00Z&end_time={date}T23:59:59Z",
|
|
"1d", "MVRV - FULL"),
|
|
Indicator(21, "nupl", Category.ONCHAIN, "coinmetrics",
|
|
"https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=CapMrktCurUSD,CapRealUSD&frequency=1d&page_size=1",
|
|
"parse_cm_nupl", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=CapMrktCurUSD,CapRealUSD&frequency=1d&start_time={date}T00:00:00Z&end_time={date}T23:59:59Z",
|
|
"1d", "NUPL - FULL"),
|
|
Indicator(22, "addr_btc", Category.ONCHAIN, "coinmetrics",
|
|
"https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=AdrActCnt&frequency=1d&page_size=1",
|
|
"parse_cm", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=AdrActCnt&frequency=1d&start_time={date}T00:00:00Z&end_time={date}T23:59:59Z",
|
|
"1d", "Active addr - FULL"),
|
|
Indicator(23, "addr_eth", Category.ONCHAIN, "coinmetrics",
|
|
"https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=eth&metrics=AdrActCnt&frequency=1d&page_size=1",
|
|
"parse_cm", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=eth&metrics=AdrActCnt&frequency=1d&start_time={date}T00:00:00Z&end_time={date}T23:59:59Z",
|
|
"1d", "ETH addr - FULL"),
|
|
Indicator(24, "txcnt", Category.ONCHAIN, "coinmetrics",
|
|
"https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=TxCnt&frequency=1d&page_size=1",
|
|
"parse_cm", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=TxCnt&frequency=1d&start_time={date}T00:00:00Z&end_time={date}T23:59:59Z",
|
|
"1d", "TX count - FULL"),
|
|
Indicator(25, "fees_btc", Category.ONCHAIN, "coinmetrics",
|
|
"https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=FeeTotUSD&frequency=1d&page_size=1",
|
|
"parse_cm", Stationarity.EPISODIC, HistoricalSupport.FULL,
|
|
"https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=FeeTotUSD&frequency=1d&start_time={date}T00:00:00Z&end_time={date}T23:59:59Z",
|
|
"1d", "BTC fees - FULL"),
|
|
Indicator(26, "fees_eth", Category.ONCHAIN, "coinmetrics",
|
|
"https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=eth&metrics=FeeTotUSD&frequency=1d&page_size=1",
|
|
"parse_cm", Stationarity.EPISODIC, HistoricalSupport.FULL,
|
|
"https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=eth&metrics=FeeTotUSD&frequency=1d&start_time={date}T00:00:00Z&end_time={date}T23:59:59Z",
|
|
"1d", "ETH fees - FULL"),
|
|
Indicator(27, "nvt", Category.ONCHAIN, "coinmetrics",
|
|
"https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=NVTAdj&frequency=1d&page_size=1",
|
|
"parse_cm", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=NVTAdj&frequency=1d&start_time={date}T00:00:00Z&end_time={date}T23:59:59Z",
|
|
"1d", "NVT - FULL"),
|
|
Indicator(28, "velocity", Category.ONCHAIN, "coinmetrics",
|
|
"https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=VelCur1yr&frequency=1d&page_size=1",
|
|
"parse_cm", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=VelCur1yr&frequency=1d&start_time={date}T00:00:00Z&end_time={date}T23:59:59Z",
|
|
"1d", "Velocity - FULL"),
|
|
Indicator(29, "sply_act", Category.ONCHAIN, "coinmetrics",
|
|
"https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=SplyAct1yr&frequency=1d&page_size=1",
|
|
"parse_cm", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=btc&metrics=SplyAct1yr&frequency=1d&start_time={date}T00:00:00Z&end_time={date}T23:59:59Z",
|
|
"1d", "Active supply - FULL"),
|
|
Indicator(30, "rcap_eth", Category.ONCHAIN, "coinmetrics",
|
|
"https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=eth&metrics=CapRealUSD&frequency=1d&page_size=1",
|
|
"parse_cm", Stationarity.TREND_UP, HistoricalSupport.FULL,
|
|
"https://community-api.coinmetrics.io/v4/timeseries/asset-metrics?assets=eth&metrics=CapRealUSD&frequency=1d&start_time={date}T00:00:00Z&end_time={date}T23:59:59Z",
|
|
"1d", "ETH rcap - FULL"),
|
|
# ONCHAIN - Blockchain.info (31-37)
|
|
Indicator(31, "hashrate", Category.ONCHAIN, "blockchain",
|
|
"https://blockchain.info/q/hashrate", "parse_bc", Stationarity.TREND_UP, HistoricalSupport.FULL,
|
|
"https://api.blockchain.info/charts/hash-rate?timespan=1days&start={date}&format=json", "1d", "Hashrate - FULL"),
|
|
Indicator(32, "difficulty", Category.ONCHAIN, "blockchain",
|
|
"https://blockchain.info/q/getdifficulty", "parse_bc", Stationarity.TREND_UP, HistoricalSupport.FULL,
|
|
"https://api.blockchain.info/charts/difficulty?timespan=1days&start={date}&format=json", "1d", "Difficulty - FULL"),
|
|
Indicator(33, "blk_int", Category.ONCHAIN, "blockchain",
|
|
"https://blockchain.info/q/interval", "parse_bc_int", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "Block int - CURRENT"),
|
|
Indicator(34, "unconf", Category.ONCHAIN, "blockchain",
|
|
"https://blockchain.info/q/unconfirmedcount", "parse_bc", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "Unconf - CURRENT"),
|
|
Indicator(35, "tx_blk", Category.ONCHAIN, "blockchain",
|
|
"https://blockchain.info/q/nperblock", "parse_bc", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://api.blockchain.info/charts/n-transactions-per-block?timespan=1days&start={date}&format=json", "1d", "TX/blk - FULL"),
|
|
Indicator(36, "total_btc", Category.ONCHAIN, "blockchain",
|
|
"https://blockchain.info/q/totalbc", "parse_bc_btc", Stationarity.TREND_UP, HistoricalSupport.FULL,
|
|
"https://api.blockchain.info/charts/total-bitcoins?timespan=1days&start={date}&format=json", "1d", "Total BTC - FULL"),
|
|
Indicator(37, "mcap_bc", Category.ONCHAIN, "blockchain",
|
|
"https://blockchain.info/q/marketcap", "parse_bc", Stationarity.TREND_UP, HistoricalSupport.FULL,
|
|
"https://api.blockchain.info/charts/market-cap?timespan=1days&start={date}&format=json", "1d", "Mcap - FULL"),
|
|
# ONCHAIN - Mempool (38-42) - ALL CURRENT
|
|
Indicator(38, "mp_cnt", Category.ONCHAIN, "mempool", "https://mempool.space/api/mempool",
|
|
"parse_mp_cnt", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "Mempool - CURRENT"),
|
|
Indicator(39, "mp_mb", Category.ONCHAIN, "mempool", "https://mempool.space/api/mempool",
|
|
"parse_mp_mb", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "Mempool MB - CURRENT"),
|
|
Indicator(40, "fee_fast", Category.ONCHAIN, "mempool", "https://mempool.space/api/v1/fees/recommended",
|
|
"parse_fee_fast", Stationarity.EPISODIC, HistoricalSupport.CURRENT, "", "", "Fast fee - CURRENT"),
|
|
Indicator(41, "fee_med", Category.ONCHAIN, "mempool", "https://mempool.space/api/v1/fees/recommended",
|
|
"parse_fee_med", Stationarity.EPISODIC, HistoricalSupport.CURRENT, "", "", "Med fee - CURRENT"),
|
|
Indicator(42, "fee_slow", Category.ONCHAIN, "mempool", "https://mempool.space/api/v1/fees/recommended",
|
|
"parse_fee_slow", Stationarity.EPISODIC, HistoricalSupport.CURRENT, "", "", "Slow fee - CURRENT"),
|
|
# DEFI - DeFi Llama (43-51)
|
|
Indicator(43, "tvl", Category.DEFI, "defillama", "https://api.llama.fi/v2/historicalChainTvl",
|
|
"parse_dl_tvl", Stationarity.TREND_UP, HistoricalSupport.FULL,
|
|
"https://api.llama.fi/v2/historicalChainTvl", "1d", "TVL - FULL (filter client-side)"),
|
|
Indicator(44, "tvl_eth", Category.DEFI, "defillama", "https://api.llama.fi/v2/historicalChainTvl/Ethereum",
|
|
"parse_dl_tvl", Stationarity.TREND_UP, HistoricalSupport.FULL,
|
|
"https://api.llama.fi/v2/historicalChainTvl/Ethereum", "1d", "ETH TVL - FULL"),
|
|
Indicator(45, "stables", Category.DEFI, "defillama", "https://stablecoins.llama.fi/stablecoins?includePrices=false",
|
|
"parse_dl_stables", Stationarity.TREND_UP, HistoricalSupport.FULL,
|
|
"https://stablecoins.llama.fi/stablecoincharts/all?stablecoin=1", "1d", "Stables - FULL"),
|
|
Indicator(46, "usdt", Category.DEFI, "defillama", "https://stablecoins.llama.fi/stablecoin/tether",
|
|
"parse_dl_single", Stationarity.TREND_UP, HistoricalSupport.FULL,
|
|
"https://stablecoins.llama.fi/stablecoincharts/all?stablecoin=1", "1d", "USDT - FULL"),
|
|
Indicator(47, "usdc", Category.DEFI, "defillama", "https://stablecoins.llama.fi/stablecoin/usd-coin",
|
|
"parse_dl_single", Stationarity.TREND_UP, HistoricalSupport.FULL,
|
|
"https://stablecoins.llama.fi/stablecoincharts/all?stablecoin=2", "1d", "USDC - FULL"),
|
|
Indicator(48, "dex_vol", Category.DEFI, "defillama",
|
|
"https://api.llama.fi/overview/dexs?excludeTotalDataChart=true&excludeTotalDataChartBreakdown=true",
|
|
"parse_dl_dex", Stationarity.EPISODIC, HistoricalSupport.PARTIAL, "", "1d", "DEX vol - PARTIAL"),
|
|
Indicator(49, "bridge", Category.DEFI, "defillama", "https://bridges.llama.fi/bridges?includeChains=false",
|
|
"parse_dl_bridge", Stationarity.EPISODIC, HistoricalSupport.PARTIAL, "", "1d", "Bridge - PARTIAL"),
|
|
Indicator(50, "yields", Category.DEFI, "defillama", "https://yields.llama.fi/pools",
|
|
"parse_dl_yields", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "Yields - CURRENT"),
|
|
Indicator(51, "fees", Category.DEFI, "defillama", "https://api.llama.fi/overview/fees?excludeTotalDataChart=true",
|
|
"parse_dl_fees", Stationarity.EPISODIC, HistoricalSupport.PARTIAL, "", "1d", "Fees - PARTIAL"),
|
|
# MACRO - FRED (52-65) - ALL FULL HISTORY (decades)
|
|
Indicator(52, "dxy", Category.MACRO, "fred", "DTWEXBGS", "parse_fred", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://api.stlouisfed.org/fred/series/observations?series_id=DTWEXBGS&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1d", "DXY - FULL"),
|
|
Indicator(53, "us10y", Category.MACRO, "fred", "DGS10", "parse_fred", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://api.stlouisfed.org/fred/series/observations?series_id=DGS10&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1d", "10Y - FULL"),
|
|
Indicator(54, "us2y", Category.MACRO, "fred", "DGS2", "parse_fred", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://api.stlouisfed.org/fred/series/observations?series_id=DGS2&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1d", "2Y - FULL"),
|
|
Indicator(55, "ycurve", Category.MACRO, "fred", "T10Y2Y", "parse_fred", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://api.stlouisfed.org/fred/series/observations?series_id=T10Y2Y&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1d", "Yield curve - FULL"),
|
|
Indicator(56, "vix", Category.MACRO, "fred", "VIXCLS", "parse_fred", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://api.stlouisfed.org/fred/series/observations?series_id=VIXCLS&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1d", "VIX - FULL"),
|
|
Indicator(57, "fedfunds", Category.MACRO, "fred", "DFF", "parse_fred", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://api.stlouisfed.org/fred/series/observations?series_id=DFF&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1d", "Fed funds - FULL"),
|
|
Indicator(58, "m2", Category.MACRO, "fred", "WM2NS", "parse_fred", Stationarity.TREND_UP, HistoricalSupport.FULL,
|
|
"https://api.stlouisfed.org/fred/series/observations?series_id=WM2NS&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1w", "M2 - FULL"),
|
|
Indicator(59, "cpi", Category.MACRO, "fred", "CPIAUCSL", "parse_fred", Stationarity.TREND_UP, HistoricalSupport.FULL,
|
|
"https://api.stlouisfed.org/fred/series/observations?series_id=CPIAUCSL&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1m", "CPI - FULL"),
|
|
Indicator(60, "sp500", Category.MACRO, "fred", "SP500", "parse_fred", Stationarity.TREND_UP, HistoricalSupport.FULL,
|
|
"https://api.stlouisfed.org/fred/series/observations?series_id=SP500&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1d", "S&P - FULL"),
|
|
Indicator(61, "gold", Category.MACRO, "fred", "GOLDAMGBD228NLBM", "parse_fred", Stationarity.TREND_UP, HistoricalSupport.FULL,
|
|
"https://api.stlouisfed.org/fred/series/observations?series_id=GOLDAMGBD228NLBM&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1d", "Gold - FULL"),
|
|
Indicator(62, "hy_spread", Category.MACRO, "fred", "BAMLH0A0HYM2", "parse_fred", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://api.stlouisfed.org/fred/series/observations?series_id=BAMLH0A0HYM2&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1d", "HY spread - FULL"),
|
|
Indicator(63, "be5y", Category.MACRO, "fred", "T5YIE", "parse_fred", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://api.stlouisfed.org/fred/series/observations?series_id=T5YIE&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1d", "Breakeven - FULL"),
|
|
Indicator(64, "nfci", Category.MACRO, "fred", "NFCI", "parse_fred", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://api.stlouisfed.org/fred/series/observations?series_id=NFCI&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1w", "NFCI - FULL"),
|
|
Indicator(65, "claims", Category.MACRO, "fred", "ICSA", "parse_fred", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://api.stlouisfed.org/fred/series/observations?series_id=ICSA&api_key={key}&file_type=json&observation_start={date}&observation_end={date}", "1w", "Claims - FULL"),
|
|
# SENTIMENT (66-72) - F&G has FULL history
|
|
Indicator(66, "fng", Category.SENTIMENT, "alternative", "https://api.alternative.me/fng/?limit=1",
|
|
"parse_fng", Stationarity.STATIONARY, HistoricalSupport.FULL,
|
|
"https://api.alternative.me/fng/?limit=1000&date_format=us", "1d", "F&G - FULL (returns history, filter)"),
|
|
Indicator(67, "fng_prev", Category.SENTIMENT, "alternative", "https://api.alternative.me/fng/?limit=2",
|
|
"parse_fng_prev", Stationarity.STATIONARY, HistoricalSupport.FULL, "", "1d", "Prev F&G"),
|
|
Indicator(68, "fng_week", Category.SENTIMENT, "alternative", "https://api.alternative.me/fng/?limit=7",
|
|
"parse_fng_week", Stationarity.STATIONARY, HistoricalSupport.FULL, "", "1d", "Week F&G"),
|
|
Indicator(69, "fng_vol", Category.SENTIMENT, "alternative", "https://api.alternative.me/fng/?limit=1",
|
|
"parse_fng", Stationarity.STATIONARY, HistoricalSupport.FULL, "", "1d", "Vol proxy"),
|
|
Indicator(70, "fng_mom", Category.SENTIMENT, "alternative", "https://api.alternative.me/fng/?limit=1",
|
|
"parse_fng", Stationarity.STATIONARY, HistoricalSupport.FULL, "", "1d", "Mom proxy"),
|
|
Indicator(71, "fng_soc", Category.SENTIMENT, "alternative", "https://api.alternative.me/fng/?limit=1",
|
|
"parse_fng", Stationarity.STATIONARY, HistoricalSupport.FULL, "", "1d", "Social proxy"),
|
|
Indicator(72, "fng_dom", Category.SENTIMENT, "alternative", "https://api.alternative.me/fng/?limit=1",
|
|
"parse_fng", Stationarity.STATIONARY, HistoricalSupport.FULL, "", "1d", "Dom proxy"),
|
|
# MICROSTRUCTURE (73-80) - Most CURRENT
|
|
Indicator(73, "imbal_btc", Category.MICROSTRUCTURE, "binance", "https://api.binance.com/api/v3/depth?symbol=BTCUSDT&limit=100",
|
|
"parse_imbal", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "Imbalance - CURRENT"),
|
|
Indicator(74, "imbal_eth", Category.MICROSTRUCTURE, "binance", "https://api.binance.com/api/v3/depth?symbol=ETHUSDT&limit=100",
|
|
"parse_imbal", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "ETH imbal - CURRENT"),
|
|
Indicator(75, "spread", Category.MICROSTRUCTURE, "binance", "https://api.binance.com/api/v3/ticker/bookTicker?symbol=BTCUSDT",
|
|
"parse_spread", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "Spread - CURRENT"),
|
|
Indicator(76, "chg24_btc", Category.MICROSTRUCTURE, "binance", "https://api.binance.com/api/v3/ticker/24hr?symbol=BTCUSDT",
|
|
"parse_chg", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "24h chg - CURRENT"),
|
|
Indicator(77, "chg24_eth", Category.MICROSTRUCTURE, "binance", "https://api.binance.com/api/v3/ticker/24hr?symbol=ETHUSDT",
|
|
"parse_chg", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "ETH 24h - CURRENT"),
|
|
Indicator(78, "vol24", Category.MICROSTRUCTURE, "binance", "https://api.binance.com/api/v3/ticker/24hr?symbol=BTCUSDT",
|
|
"parse_vol", Stationarity.EPISODIC, HistoricalSupport.FULL,
|
|
"https://api.binance.com/api/v3/klines?symbol=BTCUSDT&interval=1d&startTime={start_ms}&endTime={end_ms}&limit=1",
|
|
"1d", "Volume - FULL via klines"),
|
|
Indicator(79, "dispersion", Category.MICROSTRUCTURE, "binance", "https://api.binance.com/api/v3/ticker/24hr",
|
|
"parse_disp", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "Dispersion - CURRENT"),
|
|
Indicator(80, "correlation", Category.MICROSTRUCTURE, "binance", "https://api.binance.com/api/v3/ticker/24hr",
|
|
"parse_corr", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "Correlation - CURRENT"),
|
|
# MARKET - CoinGecko (81-85)
|
|
Indicator(81, "btc_price", Category.MACRO, "coingecko", "https://api.coingecko.com/api/v3/simple/price?ids=bitcoin&vs_currencies=usd",
|
|
"parse_cg_btc", Stationarity.TREND_UP, HistoricalSupport.FULL,
|
|
"https://api.coingecko.com/api/v3/coins/bitcoin/history?date={date_dmy}", "1d", "BTC price - FULL"),
|
|
Indicator(82, "eth_price", Category.MACRO, "coingecko", "https://api.coingecko.com/api/v3/simple/price?ids=ethereum&vs_currencies=usd",
|
|
"parse_cg_eth", Stationarity.TREND_UP, HistoricalSupport.FULL,
|
|
"https://api.coingecko.com/api/v3/coins/ethereum/history?date={date_dmy}", "1d", "ETH price - FULL"),
|
|
Indicator(83, "mcap", Category.MACRO, "coingecko", "https://api.coingecko.com/api/v3/global",
|
|
"parse_cg_mcap", Stationarity.TREND_UP, HistoricalSupport.PARTIAL, "", "1d", "Mcap - PARTIAL"),
|
|
Indicator(84, "btc_dom", Category.MACRO, "coingecko", "https://api.coingecko.com/api/v3/global",
|
|
"parse_cg_dom_btc", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "BTC dom - CURRENT"),
|
|
Indicator(85, "eth_dom", Category.MACRO, "coingecko", "https://api.coingecko.com/api/v3/global",
|
|
"parse_cg_dom_eth", Stationarity.STATIONARY, HistoricalSupport.CURRENT, "", "", "ETH dom - CURRENT"),
|
|
]
|
|
# fmt: on
|
|
|
|
N_INDICATORS = len(INDICATORS)
|
|
|
|
class StationarityTransformer:
|
|
def __init__(self, lookback: int = 10):
|
|
self.history: Dict[int, deque] = {i: deque(maxlen=lookback+1) for i in range(1, N_INDICATORS+1)}
|
|
def transform(self, ind_id: int, raw: float) -> float:
|
|
ind = INDICATORS[ind_id - 1]
|
|
hist = self.history[ind_id]
|
|
hist.append(raw)
|
|
if ind.stationarity == Stationarity.STATIONARY: return raw
|
|
if ind.stationarity == Stationarity.TREND_UP:
|
|
return (raw - hist[-2]) / abs(hist[-2]) if len(hist) >= 2 and hist[-2] != 0 else 0.0
|
|
if ind.stationarity == Stationarity.EPISODIC:
|
|
if len(hist) < 3: return 0.0
|
|
m, s = np.mean(list(hist)), np.std(list(hist))
|
|
return (raw - m) / s if s > 0 else 0.0
|
|
return raw
|
|
def transform_matrix(self, raw: np.ndarray) -> np.ndarray:
|
|
return np.array([self.transform(i+1, raw[i]) for i in range(len(raw))])
|
|
|
|
class ExternalFactorsFetcher:
|
|
def __init__(self, config: Config = None):
|
|
self.config = config or Config()
|
|
self.cache: Dict[str, Tuple[float, Any]] = {}
|
|
import time as t; self._time = t
|
|
|
|
def _build_hist_url(self, ind: Indicator, dt: datetime) -> Optional[str]:
|
|
if ind.historical == HistoricalSupport.CURRENT or not ind.hist_url: return None
|
|
url = ind.hist_url
|
|
date_str = dt.strftime("%Y-%m-%d")
|
|
date_dmy = dt.strftime("%d-%m-%Y")
|
|
start_ms = int(dt.replace(hour=0, minute=0, second=0).timestamp() * 1000)
|
|
end_ms = int(dt.replace(hour=23, minute=59, second=59).timestamp() * 1000)
|
|
key = self.config.fred_api_key or "DEMO_KEY"
|
|
return url.replace("{date}", date_str).replace("{date_dmy}", date_dmy).replace("{start_ms}", str(start_ms)).replace("{end_ms}", str(end_ms)).replace("{key}", key)
|
|
|
|
async def _fetch(self, session, url: str) -> Optional[Any]:
|
|
if url in self.cache:
|
|
ct, cd = self.cache[url]
|
|
if self._time.time() - ct < self.config.cache_ttl: return cd
|
|
try:
|
|
async with session.get(url, timeout=aiohttp.ClientTimeout(total=self.config.timeout), headers={"User-Agent": "Mozilla/5.0"}) as r:
|
|
if r.status == 200:
|
|
d = await r.json() if 'json' in r.headers.get('Content-Type', '') else await r.text()
|
|
if isinstance(d, str):
|
|
try: d = json.loads(d)
|
|
except: pass
|
|
self.cache[url] = (self._time.time(), d)
|
|
return d
|
|
except: pass
|
|
return None
|
|
|
|
def _fred_url(self, series: str) -> str:
|
|
return f"https://api.stlouisfed.org/fred/series/observations?series_id={series}&api_key={self.config.fred_api_key or 'DEMO_KEY'}&file_type=json&sort_order=desc&limit=1"
|
|
|
|
# Parsers
|
|
def parse_binance_funding(self, d): return float(d[0]['fundingRate']) if isinstance(d, list) and d else 0.0
|
|
def parse_binance_oi(self, d):
|
|
if isinstance(d, list) and d: return float(d[-1].get('sumOpenInterest', 0))
|
|
return float(d.get('openInterest', 0)) if isinstance(d, dict) else 0.0
|
|
def parse_binance_ls(self, d): return float(d[-1]['longShortRatio']) if isinstance(d, list) and d else 1.0
|
|
def parse_binance_taker(self, d): return float(d[-1]['buySellRatio']) if isinstance(d, list) and d else 1.0
|
|
def parse_binance_basis(self, d): return float(d.get('lastFundingRate', 0)) * 365 * 3 if isinstance(d, dict) else 0.0
|
|
def parse_liq_proxy(self, d): return np.tanh(float(d.get('priceChangePercent', 0)) / 10) if isinstance(d, dict) else 0.0
|
|
def parse_deribit_dvol(self, d):
|
|
if isinstance(d, dict) and 'result' in d and isinstance(d['result'], dict) and 'data' in d['result'] and d['result']['data']:
|
|
return float(d['result']['data'][-1][4]) if len(d['result']['data'][-1]) > 4 else 0.0
|
|
return 0.0
|
|
def parse_deribit_pcr(self, d):
|
|
if isinstance(d, dict) and 'result' in d:
|
|
r = d['result']
|
|
p = sum(float(o.get('volume', 0)) for o in r if '-P' in o.get('instrument_name', ''))
|
|
c = sum(float(o.get('volume', 0)) for o in r if '-C' in o.get('instrument_name', ''))
|
|
return p / c if c > 0 else 1.0
|
|
return 1.0
|
|
def parse_deribit_pcr_oi(self, d):
|
|
if isinstance(d, dict) and 'result' in d:
|
|
r = d['result']
|
|
p = sum(float(o.get('open_interest', 0)) for o in r if '-P' in o.get('instrument_name', ''))
|
|
c = sum(float(o.get('open_interest', 0)) for o in r if '-C' in o.get('instrument_name', ''))
|
|
return p / c if c > 0 else 1.0
|
|
return 1.0
|
|
def parse_deribit_oi(self, d): return sum(float(o.get('open_interest', 0)) for o in d['result']) if isinstance(d, dict) and 'result' in d else 0.0
|
|
def parse_deribit_fund(self, d):
|
|
if isinstance(d, dict) and 'result' in d:
|
|
r = d['result']
|
|
return float(r[-1].get('interest_8h', 0)) if isinstance(r, list) and r else float(r)
|
|
return 0.0
|
|
def parse_cm(self, d):
|
|
if isinstance(d, dict) and 'data' in d and d['data']:
|
|
for k, v in d['data'][-1].items():
|
|
if k not in ['asset', 'time']:
|
|
try: return float(v)
|
|
except: pass
|
|
return 0.0
|
|
def parse_cm_mvrv(self, d):
|
|
if isinstance(d, dict) and 'data' in d and d['data']:
|
|
r = d['data'][-1]
|
|
m, rc = float(r.get('CapMrktCurUSD', 0)), float(r.get('CapRealUSD', 1))
|
|
return m / rc if rc > 0 else 0.0
|
|
return 0.0
|
|
def parse_cm_nupl(self, d):
|
|
if isinstance(d, dict) and 'data' in d and d['data']:
|
|
r = d['data'][-1]
|
|
m, rc = float(r.get('CapMrktCurUSD', 0)), float(r.get('CapRealUSD', 1))
|
|
return (m - rc) / m if m > 0 else 0.0
|
|
return 0.0
|
|
def parse_bc(self, d):
|
|
if isinstance(d, (int, float)): return float(d)
|
|
if isinstance(d, str):
|
|
try: return float(d)
|
|
except: pass
|
|
if isinstance(d, dict) and 'values' in d and d['values']: return float(d['values'][-1].get('y', 0))
|
|
return 0.0
|
|
def parse_bc_int(self, d): v = self.parse_bc(d); return abs(v - 600) / 600 if v > 0 else 0.0
|
|
def parse_bc_btc(self, d): v = self.parse_bc(d); return v / 1e8 if v > 0 else 0.0
|
|
def parse_mp_cnt(self, d): return float(d.get('count', 0)) if isinstance(d, dict) else 0.0
|
|
def parse_mp_mb(self, d): return float(d.get('vsize', 0)) / 1e6 if isinstance(d, dict) else 0.0
|
|
def parse_fee_fast(self, d): return float(d.get('fastestFee', 0)) if isinstance(d, dict) else 0.0
|
|
def parse_fee_med(self, d): return float(d.get('halfHourFee', 0)) if isinstance(d, dict) else 0.0
|
|
def parse_fee_slow(self, d): return float(d.get('economyFee', 0)) if isinstance(d, dict) else 0.0
|
|
def parse_dl_tvl(self, d, target_date: datetime = None):
|
|
if isinstance(d, list) and d:
|
|
if target_date:
|
|
ts = int(target_date.timestamp())
|
|
for e in reversed(d):
|
|
if e.get('date', 0) <= ts: return float(e.get('tvl', 0))
|
|
return float(d[-1].get('tvl', 0))
|
|
return 0.0
|
|
def parse_dl_stables(self, d):
|
|
if isinstance(d, dict) and 'peggedAssets' in d:
|
|
return sum(float(a.get('circulating', {}).get('peggedUSD', 0)) for a in d['peggedAssets'])
|
|
return 0.0
|
|
def parse_dl_single(self, d):
|
|
if isinstance(d, dict) and 'tokens' in d and d['tokens']:
|
|
return float(d['tokens'][-1].get('circulating', {}).get('peggedUSD', 0))
|
|
return 0.0
|
|
def parse_dl_dex(self, d): return float(d.get('total24h', 0)) if isinstance(d, dict) else 0.0
|
|
def parse_dl_bridge(self, d):
|
|
if isinstance(d, dict) and 'bridges' in d:
|
|
return sum(float(b.get('lastDayVolume', 0)) for b in d['bridges'])
|
|
return 0.0
|
|
def parse_dl_yields(self, d):
|
|
if isinstance(d, dict) and 'data' in d:
|
|
apys = [float(p.get('apy', 0)) for p in d['data'][:100] if p.get('apy')]
|
|
return np.mean(apys) if apys else 0.0
|
|
return 0.0
|
|
def parse_dl_fees(self, d): return float(d.get('total24h', 0)) if isinstance(d, dict) else 0.0
|
|
def parse_fred(self, d):
|
|
if isinstance(d, dict) and 'observations' in d and d['observations']:
|
|
v = d['observations'][-1].get('value', '.')
|
|
if v != '.':
|
|
try: return float(v)
|
|
except: pass
|
|
return 0.0
|
|
def parse_fng(self, d): return float(d['data'][0]['value']) if isinstance(d, dict) and 'data' in d and d['data'] else 50.0
|
|
def parse_fng_prev(self, d): return float(d['data'][1]['value']) if isinstance(d, dict) and 'data' in d and len(d['data']) > 1 else 50.0
|
|
def parse_fng_week(self, d): return np.mean([float(x['value']) for x in d['data'][:7]]) if isinstance(d, dict) and 'data' in d and len(d['data']) >= 7 else 50.0
|
|
def parse_imbal(self, d):
|
|
if isinstance(d, dict):
|
|
bv = sum(float(b[1]) for b in d.get('bids', [])[:50])
|
|
av = sum(float(a[1]) for a in d.get('asks', [])[:50])
|
|
t = bv + av
|
|
return (bv - av) / t if t > 0 else 0.0
|
|
return 0.0
|
|
def parse_spread(self, d):
|
|
if isinstance(d, dict):
|
|
b, a = float(d.get('bidPrice', 0)), float(d.get('askPrice', 0))
|
|
return (a - b) / b * 10000 if b > 0 else 0.0
|
|
return 0.0
|
|
def parse_chg(self, d): return float(d.get('priceChangePercent', 0)) if isinstance(d, dict) else 0.0
|
|
def parse_vol(self, d):
|
|
if isinstance(d, dict): return float(d.get('quoteVolume', 0))
|
|
if isinstance(d, list) and d and isinstance(d[0], list): return float(d[-1][7])
|
|
return 0.0
|
|
def parse_disp(self, d):
|
|
if isinstance(d, list) and len(d) > 10:
|
|
chg = [float(t['priceChangePercent']) for t in d if t.get('symbol', '').endswith('USDT') and 'priceChangePercent' in t]
|
|
return float(np.std(chg[:50])) if len(chg) > 5 else 0.0
|
|
return 0.0
|
|
def parse_corr(self, d): disp = self.parse_disp(d); return 1 / (1 + disp) if disp > 0 else 0.5
|
|
def parse_cg_btc(self, d):
|
|
if isinstance(d, dict) and 'bitcoin' in d: return float(d['bitcoin']['usd'])
|
|
if isinstance(d, dict) and 'market_data' in d: return float(d['market_data'].get('current_price', {}).get('usd', 0))
|
|
return 0.0
|
|
def parse_cg_eth(self, d):
|
|
if isinstance(d, dict) and 'ethereum' in d: return float(d['ethereum']['usd'])
|
|
if isinstance(d, dict) and 'market_data' in d: return float(d['market_data'].get('current_price', {}).get('usd', 0))
|
|
return 0.0
|
|
def parse_cg_mcap(self, d): return float(d['data']['total_market_cap']['usd']) if isinstance(d, dict) and 'data' in d else 0.0
|
|
def parse_cg_dom_btc(self, d): return float(d['data']['market_cap_percentage']['btc']) if isinstance(d, dict) and 'data' in d else 0.0
|
|
def parse_cg_dom_eth(self, d): return float(d['data']['market_cap_percentage']['eth']) if isinstance(d, dict) and 'data' in d else 0.0
|
|
|
|
async def fetch_indicator(self, session, ind: Indicator, target_date: datetime = None) -> Tuple[int, str, float, bool]:
|
|
if target_date and ind.historical != HistoricalSupport.CURRENT:
|
|
url = self._build_hist_url(ind, target_date)
|
|
else:
|
|
url = self._fred_url(ind.url) if ind.source == "fred" else ind.url
|
|
if url is None: return (ind.id, ind.name, 0.0, False)
|
|
data = await self._fetch(session, url)
|
|
if data is None: return (ind.id, ind.name, 0.0, False)
|
|
parser = getattr(self, ind.parser, None)
|
|
if parser is None: return (ind.id, ind.name, 0.0, False)
|
|
try:
|
|
value = parser(data)
|
|
return (ind.id, ind.name, value, value != 0.0 or 'imbal' in ind.name)
|
|
except: return (ind.id, ind.name, 0.0, False)
|
|
|
|
async def fetch_all(self, target_date: datetime = None) -> Dict[str, Any]:
|
|
connector = aiohttp.TCPConnector(limit=self.config.max_concurrent)
|
|
async with aiohttp.ClientSession(connector=connector) as session:
|
|
results = await asyncio.gather(*[self.fetch_indicator(session, ind, target_date) for ind in INDICATORS])
|
|
matrix = np.zeros(N_INDICATORS)
|
|
success = 0
|
|
details = {}
|
|
for idx, name, value, ok in results:
|
|
matrix[idx - 1] = value
|
|
if ok: success += 1
|
|
details[idx] = {'name': name, 'value': value, 'success': ok}
|
|
return {'matrix': matrix, 'timestamp': (target_date or datetime.now(timezone.utc)).isoformat(), 'success_count': success, 'total': N_INDICATORS, 'details': details}
|
|
|
|
def fetch_sync(self, target_date: datetime = None) -> Dict[str, Any]:
|
|
return asyncio.run(self.fetch_all(target_date))
|
|
|
|
class ExternalFactorsMatrix:
|
|
"""DOLPHIN interface with BACKFILL. Usage: efm.update() or efm.update(datetime(2024,6,15))"""
|
|
def __init__(self, config: Config = None):
|
|
self.config = config or Config()
|
|
self.fetcher = ExternalFactorsFetcher(self.config)
|
|
self.transformer = StationarityTransformer()
|
|
self.raw_matrix: Optional[np.ndarray] = None
|
|
self.stationary_matrix: Optional[np.ndarray] = None
|
|
self.last_result: Optional[Dict] = None
|
|
|
|
def update(self, target_date: datetime = None) -> np.ndarray:
|
|
self.last_result = self.fetcher.fetch_sync(target_date)
|
|
self.raw_matrix = self.last_result['matrix']
|
|
self.stationary_matrix = self.transformer.transform_matrix(self.raw_matrix)
|
|
return self.stationary_matrix
|
|
|
|
def update_raw(self, target_date: datetime = None) -> np.ndarray:
|
|
self.last_result = self.fetcher.fetch_sync(target_date)
|
|
self.raw_matrix = self.last_result['matrix']
|
|
return self.raw_matrix
|
|
|
|
def get_indicator_names(self) -> List[str]: return [i.name for i in INDICATORS]
|
|
def get_backfillable(self) -> List[Tuple[int, str, str]]:
|
|
return [(i.id, i.name, i.hist_resolution) for i in INDICATORS if i.historical in [HistoricalSupport.FULL, HistoricalSupport.PARTIAL]]
|
|
def get_current_only(self) -> List[Tuple[int, str]]:
|
|
return [(i.id, i.name) for i in INDICATORS if i.historical == HistoricalSupport.CURRENT]
|
|
def summary(self) -> str:
|
|
if not self.last_result: return "No data."
|
|
r = self.last_result
|
|
f = sum(1 for i in INDICATORS if i.historical == HistoricalSupport.FULL)
|
|
p = sum(1 for i in INDICATORS if i.historical == HistoricalSupport.PARTIAL)
|
|
c = sum(1 for i in INDICATORS if i.historical == HistoricalSupport.CURRENT)
|
|
return f"Success: {r['success_count']}/{r['total']} | Historical: FULL={f}, PARTIAL={p}, CURRENT={c}"
|
|
|
|
if __name__ == "__main__":
|
|
print(f"EXTERNAL FACTORS v5.0 - {N_INDICATORS} indicators with BACKFILL")
|
|
f = [i for i in INDICATORS if i.historical == HistoricalSupport.FULL]
|
|
p = [i for i in INDICATORS if i.historical == HistoricalSupport.PARTIAL]
|
|
c = [i for i in INDICATORS if i.historical == HistoricalSupport.CURRENT]
|
|
print(f"\nFULL: {len(f)} | PARTIAL: {len(p)} | CURRENT: {len(c)}")
|
|
print("\nFULL HISTORY indicators:")
|
|
for i in f: print(f" {i.id:2d}. {i.name:15s} [{i.hist_resolution:3s}] {i.source}")
|
|
print("\nCURRENT ONLY:")
|
|
for i in c: print(f" {i.id:2d}. {i.name:15s} - {i.description}")
|