1148 lines
35 KiB
YAML
1148 lines
35 KiB
YAML
# Seed Sources for Sentiment Engine Catalogue
|
|
# All sources are FREE and provide substantial text content
|
|
# Curated for crypto/financial sentiment analysis
|
|
# Generated: 2026-08-31
|
|
# Includes: rate limiting, desirable query windows, backoff, concurrency
|
|
|
|
sources:
|
|
# =============================================================================
|
|
# CRYPTO NATIVE NEWS - RSS Feeds (full text or substantial excerpts)
|
|
# =============================================================================
|
|
- source_id: "rss:coindesk.com"
|
|
name: "CoinDesk"
|
|
connector_type: "rss"
|
|
base_url: "https://www.coindesk.com"
|
|
config:
|
|
feed_urls:
|
|
- "https://www.coindesk.com/arc/outboundfeeds/rss/"
|
|
- "https://www.coindesk.com/arc/outboundfeeds/rss/category/markets/"
|
|
- "https://www.coindesk.com/arc/outboundfeeds/rss/category/tech/"
|
|
- "https://www.coindesk.com/arc/outboundfeeds/rss/category/business/"
|
|
- "https://www.coindesk.com/arc/outboundfeeds/rss/category/policy/"
|
|
max_items_per_feed: 50
|
|
poll_interval_seconds: 120
|
|
base_credibility: 0.85
|
|
relevance: 0.95
|
|
tags: ["news", "crypto", "markets", "policy", "tech"]
|
|
enabled: true
|
|
# Rate limiting - be respectful to CoinDesk
|
|
rate_limit_rps: 0.5
|
|
rate_limit_rpm: 30
|
|
rate_limit_burst: 3
|
|
# Preferred query: during US trading hours + Asian morning
|
|
preferred_query_windows:
|
|
- {start_hour: 13, end_hour: 22, timezone: "UTC", weight: 1.0} # US morning/afternoon
|
|
- {start_hour: 0, end_hour: 8, timezone: "UTC", weight: 0.7} # Asian morning
|
|
avoid_query_windows:
|
|
- {start_hour: 22, end_hour: 23, timezone: "UTC", reason: "Low volume"}
|
|
query_jitter_seconds: 30
|
|
backoff_base_seconds: 5.0
|
|
backoff_max_seconds: 600.0
|
|
backoff_multiplier: 2.0
|
|
max_concurrent_requests: 1
|
|
max_latency_ms: 15000
|
|
min_success_rate: 0.85
|
|
|
|
- source_id: "rss:cointelegraph.com"
|
|
name: "CoinTelegraph"
|
|
connector_type: "rss"
|
|
base_url: "https://cointelegraph.com"
|
|
config:
|
|
feed_urls:
|
|
- "https://cointelegraph.com/rss"
|
|
- "https://cointelegraph.com/rss/category/market-analysis"
|
|
- "https://cointelegraph.com/rss/category/bitcoin"
|
|
- "https://cointelegraph.com/rss/category/ethereum"
|
|
- "https://cointelegraph.com/rss/category/defi"
|
|
- "https://cointelegraph.com/rss/category/nft"
|
|
- "https://cointelegraph.com/rss/category/regulation"
|
|
max_items_per_feed: 50
|
|
poll_interval_seconds: 120
|
|
base_credibility: 0.75
|
|
relevance: 0.90
|
|
tags: ["news", "crypto", "markets", "defi", "nft", "regulation"]
|
|
enabled: true
|
|
rate_limit_rps: 0.5
|
|
rate_limit_rpm: 30
|
|
rate_limit_burst: 3
|
|
preferred_query_windows:
|
|
- {start_hour: 12, end_hour: 23, timezone: "UTC", weight: 1.0}
|
|
- {start_hour: 0, end_hour: 8, timezone: "UTC", weight: 0.6}
|
|
avoid_query_windows:
|
|
- {start_hour: 23, end_hour: 1, timezone: "UTC", reason: "Maintenance window"}
|
|
query_jitter_seconds: 30
|
|
backoff_base_seconds: 5.0
|
|
backoff_max_seconds: 600.0
|
|
max_concurrent_requests: 1
|
|
|
|
- source_id: "rss:theblock.co"
|
|
name: "The Block"
|
|
connector_type: "rss"
|
|
base_url: "https://www.theblock.co"
|
|
config:
|
|
feed_urls:
|
|
- "https://www.theblock.co/rss"
|
|
- "https://www.theblock.co/rss/latest"
|
|
- "https://www.theblock.co/rss/markets"
|
|
- "https://www.theblock.co/rss/defi"
|
|
- "https://www.theblock.co/rss/policy"
|
|
max_items_per_feed: 50
|
|
poll_interval_seconds: 120
|
|
base_credibility: 0.85
|
|
relevance: 0.95
|
|
tags: ["news", "crypto", "markets", "defi", "policy", "research"]
|
|
enabled: true
|
|
rate_limit_rps: 0.33
|
|
rate_limit_rpm: 20
|
|
rate_limit_burst: 2
|
|
preferred_query_windows:
|
|
- {start_hour: 13, end_hour: 22, timezone: "UTC", weight: 1.0}
|
|
query_jitter_seconds: 45
|
|
backoff_base_seconds: 10.0
|
|
backoff_max_seconds: 900.0
|
|
max_concurrent_requests: 1
|
|
|
|
- source_id: "rss:decrypt.co"
|
|
name: "Decrypt"
|
|
connector_type: "rss"
|
|
base_url: "https://decrypt.co"
|
|
config:
|
|
feed_urls:
|
|
- "https://decrypt.co/feed"
|
|
- "https://decrypt.co/feed/news"
|
|
- "https://decrypt.co/feed/markets"
|
|
- "https://decrypt.co/feed/defi"
|
|
- "https://decrypt.co/feed/nfts"
|
|
- "https://decrypt.co/feed/web3"
|
|
max_items_per_feed: 50
|
|
poll_interval_seconds: 120
|
|
base_credibility: 0.75
|
|
relevance: 0.85
|
|
tags: ["news", "crypto", "markets", "defi", "nft", "web3"]
|
|
enabled: true
|
|
rate_limit_rps: 0.5
|
|
rate_limit_rpm: 30
|
|
preferred_query_windows:
|
|
- {start_hour: 12, end_hour: 22, timezone: "UTC", weight: 1.0}
|
|
- {start_hour: 0, end_hour: 8, timezone: "UTC", weight: 0.7}
|
|
max_concurrent_requests: 1
|
|
|
|
- source_id: "rss:messari.io"
|
|
name: "Messari"
|
|
connector_type: "rss"
|
|
base_url: "https://messari.io"
|
|
config:
|
|
feed_urls:
|
|
- "https://messari.io/feed"
|
|
- "https://messari.io/feed/research"
|
|
- "https://messari.io/feed/news"
|
|
max_items_per_feed: 30
|
|
poll_interval_seconds: 180
|
|
base_credibility: 0.80
|
|
relevance: 0.90
|
|
tags: ["research", "crypto", "analytics", "tokenomics"]
|
|
enabled: true
|
|
rate_limit_rps: 0.2
|
|
rate_limit_rpm: 12
|
|
rate_limit_burst: 2
|
|
preferred_query_windows:
|
|
- {start_hour: 13, end_hour: 21, timezone: "UTC", weight: 1.0} # US business hours
|
|
avoid_query_windows:
|
|
- {start_hour: 0, end_hour: 12, timezone: "UTC", reason: "Low update frequency"}
|
|
max_concurrent_requests: 1
|
|
|
|
- source_id: "rss:cryptoslate.com"
|
|
name: "CryptoSlate"
|
|
connector_type: "rss"
|
|
base_url: "https://cryptoslate.com"
|
|
config:
|
|
feed_urls:
|
|
- "https://cryptoslate.com/feed/"
|
|
- "https://cryptoslate.com/feed/news/"
|
|
- "https://cryptoslate.com/feed/market-reports/"
|
|
- "https://cryptoslate.com/feed/defi/"
|
|
max_items_per_feed: 40
|
|
poll_interval_seconds: 180
|
|
base_credibility: 0.70
|
|
relevance: 0.80
|
|
tags: ["news", "crypto", "markets", "defi"]
|
|
enabled: true
|
|
rate_limit_rps: 0.33
|
|
rate_limit_rpm: 20
|
|
preferred_query_windows:
|
|
- {start_hour: 12, end_hour: 23, timezone: "UTC", weight: 1.0}
|
|
|
|
- source_id: "rss:bitcoinmagazine.com"
|
|
name: "Bitcoin Magazine"
|
|
connector_type: "rss"
|
|
base_url: "https://bitcoinmagazine.com"
|
|
config:
|
|
feed_urls:
|
|
- "https://bitcoinmagazine.com/feed/"
|
|
- "https://bitcoinmagazine.com/feed/news/"
|
|
- "https://bitcoinmagazine.com/feed/technical/"
|
|
max_items_per_feed: 30
|
|
poll_interval_seconds: 180
|
|
base_credibility: 0.70
|
|
relevance: 0.85
|
|
tags: ["news", "bitcoin", "technical", "macro"]
|
|
enabled: true
|
|
rate_limit_rps: 0.25
|
|
rate_limit_rpm: 15
|
|
|
|
- source_id: "rss:cryptobriefing.com"
|
|
name: "Crypto Briefing"
|
|
connector_type: "rss"
|
|
base_url: "https://cryptobriefing.com"
|
|
config:
|
|
feed_urls:
|
|
- "https://cryptobriefing.com/feed/"
|
|
- "https://cryptobriefing.com/feed/news/"
|
|
- "https://cryptobriefing.com/feed/reviews/"
|
|
max_items_per_feed: 30
|
|
poll_interval_seconds: 180
|
|
base_credibility: 0.65
|
|
relevance: 0.75
|
|
tags: ["news", "crypto", "reviews", "analysis"]
|
|
enabled: true
|
|
rate_limit_rps: 0.33
|
|
rate_limit_rpm: 20
|
|
|
|
- source_id: "rss:thedefiant.io"
|
|
name: "The Defiant"
|
|
connector_type: "rss"
|
|
base_url: "https://thedefiant.io"
|
|
config:
|
|
feed_urls:
|
|
- "https://thedefiant.io/feed/"
|
|
- "https://thedefiant.io/feed/defi/"
|
|
max_items_per_feed: 30
|
|
poll_interval_seconds: 180
|
|
base_credibility: 0.80
|
|
relevance: 0.90
|
|
tags: ["defi", "crypto", "analysis", "newsletter"]
|
|
enabled: true
|
|
rate_limit_rps: 0.25
|
|
rate_limit_rpm: 15
|
|
preferred_query_windows:
|
|
- {start_hour: 13, end_hour: 21, timezone: "UTC", weight: 1.0}
|
|
|
|
- source_id: "rss:banklesshq.com"
|
|
name: "Bankless"
|
|
connector_type: "rss"
|
|
base_url: "https://banklesshq.com"
|
|
config:
|
|
feed_urls:
|
|
- "https://banklesshq.com/feed/"
|
|
max_items_per_feed: 20
|
|
poll_interval_seconds: 300
|
|
base_credibility: 0.75
|
|
relevance: 0.85
|
|
tags: ["defi", "ethereum", "macro", "newsletter"]
|
|
enabled: true
|
|
rate_limit_rps: 0.1
|
|
rate_limit_rpm: 6
|
|
rate_limit_burst: 1
|
|
preferred_query_windows:
|
|
- {start_hour: 14, end_hour: 20, timezone: "UTC", weight: 1.0} # Weekly-ish, US afternoon
|
|
|
|
- source_id: "rss:dlnews.com"
|
|
name: "DL News"
|
|
connector_type: "rss"
|
|
base_url: "https://dlnews.com"
|
|
config:
|
|
feed_urls:
|
|
- "https://dlnews.com/feed/"
|
|
- "https://dlnews.com/feed/markets/"
|
|
- "https://dlnews.com/feed/regulation/"
|
|
max_items_per_feed: 30
|
|
poll_interval_seconds: 180
|
|
base_credibility: 0.70
|
|
relevance: 0.80
|
|
tags: ["news", "crypto", "markets", "regulation"]
|
|
enabled: true
|
|
rate_limit_rps: 0.33
|
|
rate_limit_rpm: 20
|
|
|
|
- source_id: "rss:protos.com"
|
|
name: "Protos"
|
|
connector_type: "rss"
|
|
base_url: "https://protos.com"
|
|
config:
|
|
feed_urls:
|
|
- "https://protos.com/feed/"
|
|
max_items_per_feed: 30
|
|
poll_interval_seconds: 180
|
|
base_credibility: 0.65
|
|
relevance: 0.75
|
|
tags: ["news", "crypto", "investigative"]
|
|
enabled: true
|
|
rate_limit_rps: 0.33
|
|
rate_limit_rpm: 20
|
|
|
|
# =============================================================================
|
|
# EXCHANGE ANNOUNCEMENTS - RSS (critical for listing/delisting events)
|
|
# Higher frequency, stricter rate limits
|
|
# =============================================================================
|
|
- source_id: "rss:binance.com"
|
|
name: "Binance Announcements"
|
|
connector_type: "rss"
|
|
base_url: "https://www.binance.com"
|
|
config:
|
|
feed_urls:
|
|
- "https://www.binance.com/en/feed"
|
|
- "https://www.binance.com/en/feed/announcements"
|
|
- "https://www.binance.com/en/feed/new-listings"
|
|
- "https://www.binance.com/en/feed/maintenance"
|
|
max_items_per_feed: 50
|
|
poll_interval_seconds: 60
|
|
base_credibility: 0.95
|
|
relevance: 1.0
|
|
tags: ["exchange", "binance", "listings", "maintenance", "official"]
|
|
enabled: true
|
|
rate_limit_rps: 0.5
|
|
rate_limit_rpm: 30
|
|
rate_limit_burst: 5
|
|
preferred_query_windows:
|
|
- {start_hour: 0, end_hour: 24, timezone: "UTC", weight: 1.0} # 24/7 critical
|
|
query_jitter_seconds: 10
|
|
backoff_base_seconds: 2.0
|
|
backoff_max_seconds: 120.0
|
|
max_concurrent_requests: 2
|
|
max_latency_ms: 10000
|
|
min_success_rate: 0.95
|
|
|
|
- source_id: "rss:coinbase.com"
|
|
name: "Coinbase Blog"
|
|
connector_type: "rss"
|
|
base_url: "https://blog.coinbase.com"
|
|
config:
|
|
feed_urls:
|
|
- "https://blog.coinbase.com/feed"
|
|
- "https://blog.coinbase.com/tagged/listings/feed"
|
|
max_items_per_feed: 30
|
|
poll_interval_seconds: 120
|
|
base_credibility: 0.90
|
|
relevance: 0.95
|
|
tags: ["exchange", "coinbase", "listings", "official", "us-regulated"]
|
|
enabled: true
|
|
rate_limit_rps: 0.5
|
|
rate_limit_rpm: 30
|
|
preferred_query_windows:
|
|
- {start_hour: 13, end_hour: 22, timezone: "UTC", weight: 1.0}
|
|
max_concurrent_requests: 1
|
|
|
|
- source_id: "rss:kraken.com"
|
|
name: "Kraken Blog"
|
|
connector_type: "rss"
|
|
base_url: "https://blog.kraken.com"
|
|
config:
|
|
feed_urls:
|
|
- "https://blog.kraken.com/feed/"
|
|
max_items_per_feed: 20
|
|
poll_interval_seconds: 180
|
|
base_credibility: 0.85
|
|
relevance: 0.90
|
|
tags: ["exchange", "kraken", "official"]
|
|
enabled: true
|
|
rate_limit_rps: 0.33
|
|
rate_limit_rpm: 20
|
|
|
|
- source_id: "rss:bybit.com"
|
|
name: "Bybit Announcements"
|
|
connector_type: "rss"
|
|
base_url: "https://bybit.com"
|
|
config:
|
|
feed_urls:
|
|
- "https://announcements.bybit.com/rss/"
|
|
max_items_per_feed: 30
|
|
poll_interval_seconds: 120
|
|
base_credibility: 0.80
|
|
relevance: 0.85
|
|
tags: ["exchange", "bybit", "derivatives", "official"]
|
|
enabled: true
|
|
rate_limit_rps: 0.5
|
|
rate_limit_rpm: 30
|
|
preferred_query_windows:
|
|
- {start_hour: 0, end_hour: 24, timezone: "UTC", weight: 1.0}
|
|
|
|
- source_id: "rss:okx.com"
|
|
name: "OKX Announcements"
|
|
connector_type: "rss"
|
|
base_url: "https://www.okx.com"
|
|
config:
|
|
feed_urls:
|
|
- "https://www.okx.com/feed/announcements"
|
|
max_items_per_feed: 30
|
|
poll_interval_seconds: 120
|
|
base_credibility: 0.80
|
|
relevance: 0.85
|
|
tags: ["exchange", "okx", "official"]
|
|
enabled: true
|
|
rate_limit_rps: 0.5
|
|
rate_limit_rpm: 30
|
|
|
|
- source_id: "rss:kucoin.com"
|
|
name: "KuCoin Announcements"
|
|
connector_type: "rss"
|
|
base_url: "https://www.kucoin.com"
|
|
config:
|
|
feed_urls:
|
|
- "https://www.kucoin.com/rss/announcements"
|
|
max_items_per_feed: 30
|
|
poll_interval_seconds: 180
|
|
base_credibility: 0.75
|
|
relevance: 0.80
|
|
tags: ["exchange", "kucoin", "official"]
|
|
enabled: true
|
|
rate_limit_rps: 0.33
|
|
rate_limit_rpm: 20
|
|
|
|
- source_id: "rss:gate.io"
|
|
name: "Gate.io Announcements"
|
|
connector_type: "rss"
|
|
base_url: "https://www.gate.io"
|
|
config:
|
|
feed_urls:
|
|
- "https://www.gate.io/rss/announcements"
|
|
max_items_per_feed: 30
|
|
poll_interval_seconds: 180
|
|
base_credibility: 0.70
|
|
relevance: 0.75
|
|
tags: ["exchange", "gate", "official"]
|
|
enabled: true
|
|
rate_limit_rps: 0.33
|
|
rate_limit_rpm: 20
|
|
|
|
- source_id: "rss:htx.com"
|
|
name: "HTX (Huobi) Announcements"
|
|
connector_type: "rss"
|
|
base_url: "https://www.htx.com"
|
|
config:
|
|
feed_urls:
|
|
- "https://www.htx.com/feed/announcements"
|
|
max_items_per_feed: 20
|
|
poll_interval_seconds: 180
|
|
base_credibility: 0.70
|
|
relevance: 0.75
|
|
tags: ["exchange", "htx", "huobi", "official"]
|
|
enabled: true
|
|
rate_limit_rps: 0.33
|
|
rate_limit_rpm: 20
|
|
|
|
# =============================================================================
|
|
# REGULATORY / MACRO - Official Sources (FRED)
|
|
# =============================================================================
|
|
- source_id: "api:fred_vix"
|
|
name: "FRED VIX (Volatility Index)"
|
|
connector_type: "rest_api"
|
|
base_url: "https://api.stlouisfed.org/fred"
|
|
config:
|
|
endpoints:
|
|
- "series/observations"
|
|
auth_type: "api_key"
|
|
params:
|
|
series_id: "VIXCLS"
|
|
file_type: "json"
|
|
limit: 1
|
|
sort_order: "desc"
|
|
poll_interval_seconds: 300
|
|
base_credibility: 0.99
|
|
relevance: 0.85
|
|
tags: ["macro", "volatility", "vix", "fred", "official"]
|
|
enabled: true
|
|
credentials_ref: "fred_api_key"
|
|
rate_limit_rps: 0.5
|
|
rate_limit_rpm: 30
|
|
preferred_query_windows:
|
|
- {start_hour: 13, end_hour: 21, timezone: "UTC", weight: 1.0} # US market hours
|
|
avoid_query_windows:
|
|
- {start_hour: 0, end_hour: 12, timezone: "UTC", reason: "FRED updates ~daily"}
|
|
backoff_base_seconds: 10.0
|
|
backoff_max_seconds: 600.0
|
|
max_concurrent_requests: 1
|
|
max_latency_ms: 10000
|
|
|
|
- source_id: "api:fred_dxy"
|
|
name: "FRED DXY (Dollar Index)"
|
|
connector_type: "rest_api"
|
|
base_url: "https://api.stlouisfed.org/fred"
|
|
config:
|
|
endpoints:
|
|
- "series/observations"
|
|
auth_type: "api_key"
|
|
params:
|
|
series_id: "DTWEXBGS"
|
|
file_type: "json"
|
|
limit: 1
|
|
sort_order: "desc"
|
|
poll_interval_seconds: 300
|
|
base_credibility: 0.99
|
|
relevance: 0.85
|
|
tags: ["macro", "dxy", "dollar", "fred", "official"]
|
|
enabled: true
|
|
credentials_ref: "fred_api_key"
|
|
rate_limit_rps: 0.5
|
|
rate_limit_rpm: 30
|
|
|
|
- source_id: "api:fred_fedfunds"
|
|
name: "FRED Fed Funds Rate"
|
|
connector_type: "rest_api"
|
|
base_url: "https://api.stlouisfed.org/fred"
|
|
config:
|
|
endpoints:
|
|
- "series/observations"
|
|
auth_type: "api_key"
|
|
params:
|
|
series_id: "FEDFUNDS"
|
|
file_type: "json"
|
|
limit: 1
|
|
sort_order: "desc"
|
|
poll_interval_seconds: 3600
|
|
base_credibility: 0.99
|
|
relevance: 0.80
|
|
tags: ["macro", "rates", "fed", "fred", "official"]
|
|
enabled: true
|
|
credentials_ref: "fred_api_key"
|
|
rate_limit_rps: 0.2
|
|
rate_limit_rpm: 12
|
|
|
|
- source_id: "api:fred_cpi"
|
|
name: "FRED CPI (Inflation)"
|
|
connector_type: "rest_api"
|
|
base_url: "https://api.stlouisfed.org/fred"
|
|
config:
|
|
endpoints:
|
|
- "series/observations"
|
|
auth_type: "api_key"
|
|
params:
|
|
series_id: "CPIAUCSL"
|
|
file_type: "json"
|
|
limit: 1
|
|
sort_order: "desc"
|
|
poll_interval_seconds: 86400
|
|
base_credibility: 0.99
|
|
relevance: 0.80
|
|
tags: ["macro", "inflation", "cpi", "fred", "official"]
|
|
enabled: true
|
|
credentials_ref: "fred_api_key"
|
|
rate_limit_rps: 0.1
|
|
rate_limit_rpm: 6
|
|
preferred_query_windows:
|
|
- {start_hour: 12, end_hour: 14, timezone: "UTC", weight: 1.0} # CPI release time ~13:30 UTC
|
|
|
|
- source_id: "api:fred_unemployment"
|
|
name: "FRED Unemployment Rate"
|
|
connector_type: "rest_api"
|
|
base_url: "https://api.stlouisfed.org/fred"
|
|
config:
|
|
endpoints:
|
|
- "series/observations"
|
|
auth_type: "api_key"
|
|
params:
|
|
series_id: "UNRATE"
|
|
file_type: "json"
|
|
limit: 1
|
|
sort_order: "desc"
|
|
poll_interval_seconds: 86400
|
|
base_credibility: 0.99
|
|
relevance: 0.75
|
|
tags: ["macro", "unemployment", "labor", "fred", "official"]
|
|
enabled: true
|
|
credentials_ref: "fred_api_key"
|
|
rate_limit_rps: 0.1
|
|
rate_limit_rpm: 6
|
|
|
|
# =============================================================================
|
|
# TRADITIONAL FINANCE NEWS - Crypto Coverage
|
|
# =============================================================================
|
|
- source_id: "rss:bloomberg.com"
|
|
name: "Bloomberg Markets"
|
|
connector_type: "rss"
|
|
base_url: "https://www.bloomberg.com"
|
|
config:
|
|
feed_urls:
|
|
- "https://feeds.bloomberg.com/markets/news.rss"
|
|
- "https://feeds.bloomberg.com/technology/news.rss"
|
|
max_items_per_feed: 30
|
|
poll_interval_seconds: 180
|
|
base_credibility: 0.90
|
|
relevance: 0.70
|
|
tags: ["tradfi", "markets", "macro", "crypto-coverage"]
|
|
enabled: true
|
|
rate_limit_rps: 0.25
|
|
rate_limit_rpm: 15
|
|
preferred_query_windows:
|
|
- {start_hour: 12, end_hour: 22, timezone: "UTC", weight: 1.0}
|
|
- {start_hour: 0, end_hour: 8, timezone: "UTC", weight: 0.5}
|
|
|
|
- source_id: "rss:reuters.com"
|
|
name: "Reuters Business"
|
|
connector_type: "rss"
|
|
base_url: "https://www.reuters.com"
|
|
config:
|
|
feed_urls:
|
|
- "https://www.reuters.com/business/finance/rss"
|
|
- "https://www.reuters.com/technology/rss"
|
|
max_items_per_feed: 30
|
|
poll_interval_seconds: 180
|
|
base_credibility: 0.92
|
|
relevance: 0.70
|
|
tags: ["tradfi", "markets", "macro", "crypto-coverage"]
|
|
enabled: true
|
|
rate_limit_rps: 0.33
|
|
rate_limit_rpm: 20
|
|
|
|
- source_id: "rss:wsj.com"
|
|
name: "Wall Street Journal Markets"
|
|
connector_type: "rss"
|
|
base_url: "https://www.wsj.com"
|
|
config:
|
|
feed_urls:
|
|
- "https://feeds.wsj.com/wsj/marketspulse.rss"
|
|
max_items_per_feed: 20
|
|
poll_interval_seconds: 180
|
|
base_credibility: 0.90
|
|
relevance: 0.65
|
|
tags: ["tradfi", "markets", "macro"]
|
|
enabled: true
|
|
rate_limit_rps: 0.25
|
|
rate_limit_rpm: 15
|
|
|
|
- source_id: "rss:ft.com"
|
|
name: "Financial Times Markets"
|
|
connector_type: "rss"
|
|
base_url: "https://www.ft.com"
|
|
config:
|
|
feed_urls:
|
|
- "https://www.ft.com/markets?format=rss"
|
|
max_items_per_feed: 20
|
|
poll_interval_seconds: 180
|
|
base_credibility: 0.90
|
|
relevance: 0.65
|
|
tags: ["tradfi", "markets", "macro", "europe"]
|
|
enabled: true
|
|
rate_limit_rps: 0.25
|
|
rate_limit_rpm: 15
|
|
preferred_query_windows:
|
|
- {start_hour: 7, end_hour: 17, timezone: "UTC", weight: 1.0} # London hours
|
|
|
|
- source_id: "rss:marketwatch.com"
|
|
name: "MarketWatch"
|
|
connector_type: "rss"
|
|
base_url: "https://www.marketwatch.com"
|
|
config:
|
|
feed_urls:
|
|
- "https://feeds.marketwatch.com/marketwatch/topstories/"
|
|
- "https://feeds.marketwatch.com/marketwatch/cryptocurrency/"
|
|
max_items_per_feed: 30
|
|
poll_interval_seconds: 180
|
|
base_credibility: 0.80
|
|
relevance: 0.75
|
|
tags: ["tradfi", "markets", "crypto-coverage"]
|
|
enabled: true
|
|
rate_limit_rps: 0.33
|
|
rate_limit_rpm: 20
|
|
|
|
# =============================================================================
|
|
# REDDIT - High-signal subreddits
|
|
# Respect Reddit API limits: 60 req/min for OAuth, 10 req/min for unauthenticated
|
|
# =============================================================================
|
|
- source_id: "reddit:CryptoCurrency"
|
|
name: "r/CryptoCurrency"
|
|
connector_type: "reddit"
|
|
base_url: "https://reddit.com/r/CryptoCurrency"
|
|
config:
|
|
subreddits: ["CryptoCurrency"]
|
|
use_pushshift: true
|
|
poll_interval_seconds: 180
|
|
base_credibility: 0.35
|
|
relevance: 0.80
|
|
tags: ["social", "reddit", "retail", "high-volume", "sentiment"]
|
|
enabled: true
|
|
rate_limit_rps: 0.1
|
|
rate_limit_rpm: 6
|
|
rate_limit_burst: 2
|
|
preferred_query_windows:
|
|
- {start_hour: 12, end_hour: 24, timezone: "UTC", weight: 1.0} # US active
|
|
- {start_hour: 0, end_hour: 8, timezone: "UTC", weight: 0.6} # Asia/Europe
|
|
avoid_query_windows:
|
|
- {start_hour: 8, end_hour: 12, timezone: "UTC", reason: "Lower activity"}
|
|
query_jitter_seconds: 60
|
|
backoff_base_seconds: 30.0
|
|
backoff_max_seconds: 1800.0
|
|
max_concurrent_requests: 1
|
|
max_latency_ms: 20000
|
|
|
|
- source_id: "reddit:Bitcoin"
|
|
name: "r/Bitcoin"
|
|
connector_type: "reddit"
|
|
base_url: "https://reddit.com/r/Bitcoin"
|
|
config:
|
|
subreddits: ["Bitcoin"]
|
|
use_pushshift: true
|
|
poll_interval_seconds: 180
|
|
base_credibility: 0.35
|
|
relevance: 0.90
|
|
tags: ["social", "reddit", "bitcoin", "maximalist", "sentiment"]
|
|
enabled: true
|
|
rate_limit_rps: 0.1
|
|
rate_limit_rpm: 6
|
|
|
|
- source_id: "reddit:EthTrader"
|
|
name: "r/EthTrader"
|
|
connector_type: "reddit"
|
|
base_url: "https://reddit.com/r/EthTrader"
|
|
config:
|
|
subreddits: ["EthTrader"]
|
|
use_pushshift: true
|
|
poll_interval_seconds: 180
|
|
base_credibility: 0.35
|
|
relevance: 0.85
|
|
tags: ["social", "reddit", "ethereum", "trading", "sentiment"]
|
|
enabled: true
|
|
rate_limit_rps: 0.1
|
|
rate_limit_rpm: 6
|
|
|
|
- source_id: "reddit:defi"
|
|
name: "r/defi"
|
|
connector_type: "reddit"
|
|
base_url: "https://reddit.com/r/defi"
|
|
config:
|
|
subreddits: ["defi"]
|
|
use_pushshift: true
|
|
poll_interval_seconds: 300
|
|
base_credibility: 0.30
|
|
relevance: 0.85
|
|
tags: ["social", "reddit", "defi", "yield", "sentiment"]
|
|
enabled: true
|
|
rate_limit_rps: 0.1
|
|
rate_limit_rpm: 6
|
|
|
|
- source_id: "reddit:ethtrader"
|
|
name: "r/ethtrader (daily discussion)"
|
|
connector_type: "reddit"
|
|
base_url: "https://reddit.com/r/ethtrader"
|
|
config:
|
|
subreddits: ["ethtrader"]
|
|
use_pushshift: true
|
|
poll_interval_seconds: 300
|
|
base_credibility: 0.30
|
|
relevance: 0.80
|
|
tags: ["social", "reddit", "ethereum", "daily-discussion"]
|
|
enabled: true
|
|
rate_limit_rps: 0.1
|
|
rate_limit_rpm: 6
|
|
|
|
- source_id: "reddit:CryptoMarkets"
|
|
name: "r/CryptoMarkets"
|
|
connector_type: "reddit"
|
|
base_url: "https://reddit.com/r/CryptoMarkets"
|
|
config:
|
|
subreddits: ["CryptoMarkets"]
|
|
use_pushshift: true
|
|
poll_interval_seconds: 180
|
|
base_credibility: 0.35
|
|
relevance: 0.85
|
|
tags: ["social", "reddit", "trading", "technical-analysis"]
|
|
enabled: true
|
|
rate_limit_rps: 0.1
|
|
rate_limit_rpm: 6
|
|
|
|
- source_id: "reddit:BitcoinMarkets"
|
|
name: "r/BitcoinMarkets"
|
|
connector_type: "reddit"
|
|
base_url: "https://reddit.com/r/BitcoinMarkets"
|
|
config:
|
|
subreddits: ["BitcoinMarkets"]
|
|
use_pushshift: true
|
|
poll_interval_seconds: 180
|
|
base_credibility: 0.35
|
|
relevance: 0.85
|
|
tags: ["social", "reddit", "bitcoin", "trading"]
|
|
enabled: true
|
|
rate_limit_rps: 0.1
|
|
rate_limit_rpm: 6
|
|
|
|
- source_id: "reddit:altcoin"
|
|
name: "r/altcoin"
|
|
connector_type: "reddit"
|
|
base_url: "https://reddit.com/r/altcoin"
|
|
config:
|
|
subreddits: ["altcoin"]
|
|
use_pushshift: true
|
|
poll_interval_seconds: 300
|
|
base_credibility: 0.25
|
|
relevance: 0.70
|
|
tags: ["social", "reddit", "altcoins", "speculation"]
|
|
enabled: true
|
|
rate_limit_rps: 0.1
|
|
rate_limit_rpm: 6
|
|
|
|
- source_id: "reddit:Solana"
|
|
name: "r/Solana"
|
|
connector_type: "reddit"
|
|
base_url: "https://reddit.com/r/Solana"
|
|
config:
|
|
subreddits: ["Solana"]
|
|
use_pushshift: true
|
|
poll_interval_seconds: 300
|
|
base_credibility: 0.30
|
|
relevance: 0.80
|
|
tags: ["social", "reddit", "solana", "ecosystem"]
|
|
enabled: true
|
|
rate_limit_rps: 0.1
|
|
rate_limit_rpm: 6
|
|
|
|
# =============================================================================
|
|
# TELEGRAM - Public channels
|
|
# Telegram bot API: 30 req/sec per bot, but be conservative
|
|
# =============================================================================
|
|
- source_id: "telegram:binance_announcements"
|
|
name: "Binance Announcements (Telegram)"
|
|
connector_type: "telegram"
|
|
base_url: "https://t.me/binance_announcements"
|
|
config:
|
|
channel_usernames: ["binance_announcements"]
|
|
base_credibility: 0.90
|
|
relevance: 0.95
|
|
tags: ["social", "telegram", "exchange", "binance", "official", "listings"]
|
|
enabled: true
|
|
rate_limit_rps: 0.5
|
|
rate_limit_rpm: 30
|
|
rate_limit_burst: 5
|
|
preferred_query_windows:
|
|
- {start_hour: 0, end_hour: 24, timezone: "UTC", weight: 1.0}
|
|
max_concurrent_requests: 1
|
|
|
|
- source_id: "telegram:coinbase_announcements"
|
|
name: "Coinbase Announcements (Telegram)"
|
|
connector_type: "telegram"
|
|
base_url: "https://t.me/coinbase_announcements"
|
|
config:
|
|
channel_usernames: ["coinbase_announcements"]
|
|
base_credibility: 0.85
|
|
relevance: 0.90
|
|
tags: ["social", "telegram", "exchange", "coinbase", "official"]
|
|
enabled: true
|
|
rate_limit_rps: 0.5
|
|
rate_limit_rpm: 30
|
|
|
|
- source_id: "telegram:coindesk"
|
|
name: "CoinDesk (Telegram)"
|
|
connector_type: "telegram"
|
|
base_url: "https://t.me/coindesk"
|
|
config:
|
|
channel_usernames: ["coindesk"]
|
|
base_credibility: 0.75
|
|
relevance: 0.85
|
|
tags: ["social", "telegram", "news", "crypto"]
|
|
enabled: true
|
|
rate_limit_rps: 0.33
|
|
rate_limit_rpm: 20
|
|
|
|
- source_id: "telegram:cointelegraph"
|
|
name: "CoinTelegraph (Telegram)"
|
|
connector_type: "telegram"
|
|
base_url: "https://t.me/cointelegraph"
|
|
config:
|
|
channel_usernames: ["cointelegraph"]
|
|
base_credibility: 0.70
|
|
relevance: 0.80
|
|
tags: ["social", "telegram", "news", "crypto"]
|
|
enabled: true
|
|
rate_limit_rps: 0.33
|
|
rate_limit_rpm: 20
|
|
|
|
- source_id: "telegram:thedefiant"
|
|
name: "The Defiant (Telegram)"
|
|
connector_type: "telegram"
|
|
base_url: "https://t.me/thedefiant"
|
|
config:
|
|
channel_usernames: ["thedefiant"]
|
|
base_credibility: 0.75
|
|
relevance: 0.85
|
|
tags: ["social", "telegram", "defi", "newsletter"]
|
|
enabled: true
|
|
rate_limit_rps: 0.25
|
|
rate_limit_rpm: 15
|
|
|
|
- source_id: "telegram:bankless"
|
|
name: "Bankless (Telegram)"
|
|
connector_type: "telegram"
|
|
base_url: "https://t.me/banklesshq"
|
|
config:
|
|
channel_usernames: ["banklesshq"]
|
|
base_credibility: 0.70
|
|
relevance: 0.80
|
|
tags: ["social", "telegram", "defi", "ethereum", "macro"]
|
|
enabled: true
|
|
rate_limit_rps: 0.2
|
|
rate_limit_rpm: 12
|
|
|
|
- source_id: "telegram:glassnode"
|
|
name: "Glassnode Alerts (Telegram)"
|
|
connector_type: "telegram"
|
|
base_url: "https://t.me/glassnodealerts"
|
|
config:
|
|
channel_usernames: ["glassnodealerts"]
|
|
base_credibility: 0.85
|
|
relevance: 0.90
|
|
tags: ["social", "telegram", "onchain", "analytics", "alerts"]
|
|
enabled: true
|
|
rate_limit_rps: 0.2
|
|
rate_limit_rpm: 12
|
|
preferred_query_windows:
|
|
- {start_hour: 0, end_hour: 24, timezone: "UTC", weight: 1.0} # Alerts anytime
|
|
|
|
- source_id: "telegram:cryptoquant"
|
|
name: "CryptoQuant (Telegram)"
|
|
connector_type: "telegram"
|
|
base_url: "https://t.me/cryptoquant_official"
|
|
config:
|
|
channel_usernames: ["cryptoquant_official"]
|
|
base_credibility: 0.80
|
|
relevance: 0.85
|
|
tags: ["social", "telegram", "onchain", "analytics", "whale-alerts"]
|
|
enabled: true
|
|
rate_limit_rps: 0.2
|
|
rate_limit_rpm: 12
|
|
|
|
- source_id: "telegram:whale_alert"
|
|
name: "Whale Alert (Telegram)"
|
|
connector_type: "telegram"
|
|
base_url: "https://t.me/whale_alert_io"
|
|
config:
|
|
channel_usernames: ["whale_alert_io"]
|
|
base_credibility: 0.75
|
|
relevance: 0.85
|
|
tags: ["social", "telegram", "whale", "large-transactions", "onchain"]
|
|
enabled: true
|
|
rate_limit_rps: 0.2
|
|
rate_limit_rpm: 12
|
|
preferred_query_windows:
|
|
- {start_hour: 0, end_hour: 24, timezone: "UTC", weight: 1.0}
|
|
|
|
# =============================================================================
|
|
# WEB CRAWL - Sites without RSS/API (supplementary)
|
|
# Very conservative rate limits
|
|
# =============================================================================
|
|
- source_id: "web:defillama.com"
|
|
name: "DeFi Llama (Blog/Announcements)"
|
|
connector_type: "web_crawl"
|
|
base_url: "https://defillama.com"
|
|
config:
|
|
seed_urls:
|
|
- "https://defillama.com/blog"
|
|
- "https://defillama.com/chains"
|
|
allowed_domains: ["defillama.com"]
|
|
max_depth: 2
|
|
rate_limit_rps: 0.5
|
|
base_credibility: 0.80
|
|
relevance: 0.85
|
|
tags: ["defi", "tvl", "analytics", "blog"]
|
|
enabled: true
|
|
rate_limit_rps: 0.1
|
|
rate_limit_rpm: 6
|
|
rate_limit_burst: 1
|
|
preferred_query_windows:
|
|
- {start_hour: 2, end_hour: 6, timezone: "UTC", weight: 1.0} # Off-peak
|
|
query_jitter_seconds: 120
|
|
backoff_base_seconds: 60.0
|
|
backoff_max_seconds: 3600.0
|
|
max_concurrent_requests: 1
|
|
max_latency_ms: 60000
|
|
|
|
- source_id: "web:dune.com"
|
|
name: "Dune Analytics (Blog)"
|
|
connector_type: "web_crawl"
|
|
base_url: "https://dune.com"
|
|
config:
|
|
seed_urls:
|
|
- "https://dune.com/blog"
|
|
allowed_domains: ["dune.com"]
|
|
max_depth: 2
|
|
rate_limit_rps: 0.5
|
|
base_credibility: 0.75
|
|
relevance: 0.80
|
|
tags: ["analytics", "sql", "dashboards", "blog"]
|
|
enabled: true
|
|
rate_limit_rps: 0.1
|
|
rate_limit_rpm: 6
|
|
|
|
- source_id: "web:nansen.ai"
|
|
name: "Nansen (Blog/Research)"
|
|
connector_type: "web_crawl"
|
|
base_url: "https://nansen.ai"
|
|
config:
|
|
seed_urls:
|
|
- "https://nansen.ai/blog"
|
|
- "https://nansen.ai/research"
|
|
allowed_domains: ["nansen.ai"]
|
|
max_depth: 2
|
|
rate_limit_rps: 0.5
|
|
base_credibility: 0.80
|
|
relevance: 0.85
|
|
tags: ["onchain", "analytics", "research", "whale"]
|
|
enabled: true
|
|
rate_limit_rps: 0.1
|
|
rate_limit_rpm: 6
|
|
|
|
- source_id: "web:coingecko.com"
|
|
name: "CoinGecko (News/Research)"
|
|
connector_type: "web_crawl"
|
|
base_url: "https://coingecko.com"
|
|
config:
|
|
seed_urls:
|
|
- "https://coingecko.com/en/news"
|
|
- "https://coingecko.com/en/research"
|
|
allowed_domains: ["coingecko.com"]
|
|
max_depth: 2
|
|
rate_limit_rps: 0.5
|
|
base_credibility: 0.70
|
|
relevance: 0.80
|
|
tags: ["data", "news", "research", "token-info"]
|
|
enabled: true
|
|
rate_limit_rps: 0.2
|
|
rate_limit_rpm: 12
|
|
|
|
- source_id: "web:coinmarketcap.com"
|
|
name: "CoinMarketCap (News)"
|
|
connector_type: "web_crawl"
|
|
base_url: "https://coinmarketcap.com"
|
|
config:
|
|
seed_urls:
|
|
- "https://coinmarketcap.com/headlines/news/"
|
|
allowed_domains: ["coinmarketcap.com"]
|
|
max_depth: 2
|
|
rate_limit_rps: 0.5
|
|
base_credibility: 0.65
|
|
relevance: 0.75
|
|
tags: ["data", "news", "token-info"]
|
|
enabled: true
|
|
rate_limit_rps: 0.2
|
|
rate_limit_rpm: 12
|
|
|
|
# =============================================================================
|
|
# ON-CHAIN / ANALYTICS - Specialized
|
|
# =============================================================================
|
|
- source_id: "rss:glassnode.com"
|
|
name: "Glassnode Insights"
|
|
connector_type: "rss"
|
|
base_url: "https://glassnode.com"
|
|
config:
|
|
feed_urls:
|
|
- "https://insights.glassnode.com/rss/"
|
|
max_items_per_feed: 20
|
|
poll_interval_seconds: 300
|
|
base_credibility: 0.90
|
|
relevance: 0.95
|
|
tags: ["onchain", "analytics", "glassnode", "metrics"]
|
|
enabled: true
|
|
rate_limit_rps: 0.2
|
|
rate_limit_rpm: 12
|
|
preferred_query_windows:
|
|
- {start_hour: 12, end_hour: 22, timezone: "UTC", weight: 1.0}
|
|
|
|
- source_id: "rss:cryptoquant.com"
|
|
name: "CryptoQuant Quicktake"
|
|
connector_type: "rss"
|
|
base_url: "https://cryptoquant.com"
|
|
config:
|
|
feed_urls:
|
|
- "https://cryptoquant.com/feed/quicktake"
|
|
max_items_per_feed: 20
|
|
poll_interval_seconds: 300
|
|
base_credibility: 0.85
|
|
relevance: 0.90
|
|
tags: ["onchain", "analytics", "cryptoquant", "whale"]
|
|
enabled: true
|
|
rate_limit_rps: 0.2
|
|
rate_limit_rpm: 12
|
|
|
|
- source_id: "rss:coinmetrics.io"
|
|
name: "Coin Metrics"
|
|
connector_type: "rss"
|
|
base_url: "https://coinmetrics.io"
|
|
config:
|
|
feed_urls:
|
|
- "https://coinmetrics.io/feed/"
|
|
max_items_per_feed: 15
|
|
poll_interval_seconds: 300
|
|
base_credibility: 0.85
|
|
relevance: 0.85
|
|
tags: ["onchain", "analytics", "coinmetrics", "institutional"]
|
|
enabled: true
|
|
rate_limit_rps: 0.2
|
|
rate_limit_rpm: 12
|
|
|
|
# =============================================================================
|
|
# GOVERNANCE / PROTOCOL ANNOUNCEMENTS
|
|
# =============================================================================
|
|
- source_id: "rss:forum.uniswap.org"
|
|
name: "Uniswap Governance Forum"
|
|
connector_type: "rss"
|
|
base_url: "https://forum.uniswap.org"
|
|
config:
|
|
feed_urls:
|
|
- "https://forum.uniswap.org/c/governance.rss"
|
|
max_items_per_feed: 20
|
|
poll_interval_seconds: 300
|
|
base_credibility: 0.80
|
|
relevance: 0.85
|
|
tags: ["governance", "defi", "uniswap", "dao"]
|
|
enabled: true
|
|
rate_limit_rps: 0.2
|
|
rate_limit_rpm: 12
|
|
|
|
- source_id: "rss:forum.aave.com"
|
|
name: "Aave Governance Forum"
|
|
connector_type: "rss"
|
|
base_url: "https://forum.aave.com"
|
|
config:
|
|
feed_urls:
|
|
- "https://forum.aave.com/c/governance.rss"
|
|
max_items_per_feed: 20
|
|
poll_interval_seconds: 300
|
|
base_credibility: 0.80
|
|
relevance: 0.85
|
|
tags: ["governance", "defi", "aave", "dao", "lending"]
|
|
enabled: true
|
|
rate_limit_rps: 0.2
|
|
rate_limit_rpm: 12
|
|
|
|
- source_id: "rss:gov.makerdao.com"
|
|
name: "MakerDAO Governance"
|
|
connector_type: "rss"
|
|
base_url: "https://forum.makerdao.com"
|
|
config:
|
|
feed_urls:
|
|
- "https://forum.makerdao.com/c/governance.rss"
|
|
max_items_per_feed: 20
|
|
poll_interval_seconds: 300
|
|
base_credibility: 0.85
|
|
relevance: 0.85
|
|
tags: ["governance", "defi", "makerdao", "dai", "dao"]
|
|
enabled: true
|
|
rate_limit_rps: 0.2
|
|
rate_limit_rpm: 12
|
|
|
|
- source_id: "rss:gov.compound.finance"
|
|
name: "Compound Governance"
|
|
connector_type: "rss"
|
|
base_url: "https://compound.finance/governance"
|
|
config:
|
|
feed_urls:
|
|
- "https://compound.finance/governance/feed/"
|
|
max_items_per_feed: 15
|
|
poll_interval_seconds: 300
|
|
base_credibility: 0.80
|
|
relevance: 0.80
|
|
tags: ["governance", "defi", "compound", "dao", "lending"]
|
|
enabled: true
|
|
rate_limit_rps: 0.2
|
|
rate_limit_rpm: 12
|
|
|
|
# Total: ~60 sources across all categories
|
|
# Run: python scripts/populate_catalogue.py --config config/seed_sources.yaml
|