sentiment-engine/sentiment_engine/config/seed_sources.yaml

# Seed Sources for Sentiment Engine Catalogue
# All sources are FREE and provide substantial text content
# Curated for crypto/financial sentiment analysis
# Generated: 2026-08-31
# Includes: rate limiting, desirable query windows, backoff, concurrency

sources:
  # =============================================================================
  # CRYPTO NATIVE NEWS - RSS Feeds (full text or substantial excerpts)
  # =============================================================================
  - source_id: "rss:coindesk.com"
    name: "CoinDesk"
    connector_type: "rss"
    base_url: "https://www.coindesk.com"
    config:
      feed_urls:
        - "https://www.coindesk.com/arc/outboundfeeds/rss/"
        - "https://www.coindesk.com/arc/outboundfeeds/rss/category/markets/"
        - "https://www.coindesk.com/arc/outboundfeeds/rss/category/tech/"
        - "https://www.coindesk.com/arc/outboundfeeds/rss/category/business/"
        - "https://www.coindesk.com/arc/outboundfeeds/rss/category/policy/"
      max_items_per_feed: 50
      poll_interval_seconds: 120
    base_credibility: 0.85
    relevance: 0.95
    tags: ["news", "crypto", "markets", "policy", "tech"]
    enabled: true
    # Rate limiting - be respectful to CoinDesk
    rate_limit_rps: 0.5
    rate_limit_rpm: 30
    rate_limit_burst: 3
    # Preferred query: during US trading hours + Asian morning
    preferred_query_windows:
      - {start_hour: 13, end_hour: 22, timezone: "UTC", weight: 1.0}   # US morning/afternoon
      - {start_hour: 0, end_hour: 8, timezone: "UTC", weight: 0.7}     # Asian morning
    avoid_query_windows:
      - {start_hour: 22, end_hour: 23, timezone: "UTC", reason: "Low volume"}
    query_jitter_seconds: 30
    backoff_base_seconds: 5.0
    backoff_max_seconds: 600.0
    backoff_multiplier: 2.0
    max_concurrent_requests: 1
    max_latency_ms: 15000
    min_success_rate: 0.85

  - source_id: "rss:cointelegraph.com"
    name: "CoinTelegraph"
    connector_type: "rss"
    base_url: "https://cointelegraph.com"
    config:
      feed_urls:
        - "https://cointelegraph.com/rss"
        - "https://cointelegraph.com/rss/category/market-analysis"
        - "https://cointelegraph.com/rss/category/bitcoin"
        - "https://cointelegraph.com/rss/category/ethereum"
        - "https://cointelegraph.com/rss/category/defi"
        - "https://cointelegraph.com/rss/category/nft"
        - "https://cointelegraph.com/rss/category/regulation"
      max_items_per_feed: 50
      poll_interval_seconds: 120
    base_credibility: 0.75
    relevance: 0.90
    tags: ["news", "crypto", "markets", "defi", "nft", "regulation"]
    enabled: true
    rate_limit_rps: 0.5
    rate_limit_rpm: 30
    rate_limit_burst: 3
    preferred_query_windows:
      - {start_hour: 12, end_hour: 23, timezone: "UTC", weight: 1.0}
      - {start_hour: 0, end_hour: 8, timezone: "UTC", weight: 0.6}
    avoid_query_windows:
      - {start_hour: 23, end_hour: 1, timezone: "UTC", reason: "Maintenance window"}
    query_jitter_seconds: 30
    backoff_base_seconds: 5.0
    backoff_max_seconds: 600.0
    max_concurrent_requests: 1

  - source_id: "rss:theblock.co"
    name: "The Block"
    connector_type: "rss"
    base_url: "https://www.theblock.co"
    config:
      feed_urls:
        - "https://www.theblock.co/rss"
        - "https://www.theblock.co/rss/latest"
        - "https://www.theblock.co/rss/markets"
        - "https://www.theblock.co/rss/defi"
        - "https://www.theblock.co/rss/policy"
      max_items_per_feed: 50
      poll_interval_seconds: 120
    base_credibility: 0.85
    relevance: 0.95
    tags: ["news", "crypto", "markets", "defi", "policy", "research"]
    enabled: true
    rate_limit_rps: 0.33
    rate_limit_rpm: 20
    rate_limit_burst: 2
    preferred_query_windows:
      - {start_hour: 13, end_hour: 22, timezone: "UTC", weight: 1.0}
    query_jitter_seconds: 45
    backoff_base_seconds: 10.0
    backoff_max_seconds: 900.0
    max_concurrent_requests: 1

  - source_id: "rss:decrypt.co"
    name: "Decrypt"
    connector_type: "rss"
    base_url: "https://decrypt.co"
    config:
      feed_urls:
        - "https://decrypt.co/feed"
        - "https://decrypt.co/feed/news"
        - "https://decrypt.co/feed/markets"
        - "https://decrypt.co/feed/defi"
        - "https://decrypt.co/feed/nfts"
        - "https://decrypt.co/feed/web3"
      max_items_per_feed: 50
      poll_interval_seconds: 120
    base_credibility: 0.75
    relevance: 0.85
    tags: ["news", "crypto", "markets", "defi", "nft", "web3"]
    enabled: true
    rate_limit_rps: 0.5
    rate_limit_rpm: 30
    preferred_query_windows:
      - {start_hour: 12, end_hour: 22, timezone: "UTC", weight: 1.0}
      - {start_hour: 0, end_hour: 8, timezone: "UTC", weight: 0.7}
    max_concurrent_requests: 1

  - source_id: "rss:messari.io"
    name: "Messari"
    connector_type: "rss"
    base_url: "https://messari.io"
    config:
      feed_urls:
        - "https://messari.io/feed"
        - "https://messari.io/feed/research"
        - "https://messari.io/feed/news"
      max_items_per_feed: 30
      poll_interval_seconds: 180
    base_credibility: 0.80
    relevance: 0.90
    tags: ["research", "crypto", "analytics", "tokenomics"]
    enabled: true
    rate_limit_rps: 0.2
    rate_limit_rpm: 12
    rate_limit_burst: 2
    preferred_query_windows:
      - {start_hour: 13, end_hour: 21, timezone: "UTC", weight: 1.0}  # US business hours
    avoid_query_windows:
      - {start_hour: 0, end_hour: 12, timezone: "UTC", reason: "Low update frequency"}
    max_concurrent_requests: 1

  - source_id: "rss:cryptoslate.com"
    name: "CryptoSlate"
    connector_type: "rss"
    base_url: "https://cryptoslate.com"
    config:
      feed_urls:
        - "https://cryptoslate.com/feed/"
        - "https://cryptoslate.com/feed/news/"
        - "https://cryptoslate.com/feed/market-reports/"
        - "https://cryptoslate.com/feed/defi/"
      max_items_per_feed: 40
      poll_interval_seconds: 180
    base_credibility: 0.70
    relevance: 0.80
    tags: ["news", "crypto", "markets", "defi"]
    enabled: true
    rate_limit_rps: 0.33
    rate_limit_rpm: 20
    preferred_query_windows:
      - {start_hour: 12, end_hour: 23, timezone: "UTC", weight: 1.0}

  - source_id: "rss:bitcoinmagazine.com"
    name: "Bitcoin Magazine"
    connector_type: "rss"
    base_url: "https://bitcoinmagazine.com"
    config:
      feed_urls:
        - "https://bitcoinmagazine.com/feed/"
        - "https://bitcoinmagazine.com/feed/news/"
        - "https://bitcoinmagazine.com/feed/technical/"
      max_items_per_feed: 30
      poll_interval_seconds: 180
    base_credibility: 0.70
    relevance: 0.85
    tags: ["news", "bitcoin", "technical", "macro"]
    enabled: true
    rate_limit_rps: 0.25
    rate_limit_rpm: 15

  - source_id: "rss:cryptobriefing.com"
    name: "Crypto Briefing"
    connector_type: "rss"
    base_url: "https://cryptobriefing.com"
    config:
      feed_urls:
        - "https://cryptobriefing.com/feed/"
        - "https://cryptobriefing.com/feed/news/"
        - "https://cryptobriefing.com/feed/reviews/"
      max_items_per_feed: 30
      poll_interval_seconds: 180
    base_credibility: 0.65
    relevance: 0.75
    tags: ["news", "crypto", "reviews", "analysis"]
    enabled: true
    rate_limit_rps: 0.33
    rate_limit_rpm: 20

  - source_id: "rss:thedefiant.io"
    name: "The Defiant"
    connector_type: "rss"
    base_url: "https://thedefiant.io"
    config:
      feed_urls:
        - "https://thedefiant.io/feed/"
        - "https://thedefiant.io/feed/defi/"
      max_items_per_feed: 30
      poll_interval_seconds: 180
    base_credibility: 0.80
    relevance: 0.90
    tags: ["defi", "crypto", "analysis", "newsletter"]
    enabled: true
    rate_limit_rps: 0.25
    rate_limit_rpm: 15
    preferred_query_windows:
      - {start_hour: 13, end_hour: 21, timezone: "UTC", weight: 1.0}

  - source_id: "rss:banklesshq.com"
    name: "Bankless"
    connector_type: "rss"
    base_url: "https://banklesshq.com"
    config:
      feed_urls:
        - "https://banklesshq.com/feed/"
      max_items_per_feed: 20
      poll_interval_seconds: 300
    base_credibility: 0.75
    relevance: 0.85
    tags: ["defi", "ethereum", "macro", "newsletter"]
    enabled: true
    rate_limit_rps: 0.1
    rate_limit_rpm: 6
    rate_limit_burst: 1
    preferred_query_windows:
      - {start_hour: 14, end_hour: 20, timezone: "UTC", weight: 1.0}  # Weekly-ish, US afternoon

  - source_id: "rss:dlnews.com"
    name: "DL News"
    connector_type: "rss"
    base_url: "https://dlnews.com"
    config:
      feed_urls:
        - "https://dlnews.com/feed/"
        - "https://dlnews.com/feed/markets/"
        - "https://dlnews.com/feed/regulation/"
      max_items_per_feed: 30
      poll_interval_seconds: 180
    base_credibility: 0.70
    relevance: 0.80
    tags: ["news", "crypto", "markets", "regulation"]
    enabled: true
    rate_limit_rps: 0.33
    rate_limit_rpm: 20

  - source_id: "rss:protos.com"
    name: "Protos"
    connector_type: "rss"
    base_url: "https://protos.com"
    config:
      feed_urls:
        - "https://protos.com/feed/"
      max_items_per_feed: 30
      poll_interval_seconds: 180
    base_credibility: 0.65
    relevance: 0.75
    tags: ["news", "crypto", "investigative"]
    enabled: true
    rate_limit_rps: 0.33
    rate_limit_rpm: 20

  # =============================================================================
  # EXCHANGE ANNOUNCEMENTS - RSS (critical for listing/delisting events)
  # Higher frequency, stricter rate limits
  # =============================================================================
  - source_id: "rss:binance.com"
    name: "Binance Announcements"
    connector_type: "rss"
    base_url: "https://www.binance.com"
    config:
      feed_urls:
        - "https://www.binance.com/en/feed"
        - "https://www.binance.com/en/feed/announcements"
        - "https://www.binance.com/en/feed/new-listings"
        - "https://www.binance.com/en/feed/maintenance"
      max_items_per_feed: 50
      poll_interval_seconds: 60
    base_credibility: 0.95
    relevance: 1.0
    tags: ["exchange", "binance", "listings", "maintenance", "official"]
    enabled: true
    rate_limit_rps: 0.5
    rate_limit_rpm: 30
    rate_limit_burst: 5
    preferred_query_windows:
      - {start_hour: 0, end_hour: 24, timezone: "UTC", weight: 1.0}  # 24/7 critical
    query_jitter_seconds: 10
    backoff_base_seconds: 2.0
    backoff_max_seconds: 120.0
    max_concurrent_requests: 2
    max_latency_ms: 10000
    min_success_rate: 0.95

  - source_id: "rss:coinbase.com"
    name: "Coinbase Blog"
    connector_type: "rss"
    base_url: "https://blog.coinbase.com"
    config:
      feed_urls:
        - "https://blog.coinbase.com/feed"
        - "https://blog.coinbase.com/tagged/listings/feed"
      max_items_per_feed: 30
      poll_interval_seconds: 120
    base_credibility: 0.90
    relevance: 0.95
    tags: ["exchange", "coinbase", "listings", "official", "us-regulated"]
    enabled: true
    rate_limit_rps: 0.5
    rate_limit_rpm: 30
    preferred_query_windows:
      - {start_hour: 13, end_hour: 22, timezone: "UTC", weight: 1.0}
    max_concurrent_requests: 1

  - source_id: "rss:kraken.com"
    name: "Kraken Blog"
    connector_type: "rss"
    base_url: "https://blog.kraken.com"
    config:
      feed_urls:
        - "https://blog.kraken.com/feed/"
      max_items_per_feed: 20
      poll_interval_seconds: 180
    base_credibility: 0.85
    relevance: 0.90
    tags: ["exchange", "kraken", "official"]
    enabled: true
    rate_limit_rps: 0.33
    rate_limit_rpm: 20

  - source_id: "rss:bybit.com"
    name: "Bybit Announcements"
    connector_type: "rss"
    base_url: "https://bybit.com"
    config:
      feed_urls:
        - "https://announcements.bybit.com/rss/"
      max_items_per_feed: 30
      poll_interval_seconds: 120
    base_credibility: 0.80
    relevance: 0.85
    tags: ["exchange", "bybit", "derivatives", "official"]
    enabled: true
    rate_limit_rps: 0.5
    rate_limit_rpm: 30
    preferred_query_windows:
      - {start_hour: 0, end_hour: 24, timezone: "UTC", weight: 1.0}

  - source_id: "rss:okx.com"
    name: "OKX Announcements"
    connector_type: "rss"
    base_url: "https://www.okx.com"
    config:
      feed_urls:
        - "https://www.okx.com/feed/announcements"
      max_items_per_feed: 30
      poll_interval_seconds: 120
    base_credibility: 0.80
    relevance: 0.85
    tags: ["exchange", "okx", "official"]
    enabled: true
    rate_limit_rps: 0.5
    rate_limit_rpm: 30

  - source_id: "rss:kucoin.com"
    name: "KuCoin Announcements"
    connector_type: "rss"
    base_url: "https://www.kucoin.com"
    config:
      feed_urls:
        - "https://www.kucoin.com/rss/announcements"
      max_items_per_feed: 30
      poll_interval_seconds: 180
    base_credibility: 0.75
    relevance: 0.80
    tags: ["exchange", "kucoin", "official"]
    enabled: true
    rate_limit_rps: 0.33
    rate_limit_rpm: 20

  - source_id: "rss:gate.io"
    name: "Gate.io Announcements"
    connector_type: "rss"
    base_url: "https://www.gate.io"
    config:
      feed_urls:
        - "https://www.gate.io/rss/announcements"
      max_items_per_feed: 30
      poll_interval_seconds: 180
    base_credibility: 0.70
    relevance: 0.75
    tags: ["exchange", "gate", "official"]
    enabled: true
    rate_limit_rps: 0.33
    rate_limit_rpm: 20

  - source_id: "rss:htx.com"
    name: "HTX (Huobi) Announcements"
    connector_type: "rss"
    base_url: "https://www.htx.com"
    config:
      feed_urls:
        - "https://www.htx.com/feed/announcements"
      max_items_per_feed: 20
      poll_interval_seconds: 180
    base_credibility: 0.70
    relevance: 0.75
    tags: ["exchange", "htx", "huobi", "official"]
    enabled: true
    rate_limit_rps: 0.33
    rate_limit_rpm: 20

  # =============================================================================
  # REGULATORY / MACRO - Official Sources (FRED)
  # =============================================================================
  - source_id: "api:fred_vix"
    name: "FRED VIX (Volatility Index)"
    connector_type: "rest_api"
    base_url: "https://api.stlouisfed.org/fred"
    config:
      endpoints:
        - "series/observations"
      auth_type: "api_key"
      params:
        series_id: "VIXCLS"
        file_type: "json"
        limit: 1
        sort_order: "desc"
      poll_interval_seconds: 300
    base_credibility: 0.99
    relevance: 0.85
    tags: ["macro", "volatility", "vix", "fred", "official"]
    enabled: true
    credentials_ref: "fred_api_key"
    rate_limit_rps: 0.5
    rate_limit_rpm: 30
    preferred_query_windows:
      - {start_hour: 13, end_hour: 21, timezone: "UTC", weight: 1.0}  # US market hours
    avoid_query_windows:
      - {start_hour: 0, end_hour: 12, timezone: "UTC", reason: "FRED updates ~daily"}
    backoff_base_seconds: 10.0
    backoff_max_seconds: 600.0
    max_concurrent_requests: 1
    max_latency_ms: 10000

  - source_id: "api:fred_dxy"
    name: "FRED DXY (Dollar Index)"
    connector_type: "rest_api"
    base_url: "https://api.stlouisfed.org/fred"
    config:
      endpoints:
        - "series/observations"
      auth_type: "api_key"
      params:
        series_id: "DTWEXBGS"
        file_type: "json"
        limit: 1
        sort_order: "desc"
      poll_interval_seconds: 300
    base_credibility: 0.99
    relevance: 0.85
    tags: ["macro", "dxy", "dollar", "fred", "official"]
    enabled: true
    credentials_ref: "fred_api_key"
    rate_limit_rps: 0.5
    rate_limit_rpm: 30

  - source_id: "api:fred_fedfunds"
    name: "FRED Fed Funds Rate"
    connector_type: "rest_api"
    base_url: "https://api.stlouisfed.org/fred"
    config:
      endpoints:
        - "series/observations"
      auth_type: "api_key"
      params:
        series_id: "FEDFUNDS"
        file_type: "json"
        limit: 1
        sort_order: "desc"
      poll_interval_seconds: 3600
    base_credibility: 0.99
    relevance: 0.80
    tags: ["macro", "rates", "fed", "fred", "official"]
    enabled: true
    credentials_ref: "fred_api_key"
    rate_limit_rps: 0.2
    rate_limit_rpm: 12

  - source_id: "api:fred_cpi"
    name: "FRED CPI (Inflation)"
    connector_type: "rest_api"
    base_url: "https://api.stlouisfed.org/fred"
    config:
      endpoints:
        - "series/observations"
      auth_type: "api_key"
      params:
        series_id: "CPIAUCSL"
        file_type: "json"
        limit: 1
        sort_order: "desc"
      poll_interval_seconds: 86400
    base_credibility: 0.99
    relevance: 0.80
    tags: ["macro", "inflation", "cpi", "fred", "official"]
    enabled: true
    credentials_ref: "fred_api_key"
    rate_limit_rps: 0.1
    rate_limit_rpm: 6
    preferred_query_windows:
      - {start_hour: 12, end_hour: 14, timezone: "UTC", weight: 1.0}  # CPI release time ~13:30 UTC

  - source_id: "api:fred_unemployment"
    name: "FRED Unemployment Rate"
    connector_type: "rest_api"
    base_url: "https://api.stlouisfed.org/fred"
    config:
      endpoints:
        - "series/observations"
      auth_type: "api_key"
      params:
        series_id: "UNRATE"
        file_type: "json"
        limit: 1
        sort_order: "desc"
      poll_interval_seconds: 86400
    base_credibility: 0.99
    relevance: 0.75
    tags: ["macro", "unemployment", "labor", "fred", "official"]
    enabled: true
    credentials_ref: "fred_api_key"
    rate_limit_rps: 0.1
    rate_limit_rpm: 6

  # =============================================================================
  # TRADITIONAL FINANCE NEWS - Crypto Coverage
  # =============================================================================
  - source_id: "rss:bloomberg.com"
    name: "Bloomberg Markets"
    connector_type: "rss"
    base_url: "https://www.bloomberg.com"
    config:
      feed_urls:
        - "https://feeds.bloomberg.com/markets/news.rss"
        - "https://feeds.bloomberg.com/technology/news.rss"
      max_items_per_feed: 30
      poll_interval_seconds: 180
    base_credibility: 0.90
    relevance: 0.70
    tags: ["tradfi", "markets", "macro", "crypto-coverage"]
    enabled: true
    rate_limit_rps: 0.25
    rate_limit_rpm: 15
    preferred_query_windows:
      - {start_hour: 12, end_hour: 22, timezone: "UTC", weight: 1.0}
      - {start_hour: 0, end_hour: 8, timezone: "UTC", weight: 0.5}

  - source_id: "rss:reuters.com"
    name: "Reuters Business"
    connector_type: "rss"
    base_url: "https://www.reuters.com"
    config:
      feed_urls:
        - "https://www.reuters.com/business/finance/rss"
        - "https://www.reuters.com/technology/rss"
      max_items_per_feed: 30
      poll_interval_seconds: 180
    base_credibility: 0.92
    relevance: 0.70
    tags: ["tradfi", "markets", "macro", "crypto-coverage"]
    enabled: true
    rate_limit_rps: 0.33
    rate_limit_rpm: 20

  - source_id: "rss:wsj.com"
    name: "Wall Street Journal Markets"
    connector_type: "rss"
    base_url: "https://www.wsj.com"
    config:
      feed_urls:
        - "https://feeds.wsj.com/wsj/marketspulse.rss"
      max_items_per_feed: 20
      poll_interval_seconds: 180
    base_credibility: 0.90
    relevance: 0.65
    tags: ["tradfi", "markets", "macro"]
    enabled: true
    rate_limit_rps: 0.25
    rate_limit_rpm: 15

  - source_id: "rss:ft.com"
    name: "Financial Times Markets"
    connector_type: "rss"
    base_url: "https://www.ft.com"
    config:
      feed_urls:
        - "https://www.ft.com/markets?format=rss"
      max_items_per_feed: 20
      poll_interval_seconds: 180
    base_credibility: 0.90
    relevance: 0.65
    tags: ["tradfi", "markets", "macro", "europe"]
    enabled: true
    rate_limit_rps: 0.25
    rate_limit_rpm: 15
    preferred_query_windows:
      - {start_hour: 7, end_hour: 17, timezone: "UTC", weight: 1.0}  # London hours

  - source_id: "rss:marketwatch.com"
    name: "MarketWatch"
    connector_type: "rss"
    base_url: "https://www.marketwatch.com"
    config:
      feed_urls:
        - "https://feeds.marketwatch.com/marketwatch/topstories/"
        - "https://feeds.marketwatch.com/marketwatch/cryptocurrency/"
      max_items_per_feed: 30
      poll_interval_seconds: 180
    base_credibility: 0.80
    relevance: 0.75
    tags: ["tradfi", "markets", "crypto-coverage"]
    enabled: true
    rate_limit_rps: 0.33
    rate_limit_rpm: 20

  # =============================================================================
  # REDDIT - High-signal subreddits
  # Respect Reddit API limits: 60 req/min for OAuth, 10 req/min for unauthenticated
  # =============================================================================
  - source_id: "reddit:CryptoCurrency"
    name: "r/CryptoCurrency"
    connector_type: "reddit"
    base_url: "https://reddit.com/r/CryptoCurrency"
    config:
      subreddits: ["CryptoCurrency"]
      use_pushshift: true
      poll_interval_seconds: 180
    base_credibility: 0.35
    relevance: 0.80
    tags: ["social", "reddit", "retail", "high-volume", "sentiment"]
    enabled: true
    rate_limit_rps: 0.1
    rate_limit_rpm: 6
    rate_limit_burst: 2
    preferred_query_windows:
      - {start_hour: 12, end_hour: 24, timezone: "UTC", weight: 1.0}  # US active
      - {start_hour: 0, end_hour: 8, timezone: "UTC", weight: 0.6}    # Asia/Europe
    avoid_query_windows:
      - {start_hour: 8, end_hour: 12, timezone: "UTC", reason: "Lower activity"}
    query_jitter_seconds: 60
    backoff_base_seconds: 30.0
    backoff_max_seconds: 1800.0
    max_concurrent_requests: 1
    max_latency_ms: 20000

  - source_id: "reddit:Bitcoin"
    name: "r/Bitcoin"
    connector_type: "reddit"
    base_url: "https://reddit.com/r/Bitcoin"
    config:
      subreddits: ["Bitcoin"]
      use_pushshift: true
      poll_interval_seconds: 180
    base_credibility: 0.35
    relevance: 0.90
    tags: ["social", "reddit", "bitcoin", "maximalist", "sentiment"]
    enabled: true
    rate_limit_rps: 0.1
    rate_limit_rpm: 6

  - source_id: "reddit:EthTrader"
    name: "r/EthTrader"
    connector_type: "reddit"
    base_url: "https://reddit.com/r/EthTrader"
    config:
      subreddits: ["EthTrader"]
      use_pushshift: true
      poll_interval_seconds: 180
    base_credibility: 0.35
    relevance: 0.85
    tags: ["social", "reddit", "ethereum", "trading", "sentiment"]
    enabled: true
    rate_limit_rps: 0.1
    rate_limit_rpm: 6

  - source_id: "reddit:defi"
    name: "r/defi"
    connector_type: "reddit"
    base_url: "https://reddit.com/r/defi"
    config:
      subreddits: ["defi"]
      use_pushshift: true
      poll_interval_seconds: 300
    base_credibility: 0.30
    relevance: 0.85
    tags: ["social", "reddit", "defi", "yield", "sentiment"]
    enabled: true
    rate_limit_rps: 0.1
    rate_limit_rpm: 6

  - source_id: "reddit:ethtrader"
    name: "r/ethtrader (daily discussion)"
    connector_type: "reddit"
    base_url: "https://reddit.com/r/ethtrader"
    config:
      subreddits: ["ethtrader"]
      use_pushshift: true
      poll_interval_seconds: 300
    base_credibility: 0.30
    relevance: 0.80
    tags: ["social", "reddit", "ethereum", "daily-discussion"]
    enabled: true
    rate_limit_rps: 0.1
    rate_limit_rpm: 6

  - source_id: "reddit:CryptoMarkets"
    name: "r/CryptoMarkets"
    connector_type: "reddit"
    base_url: "https://reddit.com/r/CryptoMarkets"
    config:
      subreddits: ["CryptoMarkets"]
      use_pushshift: true
      poll_interval_seconds: 180
    base_credibility: 0.35
    relevance: 0.85
    tags: ["social", "reddit", "trading", "technical-analysis"]
    enabled: true
    rate_limit_rps: 0.1
    rate_limit_rpm: 6

  - source_id: "reddit:BitcoinMarkets"
    name: "r/BitcoinMarkets"
    connector_type: "reddit"
    base_url: "https://reddit.com/r/BitcoinMarkets"
    config:
      subreddits: ["BitcoinMarkets"]
      use_pushshift: true
      poll_interval_seconds: 180
    base_credibility: 0.35
    relevance: 0.85
    tags: ["social", "reddit", "bitcoin", "trading"]
    enabled: true
    rate_limit_rps: 0.1
    rate_limit_rpm: 6

  - source_id: "reddit:altcoin"
    name: "r/altcoin"
    connector_type: "reddit"
    base_url: "https://reddit.com/r/altcoin"
    config:
      subreddits: ["altcoin"]
      use_pushshift: true
      poll_interval_seconds: 300
    base_credibility: 0.25
    relevance: 0.70
    tags: ["social", "reddit", "altcoins", "speculation"]
    enabled: true
    rate_limit_rps: 0.1
    rate_limit_rpm: 6

  - source_id: "reddit:Solana"
    name: "r/Solana"
    connector_type: "reddit"
    base_url: "https://reddit.com/r/Solana"
    config:
      subreddits: ["Solana"]
      use_pushshift: true
      poll_interval_seconds: 300
    base_credibility: 0.30
    relevance: 0.80
    tags: ["social", "reddit", "solana", "ecosystem"]
    enabled: true
    rate_limit_rps: 0.1
    rate_limit_rpm: 6

  # =============================================================================
  # TELEGRAM - Public channels
  # Telegram bot API: 30 req/sec per bot, but be conservative
  # =============================================================================
  - source_id: "telegram:binance_announcements"
    name: "Binance Announcements (Telegram)"
    connector_type: "telegram"
    base_url: "https://t.me/binance_announcements"
    config:
      channel_usernames: ["binance_announcements"]
    base_credibility: 0.90
    relevance: 0.95
    tags: ["social", "telegram", "exchange", "binance", "official", "listings"]
    enabled: true
    rate_limit_rps: 0.5
    rate_limit_rpm: 30
    rate_limit_burst: 5
    preferred_query_windows:
      - {start_hour: 0, end_hour: 24, timezone: "UTC", weight: 1.0}
    max_concurrent_requests: 1

  - source_id: "telegram:coinbase_announcements"
    name: "Coinbase Announcements (Telegram)"
    connector_type: "telegram"
    base_url: "https://t.me/coinbase_announcements"
    config:
      channel_usernames: ["coinbase_announcements"]
    base_credibility: 0.85
    relevance: 0.90
    tags: ["social", "telegram", "exchange", "coinbase", "official"]
    enabled: true
    rate_limit_rps: 0.5
    rate_limit_rpm: 30

  - source_id: "telegram:coindesk"
    name: "CoinDesk (Telegram)"
    connector_type: "telegram"
    base_url: "https://t.me/coindesk"
    config:
      channel_usernames: ["coindesk"]
    base_credibility: 0.75
    relevance: 0.85
    tags: ["social", "telegram", "news", "crypto"]
    enabled: true
    rate_limit_rps: 0.33
    rate_limit_rpm: 20

  - source_id: "telegram:cointelegraph"
    name: "CoinTelegraph (Telegram)"
    connector_type: "telegram"
    base_url: "https://t.me/cointelegraph"
    config:
      channel_usernames: ["cointelegraph"]
    base_credibility: 0.70
    relevance: 0.80
    tags: ["social", "telegram", "news", "crypto"]
    enabled: true
    rate_limit_rps: 0.33
    rate_limit_rpm: 20

  - source_id: "telegram:thedefiant"
    name: "The Defiant (Telegram)"
    connector_type: "telegram"
    base_url: "https://t.me/thedefiant"
    config:
      channel_usernames: ["thedefiant"]
    base_credibility: 0.75
    relevance: 0.85
    tags: ["social", "telegram", "defi", "newsletter"]
    enabled: true
    rate_limit_rps: 0.25
    rate_limit_rpm: 15

  - source_id: "telegram:bankless"
    name: "Bankless (Telegram)"
    connector_type: "telegram"
    base_url: "https://t.me/banklesshq"
    config:
      channel_usernames: ["banklesshq"]
    base_credibility: 0.70
    relevance: 0.80
    tags: ["social", "telegram", "defi", "ethereum", "macro"]
    enabled: true
    rate_limit_rps: 0.2
    rate_limit_rpm: 12

  - source_id: "telegram:glassnode"
    name: "Glassnode Alerts (Telegram)"
    connector_type: "telegram"
    base_url: "https://t.me/glassnodealerts"
    config:
      channel_usernames: ["glassnodealerts"]
    base_credibility: 0.85
    relevance: 0.90
    tags: ["social", "telegram", "onchain", "analytics", "alerts"]
    enabled: true
    rate_limit_rps: 0.2
    rate_limit_rpm: 12
    preferred_query_windows:
      - {start_hour: 0, end_hour: 24, timezone: "UTC", weight: 1.0}  # Alerts anytime

  - source_id: "telegram:cryptoquant"
    name: "CryptoQuant (Telegram)"
    connector_type: "telegram"
    base_url: "https://t.me/cryptoquant_official"
    config:
      channel_usernames: ["cryptoquant_official"]
    base_credibility: 0.80
    relevance: 0.85
    tags: ["social", "telegram", "onchain", "analytics", "whale-alerts"]
    enabled: true
    rate_limit_rps: 0.2
    rate_limit_rpm: 12

  - source_id: "telegram:whale_alert"
    name: "Whale Alert (Telegram)"
    connector_type: "telegram"
    base_url: "https://t.me/whale_alert_io"
    config:
      channel_usernames: ["whale_alert_io"]
    base_credibility: 0.75
    relevance: 0.85
    tags: ["social", "telegram", "whale", "large-transactions", "onchain"]
    enabled: true
    rate_limit_rps: 0.2
    rate_limit_rpm: 12
    preferred_query_windows:
      - {start_hour: 0, end_hour: 24, timezone: "UTC", weight: 1.0}

  # =============================================================================
  # WEB CRAWL - Sites without RSS/API (supplementary)
  # Very conservative rate limits
  # =============================================================================
  - source_id: "web:defillama.com"
    name: "DeFi Llama (Blog/Announcements)"
    connector_type: "web_crawl"
    base_url: "https://defillama.com"
    config:
      seed_urls:
        - "https://defillama.com/blog"
        - "https://defillama.com/chains"
      allowed_domains: ["defillama.com"]
      max_depth: 2
      rate_limit_rps: 0.5
    base_credibility: 0.80
    relevance: 0.85
    tags: ["defi", "tvl", "analytics", "blog"]
    enabled: true
    rate_limit_rps: 0.1
    rate_limit_rpm: 6
    rate_limit_burst: 1
    preferred_query_windows:
      - {start_hour: 2, end_hour: 6, timezone: "UTC", weight: 1.0}  # Off-peak
    query_jitter_seconds: 120
    backoff_base_seconds: 60.0
    backoff_max_seconds: 3600.0
    max_concurrent_requests: 1
    max_latency_ms: 60000

  - source_id: "web:dune.com"
    name: "Dune Analytics (Blog)"
    connector_type: "web_crawl"
    base_url: "https://dune.com"
    config:
      seed_urls:
        - "https://dune.com/blog"
      allowed_domains: ["dune.com"]
      max_depth: 2
      rate_limit_rps: 0.5
    base_credibility: 0.75
    relevance: 0.80
    tags: ["analytics", "sql", "dashboards", "blog"]
    enabled: true
    rate_limit_rps: 0.1
    rate_limit_rpm: 6

  - source_id: "web:nansen.ai"
    name: "Nansen (Blog/Research)"
    connector_type: "web_crawl"
    base_url: "https://nansen.ai"
    config:
      seed_urls:
        - "https://nansen.ai/blog"
        - "https://nansen.ai/research"
      allowed_domains: ["nansen.ai"]
      max_depth: 2
      rate_limit_rps: 0.5
    base_credibility: 0.80
    relevance: 0.85
    tags: ["onchain", "analytics", "research", "whale"]
    enabled: true
    rate_limit_rps: 0.1
    rate_limit_rpm: 6

  - source_id: "web:coingecko.com"
    name: "CoinGecko (News/Research)"
    connector_type: "web_crawl"
    base_url: "https://coingecko.com"
    config:
      seed_urls:
        - "https://coingecko.com/en/news"
        - "https://coingecko.com/en/research"
      allowed_domains: ["coingecko.com"]
      max_depth: 2
      rate_limit_rps: 0.5
    base_credibility: 0.70
    relevance: 0.80
    tags: ["data", "news", "research", "token-info"]
    enabled: true
    rate_limit_rps: 0.2
    rate_limit_rpm: 12

  - source_id: "web:coinmarketcap.com"
    name: "CoinMarketCap (News)"
    connector_type: "web_crawl"
    base_url: "https://coinmarketcap.com"
    config:
      seed_urls:
        - "https://coinmarketcap.com/headlines/news/"
      allowed_domains: ["coinmarketcap.com"]
      max_depth: 2
      rate_limit_rps: 0.5
    base_credibility: 0.65
    relevance: 0.75
    tags: ["data", "news", "token-info"]
    enabled: true
    rate_limit_rps: 0.2
    rate_limit_rpm: 12

  # =============================================================================
  # ON-CHAIN / ANALYTICS - Specialized
  # =============================================================================
  - source_id: "rss:glassnode.com"
    name: "Glassnode Insights"
    connector_type: "rss"
    base_url: "https://glassnode.com"
    config:
      feed_urls:
        - "https://insights.glassnode.com/rss/"
      max_items_per_feed: 20
      poll_interval_seconds: 300
    base_credibility: 0.90
    relevance: 0.95
    tags: ["onchain", "analytics", "glassnode", "metrics"]
    enabled: true
    rate_limit_rps: 0.2
    rate_limit_rpm: 12
    preferred_query_windows:
      - {start_hour: 12, end_hour: 22, timezone: "UTC", weight: 1.0}

  - source_id: "rss:cryptoquant.com"
    name: "CryptoQuant Quicktake"
    connector_type: "rss"
    base_url: "https://cryptoquant.com"
    config:
      feed_urls:
        - "https://cryptoquant.com/feed/quicktake"
      max_items_per_feed: 20
      poll_interval_seconds: 300
    base_credibility: 0.85
    relevance: 0.90
    tags: ["onchain", "analytics", "cryptoquant", "whale"]
    enabled: true
    rate_limit_rps: 0.2
    rate_limit_rpm: 12

  - source_id: "rss:coinmetrics.io"
    name: "Coin Metrics"
    connector_type: "rss"
    base_url: "https://coinmetrics.io"
    config:
      feed_urls:
        - "https://coinmetrics.io/feed/"
      max_items_per_feed: 15
      poll_interval_seconds: 300
    base_credibility: 0.85
    relevance: 0.85
    tags: ["onchain", "analytics", "coinmetrics", "institutional"]
    enabled: true
    rate_limit_rps: 0.2
    rate_limit_rpm: 12

  # =============================================================================
  # GOVERNANCE / PROTOCOL ANNOUNCEMENTS
  # =============================================================================
  - source_id: "rss:forum.uniswap.org"
    name: "Uniswap Governance Forum"
    connector_type: "rss"
    base_url: "https://forum.uniswap.org"
    config:
      feed_urls:
        - "https://forum.uniswap.org/c/governance.rss"
      max_items_per_feed: 20
      poll_interval_seconds: 300
    base_credibility: 0.80
    relevance: 0.85
    tags: ["governance", "defi", "uniswap", "dao"]
    enabled: true
    rate_limit_rps: 0.2
    rate_limit_rpm: 12

  - source_id: "rss:forum.aave.com"
    name: "Aave Governance Forum"
    connector_type: "rss"
    base_url: "https://forum.aave.com"
    config:
      feed_urls:
        - "https://forum.aave.com/c/governance.rss"
      max_items_per_feed: 20
      poll_interval_seconds: 300
    base_credibility: 0.80
    relevance: 0.85
    tags: ["governance", "defi", "aave", "dao", "lending"]
    enabled: true
    rate_limit_rps: 0.2
    rate_limit_rpm: 12

  - source_id: "rss:gov.makerdao.com"
    name: "MakerDAO Governance"
    connector_type: "rss"
    base_url: "https://forum.makerdao.com"
    config:
      feed_urls:
        - "https://forum.makerdao.com/c/governance.rss"
      max_items_per_feed: 20
      poll_interval_seconds: 300
    base_credibility: 0.85
    relevance: 0.85
    tags: ["governance", "defi", "makerdao", "dai", "dao"]
    enabled: true
    rate_limit_rps: 0.2
    rate_limit_rpm: 12

  - source_id: "rss:gov.compound.finance"
    name: "Compound Governance"
    connector_type: "rss"
    base_url: "https://compound.finance/governance"
    config:
      feed_urls:
        - "https://compound.finance/governance/feed/"
      max_items_per_feed: 15
      poll_interval_seconds: 300
    base_credibility: 0.80
    relevance: 0.80
    tags: ["governance", "defi", "compound", "dao", "lending"]
    enabled: true
    rate_limit_rps: 0.2
    rate_limit_rpm: 12

# Total: ~60 sources across all categories
# Run: python scripts/populate_catalogue.py --config config/seed_sources.yaml