feat(sentiment): add 50+ social media sources for small-cap coverage

- Reddit: 15 new subreddits (altcoin, CryptoMoonShots, SatoshiStreetBets, defi, CryptoMarkets + 11 project-specific) - Twitter: 10 search queries for trade assets (ZIL, ONE, STX, ALGO, DASH, LTC, FET, XTZ, ENJ + combined) - Telegram: 11 official project channels - Discord: 11 official project servers - All configured with appropriate credibility/relevance scores for small-cap coverage
2026-09-23 10:09:57 +02:00
parent c0445e60ea
commit d69e82200a
2 changed files with 1014 additions and 4 deletions
--- a/sentiment_engine/config/sources.yaml
+++ b/sentiment_engine/config/sources.yaml
@@ -152,6 +152,206 @@ sources:
      use_pushshift: true
      max_items: 100

+  # Social - Reddit (Altcoin/Small-cap focused)
+  - source_id: "reddit:altcoin"
+    type: "reddit"
+    url: "https://api.pushshift.io/reddit/search/submission"
+    cadence_seconds: 300
+    base_credibility: 0.3
+    relevance: 0.85
+    extra_config:
+      subreddits: ["altcoin"]
+      use_pushshift: true
+      max_items: 100
+
+  - source_id: "reddit:CryptoMoonShots"
+    type: "reddit"
+    url: "https://api.pushshift.io/reddit/search/submission"
+    cadence_seconds: 300
+    base_credibility: 0.25
+    relevance: 0.9
+    extra_config:
+      subreddits: ["CryptoMoonShots"]
+      use_pushshift: true
+      max_items: 100
+
+  - source_id: "reddit:SatoshiStreetBets"
+    type: "reddit"
+    url: "https://api.pushshift.io/reddit/search/submission"
+    cadence_seconds: 300
+    base_credibility: 0.25
+    relevance: 0.85
+    extra_config:
+      subreddits: ["SatoshiStreetBets"]
+      use_pushshift: true
+      max_items: 100
+
+  - source_id: "reddit:defi"
+    type: "reddit"
+    url: "https://api.pushshift.io/reddit/search/submission"
+    cadence_seconds: 300
+    base_credibility: 0.3
+    relevance: 0.8
+    extra_config:
+      subreddits: ["defi"]
+      use_pushshift: true
+      max_items: 100
+
+  - source_id: "reddit:CryptoMarkets"
+    type: "reddit"
+    url: "https://api.pushshift.io/reddit/search/submission"
+    cadence_seconds: 300
+    base_credibility: 0.35
+    relevance: 0.8
+    extra_config:
+      subreddits: ["CryptoMarkets"]
+      use_pushshift: true
+      max_items: 100
+
+  # Social - Reddit (Project-specific for trade assets)
+  - source_id: "reddit:zilliqa"
+    type: "reddit"
+    url: "https://api.pushshift.io/reddit/search/submission"
+    cadence_seconds: 300
+    base_credibility: 0.3
+    relevance: 0.95
+    extra_config:
+      subreddits: ["zilliqa"]
+      use_pushshift: true
+      max_items: 50
+
+  - source_id: "reddit:harmonyone"
+    type: "reddit"
+    url: "https://api.pushshift.io/reddit/search/submission"
+    cadence_seconds: 300
+    base_credibility: 0.3
+    relevance: 0.95
+    extra_config:
+      subreddits: ["harmonyone"]
+      use_pushshift: true
+      max_items: 50
+
+  - source_id: "reddit:stacks"
+    type: "reddit"
+    url: "https://api.pushshift.io/reddit/search/submission"
+    cadence_seconds: 300
+    base_credibility: 0.3
+    relevance: 0.95
+    extra_config:
+      subreddits: ["stacks"]
+      use_pushshift: true
+      max_items: 50
+
+  - source_id: "reddit:algorand"
+    type: "reddit"
+    url: "https://api.pushshift.io/reddit/search/submission"
+    cadence_seconds: 300
+    base_credibility: 0.3
+    relevance: 0.95
+    extra_config:
+      subreddits: ["algorand"]
+      use_pushshift: true
+      max_items: 50
+
+  - source_id: "reddit:dashpay"
+    type: "reddit"
+    url: "https://api.pushshift.io/reddit/search/submission"
+    cadence_seconds: 300
+    base_credibility: 0.3
+    relevance: 0.95
+    extra_config:
+      subreddits: ["dashpay"]
+      use_pushshift: true
+      max_items: 50
+
+  - source_id: "reddit:litecoin"
+    type: "reddit"
+    url: "https://api.pushshift.io/reddit/search/submission"
+    cadence_seconds: 300
+    base_credibility: 0.3
+    relevance: 0.95
+    extra_config:
+      subreddits: ["litecoin"]
+      use_pushshift: true
+      max_items: 50
+
+  - source_id: "reddit:fetchai"
+    type: "reddit"
+    url: "https://api.pushshift.io/reddit/search/submission"
+    cadence_seconds: 300
+    base_credibility: 0.3
+    relevance: 0.95
+    extra_config:
+      subreddits: ["fetchai"]
+      use_pushshift: true
+      max_items: 50
+
+  - source_id: "reddit:tezos"
+    type: "reddit"
+    url: "https://api.pushshift.io/reddit/search/submission"
+    cadence_seconds: 300
+    base_credibility: 0.3
+    relevance: 0.95
+    extra_config:
+      subreddits: ["tezos"]
+      use_pushshift: true
+      max_items: 50
+
+  - source_id: "reddit:enjincoin"
+    type: "reddit"
+    url: "https://api.pushshift.io/reddit/search/submission"
+    cadence_seconds: 300
+    base_credibility: 0.3
+    relevance: 0.95
+    extra_config:
+      subreddits: ["enjincoin"]
+      use_pushshift: true
+      max_items: 50
+
+  - source_id: "reddit:stellar"
+    type: "reddit"
+    url: "https://api.pushshift.io/reddit/search/submission"
+    cadence_seconds: 300
+    base_credibility: 0.3
+    relevance: 0.95
+    extra_config:
+      subreddits: ["stellar"]
+      use_pushshift: true
+      max_items: 50
+
+  - source_id: "reddit:ethereumclassic"
+    type: "reddit"
+    url: "https://api.pushshift.io/reddit/search/submission"
+    cadence_seconds: 300
+    base_credibility: 0.3
+    relevance: 0.95
+    extra_config:
+      subreddits: ["ethereumclassic"]
+      use_pushshift: true
+      max_items: 50
+
+  - source_id: "reddit:tronix"
+    type: "reddit"
+    url: "https://api.pushshift.io/reddit/search/submission"
+    cadence_seconds: 300
+    base_credibility: 0.3
+    relevance: 0.95
+    extra_config:
+      subreddits: ["tronix"]
+      use_pushshift: true
+      max_items: 50
+
+  - source_id: "reddit:ontology"
+    type: "reddit"
+    url: "https://api.pushshift.io/reddit/search/submission"
+    cadence_seconds: 300
+    base_credibility: 0.3
+    relevance: 0.95
+    extra_config:
+      subreddits: ["ontology"]
+      use_pushshift: true
+      max_items: 50
+
  # Exchange announcements (direct)
  - source_id: "exchange:binance"
    type: "exchange_ann"
@@ -227,6 +427,379 @@ sources:
      feed_urls: ["https://seekingalpha.com/api/sa/combined/feed"]
      max_items_per_feed: 50

+  # Social - Twitter/X (Search-based for trade assets)
+  - source_id: "twitter:smallcap_search"
+    type: "twitter"
+    url: "https://api.twitter.com"
+    cadence_seconds: 180
+    base_credibility: 0.35
+    relevance: 0.85
+    extra_config:
+      bearer_token: "${TWITTER_BEARER_TOKEN}"
+      search_query: "($ZIL OR $ONG OR $ONE OR $STX OR $ALGO OR $DASH OR $LTC OR $FET OR $XTZ OR $ENJ OR $XLM OR $ETC OR $TRX) lang:en"
+      max_results: 100
+
+  - source_id: "twitter:zilliqa"
+    type: "twitter"
+    url: "https://api.twitter.com"
+    cadence_seconds: 180
+    base_credibility: 0.35
+    relevance: 0.9
+    extra_config:
+      bearer_token: "${TWITTER_BEARER_TOKEN}"
+      search_query: "($ZIL OR Zilliqa) lang:en"
+      max_results: 50
+
+  - source_id: "twitter:harmony"
+    type: "twitter"
+    url: "https://api.twitter.com"
+    cadence_seconds: 180
+    base_credibility: 0.35
+    relevance: 0.9
+    extra_config:
+      bearer_token: "${TWITTER_BEARER_TOKEN}"
+      search_query: "($ONE OR Harmony) lang:en"
+      max_results: 50
+
+  - source_id: "twitter:stacks"
+    type: "twitter"
+    url: "https://api.twitter.com"
+    cadence_seconds: 180
+    base_credibility: 0.35
+    relevance: 0.9
+    extra_config:
+      bearer_token: "${TWITTER_BEARER_TOKEN}"
+      search_query: "($STX OR Stacks) lang:en"
+      max_results: 50
+
+  - source_id: "twitter:algorand"
+    type: "twitter"
+    url: "https://api.twitter.com"
+    cadence_seconds: 180
+    base_credibility: 0.35
+    relevance: 0.9
+    extra_config:
+      bearer_token: "${TWITTER_BEARER_TOKEN}"
+      search_query: "($ALGO OR Algorand) lang:en"
+      max_results: 50
+
+  - source_id: "twitter:dash"
+    type: "twitter"
+    url: "https://api.twitter.com"
+    cadence_seconds: 180
+    base_credibility: 0.35
+    relevance: 0.9
+    extra_config:
+      bearer_token: "${TWITTER_BEARER_TOKEN}"
+      search_query: "($DASH OR Dash) lang:en"
+      max_results: 50
+
+  - source_id: "twitter:litecoin"
+    type: "twitter"
+    url: "https://api.twitter.com"
+    cadence_seconds: 180
+    base_credibility: 0.35
+    relevance: 0.9
+    extra_config:
+      bearer_token: "${TWITTER_BEARER_TOKEN}"
+      search_query: "($LTC OR Litecoin) lang:en"
+      max_results: 50
+
+  - source_id: "twitter:fetchai"
+    type: "twitter"
+    url: "https://api.twitter.com"
+    cadence_seconds: 180
+    base_credibility: 0.35
+    relevance: 0.9
+    extra_config:
+      bearer_token: "${TWITTER_BEARER_TOKEN}"
+      search_query: "($FET OR Fetch.ai) lang:en"
+      max_results: 50
+
+  - source_id: "twitter:tezos"
+    type: "twitter"
+    url: "https://api.twitter.com"
+    cadence_seconds: 180
+    base_credibility: 0.35
+    relevance: 0.9
+    extra_config:
+      bearer_token: "${TWITTER_BEARER_TOKEN}"
+      search_query: "($XTZ OR Tezos) lang:en"
+      max_results: 50
+
+  - source_id: "twitter:enjin"
+    type: "twitter"
+    url: "https://api.twitter.com"
+    cadence_seconds: 180
+    base_credibility: 0.35
+    relevance: 0.9
+    extra_config:
+      bearer_token: "${TWITTER_BEARER_TOKEN}"
+      search_query: "($ENJ OR Enjin) lang:en"
+      max_results: 50
+
+  # Social - Telegram (Official project channels)
+  - source_id: "telegram:zilliqa_official"
+    type: "telegram"
+    url: "https://t.me/zilliqa"
+    cadence_seconds: 60
+    base_credibility: 0.4
+    relevance: 0.95
+    extra_config:
+      channel_usernames: ["zilliqa"]
+      bot_token: "${TELEGRAM_BOT_TOKEN}"
+
+  - source_id: "telegram:harmony_official"
+    type: "telegram"
+    url: "https://t.me/harmonyofficial"
+    cadence_seconds: 60
+    base_credibility: 0.4
+    relevance: 0.95
+    extra_config:
+      channel_usernames: ["harmonyofficial"]
+      bot_token: "${TELEGRAM_BOT_TOKEN}"
+
+  - source_id: "telegram:stacks_official"
+    type: "telegram"
+    url: "https://t.me/stacksblockchain"
+    cadence_seconds: 60
+    base_credibility: 0.4
+    relevance: 0.95
+    extra_config:
+      channel_usernames: ["stacksblockchain"]
+      bot_token: "${TELEGRAM_BOT_TOKEN}"
+
+  - source_id: "telegram:algorand_official"
+    type: "telegram"
+    url: "https://t.me/algorand"
+    cadence_seconds: 60
+    base_credibility: 0.4
+    relevance: 0.95
+    extra_config:
+      channel_usernames: ["algorand"]
+      bot_token: "${TELEGRAM_BOT_TOKEN}"
+
+  - source_id: "telegram:dash_official"
+    type: "telegram"
+    url: "https://t.me/dashpay"
+    cadence_seconds: 60
+    base_credibility: 0.4
+    relevance: 0.95
+    extra_config:
+      channel_usernames: ["dashpay"]
+      bot_token: "${TELEGRAM_BOT_TOKEN}"
+
+  - source_id: "telegram:litecoin_official"
+    type: "telegram"
+    url: "https://t.me/litecoin"
+    cadence_seconds: 60
+    base_credibility: 0.4
+    relevance: 0.95
+    extra_config:
+      channel_usernames: ["litecoin"]
+      bot_token: "${TELEGRAM_BOT_TOKEN}"
+
+  - source_id: "telegram:fetchai_official"
+    type: "telegram"
+    url: "https://t.me/fetch_ai"
+    cadence_seconds: 60
+    base_credibility: 0.4
+    relevance: 0.95
+    extra_config:
+      channel_usernames: ["fetch_ai"]
+      bot_token: "${TELEGRAM_BOT_TOKEN}"
+
+  - source_id: "telegram:tezos_official"
+    type: "telegram"
+    url: "https://t.me/tezos"
+    cadence_seconds: 60
+    base_credibility: 0.4
+    relevance: 0.95
+    extra_config:
+      channel_usernames: ["tezos"]
+      bot_token: "${TELEGRAM_BOT_TOKEN}"
+
+  - source_id: "telegram:enjin_official"
+    type: "telegram"
+    url: "https://t.me/enjin"
+    cadence_seconds: 60
+    base_credibility: 0.4
+    relevance: 0.95
+    extra_config:
+      channel_usernames: ["enjin"]
+      bot_token: "${TELEGRAM_BOT_TOKEN}"
+
+  - source_id: "telegram:stellar_official"
+    type: "telegram"
+    url: "https://t.me/stellar"
+    cadence_seconds: 60
+    base_credibility: 0.4
+    relevance: 0.95
+    extra_config:
+      channel_usernames: ["stellar"]
+      bot_token: "${TELEGRAM_BOT_TOKEN}"
+
+  - source_id: "telegram:ethereumclassic_official"
+    type: "telegram"
+    url: "https://t.me/ethereumclassic"
+    cadence_seconds: 60
+    base_credibility: 0.4
+    relevance: 0.95
+    extra_config:
+      channel_usernames: ["ethereumclassic"]
+      bot_token: "${TELEGRAM_BOT_TOKEN}"
+
+  - source_id: "telegram:tron_official"
+    type: "telegram"
+    url: "https://t.me/tronnetwork"
+    cadence_seconds: 60
+    base_credibility: 0.4
+    relevance: 0.95
+    extra_config:
+      channel_usernames: ["tronnetwork"]
+      bot_token: "${TELEGRAM_BOT_TOKEN}"
+
+  - source_id: "telegram:ontology_official"
+    type: "telegram"
+    url: "https://t.me/ontologynetwork"
+    cadence_seconds: 60
+    base_credibility: 0.4
+    relevance: 0.95
+    extra_config:
+      channel_usernames: ["ontologynetwork"]
+      bot_token: "${TELEGRAM_BOT_TOKEN}"
+
+  # Social - Discord (Official project servers)
+  - source_id: "discord:zilliqa"
+    type: "discord"
+    url: "https://discord.gg/zilliqa"
+    cadence_seconds: 60
+    base_credibility: 0.35
+    relevance: 0.95
+    extra_config:
+      channel_ids: []
+      bot_token: "${DISCORD_BOT_TOKEN}"
+
+  - source_id: "discord:harmony"
+    type: "discord"
+    url: "https://discord.gg/harmony"
+    cadence_seconds: 60
+    base_credibility: 0.35
+    relevance: 0.95
+    extra_config:
+      channel_ids: []
+      bot_token: "${DISCORD_BOT_TOKEN}"
+
+  - source_id: "discord:stacks"
+    type: "discord"
+    url: "https://discord.gg/stacks"
+    cadence_seconds: 60
+    base_credibility: 0.35
+    relevance: 0.95
+    extra_config:
+      channel_ids: []
+      bot_token: "${DISCORD_BOT_TOKEN}"
+
+  - source_id: "discord:algorand"
+    type: "discord"
+    url: "https://discord.gg/algorand"
+    cadence_seconds: 60
+    base_credibility: 0.35
+    relevance: 0.95
+    extra_config:
+      channel_ids: []
+      bot_token: "${DISCORD_BOT_TOKEN}"
+
+  - source_id: "discord:dash"
+    type: "discord"
+    url: "https://discord.gg/dash"
+    cadence_seconds: 60
+    base_credibility: 0.35
+    relevance: 0.95
+    extra_config:
+      channel_ids: []
+      bot_token: "${DISCORD_BOT_TOKEN}"
+
+  - source_id: "discord:litecoin"
+    type: "discord"
+    url: "https://discord.gg/litecoin"
+    cadence_seconds: 60
+    base_credibility: 0.35
+    relevance: 0.95
+    extra_config:
+      channel_ids: []
+      bot_token: "${DISCORD_BOT_TOKEN}"
+
+  - source_id: "discord:fetchai"
+    type: "discord"
+    url: "https://discord.gg/fetchai"
+    cadence_seconds: 60
+    base_credibility: 0.35
+    relevance: 0.95
+    extra_config:
+      channel_ids: []
+      bot_token: "${DISCORD_BOT_TOKEN}"
+
+  - source_id: "discord:tezos"
+    type: "discord"
+    url: "https://discord.gg/tezos"
+    cadence_seconds: 60
+    base_credibility: 0.35
+    relevance: 0.95
+    extra_config:
+      channel_ids: []
+      bot_token: "${DISCORD_BOT_TOKEN}"
+
+  - source_id: "discord:enjin"
+    type: "discord"
+    url: "https://discord.gg/enjin"
+    cadence_seconds: 60
+    base_credibility: 0.35
+    relevance: 0.95
+    extra_config:
+      channel_ids: []
+      bot_token: "${DISCORD_BOT_TOKEN}"
+
+  - source_id: "discord:stellar"
+    type: "discord"
+    url: "https://discord.gg/stellar"
+    cadence_seconds: 60
+    base_credibility: 0.35
+    relevance: 0.95
+    extra_config:
+      channel_ids: []
+      bot_token: "${DISCORD_BOT_TOKEN}"
+
+  - source_id: "discord:ethereumclassic"
+    type: "discord"
+    url: "https://discord.gg/ethereumclassic"
+    cadence_seconds: 60
+    base_credibility: 0.35
+    relevance: 0.95
+    extra_config:
+      channel_ids: []
+      bot_token: "${DISCORD_BOT_TOKEN}"
+
+  - source_id: "discord:tron"
+    type: "discord"
+    url: "https://discord.gg/tron"
+    cadence_seconds: 60
+    base_credibility: 0.35
+    relevance: 0.95
+    extra_config:
+      channel_ids: []
+      bot_token: "${DISCORD_BOT_TOKEN}"
+
+  - source_id: "discord:ontology"
+    type: "discord"
+    url: "https://discord.gg/ontology"
+    cadence_seconds: 60
+    base_credibility: 0.35
+    relevance: 0.95
+    extra_config:
+      channel_ids: []
+      bot_token: "${DISCORD_BOT_TOKEN}"
+
  # Web crawl (generic)
  - source_id: "web:coindesk"
    type: "web_crawl"