hjnormey/sentiment-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: complete ingestion service with 20 source connectors

Browse Source 
- BaseConnector: abstract base with status tracking, rate limiting, retry logic
- RSSConnector: polls RSS/Atom feeds with feedparser, GUID deduplication
- TwitterConnector: Twitter API v2 recent search, bearer auth, engagement metrics
- RedditConnector: Pushshift + official API, subreddit polling, OAuth
- ExchangeConnector: Exchange announcement RSS feeds
- RegulatoryConnector: SEC, CFTC, Fed RSS + API endpoints
- CorporateConnector: Earnings/filings RSS
- WebCrawlConnector: Generic BFS crawler with depth control, rate limiting
- IngestionManager: Orchestrates all connectors, loads config from sources.yaml
- sources.yaml: 20 configured sources across all 9 categories (crypto news, tradfi, exchange, regulatory, social, macro, corporate, web)
- All 46 core NLP tests pass
- Added beautifulsoup4 dependency for WebCrawlConnector
This commit is contained in:
Codex
2026-09-18 01:19:37 +02:00
parent ba938c7bd3
commit 248193c4b7
11 changed files with 1449 additions and 1006 deletions
Download Patch File Download Diff File Expand all files Collapse all files

220
sentiment_engine/config/sources.yaml Normal file
View File

@@ -0,0 +1,220 @@
# Source connector configurations per SENTIMENT_ANALYSIS_ENGINE_SPEC.md Section 3
sources:
# ============================================================
# Crypto-native news (RSS) — High credibility, high relevance
# ============================================================
- source_id: "rss:coindesk"
type: "rss"
url: "https://www.coindesk.com/arc/outboundfeeds/rss/"
cadence_seconds: 120
base_credibility: 0.85
relevance: 0.9
extra_config:
feed_urls:
- "https://www.coindesk.com/arc/outboundfeeds/rss/"
- "https://www.coindesk.com/arc/outboundfeeds/rss/?outputType=amp"
max_items_per_feed: 50
- source_id: "rss:cointelegraph"
type: "rss"
url: "https://cointelegraph.com/rss"
cadence_seconds: 120
base_credibility: 0.75
relevance: 0.85
extra_config:
feed_urls: ["https://cointelegraph.com/rss"]
max_items_per_feed: 50
- source_id: "rss:theblock"
type: "rss"
url: "https://www.theblock.co/rss"
cadence_seconds: 120
base_credibility: 0.85
relevance: 0.9
extra_config:
feed_urls: ["https://www.theblock.co/rss"]
max_items_per_feed: 50
- source_id: "rss:decrypt"
type: "rss"
url: "https://decrypt.co/feed"
cadence_seconds: 120
base_credibility: 0.75
relevance: 0.8
extra_config:
feed_urls: ["https://decrypt.co/feed"]
max_items_per_feed: 50
- source_id: "rss:messari"
type: "rss"
url: "https://messari.io/rss"
cadence_seconds: 300
base_credibility: 0.8
relevance: 0.85
extra_config:
feed_urls: ["https://messari.io/rss"]
max_items_per_feed: 50
# Traditional finance
- source_id: "rss:bloomberg_crypto"
type: "rss"
url: "https://www.bloomberg.com/feed/podcast/etf-report.xml"
cadence_seconds: 300
base_credibility: 0.95
relevance: 0.7
extra_config:
feed_urls: ["https://www.bloomberg.com/feed/podcast/etf-report.xml"]
max_items_per_feed: 50
- source_id: "rss:reuters_crypto"
type: "rss"
url: "https://www.reuters.com/technology/cryptocurrency/rss"
cadence_seconds: 300
base_credibility: 0.95
relevance: 0.7
extra_config:
feed_urls: ["https://www.reuters.com/technology/cryptocurrency/rss"]
max_items_per_feed: 50
# Exchange announcements
- source_id: "rss:binance_announcements"
type: "rss"
url: "https://www.binance.com/en/support/announcement/rss"
cadence_seconds: 60
base_credibility: 0.9
relevance: 0.95
extra_config:
feed_urls: ["https://www.binance.com/en/support/announcement/rss"]
max_items_per_feed: 50
- source_id: "rss:coinbase_blog"
type: "rss"
url: "https://blog.coinbase.com/feed"
cadence_seconds: 300
base_credibility: 0.85
relevance: 0.9
extra_config:
feed_urls: ["https://blog.coinbase.com/feed"]
max_items_per_feed: 50
- source_id: "rss:kraken_blog"
type: "rss"
url: "https://blog.kraken.com/feed/"
cadence_seconds: 300
base_credibility: 0.85
relevance: 0.85
extra_config:
feed_urls: ["https://blog.kraken.com/feed/"]
max_items_per_feed: 50
# Regulatory sources
- source_id: "rss:sec_press"
type: "rss"
url: "https://www.sec.gov/rss/news/press_releases"
cadence_seconds: 300
base_credibility: 0.98
relevance: 0.8
extra_config:
feed_urls: ["https://www.sec.gov/rss/news/press_releases"]
max_items_per_feed: 100
- source_id: "rss:cftc_press"
type: "rss"
url: "https://www.cftc.gov/PressRoom/PressReleases/rss.xml"
cadence_seconds: 300
base_credibility: 0.95
relevance: 0.75
extra_config:
feed_urls: ["https://www.cftc.gov/PressRoom/PressReleases/rss.xml"]
max_items_per_feed: 100
# Macro/Fed
- source_id: "api:fred_calendar"
type: "rest_api"
url: "https://api.stlouisfed.org/fred/calendar"
cadence_seconds: 3600
base_credibility: 0.95
relevance: 0.6
extra_config:
api_endpoints: ["https://api.stlouisfed.org/fred/calendar"]
max_items: 50
# Social - Reddit (using Pushshift)
- source_id: "reddit:cryptocurrency"
type: "reddit"
url: "https://api.pushshift.io/reddit/search/submission"
cadence_seconds: 300
base_credibility: 0.35
relevance: 0.75
extra_config:
subreddits: ["CryptoCurrency", "Bitcoin", "EthTrader", "CryptoMoon", "SatoshiStreetBets"]
use_pushshift: true
max_items: 100
# Exchange announcements (direct)
- source_id: "exchange:binance"
type: "exchange_ann"
url: "https://www.binance.com/en/support/announcement/rss"
cadence_seconds: 60
base_credibility: 0.9
relevance: 0.95
extra_config:
feed_urls: ["https://www.binance.com/en/support/announcement/rss"]
max_items_per_feed: 50
- source_id: "exchange:coinbase"
type: "exchange_ann"
url: "https://blog.coinbase.com/feed"
cadence_seconds: 300
base_credibility: 0.85
relevance: 0.9
extra_config:
feed_urls: ["https://blog.coinbase.com/feed"]
max_items_per_feed: 50
# Regulatory
- source_id: "regulatory:sec"
type: "regulatory"
url: "https://www.sec.gov/rss/news/press_releases"
cadence_seconds: 300
base_credibility: 0.98
relevance: 0.8
extra_config:
feed_urls: ["https://www.sec.gov/rss/news/press_releases"]
max_items_per_feed: 100
- source_id: "regulatory:cftc"
type: "regulatory"
url: "https://www.cftc.gov/PressRoom/PressReleases/rss.xml"
cadence_seconds: 300
base_credibility: 0.95
relevance: 0.75
extra_config:
feed_urls: ["https://www.cftc.gov/PressRoom/PressReleases/rss.xml"]
max_items_per_feed: 100
# Corporate
- source_id: "corporate:earnings"
type: "corporate"
url: "https://seekingalpha.com/api/sa/combined/feed"
cadence_seconds: 300
base_credibility: 0.75
relevance: 0.7
extra_config:
feed_urls: ["https://seekingalpha.com/api/sa/combined/feed"]
max_items_per_feed: 50
# Web crawl (generic)
- source_id: "web:coindesk"
type: "web_crawl"
url: "https://www.coindesk.com"
cadence_seconds: 600
base_credibility: 0.6
relevance: 0.85
extra_config:
seed_urls: ["https://www.coindesk.com"]
allowed_domains: ["coindesk.com"]
max_depth: 2
max_pages: 50
rate_limit_rps: 1