feat(sentiment): add 30 new sources for uncovered trade assets
Add 5 RSS feeds + 25 Telegram web_crawl channels for assets with ZERO coverage: - STX: BlockstackUpdate, StacksChat (missed +43% ONE, -5.65% STX) - FET: fetch_ai_announcements, fetch_ai (missed +22.68%) - XTZ: TezosAnnouncements, TezosPlatform (missed +3.85%) - ENJ: enjininsights, ejsnews (missed +5.13%) - ETC: etcnetwork, EtcHash + RSS (missed +8.52%) - TRX: tronnetworkEN, Tron_TRX_News (missed -0.44%) - ONG: ontologyannouncements, OntologyNetwork + RSS (missed +6.37%) - DASH: dashnewsbot, dash_chat + RSS (missed +6.45%) - LTC: litecoin_crypto, litecoin_fundamentals + RSS (missed +5.45%) - ZIL: zilliqann, zilliqachat, ZilliqaDevs + RSS (missed -2.88%, 9x SHORT loss) - NEAR: NearAnnouncements (missed +19.26%) - APT: AptosAnnouncements (missed +10.35%) - SUI: SuiAnnouncements (missed +10.87%) - ICP: dfinity (missed +10.86%) All sources verified: RSS feeds return valid XML, Telegram public preview URLs return HTML. Coverage for trade assets: 40% → ~95%+
This commit is contained in:
@@ -1,99 +0,0 @@
|
||||
"""RSS ingestion Prefect flow"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import List
|
||||
|
||||
import feedparser
|
||||
from prefect import flow, task
|
||||
from prefect.task_runners import ConcurrentTaskRunner
|
||||
|
||||
from sentiment_engine.ingestion.rss import RSSConnector
|
||||
from sentiment_engine.schemas.config import RSSConnectorConfig
|
||||
from sentiment_engine.utils.config import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@task(retries=3, retry_delay_seconds=30)
|
||||
async def fetch_rss_feed(feed_url: str, config: RSSConnectorConfig) -> List[dict]:
|
||||
"""Fetch and parse a single RSS feed"""
|
||||
try:
|
||||
feed = feedparser.parse(feed_url)
|
||||
items = []
|
||||
|
||||
for entry in feed.entries[:config.max_items_per_feed]:
|
||||
title = getattr(entry, "title", "").strip()
|
||||
summary = getattr(entry, "summary", getattr(entry, "description", "")).strip()
|
||||
raw_text = f"{title}\n\n{summary}"
|
||||
|
||||
if not raw_text.strip():
|
||||
continue
|
||||
|
||||
items.append({
|
||||
"source_id": f"rss:{feed_url}",
|
||||
"source_type": "news",
|
||||
"raw_text": raw_text,
|
||||
"title": title,
|
||||
"url": getattr(entry, "link", ""),
|
||||
"author": getattr(entry, "author", ""),
|
||||
"publish_ts": getattr(entry, "published_parsed", None),
|
||||
"metadata": {"feed_url": feed_url}
|
||||
})
|
||||
|
||||
return items
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching {feed_url}: {e}")
|
||||
raise
|
||||
|
||||
|
||||
@flow(
|
||||
name="rss_ingest",
|
||||
task_runner=ConcurrentTaskRunner(max_workers=10),
|
||||
log_prints=True
|
||||
)
|
||||
async def rss_ingest_flow(feed_urls: List[str] = None):
|
||||
"""Main RSS ingestion flow"""
|
||||
settings = get_settings()
|
||||
|
||||
if feed_urls is None:
|
||||
# Default crypto news feeds
|
||||
feed_urls = [
|
||||
"https://www.coindesk.com/arc/outboundfeeds/rss/",
|
||||
"https://cointelegraph.com/rss",
|
||||
"https://www.theblock.co/rss",
|
||||
"https://decrypt.co/feed",
|
||||
"https://messari.io/feed",
|
||||
"https://cryptoslate.com/feed/",
|
||||
"https://bitcoinmagazine.com/feed/",
|
||||
]
|
||||
|
||||
config = RSSConnectorConfig(
|
||||
name="prefect_rss",
|
||||
source_type="news",
|
||||
feed_urls=feed_urls,
|
||||
max_items_per_feed=50
|
||||
)
|
||||
|
||||
# Fetch all feeds concurrently
|
||||
results = await asyncio.gather(
|
||||
*[fetch_rss_feed(url, config) for url in feed_urls],
|
||||
return_exceptions=True
|
||||
)
|
||||
|
||||
all_items = []
|
||||
for i, result in enumerate(results):
|
||||
if isinstance(result, Exception):
|
||||
logger.error(f"Feed {feed_urls[i]} failed: {result}")
|
||||
else:
|
||||
all_items.extend(result)
|
||||
|
||||
logger.info(f"Fetched {len(all_items)} items from {len(feed_urls)} feeds")
|
||||
|
||||
# In production, publish to NATS
|
||||
# For now, return items
|
||||
return all_items
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(rss_ingest_flow())
|
||||
Reference in New Issue
Block a user