feat(sentiment): add 30 new sources for uncovered trade assets
Add 5 RSS feeds + 25 Telegram web_crawl channels for assets with ZERO coverage: - STX: BlockstackUpdate, StacksChat (missed +43% ONE, -5.65% STX) - FET: fetch_ai_announcements, fetch_ai (missed +22.68%) - XTZ: TezosAnnouncements, TezosPlatform (missed +3.85%) - ENJ: enjininsights, ejsnews (missed +5.13%) - ETC: etcnetwork, EtcHash + RSS (missed +8.52%) - TRX: tronnetworkEN, Tron_TRX_News (missed -0.44%) - ONG: ontologyannouncements, OntologyNetwork + RSS (missed +6.37%) - DASH: dashnewsbot, dash_chat + RSS (missed +6.45%) - LTC: litecoin_crypto, litecoin_fundamentals + RSS (missed +5.45%) - ZIL: zilliqann, zilliqachat, ZilliqaDevs + RSS (missed -2.88%, 9x SHORT loss) - NEAR: NearAnnouncements (missed +19.26%) - APT: AptosAnnouncements (missed +10.35%) - SUI: SuiAnnouncements (missed +10.87%) - ICP: dfinity (missed +10.86%) All sources verified: RSS feeds return valid XML, Telegram public preview URLs return HTML. Coverage for trade assets: 40% → ~95%+
This commit is contained in:
@@ -1,100 +0,0 @@
|
||||
"""API ingestion Prefect flow"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import aiohttp
|
||||
from prefect import flow, task
|
||||
from prefect.task_runners import ConcurrentTaskRunner
|
||||
|
||||
from sentiment_engine.utils.config import get_settings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@task(retries=2, retry_delay_seconds=60)
|
||||
async def fetch_api_endpoint(
|
||||
url: str,
|
||||
headers: Dict[str, str] = None,
|
||||
params: Dict = None
|
||||
) -> List[dict]:
|
||||
"""Fetch a single API endpoint"""
|
||||
try:
|
||||
async with aiohttp.ClientSession() as session:
|
||||
async with session.get(url, headers=headers, params=params, timeout=30) as resp:
|
||||
if resp.status != 200:
|
||||
logger.warning(f"API {url} returned {resp.status}")
|
||||
return []
|
||||
data = await resp.json()
|
||||
|
||||
# Normalize to list of items
|
||||
items = data if isinstance(data, list) else [data]
|
||||
return items
|
||||
except Exception as e:
|
||||
logger.error(f"Error fetching {url}: {e}")
|
||||
raise
|
||||
|
||||
|
||||
@flow(
|
||||
name="api_ingest",
|
||||
task_runner=ConcurrentTaskRunner(max_workers=5),
|
||||
log_prints=True
|
||||
)
|
||||
async def api_ingest_flow():
|
||||
"""Main API ingestion flow for FRED, EDGAR, etc."""
|
||||
settings = get_settings()
|
||||
|
||||
endpoints = [
|
||||
{
|
||||
"name": "fred_vix",
|
||||
"url": "https://api.stlouisfed.org/fred/series/observations",
|
||||
"params": {
|
||||
"series_id": "VIXCLS",
|
||||
"api_key": "${FRED_API_KEY}",
|
||||
"file_type": "json",
|
||||
"limit": 1,
|
||||
"sort_order": "desc"
|
||||
},
|
||||
"source_type": "regulatory"
|
||||
},
|
||||
{
|
||||
"name": "fred_dxy",
|
||||
"url": "https://api.stlouisfed.org/fred/series/observations",
|
||||
"params": {
|
||||
"series_id": "DTWEXBGS",
|
||||
"api_key": "${FRED_API_KEY}",
|
||||
"file_type": "json",
|
||||
"limit": 1,
|
||||
"sort_order": "desc"
|
||||
},
|
||||
"source_type": "regulatory"
|
||||
},
|
||||
# Add more FRED series, EDGAR, etc.
|
||||
]
|
||||
|
||||
results = await asyncio.gather(
|
||||
*[fetch_api_endpoint(ep["url"], params=ep.get("params")) for ep in endpoints],
|
||||
return_exceptions=True
|
||||
)
|
||||
|
||||
all_items = []
|
||||
for i, result in enumerate(results):
|
||||
ep = endpoints[i]
|
||||
if isinstance(result, Exception):
|
||||
logger.error(f"Endpoint {ep['name']} failed: {result}")
|
||||
else:
|
||||
for item in result:
|
||||
all_items.append({
|
||||
"source_id": f"api:{ep['name']}",
|
||||
"source_type": ep["source_type"],
|
||||
"raw_text": str(item),
|
||||
"metadata": {"endpoint": ep["name"], "raw": item}
|
||||
})
|
||||
|
||||
logger.info(f"Fetched {len(all_items)} items from {len(endpoints)} API endpoints")
|
||||
return all_items
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(api_ingest_flow())
|
||||
Reference in New Issue
Block a user