Add sentiment_engine with CryptoSentimentCalibrator fixes - improved keyword lists, lowered FinBERT threshold, added neutral handling

2026-09-14 13:30:05 +02:00
parent 19a7812094
commit a276aeaded
149 changed files with 35226 additions and 0 deletions
--- a/sentiment_engine/prefect_flows/connectors/api_ingest.py
+++ b/sentiment_engine/prefect_flows/connectors/api_ingest.py
@@ -0,0 +1,100 @@
+"""API ingestion Prefect flow"""
+
+import asyncio
+import logging
+from typing import Dict, List, Optional
+
+import aiohttp
+from prefect import flow, task
+from prefect.task_runners import ConcurrentTaskRunner
+
+from sentiment_engine.utils.config import get_settings
+
+logger = logging.getLogger(__name__)
+
+
+@task(retries=2, retry_delay_seconds=60)
+async def fetch_api_endpoint(
+    url: str,
+    headers: Dict[str, str] = None,
+    params: Dict = None
+) -> List[dict]:
+    """Fetch a single API endpoint"""
+    try:
+        async with aiohttp.ClientSession() as session:
+            async with session.get(url, headers=headers, params=params, timeout=30) as resp:
+                if resp.status != 200:
+                    logger.warning(f"API {url} returned {resp.status}")
+                    return []
+                data = await resp.json()
+
+        # Normalize to list of items
+        items = data if isinstance(data, list) else [data]
+        return items
+    except Exception as e:
+        logger.error(f"Error fetching {url}: {e}")
+        raise
+
+
+@flow(
+    name="api_ingest",
+    task_runner=ConcurrentTaskRunner(max_workers=5),
+    log_prints=True
+)
+async def api_ingest_flow():
+    """Main API ingestion flow for FRED, EDGAR, etc."""
+    settings = get_settings()
+
+    endpoints = [
+        {
+            "name": "fred_vix",
+            "url": "https://api.stlouisfed.org/fred/series/observations",
+            "params": {
+                "series_id": "VIXCLS",
+                "api_key": "${FRED_API_KEY}",
+                "file_type": "json",
+                "limit": 1,
+                "sort_order": "desc"
+            },
+            "source_type": "regulatory"
+        },
+        {
+            "name": "fred_dxy",
+            "url": "https://api.stlouisfed.org/fred/series/observations",
+            "params": {
+                "series_id": "DTWEXBGS",
+                "api_key": "${FRED_API_KEY}",
+                "file_type": "json",
+                "limit": 1,
+                "sort_order": "desc"
+            },
+            "source_type": "regulatory"
+        },
+        # Add more FRED series, EDGAR, etc.
+    ]
+
+    results = await asyncio.gather(
+        *[fetch_api_endpoint(ep["url"], params=ep.get("params")) for ep in endpoints],
+        return_exceptions=True
+    )
+
+    all_items = []
+    for i, result in enumerate(results):
+        ep = endpoints[i]
+        if isinstance(result, Exception):
+            logger.error(f"Endpoint {ep['name']} failed: {result}")
+        else:
+            for item in result:
+                all_items.append({
+                    "source_id": f"api:{ep['name']}",
+                    "source_type": ep["source_type"],
+                    "raw_text": str(item),
+                    "metadata": {"endpoint": ep["name"], "raw": item}
+                })
+
+    logger.info(f"Fetched {len(all_items)} items from {len(endpoints)} API endpoints")
+    return all_items
+
+
+if __name__ == "__main__":
+    asyncio.run(api_ingest_flow())