101 lines
2.9 KiB
Python
101 lines
2.9 KiB
Python
|
"""API ingestion Prefect flow"""
|
||
|
|
|
||
|
|
import asyncio
|
||
|
|
import logging
|
||
|
|
from typing import Dict, List, Optional
|
||
|
|
|
||
|
|
import aiohttp
|
||
|
|
from prefect import flow, task
|
||
|
|
from prefect.task_runners import ConcurrentTaskRunner
|
||
|
|
|
||
|
|
from sentiment_engine.utils.config import get_settings
|
||
|
|
|
||
|
|
logger = logging.getLogger(__name__)
|
||
|
|
|
||
|
|
|
||
|
|
@task(retries=2, retry_delay_seconds=60)
|
||
|
|
async def fetch_api_endpoint(
|
||
|
|
url: str,
|
||
|
|
headers: Dict[str, str] = None,
|
||
|
|
params: Dict = None
|
||
|
|
) -> List[dict]:
|
||
|
|
"""Fetch a single API endpoint"""
|
||
|
|
try:
|
||
|
|
async with aiohttp.ClientSession() as session:
|
||
|
|
async with session.get(url, headers=headers, params=params, timeout=30) as resp:
|
||
|
|
if resp.status != 200:
|
||
|
|
logger.warning(f"API {url} returned {resp.status}")
|
||
|
|
return []
|
||
|
|
data = await resp.json()
|
||
|
|
|
||
|
|
# Normalize to list of items
|
||
|
|
items = data if isinstance(data, list) else [data]
|
||
|
|
return items
|
||
|
|
except Exception as e:
|
||
|
|
logger.error(f"Error fetching {url}: {e}")
|
||
|
|
raise
|
||
|
|
|
||
|
|
|
||
|
|
@flow(
|
||
|
|
name="api_ingest",
|
||
|
|
task_runner=ConcurrentTaskRunner(max_workers=5),
|
||
|
|
log_prints=True
|
||
|
|
)
|
||
|
|
async def api_ingest_flow():
|
||
|
|
"""Main API ingestion flow for FRED, EDGAR, etc."""
|
||
|
|
settings = get_settings()
|
||
|
|
|
||
|
|
endpoints = [
|
||
|
|
{
|
||
|
|
"name": "fred_vix",
|
||
|
|
"url": "https://api.stlouisfed.org/fred/series/observations",
|
||
|
|
"params": {
|
||
|
|
"series_id": "VIXCLS",
|
||
|
|
"api_key": "${FRED_API_KEY}",
|
||
|
|
"file_type": "json",
|
||
|
|
"limit": 1,
|
||
|
|
"sort_order": "desc"
|
||
|
|
},
|
||
|
|
"source_type": "regulatory"
|
||
|
|
},
|
||
|
|
{
|
||
|
|
"name": "fred_dxy",
|
||
|
|
"url": "https://api.stlouisfed.org/fred/series/observations",
|
||
|
|
"params": {
|
||
|
|
"series_id": "DTWEXBGS",
|
||
|
|
"api_key": "${FRED_API_KEY}",
|
||
|
|
"file_type": "json",
|
||
|
|
"limit": 1,
|
||
|
|
"sort_order": "desc"
|
||
|
|
},
|
||
|
|
"source_type": "regulatory"
|
||
|
|
},
|
||
|
|
# Add more FRED series, EDGAR, etc.
|
||
|
|
]
|
||
|
|
|
||
|
|
results = await asyncio.gather(
|
||
|
|
*[fetch_api_endpoint(ep["url"], params=ep.get("params")) for ep in endpoints],
|
||
|
|
return_exceptions=True
|
||
|
|
)
|
||
|
|
|
||
|
|
all_items = []
|
||
|
|
for i, result in enumerate(results):
|
||
|
|
ep = endpoints[i]
|
||
|
|
if isinstance(result, Exception):
|
||
|
|
logger.error(f"Endpoint {ep['name']} failed: {result}")
|
||
|
|
else:
|
||
|
|
for item in result:
|
||
|
|
all_items.append({
|
||
|
|
"source_id": f"api:{ep['name']}",
|
||
|
|
"source_type": ep["source_type"],
|
||
|
|
"raw_text": str(item),
|
||
|
|
"metadata": {"endpoint": ep["name"], "raw": item}
|
||
|
|
})
|
||
|
|
|
||
|
|
logger.info(f"Fetched {len(all_items)} items from {len(endpoints)} API endpoints")
|
||
|
|
return all_items
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
asyncio.run(api_ingest_flow())
|