262 lines
9.2 KiB
Python
262 lines
9.2 KiB
Python
|
"""E2E tests for the full sentiment engine pipeline"""
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
import asyncio
|
||
|
|
import time
|
||
|
|
import tempfile
|
||
|
|
import os
|
||
|
|
from datetime import datetime
|
||
|
|
|
||
|
|
from sentiment_engine.catalogue.manager import CatalogueManager
|
||
|
|
from sentiment_engine.ingestion.router import IngestionRouter
|
||
|
|
from sentiment_engine.utils.config import get_settings
|
||
|
|
from sentiment_engine.schemas.payload import NormalizedPayload, SourceType, AssetMention, EngagementMetrics
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
async def catalogue():
|
||
|
|
cat = CatalogueManager()
|
||
|
|
await cat.initialize()
|
||
|
|
yield cat
|
||
|
|
await cat.stop()
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
async def router(catalogue):
|
||
|
|
settings = get_settings()
|
||
|
|
router = IngestionRouter(
|
||
|
|
nats_servers=settings.nats_servers,
|
||
|
|
stream_name=settings.nats_stream_ingestion,
|
||
|
|
subject_map={
|
||
|
|
"news": "sentiment.ingest.news",
|
||
|
|
"social": "sentiment.ingest.social",
|
||
|
|
"regulatory": "sentiment.ingest.regulatory",
|
||
|
|
"exchange": "sentiment.ingest.exchange",
|
||
|
|
},
|
||
|
|
catalogue=catalogue
|
||
|
|
)
|
||
|
|
await router.connect()
|
||
|
|
yield router
|
||
|
|
await router._nc.close()
|
||
|
|
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_rss_to_nats_pipeline():
|
||
|
|
"""E2E test: RSS ingestion -> NATS -> catalogue"""
|
||
|
|
import yaml
|
||
|
|
import asyncio
|
||
|
|
import sys
|
||
|
|
sys.path.insert(0, '/mnt/dolphinng5_predict/sentiment_engine/src')
|
||
|
|
|
||
|
|
from sentiment_engine.catalogue.manager import CatalogueManager
|
||
|
|
from sentiment_engine.ingestion.router import IngestionRouter
|
||
|
|
|
||
|
|
# Load settings from YAML
|
||
|
|
with open('/mnt/dolphinng5_predict/sentiment_engine/config/settings.yaml') as f:
|
||
|
|
settings = yaml.safe_load(f)
|
||
|
|
|
||
|
|
# 1. Catalogue
|
||
|
|
cat = CatalogueManager()
|
||
|
|
await cat.initialize()
|
||
|
|
assert len(cat.catalogue.get_sources()) == 14
|
||
|
|
|
||
|
|
# 2. Ingestion Router - pass CatalogueManager, not store
|
||
|
|
router = IngestionRouter(
|
||
|
|
nats_servers=settings['nats']['servers'],
|
||
|
|
stream_name=settings['nats']['stream_ingestion'],
|
||
|
|
subject_map={
|
||
|
|
'news': 'sentiment.ingest.news',
|
||
|
|
'social': 'sentiment.ingest.social',
|
||
|
|
'regulatory': 'sentiment.ingest.regulatory',
|
||
|
|
'exchange': 'sentiment.ingest.exchange',
|
||
|
|
},
|
||
|
|
catalogue=cat # Pass CatalogueManager, not store
|
||
|
|
)
|
||
|
|
await router.connect()
|
||
|
|
|
||
|
|
# 3. Route a test payload
|
||
|
|
test_payload = NormalizedPayload(
|
||
|
|
source_id="rss:coindesk.com",
|
||
|
|
source_type=SourceType.NEWS,
|
||
|
|
source_credibility_base=0.85,
|
||
|
|
ingest_ts=time.time(),
|
||
|
|
publish_ts=time.time(),
|
||
|
|
asset_mentions=[AssetMention(asset_id="BTC", mention_span=(0, 3), confidence=0.9, source_text="BTC", mention_type="ticker")],
|
||
|
|
raw_text="BTC surges to new highs as institutional adoption accelerates. Bitcoin breaks $70k resistance with massive volume.",
|
||
|
|
title="Bitcoin Breaks $70k",
|
||
|
|
url="https://coindesk.com/btc-surge",
|
||
|
|
author="Test Author",
|
||
|
|
content_length=200,
|
||
|
|
language="en",
|
||
|
|
metadata={"test": True}
|
||
|
|
)
|
||
|
|
|
||
|
|
# Route the payload
|
||
|
|
result = await router.route(test_payload)
|
||
|
|
assert result is True
|
||
|
|
|
||
|
|
# Verify fetch was recorded
|
||
|
|
source = cat.catalogue.get_source("rss:coindesk.com")
|
||
|
|
assert source is not None
|
||
|
|
assert source.total_fetches >= 1
|
||
|
|
assert source.successful_fetches >= 1
|
||
|
|
|
||
|
|
await router._nc.close()
|
||
|
|
await cat.stop()
|
||
|
|
|
||
|
|
|
||
|
|
class TestFullIngestionFlow:
|
||
|
|
"""Test complete ingestion flow with multiple sources"""
|
||
|
|
|
||
|
|
@pytest.fixture(autouse=True)
|
||
|
|
async def setup_teardown(self):
|
||
|
|
"""Setup and teardown for each test"""
|
||
|
|
self.cat = CatalogueManager()
|
||
|
|
await self.cat.initialize()
|
||
|
|
|
||
|
|
settings = get_settings()
|
||
|
|
self.router = IngestionRouter(
|
||
|
|
nats_servers=settings.nats_servers,
|
||
|
|
stream_name=settings.nats_stream_ingestion,
|
||
|
|
subject_map={
|
||
|
|
"news": "sentiment.ingest.news",
|
||
|
|
"social": "sentiment.ingest.social",
|
||
|
|
"regulatory": "sentiment.ingest.regulatory",
|
||
|
|
"exchange": "sentiment.ingest.exchange",
|
||
|
|
},
|
||
|
|
catalogue=self.cat # Pass CatalogueManager, not store
|
||
|
|
)
|
||
|
|
await self.router.connect()
|
||
|
|
|
||
|
|
yield
|
||
|
|
|
||
|
|
await self.router._nc.close()
|
||
|
|
await self.cat.stop()
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_multi_source_ingestion(self):
|
||
|
|
"""Test ingestion from multiple source types"""
|
||
|
|
import time
|
||
|
|
|
||
|
|
payloads = [
|
||
|
|
NormalizedPayload(
|
||
|
|
source_id="rss:coindesk.com",
|
||
|
|
source_type=SourceType.NEWS,
|
||
|
|
source_credibility_base=0.85,
|
||
|
|
ingest_ts=time.time(),
|
||
|
|
publish_ts=time.time(),
|
||
|
|
asset_mentions=[AssetMention(asset_id="BTC", mention_span=(0, 3), confidence=0.9, source_text="BTC", mention_type="ticker")],
|
||
|
|
raw_text="Bitcoin reaches new all-time high as ETF inflows continue to surge.",
|
||
|
|
title="BTC New ATH",
|
||
|
|
url="https://coindesk.com/btc-ath",
|
||
|
|
author="Author 1",
|
||
|
|
content_length=150,
|
||
|
|
language="en",
|
||
|
|
metadata={}
|
||
|
|
),
|
||
|
|
NormalizedPayload(
|
||
|
|
source_id="reddit:Bitcoin",
|
||
|
|
source_type=SourceType.SOCIAL,
|
||
|
|
source_credibility_base=0.65,
|
||
|
|
ingest_ts=time.time(),
|
||
|
|
publish_ts=time.time(),
|
||
|
|
asset_mentions=[AssetMention(asset_id="ETH", mention_span=(0, 3), confidence=0.85, source_text="ETH", mention_type="ticker")],
|
||
|
|
raw_text="Ethereum merge successful, staking rewards now live.",
|
||
|
|
title="ETH Merge Complete",
|
||
|
|
url="https://reddit.com/r/bitcoin/comments/xxx",
|
||
|
|
author="redditor123",
|
||
|
|
content_length=200,
|
||
|
|
language="en",
|
||
|
|
metadata={}
|
||
|
|
),
|
||
|
|
NormalizedPayload(
|
||
|
|
source_id="api:fred_vix",
|
||
|
|
source_type=SourceType.REGULATORY,
|
||
|
|
source_credibility_base=0.95,
|
||
|
|
ingest_ts=time.time(),
|
||
|
|
publish_ts=time.time(),
|
||
|
|
asset_mentions=[AssetMention(asset_id="SPX", mention_span=(0, 3), confidence=0.8, source_text="SPX", mention_type="ticker")],
|
||
|
|
raw_text="VIX drops below 15 as market volatility decreases.",
|
||
|
|
title="VIX Low",
|
||
|
|
url="https://fred.stlouisfed.org/series/VIXCLS",
|
||
|
|
author="FRED",
|
||
|
|
content_length=100,
|
||
|
|
language="en",
|
||
|
|
metadata={}
|
||
|
|
)
|
||
|
|
]
|
||
|
|
|
||
|
|
# Route all payloads
|
||
|
|
routed = await self.router.route_batch(payloads)
|
||
|
|
assert routed == 3
|
||
|
|
|
||
|
|
# Verify all sources recorded fetches
|
||
|
|
for source_id in ["rss:coindesk.com", "reddit:Bitcoin", "api:fred_vix"]:
|
||
|
|
source = self.cat.catalogue.get_source(source_id)
|
||
|
|
assert source is not None, f"Source {source_id} not found"
|
||
|
|
assert source.total_fetches >= 1, f"No fetches recorded for {source_id}"
|
||
|
|
assert source.successful_fetches >= 1, f"No successful fetches for {source_id}"
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_duplicate_detection(self):
|
||
|
|
"""Test that duplicate payloads are detected and not re-routed"""
|
||
|
|
import time
|
||
|
|
import tempfile
|
||
|
|
import os
|
||
|
|
|
||
|
|
# Create a fresh catalogue with temporary database for this test
|
||
|
|
db_path = tempfile.mktemp(suffix='.duckdb')
|
||
|
|
cat = CatalogueManager(db_path=db_path)
|
||
|
|
await cat.initialize()
|
||
|
|
|
||
|
|
settings = get_settings()
|
||
|
|
router = IngestionRouter(
|
||
|
|
nats_servers=settings.nats_servers,
|
||
|
|
stream_name=settings.nats_stream_ingestion,
|
||
|
|
subject_map={
|
||
|
|
"news": "sentiment.ingest.news",
|
||
|
|
"social": "sentiment.ingest.social",
|
||
|
|
"regulatory": "sentiment.ingest.regulatory",
|
||
|
|
"exchange": "sentiment.ingest.exchange",
|
||
|
|
},
|
||
|
|
catalogue=cat
|
||
|
|
)
|
||
|
|
await router.connect()
|
||
|
|
|
||
|
|
payload = NormalizedPayload(
|
||
|
|
source_id="rss:coindesk.com",
|
||
|
|
source_type=SourceType.NEWS,
|
||
|
|
source_credibility_base=0.85,
|
||
|
|
ingest_ts=time.time(),
|
||
|
|
publish_ts=time.time(),
|
||
|
|
asset_mentions=[AssetMention(asset_id="BTC", mention_span=(0, 3), confidence=0.9, source_text="BTC", mention_type="ticker")],
|
||
|
|
raw_text="Duplicate test payload for deduplication check.",
|
||
|
|
title="Duplicate Test",
|
||
|
|
url="https://coindesk.com/duplicate",
|
||
|
|
author="Test",
|
||
|
|
content_length=100,
|
||
|
|
language="en",
|
||
|
|
metadata={}
|
||
|
|
)
|
||
|
|
|
||
|
|
# Route same payload twice
|
||
|
|
result1 = await router.route(payload)
|
||
|
|
result2 = await router.route(payload)
|
||
|
|
|
||
|
|
assert result1 is True
|
||
|
|
assert result2 is False # Second should be detected as duplicate
|
||
|
|
|
||
|
|
# Only one fetch should be recorded
|
||
|
|
source = cat.catalogue.get_source("rss:coindesk.com")
|
||
|
|
assert source.total_fetches == 1
|
||
|
|
assert source.successful_fetches == 1
|
||
|
|
|
||
|
|
await router._nc.close()
|
||
|
|
await cat.stop()
|
||
|
|
# Clean up temp database
|
||
|
|
try:
|
||
|
|
os.unlink(db_path)
|
||
|
|
except:
|
||
|
|
pass
|
||
|
|
|