Add sentiment_engine with CryptoSentimentCalibrator fixes - improved keyword lists, lowered FinBERT threshold, added neutral handling
This commit is contained in:
Codex
0
sentiment_engine/tests/e2e/__init__.py
Normal file
0
sentiment_engine/tests/e2e/__init__.py
Normal file
261
sentiment_engine/tests/e2e/test_full_pipeline.py
Normal file
261
sentiment_engine/tests/e2e/test_full_pipeline.py
Normal file
@@ -0,0 +1,261 @@
|
||||
"""E2E tests for the full sentiment engine pipeline"""
|
||||
|
||||
import pytest
|
||||
import asyncio
|
||||
import time
|
||||
import tempfile
|
||||
import os
|
||||
from datetime import datetime
|
||||
|
||||
from sentiment_engine.catalogue.manager import CatalogueManager
|
||||
from sentiment_engine.ingestion.router import IngestionRouter
|
||||
from sentiment_engine.utils.config import get_settings
|
||||
from sentiment_engine.schemas.payload import NormalizedPayload, SourceType, AssetMention, EngagementMetrics
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def catalogue():
|
||||
cat = CatalogueManager()
|
||||
await cat.initialize()
|
||||
yield cat
|
||||
await cat.stop()
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
async def router(catalogue):
|
||||
settings = get_settings()
|
||||
router = IngestionRouter(
|
||||
nats_servers=settings.nats_servers,
|
||||
stream_name=settings.nats_stream_ingestion,
|
||||
subject_map={
|
||||
"news": "sentiment.ingest.news",
|
||||
"social": "sentiment.ingest.social",
|
||||
"regulatory": "sentiment.ingest.regulatory",
|
||||
"exchange": "sentiment.ingest.exchange",
|
||||
},
|
||||
catalogue=catalogue
|
||||
)
|
||||
await router.connect()
|
||||
yield router
|
||||
await router._nc.close()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_rss_to_nats_pipeline():
|
||||
"""E2E test: RSS ingestion -> NATS -> catalogue"""
|
||||
import yaml
|
||||
import asyncio
|
||||
import sys
|
||||
sys.path.insert(0, '/mnt/dolphinng5_predict/sentiment_engine/src')
|
||||
|
||||
from sentiment_engine.catalogue.manager import CatalogueManager
|
||||
from sentiment_engine.ingestion.router import IngestionRouter
|
||||
|
||||
# Load settings from YAML
|
||||
with open('/mnt/dolphinng5_predict/sentiment_engine/config/settings.yaml') as f:
|
||||
settings = yaml.safe_load(f)
|
||||
|
||||
# 1. Catalogue
|
||||
cat = CatalogueManager()
|
||||
await cat.initialize()
|
||||
assert len(cat.catalogue.get_sources()) == 14
|
||||
|
||||
# 2. Ingestion Router - pass CatalogueManager, not store
|
||||
router = IngestionRouter(
|
||||
nats_servers=settings['nats']['servers'],
|
||||
stream_name=settings['nats']['stream_ingestion'],
|
||||
subject_map={
|
||||
'news': 'sentiment.ingest.news',
|
||||
'social': 'sentiment.ingest.social',
|
||||
'regulatory': 'sentiment.ingest.regulatory',
|
||||
'exchange': 'sentiment.ingest.exchange',
|
||||
},
|
||||
catalogue=cat # Pass CatalogueManager, not store
|
||||
)
|
||||
await router.connect()
|
||||
|
||||
# 3. Route a test payload
|
||||
test_payload = NormalizedPayload(
|
||||
source_id="rss:coindesk.com",
|
||||
source_type=SourceType.NEWS,
|
||||
source_credibility_base=0.85,
|
||||
ingest_ts=time.time(),
|
||||
publish_ts=time.time(),
|
||||
asset_mentions=[AssetMention(asset_id="BTC", mention_span=(0, 3), confidence=0.9, source_text="BTC", mention_type="ticker")],
|
||||
raw_text="BTC surges to new highs as institutional adoption accelerates. Bitcoin breaks $70k resistance with massive volume.",
|
||||
title="Bitcoin Breaks $70k",
|
||||
url="https://coindesk.com/btc-surge",
|
||||
author="Test Author",
|
||||
content_length=200,
|
||||
language="en",
|
||||
metadata={"test": True}
|
||||
)
|
||||
|
||||
# Route the payload
|
||||
result = await router.route(test_payload)
|
||||
assert result is True
|
||||
|
||||
# Verify fetch was recorded
|
||||
source = cat.catalogue.get_source("rss:coindesk.com")
|
||||
assert source is not None
|
||||
assert source.total_fetches >= 1
|
||||
assert source.successful_fetches >= 1
|
||||
|
||||
await router._nc.close()
|
||||
await cat.stop()
|
||||
|
||||
|
||||
class TestFullIngestionFlow:
|
||||
"""Test complete ingestion flow with multiple sources"""
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
async def setup_teardown(self):
|
||||
"""Setup and teardown for each test"""
|
||||
self.cat = CatalogueManager()
|
||||
await self.cat.initialize()
|
||||
|
||||
settings = get_settings()
|
||||
self.router = IngestionRouter(
|
||||
nats_servers=settings.nats_servers,
|
||||
stream_name=settings.nats_stream_ingestion,
|
||||
subject_map={
|
||||
"news": "sentiment.ingest.news",
|
||||
"social": "sentiment.ingest.social",
|
||||
"regulatory": "sentiment.ingest.regulatory",
|
||||
"exchange": "sentiment.ingest.exchange",
|
||||
},
|
||||
catalogue=self.cat # Pass CatalogueManager, not store
|
||||
)
|
||||
await self.router.connect()
|
||||
|
||||
yield
|
||||
|
||||
await self.router._nc.close()
|
||||
await self.cat.stop()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_multi_source_ingestion(self):
|
||||
"""Test ingestion from multiple source types"""
|
||||
import time
|
||||
|
||||
payloads = [
|
||||
NormalizedPayload(
|
||||
source_id="rss:coindesk.com",
|
||||
source_type=SourceType.NEWS,
|
||||
source_credibility_base=0.85,
|
||||
ingest_ts=time.time(),
|
||||
publish_ts=time.time(),
|
||||
asset_mentions=[AssetMention(asset_id="BTC", mention_span=(0, 3), confidence=0.9, source_text="BTC", mention_type="ticker")],
|
||||
raw_text="Bitcoin reaches new all-time high as ETF inflows continue to surge.",
|
||||
title="BTC New ATH",
|
||||
url="https://coindesk.com/btc-ath",
|
||||
author="Author 1",
|
||||
content_length=150,
|
||||
language="en",
|
||||
metadata={}
|
||||
),
|
||||
NormalizedPayload(
|
||||
source_id="reddit:Bitcoin",
|
||||
source_type=SourceType.SOCIAL,
|
||||
source_credibility_base=0.65,
|
||||
ingest_ts=time.time(),
|
||||
publish_ts=time.time(),
|
||||
asset_mentions=[AssetMention(asset_id="ETH", mention_span=(0, 3), confidence=0.85, source_text="ETH", mention_type="ticker")],
|
||||
raw_text="Ethereum merge successful, staking rewards now live.",
|
||||
title="ETH Merge Complete",
|
||||
url="https://reddit.com/r/bitcoin/comments/xxx",
|
||||
author="redditor123",
|
||||
content_length=200,
|
||||
language="en",
|
||||
metadata={}
|
||||
),
|
||||
NormalizedPayload(
|
||||
source_id="api:fred_vix",
|
||||
source_type=SourceType.REGULATORY,
|
||||
source_credibility_base=0.95,
|
||||
ingest_ts=time.time(),
|
||||
publish_ts=time.time(),
|
||||
asset_mentions=[AssetMention(asset_id="SPX", mention_span=(0, 3), confidence=0.8, source_text="SPX", mention_type="ticker")],
|
||||
raw_text="VIX drops below 15 as market volatility decreases.",
|
||||
title="VIX Low",
|
||||
url="https://fred.stlouisfed.org/series/VIXCLS",
|
||||
author="FRED",
|
||||
content_length=100,
|
||||
language="en",
|
||||
metadata={}
|
||||
)
|
||||
]
|
||||
|
||||
# Route all payloads
|
||||
routed = await self.router.route_batch(payloads)
|
||||
assert routed == 3
|
||||
|
||||
# Verify all sources recorded fetches
|
||||
for source_id in ["rss:coindesk.com", "reddit:Bitcoin", "api:fred_vix"]:
|
||||
source = self.cat.catalogue.get_source(source_id)
|
||||
assert source is not None, f"Source {source_id} not found"
|
||||
assert source.total_fetches >= 1, f"No fetches recorded for {source_id}"
|
||||
assert source.successful_fetches >= 1, f"No successful fetches for {source_id}"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_duplicate_detection(self):
|
||||
"""Test that duplicate payloads are detected and not re-routed"""
|
||||
import time
|
||||
import tempfile
|
||||
import os
|
||||
|
||||
# Create a fresh catalogue with temporary database for this test
|
||||
db_path = tempfile.mktemp(suffix='.duckdb')
|
||||
cat = CatalogueManager(db_path=db_path)
|
||||
await cat.initialize()
|
||||
|
||||
settings = get_settings()
|
||||
router = IngestionRouter(
|
||||
nats_servers=settings.nats_servers,
|
||||
stream_name=settings.nats_stream_ingestion,
|
||||
subject_map={
|
||||
"news": "sentiment.ingest.news",
|
||||
"social": "sentiment.ingest.social",
|
||||
"regulatory": "sentiment.ingest.regulatory",
|
||||
"exchange": "sentiment.ingest.exchange",
|
||||
},
|
||||
catalogue=cat
|
||||
)
|
||||
await router.connect()
|
||||
|
||||
payload = NormalizedPayload(
|
||||
source_id="rss:coindesk.com",
|
||||
source_type=SourceType.NEWS,
|
||||
source_credibility_base=0.85,
|
||||
ingest_ts=time.time(),
|
||||
publish_ts=time.time(),
|
||||
asset_mentions=[AssetMention(asset_id="BTC", mention_span=(0, 3), confidence=0.9, source_text="BTC", mention_type="ticker")],
|
||||
raw_text="Duplicate test payload for deduplication check.",
|
||||
title="Duplicate Test",
|
||||
url="https://coindesk.com/duplicate",
|
||||
author="Test",
|
||||
content_length=100,
|
||||
language="en",
|
||||
metadata={}
|
||||
)
|
||||
|
||||
# Route same payload twice
|
||||
result1 = await router.route(payload)
|
||||
result2 = await router.route(payload)
|
||||
|
||||
assert result1 is True
|
||||
assert result2 is False # Second should be detected as duplicate
|
||||
|
||||
# Only one fetch should be recorded
|
||||
source = cat.catalogue.get_source("rss:coindesk.com")
|
||||
assert source.total_fetches == 1
|
||||
assert source.successful_fetches == 1
|
||||
|
||||
await router._nc.close()
|
||||
await cat.stop()
|
||||
# Clean up temp database
|
||||
try:
|
||||
os.unlink(db_path)
|
||||
except:
|
||||
pass
|
||||
|
||||
Reference in New Issue
Block a user