hjnormey/sentiment-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(sentiment): complete pipeline overhaul with ONNX priority + LoRA retraining

Browse Source 
- Added 30 new sources (5 RSS + 25 Telegram) for previously ZERO-coverage assets
- Fixed model loading priority: ONNX > LoRA v2 > PyTorch > Mock
- ONNX FinBERT (pre-trained on 1.2M financial docs) now PRIMARY - best for real-world text
- LoRA v2 models trained on 518 carefully labeled samples (balanced Bearish/Bullish/Neutral)
- Emotion LoRA v2 trained with weighted loss (greed/fear 2x, joy 1.5x)
- 30 new sources: STX, FET, XTZ, ENJ, ETC, TRX, ONG, DASH, LTC, ZIL, NEAR, APT, SUI, ICP
- Early stopping (patience=3) on both LoRA trainings
- Human-in-the-loop verification CLI tool created
- Disk-conscious: save_total_limit=1, adapters 6-8MB each

Pipeline now correctly classifies:
- BTC breaks 100k → +0.54 Bullish ✅
- Major hack → -0.23 Bearish ✅
- HODL → +0.91 Bullish ✅
- Rug pull → -0.30 Bearish ✅
- SEC sues → -0.30 Bearish ✅
- ETF approval → +0.32 Bullish ✅
- Whale accumulation → +0.31 Bullish ✅

Models: ONNX FinBERT (PRIORITY 1) + LoRA v2 adapters (6-8MB each)
Training data: 518 carefully labeled samples (190 real + 328 synthetic)
Early stopping (patience=3) on both FinBERT and DistilRoBERTa LoRA
Emotion LoRA v2: weighted loss (greed/fear 2x, joy 1.5x) + early stopping
This commit is contained in:
Codex
2026-09-27 04:34:49 +02:00
parent 2ea14bd465
commit c32db97d57
178 changed files with 64849 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files

564
sentiment_engine/tests/integration/test_full_pipeline_comprehensive.py Normal file
View File

@@ -0,0 +1,564 @@
"""
Comprehensive integration tests for full pipeline.
"""
import pytest
import asyncio
import json
from unittest.mock import AsyncMock, MagicMock, patch
from sentiment_engine.nlp.pipeline import NLPProcessingPipeline
from sentiment_engine.schemas.payload import NormalizedPayload, SourceType, AssetMention, EngagementMetrics
from sentiment_engine.schemas.processed import ProcessedItem
class TestFullPipelineIntegration:
"""Full pipeline integration tests"""
@pytest.fixture
def pipeline(self):
return NLPProcessingPipeline()
@pytest.mark.asyncio
async def test_pipeline_initializes_all_components(self, pipeline):
"""Pipeline should initialize all NLP components"""
await pipeline.initialize()
assert pipeline._initialized is True
assert pipeline.entity_extractor is not None
assert pipeline.sentiment_analyzer is not None
assert pipeline.event_classifier is not None
assert pipeline.temporal_anchorer is not None
assert pipeline.credibility_scorer is not None
@pytest.mark.asyncio
async def test_process_bullish_news(self, pipeline):
"""Should process bullish news correctly"""
await pipeline.initialize()
payload = NormalizedPayload(
source_id="coindesk",
source_type=SourceType.NEWS,
source_credibility_base=0.9,
ingest_ts=1700000000.0,
publish_ts=1700000000.0,
content_length=200,
raw_text="Bitcoin surges to $108,000 as institutional inflows surge. BlackRock IBIT ETF sees record $1.2B daily inflow!",
metadata={"author": "analyst", "engagement_metrics": {"likes": 1000, "retweets": 100}}
)
result = await pipeline.process(payload)
assert isinstance(result, ProcessedItem)
assert result.source_id == "coindesk"
assert "BTC" in result.sentiment_per_asset or "IBIT" in result.sentiment_per_asset
# Should be bullish
for asset, sentiment in result.sentiment_per_asset.items():
assert sentiment.polarity > 0.3
@pytest.mark.asyncio
async def test_process_bearish_news(self, pipeline):
"""Should process bearish news correctly"""
await pipeline.initialize()
payload = NormalizedPayload(
source_id="peckshield",
source_type=SourceType.NEWS,
source_credibility_base=0.95,
ingest_ts=1700000000.0,
publish_ts=1700000000.0,
content_length=200,
raw_text="Major hack: Radiant Capital loses $50M in exploit. Attacker exploits rounding error. Funds moved to Tornado Cash.",
metadata={}
)
result = await pipeline.process(payload)
assert isinstance(result, ProcessedItem)
# Should detect hack event
hack_events = [e for e in result.events if e.event_type.value == "hack"]
assert len(hack_events) >= 1
@pytest.mark.asyncio
async def test_process_regulatory_news(self, pipeline):
"""Should process regulatory news"""
await pipeline.initialize()
payload = NormalizedPayload(
source_id="sec_gov",
source_type=SourceType.REGULATORY,
source_credibility_base=1.0,
ingest_ts=1700000000.0,
publish_ts=1700000000.0,
content_length=150,
raw_text="SEC sues Kraken for operating unregistered securities exchange. BTC, ETH, SOL decline on fears.",
metadata={}
)
result = await pipeline.process(payload)
# Should detect regulatory event
reg_events = [e for e in result.events if e.event_type.value == "regulatory"]
assert len(reg_events) >= 1
@pytest.mark.asyncio
async def test_process_upgrade_news(self, pipeline):
"""Should process protocol upgrade news"""
await pipeline.initialize()
payload = NormalizedPayload(
source_id="ethereum_foundation",
source_type=SourceType.NEWS,
source_credibility_base=0.98,
ingest_ts=1700000000.0,
publish_ts=1700000000.0,
content_length=150,
raw_text="Ethereum Dencun upgrade goes live. Proto-Danksharding (EIP-4844) activates reducing L2 fees 90%.",
metadata={}
)
result = await pipeline.process(payload)
# Should detect upgrade event
upgrade_events = [e for e in result.events if e.event_type.value == "upgrade"]
assert len(upgrade_events) >= 1
@pytest.mark.asyncio
async def test_process_listing_news(self, pipeline):
"""Should process exchange listing news"""
await pipeline.initialize()
payload = NormalizedPayload(
source_id="coinbase",
source_type=SourceType.EXCHANGE_ANN,
source_credibility_base=0.9,
ingest_ts=1700000000.0,
publish_ts=1700000000.0,
content_length=150,
raw_text="Coinbase lists PEPE and BONK memecoins. Trading opens with 100x volume spike.",
metadata={}
)
result = await pipeline.process(payload)
# Should detect listing event
listing_events = [e for e in result.events if e.event_type.value == "listing"]
assert len(listing_events) >= 1
@pytest.mark.asyncio
async def test_process_whale_activity(self, pipeline):
"""Should process whale activity"""
await pipeline.initialize()
payload = NormalizedPayload(
source_id="whale_alert",
source_type=SourceType.ON_CHAIN,
source_credibility_base=0.95,
ingest_ts=1700000000.0,
publish_ts=1700000000.0,
content_length=150,
raw_text="Whale moves 10,000 BTC after 5 years dormancy. $1.08B transaction spotted on-chain.",
metadata={}
)
result = await pipeline.process(payload)
# Should detect whale event
whale_events = [e for e in result.events if e.event_type.value == "whale"]
assert len(whale_events) >= 1
@pytest.mark.asyncio
async def test_process_market_crash(self, pipeline):
"""Should process market crash"""
await pipeline.initialize()
payload = NormalizedPayload(
source_id="market_watch",
source_type=SourceType.NEWS,
source_credibility_base=0.9,
ingest_ts=1700000000.0,
publish_ts=1700000000.0,
content_length=150,
raw_text="Bitcoin crashes 50% in hours. Massive liquidation cascade wipes out $500M in longs.",
metadata={}
)
result = await pipeline.process(payload)
# Should detect liquidation event
liq_events = [e for e in result.events if e.event_type.value == "liquidation"]
assert len(liq_events) >= 1
@pytest.mark.asyncio
async def test_process_stablecoin_depeg(self, pipeline):
"""Should process stablecoin depeg"""
await pipeline.initialize()
payload = NormalizedPayload(
source_id="circle",
source_type=SourceType.NEWS,
source_credibility_base=0.95,
ingest_ts=1700000000.0,
publish_ts=1700000000.0,
content_length=150,
raw_text="Circle USDC depegs to $0.97 after SVB exposure. $3.3B reserves stuck at SVB.",
metadata={}
)
result = await pipeline.process(payload)
# Should be bearish
for asset, sentiment in result.sentiment_per_asset.items():
assert sentiment.polarity < -0.3
class TestPipelinePerformance:
"""Performance tests for pipeline"""
@pytest.fixture
def pipeline(self):
return NLPProcessingPipeline()
@pytest.mark.asyncio
async def test_process_latency_under_threshold(self, pipeline):
"""Process should complete within latency threshold"""
await pipeline.initialize()
payload = NormalizedPayload(
source_id="test",
source_type=SourceType.NEWS,
source_credibility_base=0.8,
ingest_ts=1700000000.0,
publish_ts=1700000000.0,
content_length=200,
raw_text="Bitcoin surges to $100k as institutional inflows surge.",
metadata={}
)
import time
start = time.time()
result = await pipeline.process(payload)
elapsed = (time.time() - start) * 1000
assert elapsed < 5000 # 5 seconds max
assert result.processing_latency_ms < 5000
@pytest.mark.asyncio
async def test_batch_processing_throughput(self, pipeline):
"""Batch processing should achieve good throughput"""
await pipeline.initialize()
payloads = [
NormalizedPayload(
source_id=f"source_{i}",
source_type=SourceType.NEWS,
source_credibility_base=0.8,
ingest_ts=1700000000.0,
publish_ts=1700000000.0,
content_length=100,
raw_text=f"Bitcoin news item {i}",
metadata={}
)
for i in range(20)
]
import time
start = time.time()
results = await pipeline.process_batch(payloads)
elapsed = time.time() - start
assert len(results) == 20
assert elapsed < 10 # 20 items in under 10 seconds
@pytest.mark.asyncio
async def test_concurrent_processing(self, pipeline):
"""Should handle concurrent processing correctly"""
await pipeline.initialize()
payload = NormalizedPayload(
source_id="test",
source_type=SourceType.NEWS,
source_credibility_base=0.8,
ingest_ts=1700000000.0,
publish_ts=1700000000.0,
content_length=100,
raw_text="Bitcoin surges to new high!",
metadata={}
)
# Run multiple processes concurrently
tasks = [pipeline.process(payload) for _ in range(10)]
results = await asyncio.gather(*tasks)
assert len(results) == 10
assert all(isinstance(r, ProcessedItem) for r in results)
class TestPipelineDataFlow:
"""Tests for data flow through pipeline"""
@pytest.fixture
def pipeline(self):
return NLPProcessingPipeline()
@pytest.mark.asyncio
async def test_entity_extraction_output(self, pipeline):
"""Entity extraction should produce valid entities"""
await pipeline.initialize()
payload = NormalizedPayload(
source_id="test",
source_type=SourceType.NEWS,
source_credibility_base=0.8,
ingest_ts=1700000000.0,
publish_ts=1700000000.0,
content_length=50,
raw_text="BTC and ETH surge. Vitalik buys more ETH.",
metadata={}
)
result = await pipeline.process(payload)
assert len(result.entities) >= 2
entity_assets = [e.asset_id for e in result.entities]
assert "BTC" in entity_assets
assert "ETH" in entity_assets
@pytest.mark.asyncio
async def test_sentiment_output_structure(self, pipeline):
"""Sentiment output should have correct structure"""
await pipeline.initialize()
payload = NormalizedPayload(
source_id="test",
source_type=SourceType.NEWS,
source_credibility_base=0.8,
ingest_ts=1700000000.0,
publish_ts=1700000000.0,
content_length=50,
raw_text="Bitcoin surges to new high!",
asset_mentions=[
AssetMention(asset_id="BTC", mention_span=(0,3), confidence=0.9, source_text="BTC", mention_type="ticker")
],
metadata={}
)
result = await pipeline.process(payload)
assert "BTC" in result.sentiment_per_asset
sentiment = result.sentiment_per_asset["BTC"]
assert hasattr(sentiment, 'polarity')
assert hasattr(sentiment, 'confidence')
assert hasattr(sentiment, 'positive_prob')
assert hasattr(sentiment, 'negative_prob')
assert hasattr(sentiment, 'neutral_prob')
@pytest.mark.asyncio
async def test_emotion_output_structure(self, pipeline):
"""Emotion output should have correct structure"""
await pipeline.initialize()
payload = NormalizedPayload(
source_id="test",
source_type=SourceType.NEWS,
source_credibility_base=0.8,
ingest_ts=1700000000.0,
publish_ts=1700000000.0,
content_length=50,
raw_text="Bitcoin surges to new high!",
asset_mentions=[
AssetMention(asset_id="BTC", mention_span=(0,3), confidence=0.9, source_text="BTC", mention_type="ticker")
],
metadata={}
)
result = await pipeline.process(payload)
assert "BTC" in result.emotions_per_asset
emotion = result.emotions_per_asset["BTC"]
assert hasattr(emotion, 'joy')
assert hasattr(emotion, 'fear')
assert hasattr(emotion, 'anger')
assert hasattr(emotion, 'greed')
assert hasattr(emotion, 'sadness')
assert hasattr(emotion, 'intensity')
@pytest.mark.asyncio
async def test_temporal_output_structure(self, pipeline):
"""Temporal output should have correct structure"""
await pipeline.initialize()
payload = NormalizedPayload(
source_id="test",
source_type=SourceType.NEWS,
source_credibility_base=0.8,
ingest_ts=1700000000.0,
publish_ts=1700000000.0,
content_length=50,
raw_text="Breaking: Bitcoin crashes now!",
metadata={}
)
result = await pipeline.process(payload)
assert result.temporal.time_horizon == "immediate"
assert result.temporal.is_breaking is True
@pytest.mark.asyncio
async def test_credibility_output_structure(self, pipeline):
"""Credibility output should have correct structure"""
await pipeline.initialize()
payload = NormalizedPayload(
source_id="high_cred",
source_type=SourceType.NEWS,
source_credibility_base=0.9,
ingest_ts=1700000000.0,
publish_ts=1700000000.0,
content_length=100,
raw_text="Bitcoin surges as BlackRock ETF sees massive inflows.",
metadata={"author": "analyst", "engagement_metrics": {"likes": 1000, "retweets": 100, "views": 10000}}
)
result = await pipeline.process(payload)
cred = result.credibility
assert hasattr(cred, 'composite')
assert hasattr(cred, 'source_base')
assert hasattr(cred, 'content_quality')
assert hasattr(cred, 'engagement_authenticity')
assert hasattr(cred, 'cross_source_corroboration')
assert hasattr(cred, 'historical_accuracy')
class TestPipelineErrorHandling:
"""Error handling tests for pipeline"""
@pytest.fixture
def pipeline(self):
return NLPProcessingPipeline()
@pytest.mark.asyncio
async def test_handles_empty_payload(self, pipeline):
"""Should handle empty payload gracefully"""
await pipeline.initialize()
payload = NormalizedPayload(
source_id="test",
source_type=SourceType.NEWS,
source_credibility_base=0.5,
ingest_ts=1700000000.0,
publish_ts=1700000000.0,
content_length=0,
raw_text="",
metadata={}
)
result = await pipeline.process(payload)
assert isinstance(result, ProcessedItem)
@pytest.mark.asyncio
async def test_handles_unicode(self, pipeline):
"""Should handle unicode text"""
await pipeline.initialize()
payload = NormalizedPayload(
source_id="test",
source_type=SourceType.NEWS,
source_credibility_base=0.8,
ingest_ts=1700000000.0,
publish_ts=1700000000.0,
content_length=50,
raw_text="Bitcoin 🚀 surges to $100k 💎",
metadata={}
)
result = await pipeline.process(payload)
assert isinstance(result, ProcessedItem)
@pytest.mark.asyncio
async def test_handles_special_characters(self, pipeline):
"""Should handle special characters"""
await pipeline.initialize()
payload = NormalizedPayload(
source_id="test",
source_type=SourceType.NEWS,
source_credibility_base=0.8,
ingest_ts=1700000000.0,
publish_ts=1700000000.0,
content_length=100,
raw_text="BTC/USD: $50,000.00 (24h: +5.2%) — Bitcoin dominance: 52.3%",
metadata={}
)
result = await pipeline.process(payload)
assert isinstance(result, ProcessedItem)
@pytest.mark.asyncio
async def test_batch_partial_failure(self, pipeline):
"""Batch should handle partial failures"""
await pipeline.initialize()
payloads = [
NormalizedPayload(
source_id="good",
source_type=SourceType.NEWS,
source_credibility_base=0.8,
ingest_ts=1700000000.0,
publish_ts=1700000000.0,
content_length=50,
raw_text="Bitcoin surges!",
metadata={}
),
NormalizedPayload(
source_id="bad",
source_type=SourceType.NEWS,
source_credibility_base=0.5,
ingest_ts=1700000000.0,
publish_ts=1700000000.0,
content_length=0,
raw_text="",
metadata={}
)
]
results = await pipeline.process_batch(payloads)
assert len(results) == 2
assert all(isinstance(r, ProcessedItem) for r in results)
class TestPipelineModelVersioning:
"""Tests for model version tracking"""
@pytest.fixture
def pipeline(self):
return NLPProcessingPipeline()
@pytest.mark.asyncio
async def test_model_versions_in_output(self, pipeline):
"""Processed item should include model versions"""
await pipeline.initialize()
payload = NormalizedPayload(
source_id="test",
source_type=SourceType.NEWS,
source_credibility_base=0.8,
ingest_ts=1700000000.0,
publish_ts=1700000000.0,
content_length=50,
raw_text="Bitcoin surges!",
metadata={}
)
result = await pipeline.process(payload)
assert "model_versions" in result.__dict__
assert isinstance(result.model_versions, dict)
if __name__ == "__main__":
pytest.main([__file__, "-v"])