feat(sentiment): complete pipeline overhaul with ONNX priority + LoRA retraining

- Added 30 new sources (5 RSS + 25 Telegram) for previously ZERO-coverage assets - Fixed model loading priority: ONNX > LoRA v2 > PyTorch > Mock - ONNX FinBERT (pre-trained on 1.2M financial docs) now PRIMARY - best for real-world text - LoRA v2 models trained on 518 carefully labeled samples (balanced Bearish/Bullish/Neutral) - Emotion LoRA v2 trained with weighted loss (greed/fear 2x, joy 1.5x) - 30 new sources: STX, FET, XTZ, ENJ, ETC, TRX, ONG, DASH, LTC, ZIL, NEAR, APT, SUI, ICP - Early stopping (patience=3) on both LoRA trainings - Human-in-the-loop verification CLI tool created - Disk-conscious: save_total_limit=1, adapters 6-8MB each Pipeline now correctly classifies: - BTC breaks 100k → +0.54 Bullish ✅ - Major hack → -0.23 Bearish ✅ - HODL → +0.91 Bullish ✅ - Rug pull → -0.30 Bearish ✅ - SEC sues → -0.30 Bearish ✅ - ETF approval → +0.32 Bullish ✅ - Whale accumulation → +0.31 Bullish ✅ Models: ONNX FinBERT (PRIORITY 1) + LoRA v2 adapters (6-8MB each) Training data: 518 carefully labeled samples (190 real + 328 synthetic) Early stopping (patience=3) on both FinBERT and DistilRoBERTa LoRA Emotion LoRA v2: weighted loss (greed/fear 2x, joy 1.5x) + early stopping
2026-09-27 04:34:49 +02:00
parent 2ea14bd465
commit c32db97d57
178 changed files with 64849 additions and 42 deletions
--- a/sentiment_engine/tests/unit/test_nlp_pipeline.py
+++ b/sentiment_engine/tests/unit/test_nlp_pipeline.py
@@ -0,0 +1,257 @@
+"""Tests for NLP processing pipeline"""
+
+import pytest
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from sentiment_engine.nlp.entity_extraction import EntityExtractor, AssetMapper
+from sentiment_engine.nlp.sentiment_emotion import SentimentEmotionAnalyzer
+from sentiment_engine.nlp.event_classification import EventClassifier, EventType
+from sentiment_engine.nlp.temporal import TemporalAnchorer
+from sentiment_engine.nlp.credibility import CredibilityScorer
+from sentiment_engine.nlp.pipeline import NLPProcessingPipeline
+from sentiment_engine.schemas.payload import NormalizedPayload, SourceType, AssetMention
+from sentiment_engine.schemas.processed import ProcessedItem, EntityExtraction, SentimentScores, EmotionScores, EventClassification, TemporalAnchor, CredibilityScore
+
+
+class TestAssetMapper:
+    """Tests for AssetMapper"""
+
+    def test_map_known_ticker(self):
+        mapper = AssetMapper()
+        asset_id, confidence = mapper.map_ticker("BTC")
+        assert asset_id == "BTC"
+        assert confidence >= 0.9
+
+    def test_map_alias(self):
+        mapper = AssetMapper()
+        asset_id, confidence = mapper.map_ticker("VITALIK")
+        assert asset_id == "ETH"
+        assert confidence >= 0.7
+
+    def test_map_unknown_ticker(self):
+        mapper = AssetMapper()
+        asset_id, confidence = mapper.map_ticker("UNKNOWNTICKER")
+        assert asset_id == "UNKNOWNTICKER"
+        assert confidence == 0.5
+
+    def test_map_contract(self):
+        mapper = AssetMapper()
+        asset_id, confidence, chain = mapper.map_contract("0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2")
+        assert asset_id == "ETH"
+        assert confidence >= 0.9
+        assert chain == "ethereum"
+
+    def test_resolve_aliases(self):
+        mapper = AssetMapper()
+        results = mapper.resolve_alias("Vitalik Buterin says ETH will moon")
+        assert any(r[1] == "ETH" for r in results)
+
+
+class TestEntityExtractor:
+    """Tests for EntityExtractor"""
+
+    @pytest.fixture
+    def extractor(self):
+        return EntityExtractor()
+
+    def test_extract_tickers(self, extractor):
+        text = "BTC and ETH are pumping hard"
+        mentions = extractor.extract_tickers(text)
+        assert len(mentions) == 2
+        asset_ids = [m.asset_id for m in mentions]
+        assert "BTC" in asset_ids
+        assert "ETH" in asset_ids
+
+    def test_extract_tickers_filters_false_positives(self, extractor):
+        text = "THE CEO OF API COMPANY SAYS BTC"
+        mentions = extractor.extract_tickers(text)
+        asset_ids = [m.asset_id for m in mentions]
+        assert "THE" not in asset_ids
+        assert "CEO" not in asset_ids
+        assert "API" not in asset_ids
+        assert "BTC" in asset_ids
+
+    def test_extract_contracts(self, extractor):
+        text = "Send to 0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2"
+        mentions = extractor.extract_contracts(text)
+        assert len(mentions) == 1
+        assert mentions[0].asset_id == "ETH"
+
+    def test_extract_aliases(self, extractor):
+        text = "Vitalik says ETH to the moon"
+        mentions = extractor.extract_aliases(text)
+        assert len(mentions) >= 1
+        assert mentions[0].asset_id == "ETH"
+
+    def test_deduplication(self, extractor):
+        text = "BTC BTC BTC"
+        mentions = extractor.extract_tickers(text)
+        assert len(mentions) == 1
+        assert mentions[0].asset_id == "BTC"
+
+    @pytest.mark.asyncio
+    async def test_extract_all(self, extractor):
+        text = "BTC surges. Vitalik buys ETH. Send to 0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2"
+        entities = await extractor.extract_all(text)
+        asset_ids = [e.asset_id for e in entities]
+        assert "BTC" in asset_ids
+        assert "ETH" in asset_ids
+        assert asset_ids.count("ETH") >= 1
+
+
+class TestSentimentEmotionAnalyzer:
+    """Tests for SentimentEmotionAnalyzer"""
+
+    @pytest.fixture
+    def analyzer(self):
+        return SentimentEmotionAnalyzer()
+
+    def test_heuristic_emotions(self, analyzer):
+        # Test fear emotions
+        text = "Crash panic fear liquidation dump"
+        emotions = analyzer._heuristic_emotions(text)
+        assert emotions.fear > 0.5
+        assert emotions.anger == 0.0  # No anger keywords in this text
+
+        # Test greed emotions
+        text = "Buy buy buy accumulate load bag stack moon lambo hodl fomo"
+        emotions = analyzer._heuristic_emotions(text)
+        assert emotions.greed > 0.5
+        assert emotions.joy > 0
+
+    def test_compute_intensity(self, analyzer):
+        text = "CRASH!!! BTC dumping hard!!!"
+        intensity = analyzer.compute_intensity(text)
+        assert intensity > 0.5
+
+
+class TestEventClassifier:
+    """Tests for EventClassifier"""
+
+    @pytest.fixture
+    def classifier(self):
+        return EventClassifier()
+
+    def test_classify_listing(self, classifier):
+        text = "Binance will list new token ABC tomorrow"
+        events = classifier._classify_sync(text, ["ABC"])
+        assert len(events) >= 1
+        assert events[0].event_type == EventType.LISTING
+
+    def test_classify_hack(self, classifier):
+        text = "Exchange hacked, millions stolen in exploit"
+        events = classifier._classify_sync(text, ["BTC"])
+        assert len(events) >= 1
+        assert events[0].event_type == EventType.HACK
+
+    def test_classify_regulatory(self, classifier):
+        text = "SEC investigation into crypto exchange"
+        events = classifier._classify_sync(text, ["BTC"])
+        assert len(events) >= 1
+        assert events[0].event_type == EventType.REGULATORY
+
+    def test_estimate_severity(self, classifier):
+        text = "Major hack, massive exploit, emergency"
+        events = classifier._classify_sync(text, ["BTC"])
+        assert events[0].severity > 0.8
+
+
+class TestTemporalAnchorer:
+    """Tests for TemporalAnchorer"""
+
+    @pytest.fixture
+    def anchorer(self):
+        return TemporalAnchorer()
+
+    def test_detect_horizon_immediate(self, anchorer):
+        text = "Breaking: BTC just crashed now!"
+        anchor = anchorer.anchor(text, None)
+        assert anchor.time_horizon == "immediate"
+        assert anchor.is_breaking is True
+
+    def test_detect_horizon_near(self, anchorer):
+        text = "Earnings report today, expecting big move"
+        anchor = anchorer.anchor(text, None)
+        assert anchor.time_horizon == "near"
+
+    def test_detect_scheduled(self, anchorer):
+        text = "Scheduled for 2024-01-15"
+        anchor = anchorer.anchor(text, None)
+        assert anchor.is_scheduled is True
+
+    def test_compute_recency_weight(self, anchorer):
+        import time
+        now = time.time()
+        # Recent
+        weight = anchorer.compute_recency_weight(now - 60)  # 1 min ago
+        assert weight > 0.9
+        # Old
+        weight = anchorer.compute_recency_weight(now - 86400)  # 1 day ago
+        assert weight < 0.1
+
+
+class TestCredibilityScorer:
+    """Tests for CredibilityScorer"""
+
+    @pytest.fixture
+    def scorer(self):
+        return CredibilityScorer()
+
+    def test_score_source(self, scorer):
+        scorer.load_registry({"test_source": {"base_credibility": 0.8}})
+        assert scorer.score_source("test_source") == 0.8
+        assert scorer.score_source("unknown") == 0.5
+
+    def test_score_content_quality(self, scorer):
+        # Long, well-structured text (enough words to avoid penalty, needs >500 words for +0.1)
+        text = "This is a well-structured article with multiple sentences. It has proper grammar and punctuation. The content is informative and detailed. The market analysis shows strong fundamentals and technical indicators suggest bullish momentum continuing." * 15
+        score = scorer.score_content_quality(text, {})
+        assert score > 0.5
+
+        # Short, poorly structured text
+        text = "btc moon"
+        score = scorer.score_content_quality(text, {})
+        assert score < 0.5
+
+    def test_score_engagement_authenticity(self, scorer):
+        # Natural ratios
+        engagement = {"likes": 100, "retweets": 10, "replies": 5, "views": 1000}
+        score = scorer.score_engagement_authenticity(engagement, "social")
+        assert score > 0.5
+
+        # Suspicious ratios
+        engagement = {"likes": 1000, "retweets": 0, "replies": 0, "views": 100}
+        score = scorer.score_engagement_authenticity(engagement, "social")
+        assert score < 0.5
+
+    def test_compute_composite(self, scorer):
+        scorer.load_registry({"test": {"base_credibility": 0.8}})
+        credibility = scorer.compute_composite(
+            source_id="test",
+            text="Breaking news about BTC crash",
+            metadata={"source_type": "news", "engagement_metrics": {"likes": 100, "retweets": 10}},
+            asset_id="BTC",
+            event_type="hack",
+            recent_items=[{"source_id": "other"}]
+        )
+        assert 0.0 <= credibility.composite <= 1.0
+
+
+class TestNLPProcessingPipeline:
+    """Tests for NLPProcessingPipeline"""
+
+    @pytest.fixture
+    def pipeline(self):
+        return NLPProcessingPipeline()
+
+    @pytest.mark.asyncio
+    async def test_pipeline_initialization(self, pipeline):
+        await pipeline.initialize()
+        assert pipeline._initialized is True
+
+    @pytest.mark.asyncio
+    async def test_process_empty_payload(self, pipeline):
+        await pipeline.initialize()
+        # This would test the full pipeline but requires models loaded
+        # For now, just verify initialization works
+        assert pipeline._initialized is True