feat(sentiment): complete pipeline overhaul with ONNX priority + LoRA retraining

- Added 30 new sources (5 RSS + 25 Telegram) for previously ZERO-coverage assets - Fixed model loading priority: ONNX > LoRA v2 > PyTorch > Mock - ONNX FinBERT (pre-trained on 1.2M financial docs) now PRIMARY - best for real-world text - LoRA v2 models trained on 518 carefully labeled samples (balanced Bearish/Bullish/Neutral) - Emotion LoRA v2 trained with weighted loss (greed/fear 2x, joy 1.5x) - 30 new sources: STX, FET, XTZ, ENJ, ETC, TRX, ONG, DASH, LTC, ZIL, NEAR, APT, SUI, ICP - Early stopping (patience=3) on both LoRA trainings - Human-in-the-loop verification CLI tool created - Disk-conscious: save_total_limit=1, adapters 6-8MB each Pipeline now correctly classifies: - BTC breaks 100k → +0.54 Bullish ✅ - Major hack → -0.23 Bearish ✅ - HODL → +0.91 Bullish ✅ - Rug pull → -0.30 Bearish ✅ - SEC sues → -0.30 Bearish ✅ - ETF approval → +0.32 Bullish ✅ - Whale accumulation → +0.31 Bullish ✅ Models: ONNX FinBERT (PRIORITY 1) + LoRA v2 adapters (6-8MB each) Training data: 518 carefully labeled samples (190 real + 328 synthetic) Early stopping (patience=3) on both FinBERT and DistilRoBERTa LoRA Emotion LoRA v2: weighted loss (greed/fear 2x, joy 1.5x) + early stopping
2026-09-27 04:34:49 +02:00
parent 2ea14bd465
commit c32db97d57
178 changed files with 64849 additions and 42 deletions
--- a/sentiment_engine/tests/unit/test_schemas.py
+++ b/sentiment_engine/tests/unit/test_schemas.py
@@ -0,0 +1,159 @@
+"""Tests for schema validation"""
+
+import pytest
+from sentiment_engine.schemas.payload import NormalizedPayload, SourceType, AssetMention, EngagementMetrics
+from sentiment_engine.schemas.processed import ProcessedItem, EntityExtraction, SentimentScores, EmotionScores, EventClassification, EventType
+from sentiment_engine.schemas.output import AssetSentiment, MarketSentiment, PumpDumpScore, VelocityMetrics, EventFlag
+
+
+class TestPayloadSchemas:
+    """Test payload schema validation"""
+
+    def test_normalized_payload_valid(self):
+        payload = NormalizedPayload(
+            source_id="test",
+            source_type=SourceType.NEWS,
+            source_credibility_base=0.8,
+            ingest_ts=1724262300.0,
+            raw_text="BTC surges to new highs",
+            asset_mentions=[AssetMention(asset_id="BTC", mention_span=(0, 3), confidence=0.9, source_text="BTC", mention_type="ticker")],
+            content_length=25,
+            language="en"
+        )
+        assert payload.source_id == "test"
+        assert payload.has_assets is True
+        assert payload.get_assets() == ["BTC"]
+
+    def test_normalized_payload_empty_text_raises(self):
+        with pytest.raises(ValueError, match="raw_text cannot be empty"):
+            NormalizedPayload(
+                source_id="test",
+                source_type=SourceType.NEWS,
+                source_credibility_base=0.8,
+                ingest_ts=1724262300.0,
+                raw_text="",
+                content_length=0,
+                language="en"
+            )
+
+    def test_engagement_metrics_total(self):
+        metrics = EngagementMetrics(retweets=10, likes=50, replies=5, upvotes=100, comments=20)
+        assert metrics.total_engagement() == 185
+
+
+class TestProcessedSchemas:
+    """Test processed item schemas"""
+
+    def test_sentiment_scores_bounds(self):
+        scores = SentimentScores(
+            polarity=0.5,
+            confidence=0.8,
+            positive_prob=0.7,
+            negative_prob=0.1,
+            neutral_prob=0.2
+        )
+        assert -1.0 <= scores.polarity <= 1.0
+        assert 0.0 <= scores.confidence <= 1.0
+
+    def test_emotion_scores_bounds(self):
+        emotions = EmotionScores(
+            joy=0.8, fear=0.1, anger=0.05, greed=0.7, sadness=0.05, intensity=0.75
+        )
+        for val in [emotions.joy, emotions.fear, emotions.anger, emotions.greed, emotions.sadness, emotions.intensity]:
+            assert 0.0 <= val <= 1.0
+
+    def test_event_classification(self):
+        event = EventClassification(
+            event_type=EventType.LISTING,
+            confidence=0.8,
+            assets_involved=["BTC"],
+            key_details={"exchange": "Binance"},
+            severity=0.7
+        )
+        assert event.event_type == EventType.LISTING
+        assert 0.0 <= event.severity <= 1.0
+
+
+class TestOutputSchemas:
+    """Test output schemas"""
+
+    def test_asset_sentiment_acb_signals(self):
+        asset = AssetSentiment(
+            asset_id="BTC",
+            fear_state=20.0,
+            greed_state=80.0,
+            sentiment_polarity=60.0,
+            pump_dump=PumpDumpScore(asset_id="BTC", pump_score=75.0, dump_score=15.0, pump_confidence=0.8, dump_confidence=0.7, last_update_ts=1724262305.0),
+            last_update_ts=1724262305.0
+        )
+
+        # Test ACB signal extraction
+        market = MarketSentiment(
+            fear_state=25.0,
+            greed_state=75.0,
+            sentiment_index=50.0,
+            hype_velocity=65.0,
+            pub_velocity=55.0,
+            aggregate_pump_risk=75.0,
+            aggregate_dump_risk=20.0,
+            last_update_ts=1724262305.0
+        )
+
+        from sentiment_engine.schemas.output import SentimentOutput
+        output = SentimentOutput(timestamp=1724262305.0, market=market, assets={"BTC": asset})
+
+        acb = output.get_acb_signals()
+        assert "market_sentiment_state" in acb
+        assert "aggregate_pump_risk" in acb
+        assert -1.0 <= acb["market_sentiment_state"] <= 1.0
+        assert 0.0 <= acb["aggregate_pump_risk"] <= 1.0
+
+    def test_book_health_veto(self):
+        asset = AssetSentiment(
+            asset_id="BTC",
+            fear_state=20.0,
+            greed_state=80.0,
+            sentiment_polarity=60.0,
+            pump_dump=PumpDumpScore(asset_id="BTC", pump_score=80.0, dump_score=15.0, pump_confidence=0.8, dump_confidence=0.7, last_update_ts=1724262305.0),
+            last_update_ts=1724262305.0
+        )
+
+        from sentiment_engine.schemas.output import SentimentOutput, MarketSentiment
+        market = MarketSentiment(
+            fear_state=25.0, greed_state=75.0, sentiment_index=50.0,
+            hype_velocity=65.0, pub_velocity=55.0,
+            aggregate_pump_risk=75.0, aggregate_dump_risk=20.0,
+            last_update_ts=1724262305.0
+        )
+        output = SentimentOutput(timestamp=1724262305.0, market=market, assets={"BTC": asset})
+
+        veto = output.get_book_health_veto(threshold=75.0)
+        assert "BTC" in veto
+
+        veto_low = output.get_book_health_veto(threshold=85.0)
+        assert "BTC" not in veto_low
+
+    def test_exit_context(self):
+        asset = AssetSentiment(
+            asset_id="BTC",
+            fear_state=85.0,
+            greed_state=15.0,
+            sentiment_polarity=-70.0,
+            pump_dump=PumpDumpScore(asset_id="BTC", pump_score=10.0, dump_score=80.0, pump_confidence=0.8, dump_confidence=0.7, last_update_ts=1724262305.0),
+            last_update_ts=1724262305.0
+        )
+
+        from sentiment_engine.schemas.output import SentimentOutput, MarketSentiment
+        market = MarketSentiment(
+            fear_state=80.0, greed_state=20.0, sentiment_index=-60.0,
+            hype_velocity=30.0, pub_velocity=40.0,
+            aggregate_pump_risk=15.0, aggregate_dump_risk=80.0,
+            last_update_ts=1724262305.0
+        )
+        output = SentimentOutput(timestamp=1724262305.0, market=market, assets={"BTC": asset})
+
+        ctx = output.get_exit_context(dump_threshold=70.0, fear_threshold=80.0)
+        assert "BTC" in ctx["high_dump_assets"]
+        assert "BTC" in ctx["high_fear_assets"]
+        assert ctx["market_dump_risk"] == 80.0
+        assert ctx["market_fear"] == 80.0