Add sentiment_engine with CryptoSentimentCalibrator fixes - improved keyword lists, lowered FinBERT threshold, added neutral handling

sentiment_engine/tests/integration/test_labeling_pipeline_comprehensive.py

@@ -0,0 +1,426 @@
+"""
+Comprehensive integration tests for Labeling Pipeline.
+"""
+
+import pytest
+import asyncio
+import json
+import tempfile
+import os
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from labeling_pipeline import (
+    LabelingPipeline, LabelingPipelineRunner,
+    OnChainVerifier, NewsCrossReferenceVerifier, MarketDataVerifier,
+    FactVerificationEngine
+)
+from sentiment_engine.schemas.processed import SentimentScores, EmotionScores, EventType
+
+
+class TestLabelingPipeline:
+    """Tests for LabelingPipeline"""
+
+    @pytest.fixture
+    def pipeline(self):
+        return LabelingPipeline()
+
+    @pytest.mark.asyncio
+    async def test_label_text_returns_complete_structure(self, pipeline):
+        """label_text should return complete labeled structure"""
+        result = await pipeline.label_text("Bitcoin surges to $100k!")
+        
+        assert "labels" in result
+        assert "confidence" in result
+        assert "verified" in result
+        assert "verification_details" in result
+        assert "labeled_at" in result
+        
+        assert result["labels"]["sentiment"] in ["Bearish", "Bullish", "Neutral"]
+        assert result["labels"]["event_type"] in [e.value for e in EventType]
+
+    @pytest.mark.asyncio
+    async def test_label_text_with_context(self, pipeline):
+        """label_text should use context"""
+        result = await pipeline.label_text(
+            "Bitcoin surges!",
+            context={"source_id": "coindesk", "source_type": "news", "source_credibility": 0.9}
+        )
+        
+        assert result["confidence"]["verification"] > 0.0
+
+    @pytest.mark.asyncio
+    async def test_label_batch(self, pipeline):
+        """Should label multiple texts"""
+        texts = [
+            "Bitcoin surges to $100k!",
+            "Major hack on exchange!",
+            "SEC approves ETF!"
+        ]
+        
+        results = await pipeline.label_batch(texts)
+        
+        assert len(results) == 3
+        assert all("labels" in r for r in results)
+
+    def test_label_text_sync(self, pipeline):
+        """Synchronous label_text should work"""
+        result = pipeline.label_text_sync("Bitcoin surges!")
+        
+        assert "labels" in result
+
+
+class TestOnChainVerifier:
+    """Tests for OnChainVerifier"""
+
+    @pytest.fixture
+    def verifier(self):
+        return OnChainVerifier()
+
+    @pytest.mark.asyncio
+    async def test_verify_transaction(self, verifier):
+        """Should verify transaction exists"""
+        with patch('aiohttp.ClientSession.get') as mock_get:
+            mock_response = AsyncMock()
+            mock_response.status = 200
+            mock_response.json = AsyncMock(return_value={"status": "confirmed"})
+            mock_get.return_value.__aenter__.return_value = mock_response
+            
+            result = await verifier.verify("tx_hash_123")
+            
+            assert result["verified"] is True
+
+    @pytest.mark.asyncio
+    async def test_verify_address_balance(self, verifier):
+        """Should verify address balance"""
+        with patch('aiohttp.ClientSession.get') as mock_get:
+            mock_response = AsyncMock()
+            mock_response.status = 200
+            mock_response.json = AsyncMock(return_value={"balance": "1000000000000000000"})
+            mock_get.return_value.__aenter__.return_value = mock_response
+            
+            result = await verifier.verify_address("0x123...", min_balance=1e18)
+            
+            assert result["verified"] is True
+
+    @pytest.mark.asyncio
+    async def test_verify_contract_interaction(self, verifier):
+        """Should verify contract interaction"""
+        with patch('aiohttp.ClientSession.get') as mock_get:
+            mock_response = AsyncMock()
+            mock_response.status = 200
+            mock_response.json = AsyncMock(return_value={"logs": [{"address": "0x..."}]})
+            mock_get.return_value.__aenter__.return_value = mock_response
+            
+            result = await verifier.verify_contract_interaction("0x123...", "0x456...")
+            
+            assert result["verified"] is True
+
+
+class TestNewsCrossReferenceVerifier:
+    """Tests for NewsCrossReferenceVerifier"""
+
+    @pytest.fixture
+    def verifier(self):
+        return NewsCrossReferenceVerifier()
+
+    @pytest.mark.asyncio
+    async def test_cross_reference_single_source(self, verifier):
+        """Should find matching articles"""
+        with patch('aiohttp.ClientSession.get') as mock_get:
+            mock_response = AsyncMock()
+            mock_response.status = 200
+            mock_response.json = AsyncMock(return_value={
+                "articles": [
+                    {"title": "Bitcoin surges", "source": "Reuters"},
+                    {"title": "BTC hits new high", "source": "Bloomberg"}
+                ]
+            })
+            mock_get.return_value.__aenter__.return_value = mock_response
+            
+            result = await verifier.cross_reference("Bitcoin surges to new high")
+            
+            assert result["verified"] is True
+            assert result["matches"] >= 2
+
+    @pytest.mark.asyncio
+    async def test_cross_reference_no_matches(self, verifier):
+        """Should return unverified when no matches"""
+        with patch('aiohttp.ClientSession.get') as mock_get:
+            mock_response = AsyncMock()
+            mock_response.status = 200
+            mock_response.json = AsyncMock(return_value={"articles": []})
+            mock_get.return_value.__aenter__.return_value = mock_response
+            
+            result = await verifier.cross_reference("Completely unique obscure news")
+            
+            assert result["verified"] is False
+
+    @pytest.mark.asyncio
+    async def test_cross_reference_confidence(self, verifier):
+        """Should calculate confidence based on match count"""
+        with patch('aiohttp.ClientSession.get') as mock_get:
+            mock_response = AsyncMock()
+            mock_response.status = 200
+            mock_response.json = AsyncMock(return_value={
+                "articles": [
+                    {"title": "Bitcoin surges", "source": "Reuters"},
+                    {"title": "BTC new high", "source": "Bloomberg"},
+                    {"title": "Bitcoin record", "source": "CNBC"},
+                    {"title": "Crypto surges", "source": "WSJ"},
+                    {"title": "Digital gold", "source": "FT"}
+                ]
+            })
+            mock_get.return_value.__aenter__.return_value = mock_response
+            
+            result = await verifier.cross_reference("Bitcoin surges to new high")
+            
+            assert result["confidence"] > 0.8
+
+
+class TestMarketDataVerifier:
+    """Tests for MarketDataVerifier"""
+
+    @pytest.fixture
+    def verifier(self):
+        return MarketDataVerifier()
+
+    @pytest.mark.asyncio
+    async def test_verify_price_movement(self, verifier):
+        """Should verify price movement matches sentiment"""
+        with patch('aiohttp.ClientSession.get') as mock_get:
+            mock_response = AsyncMock()
+            mock_response.status = 200
+            mock_response.json = AsyncMock(return_value={
+                "price_change_24h": 0.15,  # 15% increase
+                "volume_24h": 1000000000
+            })
+            mock_get.return_value.__aenter__.return_value = mock_response
+            
+            result = await verifier.verify_price_movement("BTC", "bullish")
+            
+            assert result["verified"] is True
+
+    @pytest.mark.asyncio
+    async def test_verify_price_movement_contradiction(self, verifier):
+        """Should detect contradiction"""
+        with patch('aiohttp.ClientSession.get') as mock_get:
+            mock_response = AsyncMock()
+            mock_response.status = 200
+            mock_response.json = AsyncMock(return_value={
+                "price_change_24h": -0.20,  # 20% decrease
+                "volume_24h": 1000000000
+            })
+            mock_get.return_value.__aenter__.return_value = mock_response
+            
+            result = await verifier.verify_price_movement("BTC", "bullish")
+            
+            assert result["verified"] is False
+
+
+class TestFactVerificationEngine:
+    """Tests for FactVerificationEngine"""
+
+    @pytest.fixture
+    def engine(self):
+        return FactVerificationEngine()
+
+    @pytest.mark.asyncio
+    async def test_verify_combines_all_sources(self, engine):
+        """Should combine all verification sources"""
+        text = "Bitcoin surges to $100k as institutional inflows surge"
+        
+        with patch.object(engine.on_chain_verifier, 'verify', new_callable=AsyncMock) as mock_onchain, \
+             patch.object(engine.news_verifier, 'cross_reference', new_callable=AsyncMock) as mock_news, \
+             patch.object(engine.market_verifier, 'verify_price_movement', new_callable=AsyncMock) as mock_market:
+            
+            mock_onchain.return_value = {"verified": True, "details": "tx confirmed"}
+            mock_news.return_value = {"verified": True, "matches": 3, "confidence": 0.8}
+            mock_market.return_value = {"verified": True, "price_change": 0.15}
+            
+            result = await engine.verify(text, "BTC", "bullish")
+            
+            assert result["verified"] is True
+            assert result["confidence"] > 0.7
+            assert len(result["evidence"]) >= 3
+
+    @pytest.mark.asyncio
+    async def test_verify_partial_sources(self, engine):
+        """Should handle partial verification"""
+        text = "Some obscure news"
+        
+        with patch.object(engine.on_chain_verifier, 'verify', new_callable=AsyncMock) as mock_onchain, \
+             patch.object(engine.news_verifier, 'cross_reference', new_callable=AsyncMock) as mock_news, \
+             patch.object(engine.market_verifier, 'verify_price_movement', new_callable=AsyncMock) as mock_market:
+            
+            mock_onchain.return_value = {"verified": False, "details": "no tx"}
+            mock_news.return_value = {"verified": False, "matches": 0, "confidence": 0.0}
+            mock_market.return_value = {"verified": False, "price_change": 0.0}
+            
+            result = await engine.verify(text, "UNKNOWN", "neutral")
+            
+            assert result["verified"] is False
+            assert result["confidence"] < 0.3
+
+
+class TestLabelingPipelineRunner:
+    """Tests for LabelingPipelineRunner"""
+
+    @pytest.fixture
+    def runner(self):
+        return LabelingPipelineRunner()
+
+    @pytest.mark.asyncio
+    async def test_run_on_dataset(self, runner):
+        """Should process dataset file"""
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.jsonl', delete=False) as f:
+            for i in range(5):
+                f.write(json.dumps({
+                    "id": f"sample_{i}",
+                    "raw_text": f"Bitcoin news {i}",
+                    "source_id": "test",
+                    "source_type": "news",
+                    "credibility": 0.8
+                }) + '\n')
+            input_file = f.name
+        
+        output_file = input_file.replace('.jsonl', '_labeled.jsonl')
+        
+        try:
+            results = await runner.run_on_dataset(input_file, output_file)
+            
+            assert len(results) == 5
+            assert all("labels" in r for r in results)
+            assert os.path.exists(output_file)
+        finally:
+            os.unlink(input_file)
+            if os.path.exists(output_file):
+                os.unlink(output_file)
+
+    @pytest.mark.asyncio
+    async def test_run_on_dataset_tracks_stats(self, runner):
+        """Should track verification stats"""
+        with tempfile.NamedTemporaryFile(mode='w', suffix='.jsonl', delete=False) as f:
+            f.write(json.dumps({"id": "1", "raw_text": "Bitcoin surges!", "source_id": "test", "source_type": "news", "credibility": 0.9}) + '\n')
+            f.write(json.dumps({"id": "2", "raw_text": "Random text without crypto", "source_id": "test", "source_type": "news", "credibility": 0.5}) + '\n')
+            input_file = f.name
+        
+        output_file = input_file.replace('.jsonl', '_labeled.jsonl')
+        
+        try:
+            results = await runner.run_on_dataset(input_file, output_file)
+            
+            assert runner.total_count == 2
+            assert runner.verified_count <= 2
+        finally:
+            os.unlink(input_file)
+            if os.path.exists(output_file):
+                os.unlink(output_file)
+
+
+class TestLabelingPipelineIntegration:
+    """Integration tests for labeling pipeline"""
+
+    @pytest.mark.asyncio
+    async def test_full_labeling_workflow(self):
+        """Complete labeling workflow"""
+        pipeline = LabelingPipeline()
+        
+        # Test various crypto events
+        test_cases = [
+            ("Bitcoin surges to $100k!", "Bullish", "earnings"),
+            ("Major hack on exchange!", "Bearish", "hack"),
+            ("SEC approves ETF!", "Bullish", "regulatory"),
+            ("Ethereum upgrade activates!", "Neutral", "upgrade"),
+            ("Whale moves 10000 BTC!", "Bullish", "whale"),
+        ]
+        
+        for text, expected_sentiment, expected_event in test_cases:
+            result = await pipeline.label_text(text)
+            
+            assert result["labels"]["sentiment"] == expected_sentiment
+            assert result["labels"]["event_type"] == expected_event
+            assert result["confidence"]["overall"] > 0.0
+
+    @pytest.mark.asyncio
+    async def test_labeling_with_verification(self):
+        """Labeling with fact verification"""
+        pipeline = LabelingPipeline()
+        
+        # High credibility source
+        result = await pipeline.label_text(
+            "Bitcoin surges to $100k!",
+            context={"source_id": "bloomberg", "source_type": "news", "source_credibility": 0.95}
+        )
+        
+        assert result["confidence"]["verification"] > 0.5
+
+    @pytest.mark.asyncio
+    async def test_batch_labeling_performance(self):
+        """Batch labeling should be efficient"""
+        pipeline = LabelingPipeline()
+        
+        texts = [f"Bitcoin news {i}" for i in range(20)]
+        
+        import time
+        start = time.time()
+        results = await pipeline.label_batch(texts)
+        elapsed = time.time() - start
+        
+        assert len(results) == 20
+        assert elapsed < 30  # Should complete within 30 seconds
+
+
+class TestLabelingPipelineEdgeCases:
+    """Edge case tests for labeling pipeline"""
+
+    @pytest.fixture
+    def pipeline(self):
+        return LabelingPipeline()
+
+    @pytest.mark.asyncio
+    async def test_empty_text(self, pipeline):
+        """Should handle empty text"""
+        result = await pipeline.label_text("")
+        
+        assert result["labels"]["sentiment"] == "Neutral"
+
+    @pytest.mark.asyncio
+    async def test_very_long_text(self, pipeline):
+        """Should handle very long text"""
+        text = "Bitcoin surges! " * 1000
+        
+        result = await pipeline.label_text(text)
+        
+        assert result["labels"]["sentiment"] in ["Bullish", "Bearish", "Neutral"]
+
+    @pytest.mark.asyncio
+    async def test_unicode_text(self, pipeline):
+        """Should handle unicode"""
+        text = "Bitcoin 🚀 surges to $100k 💎"
+        
+        result = await pipeline.label_text(text)
+        
+        assert result["labels"]["sentiment"] == "Bullish"
+
+    @pytest.mark.asyncio
+    async def test_special_characters(self, pipeline):
+        """Should handle special characters"""
+        text = "BTC/USD: $50,000.00 (24h: +5.2%)"
+        
+        result = await pipeline.label_text(text)
+        
+        assert result["labels"]["sentiment"] in ["Bullish", "Bearish", "Neutral"]
+
+    @pytest.mark.asyncio
+    async def test_no_crypto_entities(self, pipeline):
+        """Should handle text without crypto entities"""
+        text = "The weather is nice today"
+        
+        result = await pipeline.label_text(text)
+        
+        assert result["labels"]["sentiment"] == "Neutral"
+        assert result["labels"]["event_type"] == "listing"  # default
+
+
+if __name__ == "__main__":
+    pytest.main([__file__, "-v"])