sentiment-engine/sentiment_engine/tests/integration/test_full_pipeline_comprehensive.py

"""
Comprehensive integration tests for full pipeline.
"""

import pytest
import asyncio
import json
from unittest.mock import AsyncMock, MagicMock, patch

from sentiment_engine.nlp.pipeline import NLPProcessingPipeline
from sentiment_engine.schemas.payload import NormalizedPayload, SourceType, AssetMention, EngagementMetrics
from sentiment_engine.schemas.processed import ProcessedItem


class TestFullPipelineIntegration:
    """Full pipeline integration tests"""

    @pytest.fixture
    def pipeline(self):
        return NLPProcessingPipeline()

    @pytest.mark.asyncio
    async def test_pipeline_initializes_all_components(self, pipeline):
        """Pipeline should initialize all NLP components"""
        await pipeline.initialize()
        
        assert pipeline._initialized is True
        assert pipeline.entity_extractor is not None
        assert pipeline.sentiment_analyzer is not None
        assert pipeline.event_classifier is not None
        assert pipeline.temporal_anchorer is not None
        assert pipeline.credibility_scorer is not None

    @pytest.mark.asyncio
    async def test_process_bullish_news(self, pipeline):
        """Should process bullish news correctly"""
        await pipeline.initialize()
        
        payload = NormalizedPayload(
            source_id="coindesk",
            source_type=SourceType.NEWS,
            source_credibility_base=0.9,
            ingest_ts=1700000000.0,
            publish_ts=1700000000.0,
            content_length=200,
            raw_text="Bitcoin surges to $108,000 as institutional inflows surge. BlackRock IBIT ETF sees record $1.2B daily inflow!",
            metadata={"author": "analyst", "engagement_metrics": {"likes": 1000, "retweets": 100}}
        )
        
        result = await pipeline.process(payload)
        
        assert isinstance(result, ProcessedItem)
        assert result.source_id == "coindesk"
        assert "BTC" in result.sentiment_per_asset or "IBIT" in result.sentiment_per_asset
        
        # Should be bullish
        for asset, sentiment in result.sentiment_per_asset.items():
            assert sentiment.polarity > 0.3

    @pytest.mark.asyncio
    async def test_process_bearish_news(self, pipeline):
        """Should process bearish news correctly"""
        await pipeline.initialize()
        
        payload = NormalizedPayload(
            source_id="peckshield",
            source_type=SourceType.NEWS,
            source_credibility_base=0.95,
            ingest_ts=1700000000.0,
            publish_ts=1700000000.0,
            content_length=200,
            raw_text="Major hack: Radiant Capital loses $50M in exploit. Attacker exploits rounding error. Funds moved to Tornado Cash.",
            metadata={}
        )
        
        result = await pipeline.process(payload)
        
        assert isinstance(result, ProcessedItem)
        
        # Should detect hack event
        hack_events = [e for e in result.events if e.event_type.value == "hack"]
        assert len(hack_events) >= 1

    @pytest.mark.asyncio
    async def test_process_regulatory_news(self, pipeline):
        """Should process regulatory news"""
        await pipeline.initialize()
        
        payload = NormalizedPayload(
            source_id="sec_gov",
            source_type=SourceType.REGULATORY,
            source_credibility_base=1.0,
            ingest_ts=1700000000.0,
            publish_ts=1700000000.0,
            content_length=150,
            raw_text="SEC sues Kraken for operating unregistered securities exchange. BTC, ETH, SOL decline on fears.",
            metadata={}
        )
        
        result = await pipeline.process(payload)
        
        # Should detect regulatory event
        reg_events = [e for e in result.events if e.event_type.value == "regulatory"]
        assert len(reg_events) >= 1

    @pytest.mark.asyncio
    async def test_process_upgrade_news(self, pipeline):
        """Should process protocol upgrade news"""
        await pipeline.initialize()
        
        payload = NormalizedPayload(
            source_id="ethereum_foundation",
            source_type=SourceType.NEWS,
            source_credibility_base=0.98,
            ingest_ts=1700000000.0,
            publish_ts=1700000000.0,
            content_length=150,
            raw_text="Ethereum Dencun upgrade goes live. Proto-Danksharding (EIP-4844) activates reducing L2 fees 90%.",
            metadata={}
        )
        
        result = await pipeline.process(payload)
        
        # Should detect upgrade event
        upgrade_events = [e for e in result.events if e.event_type.value == "upgrade"]
        assert len(upgrade_events) >= 1

    @pytest.mark.asyncio
    async def test_process_listing_news(self, pipeline):
        """Should process exchange listing news"""
        await pipeline.initialize()
        
        payload = NormalizedPayload(
            source_id="coinbase",
            source_type=SourceType.EXCHANGE_ANN,
            source_credibility_base=0.9,
            ingest_ts=1700000000.0,
            publish_ts=1700000000.0,
            content_length=150,
            raw_text="Coinbase lists PEPE and BONK memecoins. Trading opens with 100x volume spike.",
            metadata={}
        )
        
        result = await pipeline.process(payload)
        
        # Should detect listing event
        listing_events = [e for e in result.events if e.event_type.value == "listing"]
        assert len(listing_events) >= 1

    @pytest.mark.asyncio
    async def test_process_whale_activity(self, pipeline):
        """Should process whale activity"""
        await pipeline.initialize()
        
        payload = NormalizedPayload(
            source_id="whale_alert",
            source_type=SourceType.ON_CHAIN,
            source_credibility_base=0.95,
            ingest_ts=1700000000.0,
            publish_ts=1700000000.0,
            content_length=150,
            raw_text="Whale moves 10,000 BTC after 5 years dormancy. $1.08B transaction spotted on-chain.",
            metadata={}
        )
        
        result = await pipeline.process(payload)
        
        # Should detect whale event
        whale_events = [e for e in result.events if e.event_type.value == "whale"]
        assert len(whale_events) >= 1

    @pytest.mark.asyncio
    async def test_process_market_crash(self, pipeline):
        """Should process market crash"""
        await pipeline.initialize()
        
        payload = NormalizedPayload(
            source_id="market_watch",
            source_type=SourceType.NEWS,
            source_credibility_base=0.9,
            ingest_ts=1700000000.0,
            publish_ts=1700000000.0,
            content_length=150,
            raw_text="Bitcoin crashes 50% in hours. Massive liquidation cascade wipes out $500M in longs.",
            metadata={}
        )
        
        result = await pipeline.process(payload)
        
        # Should detect liquidation event
        liq_events = [e for e in result.events if e.event_type.value == "liquidation"]
        assert len(liq_events) >= 1

    @pytest.mark.asyncio
    async def test_process_stablecoin_depeg(self, pipeline):
        """Should process stablecoin depeg"""
        await pipeline.initialize()
        
        payload = NormalizedPayload(
            source_id="circle",
            source_type=SourceType.NEWS,
            source_credibility_base=0.95,
            ingest_ts=1700000000.0,
            publish_ts=1700000000.0,
            content_length=150,
            raw_text="Circle USDC depegs to $0.97 after SVB exposure. $3.3B reserves stuck at SVB.",
            metadata={}
        )
        
        result = await pipeline.process(payload)
        
        # Should be bearish
        for asset, sentiment in result.sentiment_per_asset.items():
            assert sentiment.polarity < -0.3


class TestPipelinePerformance:
    """Performance tests for pipeline"""

    @pytest.fixture
    def pipeline(self):
        return NLPProcessingPipeline()

    @pytest.mark.asyncio
    async def test_process_latency_under_threshold(self, pipeline):
        """Process should complete within latency threshold"""
        await pipeline.initialize()
        
        payload = NormalizedPayload(
            source_id="test",
            source_type=SourceType.NEWS,
            source_credibility_base=0.8,
            ingest_ts=1700000000.0,
            publish_ts=1700000000.0,
            content_length=200,
            raw_text="Bitcoin surges to $100k as institutional inflows surge.",
            metadata={}
        )
        
        import time
        start = time.time()
        result = await pipeline.process(payload)
        elapsed = (time.time() - start) * 1000
        
        assert elapsed < 5000  # 5 seconds max
        assert result.processing_latency_ms < 5000

    @pytest.mark.asyncio
    async def test_batch_processing_throughput(self, pipeline):
        """Batch processing should achieve good throughput"""
        await pipeline.initialize()
        
        payloads = [
            NormalizedPayload(
                source_id=f"source_{i}",
                source_type=SourceType.NEWS,
                source_credibility_base=0.8,
                ingest_ts=1700000000.0,
                publish_ts=1700000000.0,
                content_length=100,
                raw_text=f"Bitcoin news item {i}",
                metadata={}
            )
            for i in range(20)
        ]
        
        import time
        start = time.time()
        results = await pipeline.process_batch(payloads)
        elapsed = time.time() - start
        
        assert len(results) == 20
        assert elapsed < 10  # 20 items in under 10 seconds

    @pytest.mark.asyncio
    async def test_concurrent_processing(self, pipeline):
        """Should handle concurrent processing correctly"""
        await pipeline.initialize()
        
        payload = NormalizedPayload(
            source_id="test",
            source_type=SourceType.NEWS,
            source_credibility_base=0.8,
            ingest_ts=1700000000.0,
            publish_ts=1700000000.0,
            content_length=100,
            raw_text="Bitcoin surges to new high!",
            metadata={}
        )
        
        # Run multiple processes concurrently
        tasks = [pipeline.process(payload) for _ in range(10)]
        results = await asyncio.gather(*tasks)
        
        assert len(results) == 10
        assert all(isinstance(r, ProcessedItem) for r in results)


class TestPipelineDataFlow:
    """Tests for data flow through pipeline"""

    @pytest.fixture
    def pipeline(self):
        return NLPProcessingPipeline()

    @pytest.mark.asyncio
    async def test_entity_extraction_output(self, pipeline):
        """Entity extraction should produce valid entities"""
        await pipeline.initialize()
        
        payload = NormalizedPayload(
            source_id="test",
            source_type=SourceType.NEWS,
            source_credibility_base=0.8,
            ingest_ts=1700000000.0,
            publish_ts=1700000000.0,
            content_length=50,
            raw_text="BTC and ETH surge. Vitalik buys more ETH.",
            metadata={}
        )
        
        result = await pipeline.process(payload)
        
        assert len(result.entities) >= 2
        entity_assets = [e.asset_id for e in result.entities]
        assert "BTC" in entity_assets
        assert "ETH" in entity_assets

    @pytest.mark.asyncio
    async def test_sentiment_output_structure(self, pipeline):
        """Sentiment output should have correct structure"""
        await pipeline.initialize()
        
        payload = NormalizedPayload(
            source_id="test",
            source_type=SourceType.NEWS,
            source_credibility_base=0.8,
            ingest_ts=1700000000.0,
            publish_ts=1700000000.0,
            content_length=50,
            raw_text="Bitcoin surges to new high!",
            asset_mentions=[
                AssetMention(asset_id="BTC", mention_span=(0,3), confidence=0.9, source_text="BTC", mention_type="ticker")
            ],
            metadata={}
        )
        
        result = await pipeline.process(payload)
        
        assert "BTC" in result.sentiment_per_asset
        sentiment = result.sentiment_per_asset["BTC"]
        assert hasattr(sentiment, 'polarity')
        assert hasattr(sentiment, 'confidence')
        assert hasattr(sentiment, 'positive_prob')
        assert hasattr(sentiment, 'negative_prob')
        assert hasattr(sentiment, 'neutral_prob')

    @pytest.mark.asyncio
    async def test_emotion_output_structure(self, pipeline):
        """Emotion output should have correct structure"""
        await pipeline.initialize()
        
        payload = NormalizedPayload(
            source_id="test",
            source_type=SourceType.NEWS,
            source_credibility_base=0.8,
            ingest_ts=1700000000.0,
            publish_ts=1700000000.0,
            content_length=50,
            raw_text="Bitcoin surges to new high!",
            asset_mentions=[
                AssetMention(asset_id="BTC", mention_span=(0,3), confidence=0.9, source_text="BTC", mention_type="ticker")
            ],
            metadata={}
        )
        
        result = await pipeline.process(payload)
        
        assert "BTC" in result.emotions_per_asset
        emotion = result.emotions_per_asset["BTC"]
        assert hasattr(emotion, 'joy')
        assert hasattr(emotion, 'fear')
        assert hasattr(emotion, 'anger')
        assert hasattr(emotion, 'greed')
        assert hasattr(emotion, 'sadness')
        assert hasattr(emotion, 'intensity')

    @pytest.mark.asyncio
    async def test_temporal_output_structure(self, pipeline):
        """Temporal output should have correct structure"""
        await pipeline.initialize()
        
        payload = NormalizedPayload(
            source_id="test",
            source_type=SourceType.NEWS,
            source_credibility_base=0.8,
            ingest_ts=1700000000.0,
            publish_ts=1700000000.0,
            content_length=50,
            raw_text="Breaking: Bitcoin crashes now!",
            metadata={}
        )
        
        result = await pipeline.process(payload)
        
        assert result.temporal.time_horizon == "immediate"
        assert result.temporal.is_breaking is True

    @pytest.mark.asyncio
    async def test_credibility_output_structure(self, pipeline):
        """Credibility output should have correct structure"""
        await pipeline.initialize()
        
        payload = NormalizedPayload(
            source_id="high_cred",
            source_type=SourceType.NEWS,
            source_credibility_base=0.9,
            ingest_ts=1700000000.0,
            publish_ts=1700000000.0,
            content_length=100,
            raw_text="Bitcoin surges as BlackRock ETF sees massive inflows.",
            metadata={"author": "analyst", "engagement_metrics": {"likes": 1000, "retweets": 100, "views": 10000}}
        )
        
        result = await pipeline.process(payload)
        
        cred = result.credibility
        assert hasattr(cred, 'composite')
        assert hasattr(cred, 'source_base')
        assert hasattr(cred, 'content_quality')
        assert hasattr(cred, 'engagement_authenticity')
        assert hasattr(cred, 'cross_source_corroboration')
        assert hasattr(cred, 'historical_accuracy')


class TestPipelineErrorHandling:
    """Error handling tests for pipeline"""

    @pytest.fixture
    def pipeline(self):
        return NLPProcessingPipeline()

    @pytest.mark.asyncio
    async def test_handles_empty_payload(self, pipeline):
        """Should handle empty payload gracefully"""
        await pipeline.initialize()
        
        payload = NormalizedPayload(
            source_id="test",
            source_type=SourceType.NEWS,
            source_credibility_base=0.5,
            ingest_ts=1700000000.0,
            publish_ts=1700000000.0,
            content_length=0,
            raw_text="",
            metadata={}
        )
        
        result = await pipeline.process(payload)
        assert isinstance(result, ProcessedItem)

    @pytest.mark.asyncio
    async def test_handles_unicode(self, pipeline):
        """Should handle unicode text"""
        await pipeline.initialize()
        
        payload = NormalizedPayload(
            source_id="test",
            source_type=SourceType.NEWS,
            source_credibility_base=0.8,
            ingest_ts=1700000000.0,
            publish_ts=1700000000.0,
            content_length=50,
            raw_text="Bitcoin 🚀 surges to $100k 💎",
            metadata={}
        )
        
        result = await pipeline.process(payload)
        assert isinstance(result, ProcessedItem)

    @pytest.mark.asyncio
    async def test_handles_special_characters(self, pipeline):
        """Should handle special characters"""
        await pipeline.initialize()
        
        payload = NormalizedPayload(
            source_id="test",
            source_type=SourceType.NEWS,
            source_credibility_base=0.8,
            ingest_ts=1700000000.0,
            publish_ts=1700000000.0,
            content_length=100,
            raw_text="BTC/USD: $50,000.00 (24h: +5.2%) — Bitcoin dominance: 52.3%",
            metadata={}
        )
        
        result = await pipeline.process(payload)
        assert isinstance(result, ProcessedItem)

    @pytest.mark.asyncio
    async def test_batch_partial_failure(self, pipeline):
        """Batch should handle partial failures"""
        await pipeline.initialize()
        
        payloads = [
            NormalizedPayload(
                source_id="good",
                source_type=SourceType.NEWS,
                source_credibility_base=0.8,
                ingest_ts=1700000000.0,
                publish_ts=1700000000.0,
                content_length=50,
                raw_text="Bitcoin surges!",
                metadata={}
            ),
            NormalizedPayload(
                source_id="bad",
                source_type=SourceType.NEWS,
                source_credibility_base=0.5,
                ingest_ts=1700000000.0,
                publish_ts=1700000000.0,
                content_length=0,
                raw_text="",
                metadata={}
            )
        ]
        
        results = await pipeline.process_batch(payloads)
        
        assert len(results) == 2
        assert all(isinstance(r, ProcessedItem) for r in results)


class TestPipelineModelVersioning:
    """Tests for model version tracking"""

    @pytest.fixture
    def pipeline(self):
        return NLPProcessingPipeline()

    @pytest.mark.asyncio
    async def test_model_versions_in_output(self, pipeline):
        """Processed item should include model versions"""
        await pipeline.initialize()
        
        payload = NormalizedPayload(
            source_id="test",
            source_type=SourceType.NEWS,
            source_credibility_base=0.8,
            ingest_ts=1700000000.0,
            publish_ts=1700000000.0,
            content_length=50,
            raw_text="Bitcoin surges!",
            metadata={}
        )
        
        result = await pipeline.process(payload)
        
        assert "model_versions" in result.__dict__
        assert isinstance(result.model_versions, dict)


if __name__ == "__main__":
    pytest.main([__file__, "-v"])