""" Comprehensive tests for NLPProcessingPipeline integration. """ import pytest import asyncio from unittest.mock import AsyncMock, MagicMock, patch from sentiment_engine.nlp.pipeline import NLPProcessingPipeline from sentiment_engine.nlp.entity_extraction import EntityExtractor, AssetMapper from sentiment_engine.nlp.sentiment_emotion import SentimentEmotionAnalyzer from sentiment_engine.nlp.event_classification import EventClassifier from sentiment_engine.nlp.temporal import TemporalAnchorer from sentiment_engine.nlp.credibility import CredibilityScorer from sentiment_engine.schemas.payload import NormalizedPayload, SourceType, AssetMention, EngagementMetrics from sentiment_engine.schemas.processed import ProcessedItem, EntityExtraction, SentimentScores, EmotionScores, EventClassification, TemporalAnchor, CredibilityScore class TestNLPProcessingPipeline: """Tests for NLPProcessingPipeline""" @pytest.fixture def pipeline(self): return NLPProcessingPipeline() @pytest.mark.asyncio async def test_initialize_all_components(self, pipeline): """Should initialize all components""" await pipeline.initialize() assert pipeline._initialized is True assert pipeline.entity_extractor is not None assert pipeline.sentiment_analyzer is not None assert pipeline.event_classifier is not None assert pipeline.temporal_anchorer is not None assert pipeline.credibility_scorer is not None @pytest.mark.asyncio async def test_process_returns_processed_item(self, pipeline): """Process should return complete ProcessedItem""" await pipeline.initialize() payload = NormalizedPayload( source_id="test_source", source_type=SourceType.NEWS, source_credibility_base=0.8, ingest_ts=1700000000.0, publish_ts=1700000000.0, content_length=100, raw_text="Bitcoin surges to $100k as institutional inflows surge!", metadata={} ) result = await pipeline.process(payload) assert isinstance(result, ProcessedItem) assert result.payload_id is not None assert result.source_id == "test_source" assert len(result.entities) >= 0 assert isinstance(result.sentiment_per_asset, dict) assert isinstance(result.emotions_per_asset, dict) assert isinstance(result.events, list) assert result.temporal is not None assert result.credibility is not None assert result.processing_latency_ms > 0 @pytest.mark.asyncio async def test_process_with_asset_mentions(self, pipeline): """Process should use asset mentions from payload""" await pipeline.initialize() payload = NormalizedPayload( source_id="test_source", source_type=SourceType.NEWS, source_credibility_base=0.8, ingest_ts=1700000000.0, publish_ts=1700000000.0, content_length=100, raw_text="BTC surges to new high!", asset_mentions=[ AssetMention( asset_id="BTC", mention_span=(0, 3), confidence=0.9, source_text="BTC", mention_type="ticker" ) ], metadata={} ) result = await pipeline.process(payload) # Should have sentiment for BTC assert "BTC" in result.sentiment_per_asset @pytest.mark.asyncio async def test_process_batch(self, pipeline): """Process batch should handle multiple payloads""" await pipeline.initialize() payloads = [ NormalizedPayload( source_id=f"source_{i}", source_type=SourceType.NEWS, source_credibility_base=0.8, ingest_ts=1700000000.0, publish_ts=1700000000.0, content_length=100, raw_text=f"Bitcoin news {i}", metadata={} ) for i in range(5) ] results = await pipeline.process_batch(payloads) assert len(results) == 5 assert all(isinstance(r, ProcessedItem) for r in results) @pytest.mark.asyncio async def test_process_batch_concurrency_limit(self, pipeline): """Process batch should respect semaphore limit""" await pipeline.initialize() payloads = [ NormalizedPayload( source_id=f"source_{i}", source_type=SourceType.NEWS, source_credibility_base=0.8, ingest_ts=1700000000.0, publish_ts=1700000000.0, content_length=100, raw_text=f"Bitcoin news {i}", metadata={} ) for i in range(20) ] # Should complete without errors results = await pipeline.process_batch(payloads) assert len(results) == 20 @pytest.mark.asyncio async def test_process_handles_errors_gracefully(self, pipeline): """Process should handle component errors gracefully""" await pipeline.initialize() # Create a payload that might cause issues payload = NormalizedPayload( source_id="test_source", source_type=SourceType.NEWS, source_credibility_base=0.8, ingest_ts=1700000000.0, publish_ts=1700000000.0, content_length=0, raw_text="", metadata={} ) # Should not crash even with empty text try: result = await pipeline.process(payload) assert isinstance(result, ProcessedItem) except Exception: # If it raises, that's also acceptable behavior pass def test_get_model_versions(self, pipeline): """Should return model versions""" versions = pipeline.get_model_versions() assert isinstance(versions, dict) class TestPipelineComponentInteraction: """Tests for component interactions within pipeline""" @pytest.fixture def pipeline(self): return NLPProcessingPipeline() @pytest.mark.asyncio async def test_entity_extraction_feeds_sentiment(self, pipeline): """Entities extracted should feed into sentiment analysis""" await pipeline.initialize() payload = NormalizedPayload( source_id="test", source_type=SourceType.NEWS, source_credibility_base=0.8, ingest_ts=1700000000.0, publish_ts=1700000000.0, content_length=50, raw_text="BTC and ETH both surge", asset_mentions=[ AssetMention(asset_id="BTC", mention_span=(0, 3), confidence=0.9, source_text="BTC", mention_type="ticker"), AssetMention(asset_id="ETH", mention_span=(8, 11), confidence=0.9, source_text="ETH", mention_type="ticker"), ], metadata={} ) result = await pipeline.process(payload) # Both assets should have sentiment assert "BTC" in result.sentiment_per_asset assert "ETH" in result.sentiment_per_asset @pytest.mark.asyncio async def test_event_classification_uses_entities(self, pipeline): """Event classification should use extracted entities""" await pipeline.initialize() payload = NormalizedPayload( source_id="test", source_type=SourceType.NEWS, source_credibility_base=0.8, ingest_ts=1700000000.0, publish_ts=1700000000.0, content_length=50, raw_text="SEC approves Bitcoin ETF for trading", asset_mentions=[ AssetMention(asset_id="BTC", mention_span=(20, 23), confidence=0.9, source_text="BTC", mention_type="ticker"), ], metadata={} ) result = await pipeline.process(payload) # Should detect regulatory event involving BTC regulatory_events = [e for e in result.events if e.event_type.value == "regulatory"] assert len(regulatory_events) >= 1 assert "BTC" in regulatory_events[0].assets_involved @pytest.mark.asyncio async def test_temporal_anchoring_uses_publish_ts(self, pipeline): """Temporal anchoring should use publish timestamp""" await pipeline.initialize() publish_ts = 1700000000.0 payload = NormalizedPayload( source_id="test", source_type=SourceType.NEWS, source_credibility_base=0.8, ingest_ts=1700000000.0, publish_ts=publish_ts, content_length=50, raw_text="Breaking: BTC crashes now!", metadata={} ) result = await pipeline.process(payload) assert result.temporal.time_horizon == "immediate" assert result.temporal.is_breaking is True @pytest.mark.asyncio async def test_credibility_scoring_uses_all_factors(self, pipeline): """Credibility should combine all factors""" await pipeline.initialize() payload = NormalizedPayload( source_id="high_cred_source", source_type=SourceType.NEWS, source_credibility_base=0.9, ingest_ts=1700000000.0, publish_ts=1700000000.0, content_length=200, raw_text="Bitcoin surges to $100k as institutional inflows surge. BlackRock IBIT sees record inflows.", metadata={ "author": "analyst", "engagement_metrics": {"likes": 1000, "retweets": 100, "replies": 50, "views": 10000} } ) result = await pipeline.process(payload) # High credibility source + good content + good engagement = high composite assert result.credibility.composite > 0.5 class TestPipelineEdgeCases: """Edge case tests for pipeline""" @pytest.fixture def pipeline(self): return NLPProcessingPipeline() @pytest.mark.asyncio async def test_process_empty_text(self, pipeline): """Should handle empty text""" await pipeline.initialize() payload = NormalizedPayload( source_id="test", source_type=SourceType.NEWS, source_credibility_base=0.5, ingest_ts=1700000000.0, publish_ts=1700000000.0, content_length=0, raw_text="", metadata={} ) result = await pipeline.process(payload) assert isinstance(result, ProcessedItem) @pytest.mark.asyncio async def test_process_very_long_text(self, pipeline): """Should handle very long text""" await pipeline.initialize() long_text = "Bitcoin surges. " * 1000 payload = NormalizedPayload( source_id="test", source_type=SourceType.NEWS, source_credibility_base=0.5, ingest_ts=1700000000.0, publish_ts=1700000000.0, content_length=len(long_text), raw_text=long_text, metadata={} ) result = await pipeline.process(payload) assert isinstance(result, ProcessedItem) assert result.processing_latency_ms < 30000 # Should complete within 30s @pytest.mark.asyncio async def test_process_unicode(self, pipeline): """Should handle unicode text""" await pipeline.initialize() payload = NormalizedPayload( source_id="test", source_type=SourceType.NEWS, source_credibility_base=0.5, ingest_ts=1700000000.0, publish_ts=1700000000.0, content_length=50, raw_text="Bitcoin 🚀 surges to $100k 💎", metadata={} ) result = await pipeline.process(payload) assert isinstance(result, ProcessedItem) @pytest.mark.asyncio async def test_process_special_characters(self, pipeline): """Should handle special characters""" await pipeline.initialize() payload = NormalizedPayload( source_id="test", source_type=SourceType.NEWS, source_credibility_base=0.5, ingest_ts=1700000000.0, publish_ts=1700000000.0, content_length=50, raw_text="BTC/USD: $50,000.00 (24h: +5.2%)", metadata={} ) result = await pipeline.process(payload) assert isinstance(result, ProcessedItem) if __name__ == "__main__": pytest.main([__file__, "-v"])