427 lines
16 KiB
Python
427 lines
16 KiB
Python
|
"""
|
||
|
|
Comprehensive integration tests for Labeling Pipeline.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import pytest
|
||
|
|
import asyncio
|
||
|
|
import json
|
||
|
|
import tempfile
|
||
|
|
import os
|
||
|
|
from unittest.mock import AsyncMock, MagicMock, patch
|
||
|
|
|
||
|
|
from labeling_pipeline import (
|
||
|
|
LabelingPipeline, LabelingPipelineRunner,
|
||
|
|
OnChainVerifier, NewsCrossReferenceVerifier, MarketDataVerifier,
|
||
|
|
FactVerificationEngine
|
||
|
|
)
|
||
|
|
from sentiment_engine.schemas.processed import SentimentScores, EmotionScores, EventType
|
||
|
|
|
||
|
|
|
||
|
|
class TestLabelingPipeline:
|
||
|
|
"""Tests for LabelingPipeline"""
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def pipeline(self):
|
||
|
|
return LabelingPipeline()
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_label_text_returns_complete_structure(self, pipeline):
|
||
|
|
"""label_text should return complete labeled structure"""
|
||
|
|
result = await pipeline.label_text("Bitcoin surges to $100k!")
|
||
|
|
|
||
|
|
assert "labels" in result
|
||
|
|
assert "confidence" in result
|
||
|
|
assert "verified" in result
|
||
|
|
assert "verification_details" in result
|
||
|
|
assert "labeled_at" in result
|
||
|
|
|
||
|
|
assert result["labels"]["sentiment"] in ["Bearish", "Bullish", "Neutral"]
|
||
|
|
assert result["labels"]["event_type"] in [e.value for e in EventType]
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_label_text_with_context(self, pipeline):
|
||
|
|
"""label_text should use context"""
|
||
|
|
result = await pipeline.label_text(
|
||
|
|
"Bitcoin surges!",
|
||
|
|
context={"source_id": "coindesk", "source_type": "news", "source_credibility": 0.9}
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result["confidence"]["verification"] > 0.0
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_label_batch(self, pipeline):
|
||
|
|
"""Should label multiple texts"""
|
||
|
|
texts = [
|
||
|
|
"Bitcoin surges to $100k!",
|
||
|
|
"Major hack on exchange!",
|
||
|
|
"SEC approves ETF!"
|
||
|
|
]
|
||
|
|
|
||
|
|
results = await pipeline.label_batch(texts)
|
||
|
|
|
||
|
|
assert len(results) == 3
|
||
|
|
assert all("labels" in r for r in results)
|
||
|
|
|
||
|
|
def test_label_text_sync(self, pipeline):
|
||
|
|
"""Synchronous label_text should work"""
|
||
|
|
result = pipeline.label_text_sync("Bitcoin surges!")
|
||
|
|
|
||
|
|
assert "labels" in result
|
||
|
|
|
||
|
|
|
||
|
|
class TestOnChainVerifier:
|
||
|
|
"""Tests for OnChainVerifier"""
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def verifier(self):
|
||
|
|
return OnChainVerifier()
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_verify_transaction(self, verifier):
|
||
|
|
"""Should verify transaction exists"""
|
||
|
|
with patch('aiohttp.ClientSession.get') as mock_get:
|
||
|
|
mock_response = AsyncMock()
|
||
|
|
mock_response.status = 200
|
||
|
|
mock_response.json = AsyncMock(return_value={"status": "confirmed"})
|
||
|
|
mock_get.return_value.__aenter__.return_value = mock_response
|
||
|
|
|
||
|
|
result = await verifier.verify("tx_hash_123")
|
||
|
|
|
||
|
|
assert result["verified"] is True
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_verify_address_balance(self, verifier):
|
||
|
|
"""Should verify address balance"""
|
||
|
|
with patch('aiohttp.ClientSession.get') as mock_get:
|
||
|
|
mock_response = AsyncMock()
|
||
|
|
mock_response.status = 200
|
||
|
|
mock_response.json = AsyncMock(return_value={"balance": "1000000000000000000"})
|
||
|
|
mock_get.return_value.__aenter__.return_value = mock_response
|
||
|
|
|
||
|
|
result = await verifier.verify_address("0x123...", min_balance=1e18)
|
||
|
|
|
||
|
|
assert result["verified"] is True
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_verify_contract_interaction(self, verifier):
|
||
|
|
"""Should verify contract interaction"""
|
||
|
|
with patch('aiohttp.ClientSession.get') as mock_get:
|
||
|
|
mock_response = AsyncMock()
|
||
|
|
mock_response.status = 200
|
||
|
|
mock_response.json = AsyncMock(return_value={"logs": [{"address": "0x..."}]})
|
||
|
|
mock_get.return_value.__aenter__.return_value = mock_response
|
||
|
|
|
||
|
|
result = await verifier.verify_contract_interaction("0x123...", "0x456...")
|
||
|
|
|
||
|
|
assert result["verified"] is True
|
||
|
|
|
||
|
|
|
||
|
|
class TestNewsCrossReferenceVerifier:
|
||
|
|
"""Tests for NewsCrossReferenceVerifier"""
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def verifier(self):
|
||
|
|
return NewsCrossReferenceVerifier()
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_cross_reference_single_source(self, verifier):
|
||
|
|
"""Should find matching articles"""
|
||
|
|
with patch('aiohttp.ClientSession.get') as mock_get:
|
||
|
|
mock_response = AsyncMock()
|
||
|
|
mock_response.status = 200
|
||
|
|
mock_response.json = AsyncMock(return_value={
|
||
|
|
"articles": [
|
||
|
|
{"title": "Bitcoin surges", "source": "Reuters"},
|
||
|
|
{"title": "BTC hits new high", "source": "Bloomberg"}
|
||
|
|
]
|
||
|
|
})
|
||
|
|
mock_get.return_value.__aenter__.return_value = mock_response
|
||
|
|
|
||
|
|
result = await verifier.cross_reference("Bitcoin surges to new high")
|
||
|
|
|
||
|
|
assert result["verified"] is True
|
||
|
|
assert result["matches"] >= 2
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_cross_reference_no_matches(self, verifier):
|
||
|
|
"""Should return unverified when no matches"""
|
||
|
|
with patch('aiohttp.ClientSession.get') as mock_get:
|
||
|
|
mock_response = AsyncMock()
|
||
|
|
mock_response.status = 200
|
||
|
|
mock_response.json = AsyncMock(return_value={"articles": []})
|
||
|
|
mock_get.return_value.__aenter__.return_value = mock_response
|
||
|
|
|
||
|
|
result = await verifier.cross_reference("Completely unique obscure news")
|
||
|
|
|
||
|
|
assert result["verified"] is False
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_cross_reference_confidence(self, verifier):
|
||
|
|
"""Should calculate confidence based on match count"""
|
||
|
|
with patch('aiohttp.ClientSession.get') as mock_get:
|
||
|
|
mock_response = AsyncMock()
|
||
|
|
mock_response.status = 200
|
||
|
|
mock_response.json = AsyncMock(return_value={
|
||
|
|
"articles": [
|
||
|
|
{"title": "Bitcoin surges", "source": "Reuters"},
|
||
|
|
{"title": "BTC new high", "source": "Bloomberg"},
|
||
|
|
{"title": "Bitcoin record", "source": "CNBC"},
|
||
|
|
{"title": "Crypto surges", "source": "WSJ"},
|
||
|
|
{"title": "Digital gold", "source": "FT"}
|
||
|
|
]
|
||
|
|
})
|
||
|
|
mock_get.return_value.__aenter__.return_value = mock_response
|
||
|
|
|
||
|
|
result = await verifier.cross_reference("Bitcoin surges to new high")
|
||
|
|
|
||
|
|
assert result["confidence"] > 0.8
|
||
|
|
|
||
|
|
|
||
|
|
class TestMarketDataVerifier:
|
||
|
|
"""Tests for MarketDataVerifier"""
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def verifier(self):
|
||
|
|
return MarketDataVerifier()
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_verify_price_movement(self, verifier):
|
||
|
|
"""Should verify price movement matches sentiment"""
|
||
|
|
with patch('aiohttp.ClientSession.get') as mock_get:
|
||
|
|
mock_response = AsyncMock()
|
||
|
|
mock_response.status = 200
|
||
|
|
mock_response.json = AsyncMock(return_value={
|
||
|
|
"price_change_24h": 0.15, # 15% increase
|
||
|
|
"volume_24h": 1000000000
|
||
|
|
})
|
||
|
|
mock_get.return_value.__aenter__.return_value = mock_response
|
||
|
|
|
||
|
|
result = await verifier.verify_price_movement("BTC", "bullish")
|
||
|
|
|
||
|
|
assert result["verified"] is True
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_verify_price_movement_contradiction(self, verifier):
|
||
|
|
"""Should detect contradiction"""
|
||
|
|
with patch('aiohttp.ClientSession.get') as mock_get:
|
||
|
|
mock_response = AsyncMock()
|
||
|
|
mock_response.status = 200
|
||
|
|
mock_response.json = AsyncMock(return_value={
|
||
|
|
"price_change_24h": -0.20, # 20% decrease
|
||
|
|
"volume_24h": 1000000000
|
||
|
|
})
|
||
|
|
mock_get.return_value.__aenter__.return_value = mock_response
|
||
|
|
|
||
|
|
result = await verifier.verify_price_movement("BTC", "bullish")
|
||
|
|
|
||
|
|
assert result["verified"] is False
|
||
|
|
|
||
|
|
|
||
|
|
class TestFactVerificationEngine:
|
||
|
|
"""Tests for FactVerificationEngine"""
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def engine(self):
|
||
|
|
return FactVerificationEngine()
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_verify_combines_all_sources(self, engine):
|
||
|
|
"""Should combine all verification sources"""
|
||
|
|
text = "Bitcoin surges to $100k as institutional inflows surge"
|
||
|
|
|
||
|
|
with patch.object(engine.on_chain_verifier, 'verify', new_callable=AsyncMock) as mock_onchain, \
|
||
|
|
patch.object(engine.news_verifier, 'cross_reference', new_callable=AsyncMock) as mock_news, \
|
||
|
|
patch.object(engine.market_verifier, 'verify_price_movement', new_callable=AsyncMock) as mock_market:
|
||
|
|
|
||
|
|
mock_onchain.return_value = {"verified": True, "details": "tx confirmed"}
|
||
|
|
mock_news.return_value = {"verified": True, "matches": 3, "confidence": 0.8}
|
||
|
|
mock_market.return_value = {"verified": True, "price_change": 0.15}
|
||
|
|
|
||
|
|
result = await engine.verify(text, "BTC", "bullish")
|
||
|
|
|
||
|
|
assert result["verified"] is True
|
||
|
|
assert result["confidence"] > 0.7
|
||
|
|
assert len(result["evidence"]) >= 3
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_verify_partial_sources(self, engine):
|
||
|
|
"""Should handle partial verification"""
|
||
|
|
text = "Some obscure news"
|
||
|
|
|
||
|
|
with patch.object(engine.on_chain_verifier, 'verify', new_callable=AsyncMock) as mock_onchain, \
|
||
|
|
patch.object(engine.news_verifier, 'cross_reference', new_callable=AsyncMock) as mock_news, \
|
||
|
|
patch.object(engine.market_verifier, 'verify_price_movement', new_callable=AsyncMock) as mock_market:
|
||
|
|
|
||
|
|
mock_onchain.return_value = {"verified": False, "details": "no tx"}
|
||
|
|
mock_news.return_value = {"verified": False, "matches": 0, "confidence": 0.0}
|
||
|
|
mock_market.return_value = {"verified": False, "price_change": 0.0}
|
||
|
|
|
||
|
|
result = await engine.verify(text, "UNKNOWN", "neutral")
|
||
|
|
|
||
|
|
assert result["verified"] is False
|
||
|
|
assert result["confidence"] < 0.3
|
||
|
|
|
||
|
|
|
||
|
|
class TestLabelingPipelineRunner:
|
||
|
|
"""Tests for LabelingPipelineRunner"""
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def runner(self):
|
||
|
|
return LabelingPipelineRunner()
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_run_on_dataset(self, runner):
|
||
|
|
"""Should process dataset file"""
|
||
|
|
with tempfile.NamedTemporaryFile(mode='w', suffix='.jsonl', delete=False) as f:
|
||
|
|
for i in range(5):
|
||
|
|
f.write(json.dumps({
|
||
|
|
"id": f"sample_{i}",
|
||
|
|
"raw_text": f"Bitcoin news {i}",
|
||
|
|
"source_id": "test",
|
||
|
|
"source_type": "news",
|
||
|
|
"credibility": 0.8
|
||
|
|
}) + '\n')
|
||
|
|
input_file = f.name
|
||
|
|
|
||
|
|
output_file = input_file.replace('.jsonl', '_labeled.jsonl')
|
||
|
|
|
||
|
|
try:
|
||
|
|
results = await runner.run_on_dataset(input_file, output_file)
|
||
|
|
|
||
|
|
assert len(results) == 5
|
||
|
|
assert all("labels" in r for r in results)
|
||
|
|
assert os.path.exists(output_file)
|
||
|
|
finally:
|
||
|
|
os.unlink(input_file)
|
||
|
|
if os.path.exists(output_file):
|
||
|
|
os.unlink(output_file)
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_run_on_dataset_tracks_stats(self, runner):
|
||
|
|
"""Should track verification stats"""
|
||
|
|
with tempfile.NamedTemporaryFile(mode='w', suffix='.jsonl', delete=False) as f:
|
||
|
|
f.write(json.dumps({"id": "1", "raw_text": "Bitcoin surges!", "source_id": "test", "source_type": "news", "credibility": 0.9}) + '\n')
|
||
|
|
f.write(json.dumps({"id": "2", "raw_text": "Random text without crypto", "source_id": "test", "source_type": "news", "credibility": 0.5}) + '\n')
|
||
|
|
input_file = f.name
|
||
|
|
|
||
|
|
output_file = input_file.replace('.jsonl', '_labeled.jsonl')
|
||
|
|
|
||
|
|
try:
|
||
|
|
results = await runner.run_on_dataset(input_file, output_file)
|
||
|
|
|
||
|
|
assert runner.total_count == 2
|
||
|
|
assert runner.verified_count <= 2
|
||
|
|
finally:
|
||
|
|
os.unlink(input_file)
|
||
|
|
if os.path.exists(output_file):
|
||
|
|
os.unlink(output_file)
|
||
|
|
|
||
|
|
|
||
|
|
class TestLabelingPipelineIntegration:
|
||
|
|
"""Integration tests for labeling pipeline"""
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_full_labeling_workflow(self):
|
||
|
|
"""Complete labeling workflow"""
|
||
|
|
pipeline = LabelingPipeline()
|
||
|
|
|
||
|
|
# Test various crypto events
|
||
|
|
test_cases = [
|
||
|
|
("Bitcoin surges to $100k!", "Bullish", "earnings"),
|
||
|
|
("Major hack on exchange!", "Bearish", "hack"),
|
||
|
|
("SEC approves ETF!", "Bullish", "regulatory"),
|
||
|
|
("Ethereum upgrade activates!", "Neutral", "upgrade"),
|
||
|
|
("Whale moves 10000 BTC!", "Bullish", "whale"),
|
||
|
|
]
|
||
|
|
|
||
|
|
for text, expected_sentiment, expected_event in test_cases:
|
||
|
|
result = await pipeline.label_text(text)
|
||
|
|
|
||
|
|
assert result["labels"]["sentiment"] == expected_sentiment
|
||
|
|
assert result["labels"]["event_type"] == expected_event
|
||
|
|
assert result["confidence"]["overall"] > 0.0
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_labeling_with_verification(self):
|
||
|
|
"""Labeling with fact verification"""
|
||
|
|
pipeline = LabelingPipeline()
|
||
|
|
|
||
|
|
# High credibility source
|
||
|
|
result = await pipeline.label_text(
|
||
|
|
"Bitcoin surges to $100k!",
|
||
|
|
context={"source_id": "bloomberg", "source_type": "news", "source_credibility": 0.95}
|
||
|
|
)
|
||
|
|
|
||
|
|
assert result["confidence"]["verification"] > 0.5
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_batch_labeling_performance(self):
|
||
|
|
"""Batch labeling should be efficient"""
|
||
|
|
pipeline = LabelingPipeline()
|
||
|
|
|
||
|
|
texts = [f"Bitcoin news {i}" for i in range(20)]
|
||
|
|
|
||
|
|
import time
|
||
|
|
start = time.time()
|
||
|
|
results = await pipeline.label_batch(texts)
|
||
|
|
elapsed = time.time() - start
|
||
|
|
|
||
|
|
assert len(results) == 20
|
||
|
|
assert elapsed < 30 # Should complete within 30 seconds
|
||
|
|
|
||
|
|
|
||
|
|
class TestLabelingPipelineEdgeCases:
|
||
|
|
"""Edge case tests for labeling pipeline"""
|
||
|
|
|
||
|
|
@pytest.fixture
|
||
|
|
def pipeline(self):
|
||
|
|
return LabelingPipeline()
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_empty_text(self, pipeline):
|
||
|
|
"""Should handle empty text"""
|
||
|
|
result = await pipeline.label_text("")
|
||
|
|
|
||
|
|
assert result["labels"]["sentiment"] == "Neutral"
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_very_long_text(self, pipeline):
|
||
|
|
"""Should handle very long text"""
|
||
|
|
text = "Bitcoin surges! " * 1000
|
||
|
|
|
||
|
|
result = await pipeline.label_text(text)
|
||
|
|
|
||
|
|
assert result["labels"]["sentiment"] in ["Bullish", "Bearish", "Neutral"]
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_unicode_text(self, pipeline):
|
||
|
|
"""Should handle unicode"""
|
||
|
|
text = "Bitcoin 🚀 surges to $100k 💎"
|
||
|
|
|
||
|
|
result = await pipeline.label_text(text)
|
||
|
|
|
||
|
|
assert result["labels"]["sentiment"] == "Bullish"
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_special_characters(self, pipeline):
|
||
|
|
"""Should handle special characters"""
|
||
|
|
text = "BTC/USD: $50,000.00 (24h: +5.2%)"
|
||
|
|
|
||
|
|
result = await pipeline.label_text(text)
|
||
|
|
|
||
|
|
assert result["labels"]["sentiment"] in ["Bullish", "Bearish", "Neutral"]
|
||
|
|
|
||
|
|
@pytest.mark.asyncio
|
||
|
|
async def test_no_crypto_entities(self, pipeline):
|
||
|
|
"""Should handle text without crypto entities"""
|
||
|
|
text = "The weather is nice today"
|
||
|
|
|
||
|
|
result = await pipeline.label_text(text)
|
||
|
|
|
||
|
|
assert result["labels"]["sentiment"] == "Neutral"
|
||
|
|
assert result["labels"]["event_type"] == "listing" # default
|
||
|
|
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
pytest.main([__file__, "-v"])
|