import sys sys.path.insert(0, '/mnt/dolphinng5_predict/sentiment_engine/src') import asyncio from datetime import datetime from sentiment_engine.ingestion.rss import RSSConnector from sentiment_engine.ingestion.telegram_preview import TelegramPreviewConnector from sentiment_engine.ingestion.base import ConnectorConfig, ConnectorType from sentiment_engine.nlp.pipeline import NLPProcessingPipeline from sentiment_engine.nlp.entity_extraction import EntityExtractor, AssetMapper from sentiment_engine.schemas.payload import NormalizedPayload, SourceType, AssetMention, EngagementMetrics async def main(): print("=== LIVE SENTIMENT ANALYSIS ===") # RSS sources rss_sources = [ {'source_id': 'rss:coindesk', 'url': 'https://www.coindesk.com/arc/outboundfeeds/rss/', 'cred': 0.85, 'feed_urls': ['https://www.coindesk.com/arc/outboundfeeds/rss/']}, {'source_id': 'rss:cointelegraph', 'url': 'https://cointelegraph.com/rss', 'cred': 0.75, 'feed_urls': ['https://cointelegraph.com/rss']}, {'source_id': 'rss:decrypt', 'url': 'https://decrypt.co/feed', 'cred': 0.75, 'feed_urls': ['https://decrypt.co/feed']}, {'source_id': 'rss:glassnode', 'url': 'https://insights.glassnode.com/rss/', 'cred': 0.85, 'feed_urls': ['https://insights.glassnode.com/rss/']}, {'source_id': 'rss:wsj_crypto', 'url': 'https://feeds.a.dj.com/rss/RSSMarketsMain.xml', 'cred': 0.85, 'feed_urls': ['https://feeds.a.dj.com/rss/RSSMarketsMain.xml']}, ] # Telegram preview sources telegram_channels = [ 'harmony_announcements', 'AlgorandFoundation', 'zilliqa', 'SolanaAnnouncements', 'AvalancheOfficial', 'StarkNetOfficial', 'CosmosAnnouncements', 'PolkadotAnnouncements', 'KusamaAnnouncements', 'CardanoAnnouncements', 'OfficialTether', 'BaseAnnouncements', 'ScrollAnnouncements', 'AptosAnnouncements', 'SuiAnnouncements', 'BitcoinNews', 'EthereumFoundation', 'PolygonAnnouncements', 'ArbitrumAnnouncements', 'OptimismAnnouncements', 'BaseAnnouncements', 'ScrollAnnouncements', 'StarkNetAnnouncements', 'NearAnnouncements', 'InjectiveAnnouncements', 'CelestiaAnnouncements', 'SeiAnnouncements', 'AptosAnnouncements', 'SuiAnnouncements', 'InjectiveAnnouncements', 'CelestiaAnnouncements', 'SeiAnnouncements', 'AptosOfficial', 'SuiOfficial', 'InjectiveOfficial', 'CelestiaOfficial', 'SeiOfficial', 'NearProtocol', 'NearAnnouncements', 'CosmosAnnouncements', 'CosmosOfficial', 'PolkadotAnnouncements', 'PolkadotOfficial', 'KusamaAnnouncements', 'KusamaOfficial', 'CardanoAnnouncements', 'CardanoOfficial', 'XRPAnnouncements', 'XRPLAnnouncements', 'RippleOfficial', 'Dogecoin', 'DogecoinOfficial', 'SHIBAnnouncements', 'ShibaInuOfficial', 'PepeAnnouncements', 'PepeOfficial', 'Bonkofficial', 'WIFAnnouncements', 'OfficialTether', 'USDCAnnouncements', 'CircleOfficial' ] print('Fetching RSS sources...') all_payloads = [] for src in [ {'source_id': 'rss:coindesk', 'url': 'https://www.coindesk.com/arc/outboundfeeds/rss/', 'cred': 0.85, 'feed_urls': ['https://www.coindesk.com/arc/outboundfeeds/rss/']}, {'source_id': 'rss:cointelegraph', 'url': 'https://cointelegraph.com/rss', 'cred': 0.75, 'feed_urls': ['https://cointelegraph.com/rss']}, {'source_id': 'rss:decrypt', 'url': 'https://decrypt.co/feed', 'cred': 0.75, 'feed_urls': ['https://decrypt.co/feed']}, {'source_id': 'rss:glassnode', 'url': 'https://insights.glassnode.com/rss/', 'cred': 0.85, 'feed_urls': ['https://insights.glassnode.com/rss/']}, {'source_id': 'rss:wsj_crypto', 'url': 'https://feeds.a.dj.com/rss/RSSMarketsMain.xml', 'cred': 0.85, 'feed_urls': ['https://feeds.a.dj.com/rss/RSSMarketsMain.xml']}, ]: config = ConnectorConfig( source_id=src['source_id'], connector_type=ConnectorType.RSS, base_url=src['url'], cadence_seconds=300, base_credibility=src['cred'], relevance=0.9, extra_config={'feed_urls': src['feed_urls'], 'max_items_per_feed': 30}, timeout_seconds=30 ) connector = RSSConnector(config) await connector.initialize() payloads = await connector.poll() print(f' {src["source_id"]}: {len(payloads)} items') all_payloads.extend(payloads) await connector.close() # Telegram preview sources telegram_channels = [ 'harmony_announcements', 'AlgorandFoundation', 'zilliqa', 'SolanaAnnouncements', 'AvalancheOfficial', 'StarkNetOfficial', 'CosmosAnnouncements', 'PolkadotAnnouncements', 'KusamaAnnouncements', 'CardanoAnnouncements', 'OfficialTether', 'BaseAnnouncements', 'ScrollAnnouncements', 'AptosAnnouncements', 'SuiAnnouncements', 'BitcoinNews', 'EthereumFoundation', 'PolygonAnnouncements', 'ArbitrumAnnouncements', 'OptimismAnnouncements', 'BaseAnnouncements', 'ScrollAnnouncements', 'StarkNetAnnouncements', 'NearAnnouncements', 'InjectiveAnnouncements', 'CelestiaAnnouncements', 'SeiAnnouncements', 'AptosAnnouncements', 'SuiAnnouncements', 'InjectiveAnnouncements', 'CelestiaAnnouncements', 'SeiAnnouncements', 'AptosOfficial', 'SuiOfficial', 'InjectiveOfficial', 'CelestiaOfficial', 'SeiOfficial', 'NearProtocol', 'NearAnnouncements', 'CosmosAnnouncements', 'CosmosOfficial', 'PolkadotAnnouncements', 'PolkadotOfficial', 'KusamaAnnouncements', 'KusamaOfficial', 'CardanoAnnouncements', 'CardanoOfficial', 'XRPAnnouncements', 'XRPLAnnouncements', 'RippleOfficial', 'Dogecoin', 'DogecoinOfficial', 'SHIBAnnouncements', 'ShibaInuOfficial', 'PepeAnnouncements', 'PepeOfficial', 'Bonkofficial', 'WIFAnnouncements', 'OfficialTether', 'USDCAnnouncements', 'CircleOfficial' ] print('Fetching Telegram preview sources...') for ch in telegram_channels: config = ConnectorConfig( source_id=f'web:telegram:{ch}', connector_type='web_crawl', base_url='https://t.me/s/', cadence_seconds=300, base_credibility=0.8, relevance=0.95, extra_config={'channels': [ch], 'max_messages_per_channel': 20}, timeout_seconds=30 ) connector = TelegramPreviewConnector(config) await connector.initialize() payloads = await connector.poll() if payloads: print(f' @{ch}: {len(payloads)} messages') all_payloads.extend(payloads) await connector.close() print(f'\nTotal payloads: {len(all_payloads)}') # Entity extraction from sentiment_engine.nlp.entity_extraction import EntityExtractor, AssetMapper entity_extractor = EntityExtractor() await entity_extractor.initialize() trade_assets = ['ZIL', 'ONG', 'ONE', 'STX', 'ALGO', 'DASH', 'LTC', 'FET', 'XTZ', 'LINK', 'ENJ', 'DOGE', 'XLM', 'ETC', 'TRX', 'BTC', 'ETH', 'SOL', 'BNB', 'XRP', 'ADA', 'AVAX', 'DOT', 'MATIC', 'KSM', 'ATOM', 'APT', 'SUI', 'NEAR', 'ICP'] matched_payloads = [] for p in all_payloads: entities = await entity_extractor.extract_all(p.raw_text) asset_ids = [e.asset_id for e in entities if e.asset_id in trade_assets] if asset_ids: matched_payloads.append({'payload': p, 'assets': asset_ids}) # Run sentiment pipeline from sentiment_engine.nlp.pipeline import NLPProcessingPipeline from sentiment_engine.schemas.payload import NormalizedPayload, SourceType, AssetMention, EngagementMetrics pipeline = NLPProcessingPipeline() await pipeline.initialize() asset_sentiments = {} for item in matched_payloads: p = item['payload'] for asset in item['assets']: asset_mention = AssetMention(asset_id=asset, mention_span=(0, len(asset)), confidence=0.9, source_text=asset, mention_type='ticker') np = NormalizedPayload( source_id=p.source_id, source_type=SourceType.NEWS, source_credibility_base=p.metadata.get('source_credibility', 0.5), ingest_ts=datetime.now().timestamp(), publish_ts=p.publish_ts or datetime.now().timestamp(), asset_mentions=[asset_mention], raw_text=p.raw_text, title=p.title, url=p.url, author=None, engagement_metrics=EngagementMetrics(), content_length=len(p.raw_text), language='en', metadata={} ) try: processed = await pipeline.process(np) sent = processed.sentiment_per_asset.get(asset) if sent: if asset not in asset_sentiments: asset_sentiments[asset] = [] asset_sentiments[asset].append({ 'polarity': sent.polarity, 'confidence': sent.confidence, 'label': 'POSITIVE' if sent.polarity > 0.1 else 'NEGATIVE' if sent.polarity < -0.1 else 'NEUTRAL', 'source': p.source_id }) except: pass # Results print('\n=== FINAL COMPREHENSIVE SENTIMENT ANALYSIS ===') trade_assets = ['ZIL', 'ONG', 'ONE', 'STX', 'ALGO', 'DASH', 'LTC', 'FET', 'XTZ', 'LINK', 'ENJ', 'DOGE', 'XLM', 'ETC', 'TRX'] for asset in ['ZIL', 'ONG', 'ONE', 'STX', 'ALGO', 'DASH', 'LTC', 'FET', 'XTZ', 'LINK', 'ENJ', 'DOGE', 'XLM', 'ETC', 'TRX']: if asset in asset_sentiments: sents = asset_sentiments[asset] avg_pol = sum(s['polarity'] for s in sents) / len(sents) avg_conf = sum(s['confidence'] for s in sents) / len(sents) pos = sum(1 for s in sents if s['polarity'] > 0.1) neg = sum(1 for s in sents if s['polarity'] < -0.1) neu = sum(1 for s in sents if -0.1 <= s['polarity'] <= 0.1) signal = 'BULLISH' if avg_pol > 0.15 else 'MILD_BULL' if avg_pol > 0.05 else 'BEARISH' if avg_pol < -0.15 else 'MILD_BEAR' if avg_pol < -0.05 else 'NEUTRAL' print(f'{asset}: {signal} | pol={avg_pol:+.3f} conf={avg_conf:.3f} | {len(sents)} items (P:{pos} N:{neg} U:{neu})') else: print(f'{asset}: NO COVERAGE') # Market context print() for asset in ['BTC', 'ETH', 'SOL', 'BNB', 'XRP', 'ADA', 'AVAX', 'DOT', 'MATIC', 'KSM', 'ATOM', 'APT', 'SUI', 'NEAR', 'ICP']: if asset in asset_sentiments: sents = asset_sentiments[asset] avg_pol = sum(s['polarity'] for s in sents) / len(sents) avg_conf = sum(s['confidence'] for s in sents) / len(sents) pos = sum(1 for s in sents if s['polarity'] > 0.1) neg = sum(1 for s in sents if s['polarity'] < -0.1) neu = sum(1 for s in sents if -0.1 <= s['polarity'] <= 0.1) print(f'{asset}: pol={avg_pol:+.3f} conf={avg_conf:.3f} | {len(sents)} items (P:{pos} N:{neg} U:{neu})') import sys sys.exit(0)