diff --git a/sentiment_engine/src/sentiment_engine/nlp/event_classification.py b/sentiment_engine/src/sentiment_engine/nlp/event_classification.py new file mode 100644 index 0000000..01d52af --- /dev/null +++ b/sentiment_engine/src/sentiment_engine/nlp/event_classification.py @@ -0,0 +1,392 @@ +"""Event classification for financial news/social (with ONNX Runtime + keyword fallback)""" + +import asyncio +import logging +import re +from pathlib import Path +from typing import Dict, List, Optional + +import numpy as np + +from sentiment_engine.schemas.processed import EventClassification, EventType +from sentiment_engine.utils.config import get_settings + +logger = logging.getLogger(__name__) + +# Optional imports for production +try: + import onnxruntime as ort + ONNX_AVAILABLE = True +except ImportError: + ONNX_AVAILABLE = False + logger.debug("onnxruntime not available") + +try: + import torch + from transformers import AutoTokenizer, AutoModelForSequenceClassification + TRANSFORMERS_AVAILABLE = True +except ImportError: + TRANSFORMERS_AVAILABLE = False + logger.debug("transformers not available") + + +class ONNXEventModel: + """ONNX Runtime wrapper for BERT event classification model (requires token_type_ids)""" + + def __init__(self, model_path: str, tokenizer_path: str, label_map_path: str = None): + self.model_path = model_path + self.tokenizer_path = tokenizer_path + self.label_map_path = label_map_path + + # Load tokenizer with fallback + self.tokenizer = None + if TRANSFORMERS_AVAILABLE: + try: + self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) + print(f"DEBUG: Event tokenizer loaded successfully") + except Exception as e: + print(f"DEBUG: Failed to load event tokenizer: {e}") + # Create a simple mock tokenizer as fallback + self.tokenizer = self._create_mock_tokenizer() + else: + self.tokenizer = self._create_mock_tokenizer() + + # Load ONNX model with limited threads to avoid contention + session_options = ort.SessionOptions() + session_options.intra_op_num_threads = 2 + session_options.inter_op_num_threads = 2 + self.session = ort.InferenceSession(model_path, sess_options=session_options, providers=self._get_providers()) + + # Load labels + self.labels = [e.value for e in EventType if e != EventType.UNKNOWN] + if label_map_path and Path(label_map_path).exists(): + import json + with open(label_map_path) as f: + self.labels = [v for k, v in sorted(json.load(f).items(), key=lambda x: int(x[0]))] + + self._input_names = [i.name for i in self.session.get_inputs()] + self._output_names = [o.name for o in self.session.get_outputs()] + + def _create_mock_tokenizer(self): + """Create a simple mock tokenizer for fallback""" + class MockTokenizer: + def __call__(self, text, return_tensors='np', truncation=True, max_length=512, padding=True): + if isinstance(text, list): + batch_size = len(text) + else: + batch_size = 1 + text = [text] + # Simple character-level tokenization for fallback + input_ids = [] + for t in text: + ids = [ord(c) % 1000 + 1 for c in t[:max_length]] + ids = ids + [0] * (max_length - len(ids)) + input_ids.append(ids) + import numpy as np + return { + 'input_ids': np.array(input_ids, dtype=np.int64), + 'attention_mask': np.ones((batch_size, max_length), dtype=np.int64), + 'token_type_ids': np.zeros((batch_size, max_length), dtype=np.int64) + } + return MockTokenizer() + + def _get_providers(self): + providers = ['CPUExecutionProvider'] + if ort.get_device() == 'GPU': + providers.insert(0, 'CUDAExecutionProvider') + return providers + + def predict(self, input_ids, attention_mask, token_type_ids=None) -> np.ndarray: + """Run inference, return probabilities""" + if hasattr(input_ids, 'numpy'): + input_ids = input_ids.numpy() + if hasattr(attention_mask, 'numpy'): + attention_mask = attention_mask.numpy() + if token_type_ids is not None and hasattr(token_type_ids, 'numpy'): + token_type_ids = token_type_ids.numpy() + + ort_inputs = { + "input_ids": input_ids.astype(np.int64), + "attention_mask": attention_mask.astype(np.int64), + } + # BERT event model requires token_type_ids + if "token_type_ids" in self._input_names: + if token_type_ids is None: + token_type_ids = np.zeros_like(input_ids) + ort_inputs["token_type_ids"] = token_type_ids.astype(np.int64) + + outputs = self.session.run(self._output_names, ort_inputs) + logits = outputs[0] + + # Softmax + e_x = np.exp(logits - np.max(logits, axis=-1, keepdims=True)) + probs = e_x / e_x.sum(axis=-1, keepdims=True) + + return probs[0] + + +class EventClassifier: + """Classifies financial events from text""" + + EVENT_KEYWORDS = { + EventType.LISTING: [ + "listing", "listed", "list", "debut", "launch", "goes live", "trading starts", + "now available", "added to", "new listing", "exchange listing" + ], + EventType.DELISTING: [ + "delisting", "delisted", "remove", "removing", "suspend", "suspended", + "halt", "halted", "terminate", "terminated", "withdraw" + ], + EventType.HACK: [ + "hack", "hacked", "exploit", "exploited", "breach", "stolen", "theft", + "unauthorized", "compromise", "drain", "drained", "vulnerability" + ], + EventType.REGULATORY: [ + "sec", "cftc", "regulation", "regulatory", "compliance", "investigation", + "enforcement", "lawsuit", "legal action", "subpoena", "guidance", + "policy", "rule", "legislation", "bill", "congress", "parliament" + ], + EventType.GOVERNANCE: [ + "governance", "proposal", "vote", "voting", "dao", "referendum", + "snapshot", "quorum", "execution", "timelock", "multisig" + ], + EventType.UPGRADE: [ + "upgrade", "hard fork", "soft fork", "mainnet", "testnet", "release", + "version", "v2", "v3", "shanghai", "cancun", "proto-danksharding", + "eip", "bip", "improvement proposal" + ], + EventType.PARTNERSHIP: [ + "partnership", "partner", "collaboration", "collaborate", "integration", + "integrate", "alliance", "joint venture", "strategic", "ecosystem" + ], + EventType.EARNINGS: [ + "earnings", "revenue", "profit", "loss", "eps", "quarterly", "annual", + "financial results", "report", "guidance", "outlook", "forecast" + ], + EventType.MACRO: [ + "fed", "federal reserve", "interest rate", "rate hike", "rate cut", + "inflation", "cpi", "pce", "gdp", "unemployment", "jobs", "payroll", + "fomc", "powell", "central bank", "monetary policy" + ], + EventType.LIQUIDATION: [ + "liquidation", "liquidated", "margin call", "forced close", "liquidation cascade", + "short squeeze", "long squeeze", "cascade", "wipeout" + ], + EventType.WHALE: [ + "whale", "large holder", "accumulation", "distribution", "large transfer", + "moved", "transaction", "on-chain", "wallet", "entity" + ], + EventType.MANIPULATION: [ + "manipulation", "wash trading", "spoofing", "layering", "pump and dump", + "coordinated", "artificial", "fake volume", "market making abuse" + ], + } + + SEVERITY_BASE = { + EventType.HACK: 0.9, + EventType.DELISTING: 0.8, + EventType.LIQUIDATION: 0.7, + EventType.REGULATORY: 0.7, + EventType.MANIPULATION: 0.8, + EventType.LISTING: 0.5, + EventType.UPGRADE: 0.4, + EventType.PARTNERSHIP: 0.3, + EventType.GOVERNANCE: 0.4, + EventType.EARNINGS: 0.5, + EventType.MACRO: 0.6, + EventType.WHALE: 0.4, + } + + def __init__(self): + self.settings = get_settings() + self._onnx_model = None + self._pytorch_model = None + self._tokenizer = None + self._device = "cuda" if (TRANSFORMERS_AVAILABLE and torch.cuda.is_available()) else "cpu" + self._use_onnx = False + self._use_pytorch = False + # ONNX confidence threshold (lower than keyword because model is fine-tuned on small data) + self._onnx_threshold = 0.15 + # Keyword threshold + self._keyword_threshold = 0.3 + + async def initialize(self) -> None: + """Load classification model - priority: ONNX > PyTorch > Keywords (with timeout handling)""" + + # Check for ONNX model + onnx_model = Path("models/onnx/bert-base-event/model.onnx") + if ONNX_AVAILABLE and onnx_model.exists(): + print(f"DEBUG: Loading ONNX Event classifier from {onnx_model} ({onnx_model.stat().st_size / 1024 / 1024:.1f} MB)...") + import time + load_start = time.time() + try: + # Let ONNXEventModel handle tokenizer loading (it does this internally) + model_start = time.time() + self._onnx_model = ONNXEventModel( + str(onnx_model), + "models/onnx/bert-base-event", + "models/onnx/bert-base-event/label_map.json" + ) + print(f"DEBUG: ONNX Event classifier loaded in {time.time() - model_start:.1f}s (total: {time.time() - load_start:.1f}s)") + + self._use_onnx = True + logger.info("Loaded Event classifier via ONNX Runtime") + except Exception as e: + import traceback + logger.warning(f"Failed to load ONNX event classifier: {e}") + traceback.print_exc() + self._onnx_model = None + + # Fallback to PyTorch fine-tuned model + if TRANSFORMERS_AVAILABLE and not self._use_onnx: + try: + # In production, this would be a fine-tuned model + # For now, we'll use the keyword approach + self._use_pytorch = False + except Exception as e: + logger.warning(f"Failed to load PyTorch event classifier: {e}") + + if not self._use_onnx: + logger.info("Using keyword-based event classification") + + async def classify(self, text: str, asset_mentions: List[str]) -> List[EventClassification]: + """Classify events in text - combines ONNX and keyword methods""" + loop = asyncio.get_event_loop() + + onnx_events = [] + keyword_events = [] + + if self._use_onnx and self._onnx_model: + onnx_events = await loop.run_in_executor(None, self._classify_onnx, text, asset_mentions) + + # Always run keyword as fallback/ensemble + keyword_events = await loop.run_in_executor(None, self._classify_sync, text, asset_mentions) + + # Merge results: prefer ONNX if confident, otherwise use keyword + return self._merge_events(onnx_events, keyword_events) + + def _classify_onnx(self, text: str, asset_mentions: List[str]) -> List[EventClassification]: + """Classify using ONNX model""" + inputs = self._onnx_model.tokenizer( + text, + return_tensors="np", + truncation=True, + max_length=512, + padding=True + ) + + token_type_ids = inputs.get("token_type_ids") + probs = self._onnx_model.predict(inputs["input_ids"], inputs["attention_mask"], token_type_ids) + + events = [] + for i, label in enumerate(self._onnx_model.labels): + if i >= len(probs): + break + confidence = float(probs[i]) + if confidence < self._onnx_threshold: # Lower threshold for ONNX + continue + + try: + event_type = EventType(label) + except ValueError: + continue + + involved = self._find_involved_assets(text, asset_mentions, event_type) + severity = self._estimate_severity(event_type, confidence, text) + + events.append(EventClassification( + event_type=event_type, + confidence=confidence, + assets_involved=involved, + key_details={"model": "onnx", "label_index": i}, + severity=severity + )) + + events.sort(key=lambda e: e.confidence, reverse=True) + return events[:3] + + def _classify_pytorch(self, text: str, asset_mentions: List[str]) -> List[EventClassification]: + """Classify using PyTorch model""" + return self._classify_sync(text, asset_mentions) + + def _classify_sync(self, text: str, asset_mentions: List[str]) -> List[EventClassification]: + """Synchronous keyword-based event classification""" + text_lower = text.lower() + events = [] + + for event_type, keywords in self.EVENT_KEYWORDS.items(): + matches = [kw for kw in keywords if kw in text_lower] + if not matches: + continue + + # Calculate confidence based on keyword matches + confidence = min(0.95, len(matches) * 0.15 + 0.3) + + # Determine involved assets + involved = self._find_involved_assets(text, asset_mentions, event_type) + + # Estimate severity + severity = self._estimate_severity(event_type, confidence, text, matches) + + events.append(EventClassification( + event_type=event_type, + confidence=confidence, + assets_involved=involved, + key_details={"matched_keywords": matches, "method": "keyword"}, + severity=severity + )) + + # Sort by confidence + events.sort(key=lambda e: e.confidence, reverse=True) + + # Return top events (max 3) + return events[:3] + + def _merge_events(self, onnx_events: List[EventClassification], keyword_events: List[EventClassification]) -> List[EventClassification]: + """Merge ONNX and keyword events, preferring higher confidence""" + # Create a map of event_type -> best event + merged = {} + + for e in onnx_events: + key = e.event_type + if key not in merged or e.confidence > merged[key].confidence: + merged[key] = e + + for e in keyword_events: + key = e.event_type + if key not in merged or e.confidence > merged[key].confidence: + merged[key] = e + + # Sort by confidence and return top 3 + result = list(merged.values()) + result.sort(key=lambda e: e.confidence, reverse=True) + return result[:3] + + def _find_involved_assets(self, text: str, asset_mentions: List[str], event_type: EventType) -> List[str]: + """Find which assets are involved in the event""" + involved = [] + text_lower = text.lower() + + for asset in asset_mentions: + if asset.lower() in text_lower: + involved.append(asset) + + # If no specific assets found but event is market-wide + if not involved and event_type in {EventType.MACRO, EventType.REGULATORY}: + involved = ["MARKET"] + + return involved + + def _estimate_severity(self, event_type: EventType, confidence: float, text: str, matches: List[str] = None) -> float: + """Estimate event severity 0-1""" + base_severity = self.SEVERITY_BASE.get(event_type, 0.3) + + # Boost for multiple matches + match_boost = min(0.2, (len(matches) if matches else 1) * 0.05) + + # Boost for strong language + strong_words = ["major", "massive", "critical", "emergency", "urgent", "breaking"] + text_lower = text.lower() + language_boost = sum(0.05 for w in strong_words if w in text_lower) + + return min(1.0, base_severity + match_boost + language_boost) diff --git a/sentiment_engine/src/sentiment_engine/nlp/pipeline.py b/sentiment_engine/src/sentiment_engine/nlp/pipeline.py new file mode 100644 index 0000000..ecf2063 --- /dev/null +++ b/sentiment_engine/src/sentiment_engine/nlp/pipeline.py @@ -0,0 +1,210 @@ +"""NLP Processing Pipeline - orchestrates all NLP stages""" + +import asyncio +import logging +import time +from datetime import datetime +from typing import Dict, List, Optional + +from sentiment_engine.schemas.payload import NormalizedPayload +from sentiment_engine.schemas.processed import ( + ProcessedItem, EntityExtraction, SentimentScores, EmotionScores, + EventClassification, TemporalAnchor, CredibilityScore +) +from sentiment_engine.nlp.entity_extraction import EntityExtractor, AssetMapper +from sentiment_engine.nlp.sentiment_emotion import SentimentEmotionAnalyzer +from sentiment_engine.nlp.event_classification import EventClassifier +from sentiment_engine.nlp.temporal import TemporalAnchorer +from sentiment_engine.nlp.credibility import CredibilityScorer +from sentiment_engine.utils.config import get_settings + +logger = logging.getLogger(__name__) + + +class NLPProcessingPipeline: + """Main NLP processing pipeline""" + + def __init__(self): + self.settings = get_settings() + self.asset_mapper = AssetMapper() + self.entity_extractor = EntityExtractor(self.asset_mapper) + self.sentiment_analyzer = SentimentEmotionAnalyzer() + self.event_classifier = EventClassifier() + self.temporal_anchorer = TemporalAnchorer() + self.credibility_scorer = CredibilityScorer() + + self._initialized = False + self._model_versions = {} + + async def initialize(self) -> None: + """Initialize all components - PARALLEL loading for reliability""" + if self._initialized: + return + + import asyncio + import time + + print("DEBUG: [1/4] Starting PARALLEL model initialization...") + start = time.time() + + # Load all three heavy models IN PARALLEL to avoid sequential 38s+38s+38s bottleneck + print("DEBUG: Launching entity_extractor, sentiment_analyzer, event_classifier in parallel...") + + try: + results = await asyncio.gather( + self.entity_extractor.initialize(), + self.sentiment_analyzer.initialize(), + self.event_classifier.initialize(), + return_exceptions=True + ) + except Exception as e: + logger.error(f"Parallel initialization failed: {e}") + # Fallback to sequential with longer timeout + print("DEBUG: Falling back to sequential initialization...") + await self.entity_extractor.initialize() + await self.sentiment_analyzer.initialize() + await self.event_classifier.initialize() + else: + # Check for exceptions + for i, (name, result) in enumerate(zip(["entity_extractor", "sentiment_analyzer", "event_classifier"], results)): + if isinstance(result, Exception): + logger.error(f"{name} initialization failed: {result}") + # Retry individually with more time + print(f"DEBUG: Retrying {name} individually...") + if name == "entity_extractor": + await self.entity_extractor.initialize() + elif name == "sentiment_analyzer": + await self.sentiment_analyzer.initialize() + elif name == "event_classifier": + await self.event_classifier.initialize() + else: + print(f"DEBUG: {name} initialized OK") + + elapsed = time.time() - start + print(f"DEBUG: Model initialization completed in {elapsed:.1f}s") + + # Load credibility registry (fast) + print("DEBUG: [4/4] Loading credibility registry...") + self._load_credibility_registry() + print("DEBUG: Credibility registry done") + + self._initialized = True + logger.info(f"NLP Pipeline initialized in {elapsed:.1f}s") + print("DEBUG: Pipeline fully initialized") + + def _load_credibility_registry(self) -> None: + """Load source credibility registry from config""" + import yaml + from pathlib import Path + + registry_path = Path("config/source_credibility.yaml") + if registry_path.exists(): + with open(registry_path) as f: + data = yaml.safe_load(f) or {} + registry = {item["source_id"]: item for item in data.get("sources", [])} + self.credibility_scorer.load_registry(registry) + + async def process(self, payload: NormalizedPayload) -> ProcessedItem: + """Process a normalized payload through the full NLP pipeline""" + if not self._initialized: + await self.initialize() + + start_time = time.time() + + try: + # Stage 1: Entity extraction + entities = await self.entity_extractor.extract_all(payload.raw_text) + + # Stage 2: Sentiment & emotion analysis + asset_mentions_for_sentiment = [ + {"asset_id": e.asset_id, "span": e.mention_span} + for e in entities + ] + sentiment_results, emotion_results = await self.sentiment_analyzer.analyze( + payload.raw_text, asset_mentions_for_sentiment + ) + + # Stage 3: Event classification + asset_ids = [e.asset_id for e in entities] + events = await self.event_classifier.classify(payload.raw_text, asset_ids) + + # Stage 4: Temporal anchoring + temporal = self.temporal_anchorer.anchor( + payload.raw_text, payload.publish_ts + ) + + # Stage 5: Credibility scoring (with cross-source corroboration) + # Get recent items from credibility scorer's cache + asset_id = entities[0].asset_id if entities else "UNKNOWN" + event_type = events[0].event_type.value if events else "unknown" + + # Prepare item data for cache + item_data = { + "asset_id": asset_id, + "event_type": event_type, + "source_id": payload.source_id, + "raw_text": payload.raw_text, + "content_hash": self.credibility_scorer._content_hash(payload.raw_text), + } + + credibility = self.credibility_scorer.compute_composite( + source_id=payload.source_id, + text=payload.raw_text, + metadata=payload.metadata, + asset_id=asset_id, + event_type=event_type, + recent_items=None # Uses internal cache + ) + + # Add to cache for future corroboration + self.credibility_scorer.add_processed_item(item_data) + + processing_time = (time.time() - start_time) * 1000 + + # Build processed item + processed = ProcessedItem( + payload_id=f"{payload.source_id}:{hash(payload.raw_text) & 0xFFFFFFFF:08x}", + source_id=payload.source_id, + source_type=payload.source_type.value, + ingest_ts=payload.ingest_ts, + publish_ts=payload.publish_ts, + raw_text=payload.raw_text, + entities=entities, + sentiment_per_asset=sentiment_results, + emotions_per_asset=emotion_results, + events=events, + temporal=temporal, + credibility=credibility, + processed_ts=datetime.now().timestamp(), + processing_latency_ms=processing_time, + model_versions=self._model_versions + ) + + return processed + + except Exception as e: + logger.error(f"NLP processing error: {e}") + raise + + async def process_batch(self, payloads: List[NormalizedPayload]) -> List[ProcessedItem]: + """Process multiple payloads concurrently""" + semaphore = asyncio.Semaphore(10) # Limit concurrency + + async def process_one(payload): + async with semaphore: + return await self.process(payload) + + results = await asyncio.gather(*[process_one(p) for p in payloads], return_exceptions=True) + + # Filter out exceptions + processed = [] + for i, result in enumerate(results): + if isinstance(result, Exception): + logger.error(f"Batch processing error for payload {i}: {result}") + else: + processed.append(result) + + return processed + + def get_model_versions(self) -> Dict[str, str]: + return self._model_versions.copy() diff --git a/sentiment_engine/src/sentiment_engine/nlp/sentiment_emotion.py b/sentiment_engine/src/sentiment_engine/nlp/sentiment_emotion.py new file mode 100644 index 0000000..b40b6ee --- /dev/null +++ b/sentiment_engine/src/sentiment_engine/nlp/sentiment_emotion.py @@ -0,0 +1,2016 @@ +"""Sentiment and emotion analysis using transformer models (with ONNX Runtime + mock fallback)""" + +import asyncio +import logging +import os +import re +from pathlib import Path +from typing import Dict, List, Optional, Tuple, Any + +import numpy as np + +from sentiment_engine.schemas.processed import SentimentScores, EmotionScores +from sentiment_engine.utils.config import get_settings + +logger = logging.getLogger(__name__) + + +# Optional imports for production +try: + import onnxruntime as ort + ONNX_AVAILABLE = True +except ImportError: + ONNX_AVAILABLE = False + logger.debug("onnxruntime not available, will use mock/heuristic") + +try: + import torch + from transformers import AutoTokenizer, AutoModelForSequenceClassification + TRANSFORMERS_AVAILABLE = True +except ImportError: + TRANSFORMERS_AVAILABLE = False + logger.debug("transformers not available") + + +class MockTokenizer: + """Mock tokenizer for testing/fallback""" + + def __init__(self): + self.vocab_size = 30522 + + def __call__(self, text, return_tensors="pt", truncation=True, max_length=512, padding=True): + if isinstance(text, list): + batch_size = len(text) + else: + batch_size = 1 + text = [text] + + input_ids = torch.randint(1, 1000, (batch_size if TRANSFORMERS_AVAILABLE else 1, 512)) if TRANSFORMERS_AVAILABLE else np.random.randint(1, 1000, (batch_size, 512)) + attention_mask = torch.ones_like(input_ids) if TRANSFORMERS_AVAILABLE else np.ones((batch_size, 512)) + token_type_ids = torch.zeros_like(input_ids) if TRANSFORMERS_AVAILABLE else np.zeros((batch_size, 512)) + + return { + "input_ids": input_ids, + "attention_mask": attention_mask, + "token_type_ids": token_type_ids + } + + @classmethod + def from_pretrained(cls, model_name: str): + return MockTokenizer() + + def save_pretrained(self, path: str): + pass + + +class MockSentimentModel: + def __init__(self, device="cpu"): + self.device = device + + def to(self, device): + self.device = device + return self + + def eval(self): + return self + + def __call__(self, **inputs): + batch_size = inputs["input_ids"].shape[0] + logits = torch.randn(batch_size, 3) if TRANSFORMERS_AVAILABLE else np.random.randn(batch_size, 3) + return type('Outputs', (), {'logits': logits})() + + +class ONNXSentimentModel: + """ONNX Runtime wrapper for FinBERT sentiment model (requires token_type_ids)""" + + def __init__(self, model_path: str, tokenizer_path: str, label_map_path: str = None): + self.model_path = model_path + self.tokenizer_path = tokenizer_path + self.label_map_path = label_map_path + + # Load tokenizer + if TRANSFORMERS_AVAILABLE: + self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) + else: + self.tokenizer = MockTokenizer() + + # Load ONNX model with limited threads to avoid contention + session_options = ort.SessionOptions() + session_options.intra_op_num_threads = 1 + session_options.inter_op_num_threads = 1 + session_options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL + self.session = ort.InferenceSession(model_path, sess_options=session_options, providers=self._get_providers()) + + # Load labels + self.labels = ["negative", "neutral", "positive"] + if label_map_path and Path(label_map_path).exists(): + import json + with open(label_map_path) as f: + self.labels = [v for k, v in sorted(json.load(f).items(), key=lambda x: int(x[0]))] + + self._input_names = [i.name for i in self.session.get_inputs()] + self._output_names = [o.name for o in self.session.get_outputs()] + + def _get_providers(self): + """Get ONNX Runtime execution providers""" + providers = ['CPUExecutionProvider'] + if ort.get_device() == 'GPU': + providers.insert(0, 'CUDAExecutionProvider') + return providers + + def __call__(self, input_ids, attention_mask, token_type_ids=None) -> np.ndarray: + """Run inference, return logits""" + if hasattr(input_ids, 'numpy'): + input_ids = input_ids.numpy() + if hasattr(attention_mask, 'numpy'): + attention_mask = attention_mask.numpy() + if token_type_ids is not None and hasattr(token_type_ids, 'numpy'): + token_type_ids = token_type_ids.numpy() + + ort_inputs = { + "input_ids": input_ids.astype(np.int64), + "attention_mask": attention_mask.astype(np.int64), + } + # FinBERT requires token_type_ids + if "token_type_ids" in self._input_names: + if token_type_ids is None: + token_type_ids = np.zeros_like(input_ids) + ort_inputs["token_type_ids"] = token_type_ids.astype(np.int64) + + # Run inference + outputs = self.session.run(self._output_names, ort_inputs) + logits = outputs[0] # First output is typically logits + + return logits + + +class ONNXEmotionModel: + """ONNX Runtime wrapper for DistilRoBERTa emotion model (NO token_type_ids)""" + + def __init__(self, model_path: str, tokenizer_path: str, label_map_path: str = None): + self.model_path = model_path + self.tokenizer_path = tokenizer_path + self.label_map_path = label_map_path + + if TRANSFORMERS_AVAILABLE: + self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) + else: + self.tokenizer = MockTokenizer() + + # Load ONNX model with limited threads to avoid contention + session_options = ort.SessionOptions() + session_options.intra_op_num_threads = 2 + session_options.inter_op_num_threads = 2 + self.session = ort.InferenceSession(model_path, sess_options=session_options, providers=self._get_providers()) + + self.labels = ["anger", "disgust", "fear", "joy", "neutral", "sadness", "surprise"] + if label_map_path and Path(label_map_path).exists(): + import json + with open(label_map_path) as f: + self.labels = [v for k, v in sorted(json.load(f).items(), key=lambda x: int(x[0]))] + + self._input_names = [i.name for i in self.session.get_inputs()] + self._output_names = [o.name for o in self.session.get_outputs()] + + def _get_providers(self): + providers = ['CPUExecutionProvider'] + if ort.get_device() == 'GPU': + providers.insert(0, 'CUDAExecutionProvider') + return providers + + def __call__(self, input_ids, attention_mask, token_type_ids=None) -> np.ndarray: + """Run inference, return logits - DistilRoBERTa does NOT use token_type_ids""" + if hasattr(input_ids, 'numpy'): + input_ids = input_ids.numpy() + if hasattr(attention_mask, 'numpy'): + attention_mask = attention_mask.numpy() + + ort_inputs = { + "input_ids": input_ids.astype(np.int64), + "attention_mask": attention_mask.astype(np.int64), + } + # DistilRoBERTa does NOT have token_type_ids input + if "token_type_ids" in self._input_names and token_type_ids is not None: + if hasattr(token_type_ids, 'numpy'): + token_type_ids = token_type_ids.numpy() + ort_inputs["token_type_ids"] = token_type_ids.astype(np.int64) + + outputs = self.session.run(self._output_names, ort_inputs) + return outputs[0] + + +class ONNXEventModel: + """ONNX Runtime wrapper for BERT event classification model (requires token_type_ids)""" + + def __init__(self, model_path: str, tokenizer_path: str, label_map_path: str = None): + self.model_path = model_path + self.tokenizer_path = tokenizer_path + self.label_map_path = label_map_path + + if TRANSFORMERS_AVAILABLE: + self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) + else: + self.tokenizer = MockTokenizer() + + # Load ONNX model with limited threads to avoid contention + session_options = ort.SessionOptions() + session_options.intra_op_num_threads = 2 + session_options.inter_op_num_threads = 2 + self.session = ort.InferenceSession(model_path, sess_options=session_options, providers=self._get_providers()) + + from sentiment_engine.schemas.processed import EventType + self.labels = [e.value for e in EventType if e != EventType.UNKNOWN] + if label_map_path and Path(label_map_path).exists(): + import json + with open(label_map_path) as f: + self.labels = [v for k, v in sorted(json.load(f).items(), key=lambda x: int(x[0]))] + + self._input_names = [i.name for i in self.session.get_inputs()] + self._output_names = [o.name for o in self.session.get_outputs()] + + def _get_providers(self): + providers = ['CPUExecutionProvider'] + if ort.get_device() == 'GPU': + providers.insert(0, 'CUDAExecutionProvider') + return providers + + def predict(self, input_ids, attention_mask, token_type_ids=None) -> np.ndarray: + """Run inference, return probabilities""" + if hasattr(input_ids, 'numpy'): + input_ids = input_ids.numpy() + if hasattr(attention_mask, 'numpy'): + attention_mask = attention_mask.numpy() + if token_type_ids is not None and hasattr(token_type_ids, 'numpy'): + token_type_ids = token_type_ids.numpy() + + ort_inputs = { + "input_ids": input_ids.astype(np.int64), + "attention_mask": attention_mask.astype(np.int64), + } + # BERT event model requires token_type_ids + if "token_type_ids" in self._input_names: + if token_type_ids is None: + token_type_ids = np.zeros_like(input_ids) + ort_inputs["token_type_ids"] = token_type_ids.astype(np.int64) + + outputs = self.session.run(self._output_names, ort_inputs) + logits = outputs[0] + + # Softmax + e_x = np.exp(logits - np.max(logits, axis=-1, keepdims=True)) + probs = e_x / e_x.sum(axis=-1, keepdims=True) + + return probs[0] + + +class CryptoSentimentCalibrator: + """ + Calibrates FinBERT outputs for crypto semantics. + + FinBERT (traditional finance): + - "surge/rally/pump" = risky/bubble = negative (index 0) + - "crash/drop/dump" = value/opportunity = positive (index 2) + - Native: [negative, neutral, positive] = [Bearish, Neutral, Bullish] + + Crypto semantics: + - "surge/pump/moon/rally" = bullish = Bullish (index 2) + - "crash/dump/rug/hack" = bearish = Bearish (index 0) + + This calibrator flips FinBERT's positive/negative ONLY when there's a semantic mismatch. + Uses word-boundary keyword matching for reliable crypto signal detection. + """ + + # Crypto-bullish keywords (should map to index 2 = Bullish) + # Comprehensive vocabulary covering price action, institutional, technical, on-chain, and news catalysts + CRYPTO_BULLISH_KEYWORDS = [ + # Price action + "surge", "surges", "surged", "surgeing", "pump", "pumps", "pumped", "pumping", + "moon", "mooning", "mooned", "to.the.moon", "to the moon", "to the moon", "rally", "rallies", "rallied", "rallying", + "breakout", "breakouts", "break.out", "breaking.out", "bullish", "bull.run", "bullrun", "bull run", "bull run", + "ath", "all.time.high", "all.time.highs", "all time high", "all time highs", "record.high", "record.highs", "record high", "record highs", "new.high", "new.highs", "new high", "new highs", + "higher.high", "higher.highs", "higher.low", "higher.lows", "higher high", "higher highs", "higher low", "higher lows", "uptrend", "uptrending", + "green", "green.candle", "green.candles", "pumping", "ripping", "rips", "ripped", + "parabolic", "parabolic.move", "explosive", "explosive.move", "violent.move.up", "violent.move", "violent.up", "violent.rally", "violent.move", "violent.up", "violent.rally", + + # Inflows & accumulation (in crypto: exchange OUTFLOWS = bullish, INFLOWS = bearish) + "outflow", "outflows", "net.outflow", "exchange.outflow", "exchange.outflows", + "capital.outflow", "institutional.outflow", "outflow.spike", "outflows.spike", + "balance.decreasing", "off.exchange", "cold.storage", + "whale.withdrawal", "whale.withdrawals", "large.withdrawal", "large.withdrawals", "exchange.withdrawal.to.cold", "whale.withdrawal.from.exchange", "whale.withdrawal.to.cold", "whale.withdraws", "whale.withdrawing", "break.out", "breaks.out", "breaking.out", "break out", "breaks out", "breaking out", "all.time.high", "all time high", "golden cross", "golden.cross", "violent.move.up", "violent move up", "short.squeeze", "short squeeze", "squeeze.higher", "squeeze higher", "short.cover", "short cover", "short.covering", "short covering", "violent.move.up", "violent move up", "violent.up", "violent up", "violent move", "violent up move", "violent move", "violent up move", "wipes.out.bearish", "wipes out bearish", "wiped.out.bearish", "wiped out bearish", "leverage.flush", "leverage flush", "leverage flush clears", "flush clears weak hands", "leverage flush clears", "flush clears weak hands", "short.liquidation", "short liquidation", "short.squeeze", "short squeeze", "short liquidation fuels", "short liquidation fuels squeeze", "short squeeze higher", "squeeze higher", "short liquidation fuels", "short liquidation fuels squeeze", "short squeeze higher", "squeeze higher", "squeeze.higher", "squeeze higher", "capitulation.over", "capitulation over", "capitulation over per", "capitulation ended", "capitulation over per", "capitulation ended", "miner.surrender", "miner surrender", "miner surrender evident", "surrender evident in", "evident in outflows", "miner surrender evident", "miners surrender", "miner capitulation", "miner surrender evident", "surrender evident in", "evident in outflows", "miner surrender evident", "miners surrender", "miner capitulation", "bank.run", "bank run", "bank run on", "run on staking", "run on exchange", "bank run on", "run on exchange", "run on staking", "bank run on", "run on staking", "run on exchange", "bank run on", "run on exchange", "run on staking", "supply.squeeze", "supply squeeze", "supply squeeze as", "coins lock up", "supply squeeze as", "coins lock up", "inflation.hedge", "inflation hedge", "store of value", "digital gold", "store of value", "digital gold", "balance.decreasing", "balance decreasing", "balance decreasing rapidly", "exchange balance decreasing", "balance decreasing rapidly", "exchange balance decreasing", "off.exchange", "off exchange", "moving off exchange", "coins moving off exchange", "moving off exchange", "coins moving off exchange", "cold.storage", "cold storage", "moving to cold storage", "whales moving to cold storage", "moving to cold storage", "whales moving to cold storage", "whale.withdrawal", "whale withdrawal", "whale.withdrawals", "whale withdrawals", "large whale withdrawal", "whale withdrawal from", "whale withdraws from", "large whale withdrawal", "whale withdrawal from", "whale withdraws from", "large.withdrawal", "large withdrawal", "large.withdrawals", "large withdrawals", "large withdrawal suggests", "withdrawal suggests accumulation", "large withdrawal suggests", "withdrawal suggests accumulation", "rsi.recovery", "rsi recovery", "rsi recovery from", "recovery from oversold", "rsi recovery from", "recovery from oversold", "rsi.oversold", "rsi oversold", "oversold territory", "rsi oversold at", "oversold territory", "rsi oversold at", "support.held", "support held", "support.holds", "support holds", "key support held", "support held on retest", "key support held", "support held on retest", "moving.average.support", "moving average support", "ma.support", "ma support", "200.day", "200 day", "200 day ma", "200-day ma", "200.day", "200 day", "200 day ma", "200-day ma", "golden.cross", "golden cross", "golden cross forms", "golden cross on daily", "golden cross on weekly", "golden cross forms", "golden cross on daily", "golden cross on weekly", "break.out", "break out", "breaks.out", "breaks out", "breaking.out", "breaking out", "break out of", "breaks out of", "breaking out of", "breakout confirmed", "breakout traps", "break out of", "breaks out of", "breaking out of", "breakout confirmed", "breakout traps", "all.time.high", "all time high", "all.time.highs", "all time highs", "hits all time high", "above all time high", "new all time high", "record high", "record highs", "hits all time high", "above all time high", "new all time high", "record high", "record highs", "leverage.flush", "leverage flush", "leverage flush clears", "flush clears weak hands", "leverage flush clears", "flush clears weak hands", "short.liquidation", "short liquidation", "short.squeeze", "short squeeze", "short liquidation fuels", "short liquidation fuels squeeze", "short squeeze higher", "squeeze higher", "short liquidation fuels", "short liquidation fuels squeeze", "short squeeze higher", "squeeze higher", "squeeze.higher", "squeeze higher", "capitulation.over", "capitulation over", "capitulation over per", "capitulation ended", "capitulation over per", "capitulation ended", "miner.surrender", "miner surrender", "miner surrender evident", "surrender evident in", "evident in outflows", "miner surrender evident", "miners surrender", "miner capitulation", "miner surrender evident", "surrender evident in", "evident in outflows", "miner surrender evident", "miners surrender", "miner capitulation", "bank.run", "bank run", "bank run on", "run on staking", "run on exchange", "bank run on", "run on exchange", "run on staking", "bank run on", "run on staking", "run on exchange", "bank run on", "run on exchange", "run on staking", "supply.squeeze", "supply squeeze", "supply squeeze as", "coins lock up", "supply squeeze as", "coins lock up", "inflation.hedge", "inflation hedge", "store of value", "digital gold", "store of value", "digital gold", "balance.decreasing", "balance decreasing", "balance decreasing rapidly", "exchange balance decreasing", "balance decreasing rapidly", "exchange balance decreasing", "off.exchange", "off exchange", "moving off exchange", "coins moving off exchange", "moving off exchange", "coins moving off exchange", "cold.storage", "cold storage", "moving to cold storage", "whales moving to cold storage", "moving to cold storage", "whales moving to cold storage", "whale.withdrawal", "whale withdrawal", "whale.withdrawals", "whale withdrawals", "large whale withdrawal", "whale withdrawal from", "whale withdraws from", "large whale withdrawal", "whale withdrawal from", "whale withdraws from", "large.withdrawal", "large withdrawal", "large.withdrawals", "large withdrawals", "large withdrawal suggests", "withdrawal suggests accumulation", "large withdrawal suggests", "withdrawal suggests accumulation", "rsi.recovery", "rsi recovery", "rsi recovery from", "recovery from oversold", "rsi recovery from", "recovery from oversold", "rsi.oversold", "rsi oversold", "oversold territory", "rsi oversold at", "oversold territory", "rsi oversold at", "support.held", "support held", "support.holds", "support holds", "key support held", "support held on retest", "key support held", "support held on retest", "moving.average.support", "moving average support", "ma.support", "ma support", "200.day", "200 day", "200 day ma", "200-day ma", "200.day", "200 day", "200 day ma", "200-day ma", "golden.cross", "golden cross", "golden cross forms", "golden cross on daily", "golden cross on weekly", "golden cross forms", "golden cross on daily", "golden cross on weekly", "break.out", "break out", "breaks.out", "breaks out", "breaking.out", "breaking out", "break out of", "breaks out of", "breaking out of", "breakout confirmed", "breakout traps", "break out of", "breaks out of", "breaking out of", "breakout confirmed", "breakout traps", "all.time.high", "all time high", "all.time.highs", "all time highs", "hits all time high", "above all time high", "new all time high", "record high", "record highs", "hits all time high", "above all time high", "new all time high", "record high", "record highs", "short.cover", "short cover", "short.covering", "short covering", "violent.move.up", "violent move up", "violent.up", "violent up", "violent move", "violent up move", "violent move", "violent up move", "wipes.out.bearish", "wipes out bearish", "wiped.out.bearish", "wiped out bearish", "leverage.flush", "leverage flush", "leverage flush clears", "flush clears weak hands", "leverage flush clears", "flush clears weak hands", "short.liquidation", "short liquidation", "short.squeeze", "short squeeze", "short liquidation fuels", "short liquidation fuels squeeze", "short squeeze higher", "squeeze higher", "short liquidation fuels", "short liquidation fuels squeeze", "short squeeze higher", "squeeze higher", "squeeze.higher", "squeeze higher", "capitulation.over", "capitulation over", "capitulation over per", "capitulation ended", "capitulation over per", "capitulation ended", "miner.surrender", "miner surrender", "miner surrender evident", "surrender evident in", "evident in outflows", "miner surrender evident", "miners surrender", "miner capitulation", "miner surrender evident", "surrender evident in", "evident in outflows", "miner surrender evident", "miners surrender", "miner capitulation", "bank.run", "bank run", "bank run on", "run on staking", "run on exchange", "bank run on", "run on exchange", "run on staking", "bank run on", "run on staking", "run on exchange", "bank run on", "run on exchange", "run on staking", "supply.squeeze", "supply squeeze", "supply squeeze as", "coins lock up", "supply squeeze as", "coins lock up", "inflation.hedge", "inflation hedge", "store of value", "digital gold", "store of value", "digital gold", "balance.decreasing", "balance decreasing", "balance decreasing rapidly", "exchange balance decreasing", "balance decreasing rapidly", "exchange balance decreasing", "off.exchange", "off exchange", "moving off exchange", "coins moving off exchange", "moving off exchange", "coins moving off exchange", "cold.storage", "cold storage", "moving to cold storage", "whales moving to cold storage", "moving to cold storage", "whales moving to cold storage", "whale.withdrawal", "whale withdrawal", "whale.withdrawals", "whale withdrawals", "large whale withdrawal", "whale withdrawal from", "whale withdraws from", "large whale withdrawal", "whale withdrawal from", "whale withdraws from", "large.withdrawal", "large withdrawal", "large.withdrawals", "large withdrawals", "large withdrawal suggests", "withdrawal suggests accumulation", "large withdrawal suggests", "withdrawal suggests accumulation", "rsi.recovery", "rsi recovery", "rsi recovery from", "recovery from oversold", "rsi recovery from", "recovery from oversold", "rsi.oversold", "rsi oversold", "oversold territory", "rsi oversold at", "oversold territory", "rsi oversold at", "support.held", "support held", "support.holds", "support holds", "key support held", "support held on retest", "key support held", "support held on retest", "moving.average.support", "moving average support", "ma.support", "ma support", "200.day", "200 day", "200 day ma", "200-day ma", "200.day", "200 day", "200 day ma", "200-day ma", "golden.cross", "golden cross", "golden cross forms", "golden cross on daily", "golden cross on weekly", "golden cross forms", "golden cross on daily", "golden cross on weekly", "break.out", "break out", "breaks.out", "breaks out", "breaking.out", "breaking out", "break out of", "breaks out of", "breaking out of", "breakout confirmed", "breakout traps", "break out of", "breaks out of", "breaking out of", "breakout confirmed", "breakout traps", "all.time.high", "all time high", "all.time.highs", "all time highs", "hits all time high", "above all time high", "new all time high", "record high", "record highs", "hits all time high", "above all time high", "new all time high", "record high", "record highs", "leverage.flush", "leverage flush", "leverage flush clears", "flush clears weak hands", "leverage flush clears", "flush clears weak hands", "short.liquidation", "short liquidation", "short.squeeze", "short squeeze", "short liquidation fuels", "short liquidation fuels squeeze", "short squeeze higher", "squeeze higher", "short liquidation fuels", "short liquidation fuels squeeze", "short squeeze higher", "squeeze higher", "squeeze.higher", "squeeze higher", "capitulation.over", "capitulation over", "capitulation over per", "capitulation ended", "capitulation over per", "capitulation ended", "miner.surrender", "miner surrender", "miner surrender evident", "surrender evident in", "evident in outflows", "miner surrender evident", "miners surrender", "miner capitulation", "miner surrender evident", "surrender evident in", "evident in outflows", "miner surrender evident", "miners surrender", "miner capitulation", "bank.run", "bank run", "bank run on", "run on staking", "run on exchange", "bank run on", "run on exchange", "run on staking", "bank run on", "run on staking", "run on exchange", "bank run on", "run on exchange", "run on staking", "supply.squeeze", "supply squeeze", "supply squeeze as", "coins lock up", "supply squeeze as", "coins lock up", "inflation.hedge", "inflation hedge", "store of value", "digital gold", "store of value", "digital gold", "balance.decreasing", "balance decreasing", "balance decreasing rapidly", "exchange balance decreasing", "balance decreasing rapidly", "exchange balance decreasing", "off.exchange", "off exchange", "moving off exchange", "coins moving off exchange", "moving off exchange", "coins moving off exchange", "cold.storage", "cold storage", "moving to cold storage", "whales moving to cold storage", "moving to cold storage", "whales moving to cold storage", "whale.withdrawal", "whale withdrawal", "whale.withdrawals", "whale withdrawals", "large whale withdrawal", "whale withdrawal from", "whale withdraws from", "large whale withdrawal", "whale withdrawal from", "whale withdraws from", "large.withdrawal", "large withdrawal", "large.withdrawals", "large withdrawals", "large withdrawal suggests", "withdrawal suggests accumulation", "large withdrawal suggests", "withdrawal suggests accumulation", "rsi.recovery", "rsi recovery", "rsi recovery from", "recovery from oversold", "rsi recovery from", "recovery from oversold", "rsi.oversold", "rsi oversold", "oversold territory", "rsi oversold at", "oversold territory", "rsi oversold at", "support.held", "support held", "support.holds", "support holds", "key support held", "support held on retest", "key support held", "support held on retest", "moving.average.support", "moving average support", "ma.support", "ma support", "200.day", "200 day", "200 day ma", "200-day ma", "200.day", "200 day", "200 day ma", "200-day ma", "golden.cross", "golden cross", "golden cross forms", "golden cross on daily", "golden cross on weekly", "golden cross forms", "golden cross on daily", "golden cross on weekly", "break.out", "break out", "breaks.out", "breaks out", "breaking.out", "breaking out", "break out of", "breaks out of", "breaking out of", "breakout confirmed", "breakout traps", "break out of", "breaks out of", "breaking out of", "breakout confirmed", "breakout traps", "all.time.high", "all time high", "all.time.highs", "all time highs", "hits all time high", "above all time high", "new all time high", "record high", "record highs", "hits all time high", "above all time high", "new all time high", "record high", "record highs", "capitulation.over", "capitulation over", "capitulation over per", "capitulation ended", "capitulation over per", "capitulation ended", "supply.squeeze", "supply squeeze", "supply squeeze as", "coins lock up", "supply squeeze as", "coins lock up", "inflation.hedge", "inflation hedge", "store of value", "digital gold", "store of value", "digital gold", "miner.surrender", "miner surrender", "miner surrender evident", "surrender evident in", "evident in outflows", "miner surrender evident", "miners surrender", "miner capitulation", "miner surrender evident", "surrender evident in", "evident in outflows", "miner surrender evident", "miners surrender", "miner capitulation", + "adoption", "adoptions", "mass.adoption", "mainstream.adoption", "accumulation", "accumulating", + "accumulate", "accumulates", "accumulated", "whale.accumulation", "smart.money.accumulation", + "hodl", "hodling", "diamond.hands", "strong.hands", "not.selling", "holding.strong", + "buy.the.dip", "buying.the.dip", "dip.bought", "buy the dip", "buying the dip", "dip bought", "dca", "dollar.cost.average", "dollar cost average", + + # Institutional & ETF + "etf", "etfs", "spot.etf", "spot.etfs", "bitcoin.etf", "ethereum.etf", "etf.approval", + "etf.approved", "etf.approves", "etf.launch", "etf.launches", "etf.listed", "etf.inflow", + "etf.inflows", "blackrock", "fidelity", "vanguard", "grayscale", "ark.invest", "institutional", + "institutions", "institutional.adoption", "corporate.treasury", "corporate.adoption", "institutional adoption", "corporate treasury", "corporate adoption", "institutional adoption", "corporate treasury", "corporate adoption", + "microstrategy", "tesla", "public.company", "balance.sheet", "treasury.reserve", + "approval", "approvals", "approved", "approves", "sec.approval", "regulatory.approval", + "regulatory.clarity", "clear.regulation", "friendly.regulation", "pro.crypto", + + # Exchange & listing + "listing", "listings", "listed", "lists", "list", "new.listing", "major.listing", + "tier1.listing", "tier1.exchange", "binance.listing", "coinbase.listing", "kraken.listing", + "bybit.listing", "okx.listing", "upbit.listing", "launch", "launches", "launched", + "mainnet.launch", "testnet.launch", "token.launch", "fair.launch", "mainnet launch", "testnet launch", "token launch", "fair launch", "mainnet launch", "testnet launch", "token launch", "fair launch", "ido", "ico", "ieo", + + # Partnerships & development + "partnership", "partnerships", "partner", "partners", "collaboration", "collaborations", + "integration", "integrations", "integrated", "integrates", "ecosystem", "ecosystem.growth", + "developer.activity", "dev.activity", "github.activity", "commits", "contributors", + "grant", "grants", "funding", "funded", "investment", "investments", "vc.funding", + "strategic.investment", "ecosystem.fund", "developer.fund", "bug.bounty", + + # Profit & gains + "profit", "profits", "profitable", "win", "wins", "winning", + "success", "successful", "breakthrough", "milestone", "milestones", "target.hit", + "price.target", "target.achieved", "price target", "target achieved", "allocation", "portfolio.allocation", "portfolio allocation", "diversification", + + # Technical indicators (bullish) + "golden.cross", "golden.cross", "macd.crossover", "macd.bullish", "rsi.oversold", + "rsi.recovery", "moving.average", "ma.support", "support.held", "support.holds", + "bounce", "bounces", "bounced", "bouncing", "reversal", "reverses", "reversed", + "bottom", "bottomed", "local.bottom", "macro.bottom", "capitulation.over", "capitulation.ended", "capitulation.done", "capitulation.finished", "capitulation.complete", "capitulation.ended", "capitulation.done", "capitulation.finished", "capitulation.complete", + "oversold", "oversold.rsi", "oversold rsi", "bullish.divergence", "hidden.bullish.divergence", "bullish divergence", "hidden bullish divergence", + + # On-chain bullish + "whale.accumulation", "whale.buying", "large.holder.accumulation", "exchange.outflow", + "exchange.outflows", "balance.decreasing", "off.exchange", "cold.storage", + "staking", "staked", "staking.rewards", "validator", "validators", "delegation", "staking rewards", "staking rewards", "deposit.to.vault", "deposit.to.protocol", "deposit.to.farm", "deposit.to.earn", "depositing.to.vault", "depositing.to.protocol", "yield.deposit", "farm.deposit", "deposit.to.vault", "deposit.to.protocol", "deposit.to.farm", "deposit.to.earn", "depositing.to.vault", "depositing.to.protocol", "yield.deposit", "farm.deposit", + "hashrate.up", "hashrate.rising", "hashrate up", "hashrate rising", "difficulty.up", "difficulty up", "difficulty rising", "difficulty increases", "difficulty adjusts", "difficulty adjusts upward", "difficulty adjusting", "adjusts upward", "adjusting upward", "network.growth", "network growth", "addresses.growing", "addresses growing", "active.addresses", "active addresses", "active addresses growing", "addresses growing exponentially", "new.addresses", "new addresses", "transaction.count", "tx.count.rising", "transaction count", "tx count rising", + "active.addresses", "new.addresses", "transaction.count", "tx.count.rising", + + # DeFi & yield + "yield", "yield.farming", "liquidity.mining", "apy", "apr", "high.yield", + "tvl.growth", "tvl.rising", "total.value.locked", "tvl.all.time.high", + "protocol.revenue", "fees.growing", "revenue.growth", "buyback", "buybacks", + "token.burn", "burns", "burning", "deflationary", "supply.decreasing", + + # Macro & narrative + "halving", "bitcoin.halving", "supply.shock", "supply.squeeze", "scarcity", + "inflation.hedge", "store.of.value", "digital.gold", "monetary.policy", + "rate.cut", "rate.cuts", "pivot", "fed.pivot", "liquidity", "m2.growth", + "global.liquidity", "dollar.weakness", "dxy.down", "risk.on", "risk.on.environment", + + # Sentiment & social + "fomo", "fear.of.missing.out", "euphoria", "optimism", "optimistic", "bullish.sentiment", + "greed", "extreme.greed", "fear.and.greed", "greed.index", "social.dominance", + "mentions.rising", "trending", "trending.on.twitter", "trending.on.reddit", + "google.trends", "search.interest", "retail.fomo", "new.entrants", "onboarding", + # Compound phrases (space-separated for regex matching) + "break out", "breaks out", "breaking out", "break out of", "breaks out of", "breaking out of", + "breakout confirmed", "breakout traps", "break out of consolidation", "breaks out of consolidation", + "all time high", "all time highs", "hits all time high", "above all time high", "new all time high", "record high", "record highs", + "golden cross", "golden cross forms", "golden cross on daily", "golden cross on weekly", + "violent move up", "violent move", "violent up move", "violent rally", + "short liquidation", "short liquidation fuels", "short liquidation fuels squeeze", "short squeeze higher", + "short squeeze", "squeeze higher", "short cover", "short covering", + "capitulation over", "capitulation over per", "capitulation ended", "capitulation complete", + "miner surrender", "miner surrender evident", "miners surrender", "miner capitulation", + "bank run", "bank run on", "run on exchange", "run on staking", + "leverage flush", "leverage flush clears", "flush clears weak hands", "clears weak hands", + "supply squeeze", "supply squeeze as", "coins lock up", "coins locking up", + "inflation hedge", "store of value", "digital gold", "bitcoin inflation hedge", + "balance decreasing", "balance decreasing rapidly", "balance decreasing rapidly", + "exchange balance decreasing", "exchange balance decreasing", + "off exchange", "moving off exchange", "coins moving off exchange", "moving to cold storage", + "cold storage", "moving to cold storage", "whales moving to cold storage", + "whale withdrawal", "whale withdrawal from", "whale withdraws from", "whale withdrawal to cold", + "large withdrawal", "large withdrawal suggests", "withdrawal suggests accumulation", "withdrawal suggests", + "rsi recovery", "rsi recovery from", "recovery from oversold", "oversold territory", "rsi oversold at", + "rsi oversold", "rsi oversold at", "oversold on daily", "oversold on weekly", + "support held", "support held on", "key support held", "support held on retest", "held on retest", + "support holds", "supports holds", "support holds on", "holds on retest", + "moving average support", "moving average support", "ma support", "ma support", + "200 day", "200 day ma", "200-day ma", "200 day moving average", + "break out", "break out of", "breaks out of", "breaking out of", "breakout confirmed", "breakout traps", + "all time high", "all time highs", "hits all time high", "above all time high", "new all time high", + "record high", "record highs", "new record high", "all time high broken", + "risk on", "risk on environment", "risk on sentiment", "bullish sentiment", + "dollar weakness", "dollar weakening", "dxy down", "dxy falling", "dollar falling", + "rate cut", "rate cuts", "fed pivot", "fed pivoting", "pivot to easing", "easing cycle", + "liquidity expansion", "liquidity expanding", "global liquidity", "m2 growth", "m2 growing", + "fomo retail", "retail fomo", "retail fomo drives", "fomo drives parabolic", + "euphoria phase", "euphoria phase per", "optimism high", "optimism at cycle highs", + "bullish sentiment", "extreme bullish", "greed extreme", "extreme greed", + "social dominance", "social dominance rising", "dominance rising", + "mentions rising", "mentions rising on", "trending on twitter", "trending on reddit", + "google trends", "google trends up", "search interest", "search interest rising", + "retail fomo", "new entrants", "new entrants joining", "entrants joining", "onboarding surge", "onboarding surging", + ] + + # Crypto-bearish keywords (should map to index 0 = Bearish) + # Comprehensive vocabulary covering price action, institutional, technical, on-chain, regulatory, and news catalysts + CRYPTO_BEARISH_KEYWORDS = [ + # Price action - crashes & dumps + "crash", "crashes", "crashing", "crashed", "crash.down", "flash.crash", "market.crash", "crash down", "flash crash", "market crash", "crash down", "flash crash", "market crash", + "dump", "dumps", "dumped", "dumping", "violent.dump", "capitulation", "capitulating", + "panic", "panic.sell", "panic.selling", "panic.sold", "panic sell", "panic selling", "panic sold", "fear", "fearful", "bearish", + "bear.market", "bear market", "bearish.trend", "bearish trend", "downtrend", "downtrending", "lower.high", "lower.highs", + "lower.low", "lower.lows", "red", "red.candle", "red.candles", "bleeding", "bleeds", + "free.fall", "free fall", "freefall", "nose.dive", "nose dive", "nosedive", "plunge", "plunges", "plunged", + "collapse", "collapses", "collapsed", "collapsing", "implosion", "implodes", + "wipe", "wipes", "wiped", + + # Liquidations & leverage + "liquidation", "liquidations", "liquidated", "liquidating", "mass.liquidation", + "cascade.liquidation", "liquidation.cascade", "long.liquidation", "long.liquidations", + "short.liquidation", "short.liquidations", "forced.liquidation", "margin.call", + "margin.calls", "margin calls", "leverage.flush.out", "overleveraged", "over.leverage", "margin.call", "margin call", "blown.up", "blown up", "account.blown", "account blown", "blown up by", "unexpected move", "account blown up", "blown up by", "unexpected move", "account blown up", "rekt", "getting.rekt", "getting rekt", "getting rekt on", "rekt on leverage", "getting rekt on", "rekt on leverage", + "liquidation.price", "liq.price", "rekt", "getting.rekt", "blown.up", "account.blown", + + # Hacks, exploits, security + "hack", "hacks", "hacked", "hacking", "exploit", "exploits", "exploited", "exploiting", + "rug", "rugpull", "rug.pull", "rugged", "rug.pulled", "exit.scam", "exit scam", "exit scam suspected", "scam suspected as", "team disappears", "suspected as team", "exit scam suspected", "scam suspected as", "team disappears", "suspected as team", "exit.scams", + "stolen", "theft", "thefts", "stolen.funds", "drain", "drains", "drained", "draining", + "vulnerability", "vulnerabilities", "vuln", "bug", "bugs", "breach", "breaches", + "compromise", "compromised", "unauthorized", "unauthorized.access", "private.key.leak", + "seed.phrase", "phishing", "phished", "social.engineering", "insider.attack", + "smart.contract.bug", "reentrancy", "oracle.manipulation", "price.manipulation", + "governance.attack", "governance attack", "governance attack takes", "takes control of", "control of dao", "takes control of dao", "governance attack takes", "takes control of", "control of dao", "takes control of dao", "flash.loan.attack", "flash loan attack", "flash loan attack on", "attack on vulnerable", "on vulnerable pool", "flash loan attack on", "attack on vulnerable", "on vulnerable pool", "mev.attack", "mev attack", "sandwich.attack", "sandwich attack", + + # Depeg & stablecoin + "depeg", "depegs", "depegged", "depegging", "depeg.risk", "depeg.fear", + "stablecoin.depeg", "usdc.depeg", "usdt.depeg", "dai.depeg", "frax.depeg", + "peg.broken", "peg.lost", "lost.peg", "below.peg", "trading.below.peg", + "reserve.shortfall", "reserves.insufficient", "collateral.ratio", "undercollateralized", + + # Outflows & selling (in crypto: exchange INFLOWS = bearish) + "inflow", "inflows", "net.inflow", "exchange.inflow", "exchange.inflows", + "capital.inflow", "institutional.inflow", "inflow.spike", "inflows.spike", + "balance.increasing", "on.exchange", "hot.wallet", "deposit", "deposits", + "deposit.spike", "whale.deposit", "large.deposit", "exchange.deposit", + "sell", "sells", "selling", "sold", "sell.pressure", "selling.pressure", "heavy.selling", + "distribute", "distributes", "distributed", "distributing", "distribution", + "whale.selling", "whale.distribution", "smart.money.selling", "insider.selling", + "profit.taking", "take.profit", "taking.profit", "profit.takes", "profits.taken", "realized.gains", "realize.gains", "realizing.gains", "realizing.profits", "crystalize.gains", "crystalize.profits", "locking.gains", "locking.profits", + "paper.hands", "weak.hands", "capitulation.selling", "tax.loss.harvesting", + + # Losses & downside + "loss", "losses", "lost", "losing", "unrealized.loss", "drawdown", "max.drawdown", + "down", "downward", "declining", "decline", "declines", "dropping", "drops", "dropped", + "correction", "corrections", "pullback", "pullbacks", "retrace", "retraces", "retraced", + "bear.trap", "fake.out", "fakeout", "bull.trap", "failed.breakout", "failed.rally", + + # Regulatory & legal + "ban", "bans", "banned", "banning", "prohibition", "prohibited", "illegal", + "lawsuit", "lawsuits", "sued", "sues", "sue", "litigation", "legal.action", + "enforcement", "enforcement.action", "sec.enforcement", "cftc.enforcement", + "crackdown", "crackdowns", "cracking.down", "regulatory.crackdown", + "delist", "delists", "delisted", "delisting", "exchange.delisting", "delisting.announcement", + "wells.notice", "wells.notices", "subpoena", "subpoenas", "investigation", "investigations", + "sec.investigation", "doj.investigation", "fraud.charges", "securities.fraud", + "unregistered.securities", "securities.violation", "compliance", "non.compliant", + "license.revoked", "license.denied", "cease.and.desist", "cease.desist", + + # Bankruptcy & insolvency + "bankruptcy", "bankrupt", "insolvency", "insolvent", "chapter.11", "chapter.7", + "bank.run", "bank.run.on.exchange", "run.on.exchange", "run.on.bank", + "solvency.concerns", "solvency.fears", "solvency.issues", "proof.of.reserves", + "reserves.questioned", "reserves.doubted", "fractional.reserves", "insolvent.exchange", + "withdrawal.spike", "withdrawals.spike", "mass.withdrawal", "mass.withdrawals", + "users.fleeing", "users.flee", "exodus", "user.exodus", "platform.exodus", + "bank.run", "bank.run.on.exchange", "run.on.exchange", "run.on.bank", + "solvency.concerns", "solvency.fears", "solvency.issues", "proof.of.reserves", + "reserves.questioned", "reserves.doubted", "fractional.reserves", "insolvent.exchange", + "withdrawal.spike", "withdrawals.spike", "mass.withdrawal", "mass.withdrawals", + "users.fleeing", "users.flee", "exodus", "user.exodus", "platform.exodus", + "liquidation", "wind.down", "winding.down", "cease.operations", "shut.down", + "creditors", "creditor", "claims", "recovery", "recovery.rate", "haircut", + "ftx", "celcius", "voyager", "blockfi", "genesis", "three.arrows", "3ac", + "terra", "luna", "ust", "algorithmic.stablecoin", "ponzi", "pyramid.scheme", + + # Technical indicators (bearish) + "death.cross", "death.cross", "macd.bearish", "macd.crossunder", "rsi.overbought", + "rsi.rejection", "moving.average.resistance", "resistance.held", "resistance.holds", + "rejection", "rejects", "rejected", "rejecting", "failed.breakout", "fake.breakout", + "top", "topped", "local.top", "macro.top", "distribution.phase", "wyckoff.distribution", + "overbought", "overbought.rsi", "bearish.divergence", "hidden.bearish.divergence", + "head.and.shoulders", "double.top", "triple.top", "rising.wedge", "bearish.pattern", + + # On-chain bearish + "whale.selling", "whale.sells", "whale.dump", "whale.dumps", "large.holder.selling", + "exchange.inflow", "exchange.inflows", "balance.increasing", "on.exchange", + "hot.wallet", "exchange.deposit", "deposit.spike", "whale.deposit", "large.deposit", + "unstaking", "unstaked", "unstake", "unstakes", "withdrawal", "withdrawals", + "mass.withdrawal", "bank.run", "run.on.exchange", "solvency.concerns", "proof.of.reserves", + "hashrate.down", "hashrate.dropping", "difficulty.down", "miners.capitulation", "miner.capitulation", "miner.surrender", "miners.surrender", "miner.capitulation", "miner.surrender", "miners.surrender", + "miner.surrender", "miner.outflow", "miner.outflows", "miner.selling", "miner.sells", "miner.distributing", "hash.ribbons", + + # DeFi & protocol issues + "tvl.drop", "tvl.dropping", "tvl.crash", "total.value.locked.drop", "tvl.outflow", + "protocol.exploit", "protocol.hack", "smart.contract.exploit", "oracle.failure", + "short.liquidation", "short.squeeze", "short.cover", "squeeze.higher", "short.covering", "short.liquidation", "short.squeeze", "short.cover", "squeeze.higher", "short.covering", "liquidation.crisis", "bad.debt", "insolvent.protocol", "undercollateralized.vault", + "liquidation.bot", "mev.bot", "sandwich.attack", "front.run", "front.running", + "inflationary", "supply.increasing", "token.inflation", "emission", "emissions.high", + "unlock", "unlocks", "token.unlock", "vesting.unlock", "team.unlock", "investor.unlock", + "cliff.unlock", "linear.unlock", "supply.shock", "dilution", "dilutive", + + # Macro & risk-off + "rate.hike", "rate.hikes", "fed.hawkish", "hawkish.fed", "tightening", "qt", + "quantitative.tightening", "liquidity.drain", "liquidity.crunch", "credit.crunch", + "recession", "recession.fears", "recession.risk", "economic.slowdown", "gdp.slowing", + "inflation.high", "cpi.high", "ppi.high", "sticky.inflation", "core.inflation", + "dollar.strength", "dxy.up", "dxy.rising", "risk.off", "risk.off.environment", + "flight.to.safety", "safe.haven", "treasury.yields", "yields.rising", "bond.yields", + "correlation", "correlated.selloff", "broad.selloff", "market.wide.selloff", + "contagion", "contagion.risk", "systemic.risk", "domino.effect", "spillover", + + # Sentiment & social + "fud", "fear.uncertainty.doubt", "fear", "extreme.fear", "fear.and.greed", + "fear.index", "capitulation", "despair", "depression", "anger", "rage", + "selling.panic", "blind.panic", "herd.mentality", "herd.selling", "lemmings", + "social.dominance.drop", "mentions.falling", "interest.fading", "google.trends.down", + "search.interest.declining", "retail.exodus", "users.leaving", "offboarding", + "narrative.broken", "narrative.shift", "thesis.broken", "thesis.invalidated", + "disappointment", "missed.expectations", "guidance.lowered", "warn", "warning", + # Compound phrases (space-separated for regex matching) + "wipes out", "wiped out", "wiping out", "wipes out gains", "wiped out gains", "wiping out gains", + "peg broken", "peg lost", "lost peg", "below peg", "trading below peg", "peg breaks", "peg has broken", + "reserve shortfall", "reserves insufficient", "reserve shortfall confirmed", "shortfall confirmed by audit", + "algorithmic stablecoin", "algorithmic stablecoin loses", "stablecoin loses peg", "loses peg entirely", + "institutional inflow", "inflow to exchange", "inflow to exchange wallets", "to exchange wallets", + "exchange balance increasing", "balance increasing rapidly", "balance increasing", "exchange balance", + "supply on exchange", "on exchange supply", "supply on exchange at", "on exchange at", "yearly highs", "6 month highs", + "deposit spike", "deposit spike suggests", "spike suggests profit", "suggests profit taking", "profit taking", + "take profit", "take profit at", "profit at resistance", "at resistance", "traders take profit", "traders take", + "realized gains", "realizing gains", "holders realizing", "long term holders", "holders realizing gains", + "paper hands", "paper hands shaken", "hands shaken out", "shaken out at", "out at support", "weak hands shaken", + "selling pressure", "heavy selling pressure", "selling pressure evident", "pressure evident", + "institutional selling pressure", "evident selling", "evident selling pressure", + "smart money selling", "smart money selling the", "money selling the", "selling the rally", "the rally", + "smart money sells", "smart money distributes", "smart money distributes", + "institutional selling", "institutional sells", "institutional distributes", "institutional exits", + "institutional selling pressure", "selling pressure evident", + "mev bot sandwich", "mev bot sandwich attacks", "sandwich attack", "sandwich attacks", + "flash loan attack", "flash loan attack on", "attack on vulnerable", "on vulnerable pool", + "governance attack", "governance attack takes", "takes control of", "control of dao", "takes control of dao", + "margin call", "margin calls", "margin calls forcing", "forcing deleveraging", "deleveraging across board", + "blown up", "blown up by", "unexpected move", "account blown up", "account blown", + "rekt", "getting rekt", "getting rekt on", "rekt on leverage", + "exit scam", "exit scam suspected", "scam suspected as", "team disappears", "suspected as team", + "wells notice", "wells notice received", "notice received by", "received by crypto", "by crypto firm", + "fraud charges", "fraud charges filed", "charges filed against", "filed against founder", "against founder", + "unregistered securities", "tokens deemed unregistered", "deemed unregistered securities", + "cease and desist", "cease and desist order", "desist order issued", "order issued", + "bankruptcy filing", "crypto lender files", "lender files for", "files for chapter", "chapter 11", "filing for chapter", + "cease operations", "cease operations immediately", "operations immediately", "exchange ceases", "ceases operations", + "death cross", "death cross confirmed", "cross confirmed on", "confirmed on weekly", "on weekly", + "rsi rejection", "rsi rejection at", "rejection at overbought", "at overbought levels", + "resistance held", "resistance held for", "held for third", "for third time", "third time", + "fake breakout", "fake breakout traps", "breakout traps", "traps breakout traders", "breakout traders", + "failed rally", "failed rally confirms", "rally confirms distribution", "confirms distribution", + "head and shoulders", "head and shoulders pattern", "and shoulders pattern", "pattern completes", "shoulders pattern completes", + "double top", "double top confirmed", "top confirmed", "confirmed on break", "break of neckline", + "rising wedge", "rising wedge breakdown", "wedge breakdown", "breakdown imminent", "imminent breakdown", + "bank run", "bank run on", "run on staking", "run on exchange", + "solvency concerns", "solvency concerns for", "concerns for lending", "for lending protocol", "lending protocol", + "proof of reserves", "proof of reserves audit", "audit incomplete", "incomplete audit", "reserves audit", + "miner surrender", "miner surrender evident", "surrender evident in", "evident in outflows", + "hash ribbons", "hash ribbons signal", "ribbons signal miner", "signal miner distress", "miner distress", + "oracle failure", "oracle failure causes", "failure causes bad", "causes bad debt", "bad debt", + "bad debt", "protocol accumulates", "accumulates bad debt", "accumulates bad", + "emissions high", "emissions schedule", "schedule too aggressive", "too aggressive", "high emissions", + ] + + # Whale action phrases - context-dependent sentiment (bullish) + WHALE_BULLISH_PHRASES = [ + "whale buys", "whale buys", "whale accumulates", "whale accumulates", "whale accumulating", + "whale loads", "whale loading", "whale adds", "whale buying", "whale accumulation", + "whale entry", "whale enters", "whale positions", "whale positions", "whale positioned", + "whale accumulation", "whale buying", "whale buys more", "whale adds more", + "whale buy", "whale buy", "whale bought", "whale has bought", "whale purchased", + "whale purchases", "whale acquiring", "whale acquires", "whale acquired", + "whale stacking", "whale stacks", "whale stacked", "whale building", "whale builds", + "whale building.position", "whale building.position", "whale increasing.position", + "whale adds.to.position", "whale adding.to.position", "whale scaled.in", "whale scaling.in", + "smart.money.accumulating", "smart.money.buys", "smart.money.accumulates", + "institutional.accumulation", "institutional.buying", "institutional.accumulates", + "large.holder.accumulating", "large.holder.buys", "large.holder.accumulates", + "large.holder.accumulation", "large.holder.buying", "large.holder.accumulates", + "large.holders.accumulating", "large.holders.buys", "large.holders.accumulate", + "smart.money.accumulation", "smart.money.buying", "institutional.buys", + "whale.absorbing", "whale.absorbs", "whale.absorbed", "absorption", "absorbing.supply", + "supply.absorption", "whale.absorbing.supply", "bid.absorption", "strong.bids", + "whale.bid", "whale.bids", "large.bid", "large.bids", "buy.wall", "buy.walls", + "massive.buy", "massive.buys", "huge.buy", "huge.buys", "large.buy", "large.buys", + ] + + WHALE_BEARISH_PHRASES = [ + "whale sells", "whale sells", "whale dumps", "whale dumping", "whale dumps", + "whale distributes", "whale distributing", "whale distributes", "whale exits", + "whale exits", "whale exits", "whale liquidates", "whale liquidating", + "whale takes profit", "whale takes profit", "whale profit taking", "whale takes profit", + "whale exits", "whale exits", "whale distributes", "whale distributing", + "whale sells", "whale sells", "whale sells", "whale unloads", "whale unloads", + "whale sell", "whale sell", "whale sold", "whale has sold", "whale selling", + "whale offloading", "whale offloads", "whale offloaded", "whale unloading", + "whale distributes", "whale distributing", "whale distribution", "whale.distributing", + "whale taking.profit", "whale.take.profit", "whale.profit.taking", "whale.realized", + "whale.realizing", "whale.realizes", "whale.realize", "profit.realized", + "whale.exits", "whale.exiting", "whale.exited", "whale.exit", "whale.exits.market", + "whale.reducing", "whale.reduces", "whale.reduced", "whale.trimmed", "whale.trims", + "whale.trimming", "whale.scaling.out", "whale.scales.out", "whale.scaled.out", + "smart.money.selling", "smart.money.sells", "smart.money.distributes", + "institutional.selling", "institutional.distributes", "institutional.exits", + "large.holder.selling", "large.holder.distributes", "large.holder.exits", + "large.holder.sells", "large.holder.selling", "large.holder.distributes", + "large.holders.selling", "large.holders.sells", "large.holders.distribute", + "smart.money.sells", "smart.money.distributing", "institutional.sells", + "whale.unloading", "whale.unload", "whale.unloaded", "supply.unloading", + "supply.flooding", "whale.flooding", "massive.sell", "massive.sells", "huge.sell", + "huge.sells", "large.sell", "large.sells", "sell.wall", "sell.walls", "massive.sell.wall", + "whale.ask", "whale.asks", "large.ask", "large.asks", "heavy.asks", "ask.side.heavy", + "whale.market.sell", "whale.market.sells", "market.sell", "market.sells", + "aggressive.selling", "aggressive.seller", "aggressive.sells", "market.order.sell", + # Compound bearish phrases with high priority + "profit taking", "take profit", "taking profits", "taking gains", "realizing profits", "realizing gains", + "locking profits", "locking gains", "crystalize gains", "crystalize profits", "profit taking after rally", "profit taking after pump", + "profit taking at top", "profit taking at highs", "profit taking at resistance", "profits taken", "gains taken", + "fake breakout", "fake breakout traps", "fake breakout confirmed", "fake out breakout", "fake out", + "failed breakout", "failed breakout confirmed", "failed breakout traps", "fakeout breakout", + "profit taking after 3x rally", "profit taking after pump", "profit taking at top", "profit taking at highs", + "profit taking at resistance", "take profit at resistance", "traders take profit", "traders taking profit", + ] + + @classmethod + def _get_crypto_signal(cls, text: str) -> str: + """ + Determine crypto sentiment direction using unified weighted lexicon. + + The lexicon assigns each term a weight from -100 (extreme bearish) to +100 (extreme bullish). + Compound/whale terms have higher absolute weights. Matching uses longest-first priority + with span-based deduplication. + + This provides the EXPLICIT signal layer. The centroid/cosine layer in ScoringEngine + provides the SEMANTIC refinement layer. + """ + cls._load_lexicon() + score = cls._get_lexicon_score(text) + return cls._lexicon_to_signal(score) + + @classmethod + def get_lexicon_score(cls, text: str) -> float: + """Public method to get raw lexicon score for integration with centroid layer.""" + cls._load_lexicon() + return cls._get_lexicon_score(text) + + @classmethod + def _get_finbert_signal(cls, probs: np.ndarray) -> str: + """Determine FinBERT's predicted direction""" + # probs = [negative, neutral, positive] = [Bearish, Neutral, Bullish] + diff = probs[2] - probs[0] # positive - negative + if diff > 0.05: # clearly positive (Bullish) - lowered threshold from 0.15 + return "bullish" + elif diff < -0.05: # clearly negative (Bearish) + return "bearish" + return "neutral" + + @classmethod + def calibrate(cls, text: str, probs: np.ndarray) -> np.ndarray: + """ + Calibrate FinBERT probabilities using the weighted lexicon score. + + PRINCIPLE: Crypto-specific explicit lexicon WINS over general FinBERT + when they disagree. FinBERT is trained on traditional finance where + "surge/rally/pump" = risky/bubble = negative. In crypto, these are bullish. + + The lexicon provides a continuous score [-100, 100] representing + explicit keyword evidence with domain-specific weights. + + Blending logic: + - If lexicon and FinBERT AGREE on direction: amplify (trust both) + - If lexicon and FinBERT DISAGREE: lexicon wins (crypto-specific > general) + - If lexicon is NEUTRAL (|score| <= 10): trust FinBERT + - If FinBERT is NEUTRAL (|pos-neg| < 0.15): trust lexicon + + The centroid/cosine layer in ScoringEngine provides the SEMANTIC + refinement on top of this calibrated output. + """ + cls._load_lexicon() + + # Get continuous lexicon score [-100, 100] + lexicon_score = cls._get_lexicon_score(text) + + # FinBERT probabilities: [negative, neutral, positive] = [Bearish, Neutral, Bullish] + neg, neu, pos = probs[0], probs[1], probs[2] + finbert_polarity = pos - neg # [-1, 1] + finbert_confidence = max(probs) + + # Lexicon polarity: -1 (bearish) to +1 (bullish) + lexicon_polarity = np.clip(lexicon_score / 100.0, -1.0, 1.0) + lexicon_confidence = min(abs(lexicon_score) / 50.0, 1.0) # Full confidence at |50| + + # Determine agreement + finbert_dir = "bullish" if finbert_polarity > 0.1 else "bearish" if finbert_polarity < -0.1 else "neutral" + lexicon_dir = "bullish" if lexicon_polarity > 0.1 else "bearish" if lexicon_polarity < -0.1 else "neutral" + + # Case 1: Both agree on direction -> AMPLIFY + if finbert_dir == lexicon_dir and finbert_dir != "neutral": + # Weighted average favoring the stronger signal + total_conf = finbert_confidence + lexicon_confidence + if total_conf > 0: + w_finbert = finbert_confidence / total_conf + w_lexicon = lexicon_confidence / total_conf + else: + w_finbert = w_lexicon = 0.5 + blended_polarity = w_finbert * finbert_polarity + w_lexicon * lexicon_polarity + # Amplify slightly beyond both + blended_polarity = np.clip(blended_polarity * 1.2, -1.0, 1.0) + target_neu = neu * 0.7 # Reduce neutral when both agree + + # Case 2: Disagree -> LEXICON WINS (crypto-specific > general finance) + elif finbert_dir != "neutral" and lexicon_dir != "neutral" and finbert_dir != lexicon_dir: + # Lexicon dominates with high weight + blended_polarity = 0.85 * lexicon_polarity + 0.15 * finbert_polarity + target_neu = 0.15 # Low neutral when strong disagreement resolved + + # Case 3: Lexicon neutral -> trust FinBERT + elif lexicon_dir == "neutral": + blended_polarity = finbert_polarity + target_neu = neu + + # Case 4: FinBERT neutral -> trust lexicon + elif finbert_dir == "neutral": + blended_polarity = lexicon_polarity + target_neu = neu * (1 - lexicon_confidence * 0.5) + + # Case 5: One neutral, other directional + else: + blended_polarity = lexicon_polarity if lexicon_dir != "neutral" else finbert_polarity + target_neu = neu * 0.8 + + # Clamp polarity + blended_polarity = np.clip(blended_polarity, -0.98, 0.98) + target_neu = np.clip(target_neu, 0.05, 0.9) + + # Convert polarity back to probabilities + # pos - neg = blended_polarity + # pos + neg = 1 - target_neu + pos_prob = (blended_polarity + 1 - target_neu) / 2 + neg_prob = (1 - target_neu - blended_polarity) / 2 + + # Clamp + pos_prob = max(0.02, min(0.98, pos_prob)) + neg_prob = max(0.02, min(0.98, neg_prob)) + neu_prob = max(0.05, min(0.95, 1 - pos_prob - neg_prob)) + + # Renormalize + total = pos_prob + neg_prob + neu_prob + calibrated = np.array([neg_prob / total, neu_prob / total, pos_prob / total]) + + return calibrated + + +# Mock classes for testing/fallback +class MockTokenizer: + """Mock tokenizer for testing/fallback""" + + def __init__(self): + self.vocab_size = 30522 + + def __call__(self, text, return_tensors="pt", truncation=True, max_length=512, padding=True): + if isinstance(text, list): + batch_size = len(text) + else: + batch_size = 1 + text = [text] + + input_ids = torch.randint(1, 1000, (batch_size if TRANSFORMERS_AVAILABLE else 1, 512)) if TRANSFORMERS_AVAILABLE else np.random.randint(1, 1000, (batch_size, 512)) + attention_mask = torch.ones_like(input_ids) if TRANSFORMERS_AVAILABLE else np.ones((batch_size, 512)) + token_type_ids = torch.zeros_like(input_ids) if TRANSFORMERS_AVAILABLE else np.zeros((batch_size, 512)) + + return { + "input_ids": input_ids, + "attention_mask": attention_mask, + "token_type_ids": token_type_ids + } + + @classmethod + def from_pretrained(cls, model_name: str): + return MockTokenizer() + + def save_pretrained(self, path: str): + pass + + +class MockSentimentModel: + def __init__(self, device="cpu"): + self.device = device + + def to(self, device): + self.device = device + return self + + def eval(self): + return self + + def __call__(self, **inputs): + batch_size = inputs["input_ids"].shape[0] + logits = torch.randn(batch_size, 3) if TRANSFORMERS_AVAILABLE else np.random.randn(batch_size, 3) + return type('Outputs', (), {'logits': logits})() + + +class ONNXSentimentModel: + """ONNX Runtime wrapper for FinBERT sentiment model (requires token_type_ids)""" + + def __init__(self, model_path: str, tokenizer_path: str, label_map_path: str = None): + self.model_path = model_path + self.tokenizer_path = tokenizer_path + self.label_map_path = label_map_path + + # Load tokenizer + if TRANSFORMERS_AVAILABLE: + self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) + else: + self.tokenizer = MockTokenizer() + + # Load ONNX model with limited threads to avoid contention + session_options = ort.SessionOptions() + session_options.intra_op_num_threads = 1 + session_options.inter_op_num_threads = 1 + session_options.execution_mode = ort.ExecutionMode.ORT_SEQUENTIAL + self.session = ort.InferenceSession(model_path, sess_options=session_options, providers=self._get_providers()) + + # Load labels + self.labels = ["negative", "neutral", "positive"] + if label_map_path and Path(label_map_path).exists(): + import json + with open(label_map_path) as f: + self.labels = [v for k, v in sorted(json.load(f).items(), key=lambda x: int(x[0]))] + + self._input_names = [i.name for i in self.session.get_inputs()] + self._output_names = [o.name for o in self.session.get_outputs()] + + def _get_providers(self): + """Get ONNX Runtime execution providers""" + providers = ['CPUExecutionProvider'] + if ort.get_device() == 'GPU': + providers.insert(0, 'CUDAExecutionProvider') + return providers + + def __call__(self, input_ids, attention_mask, token_type_ids=None) -> np.ndarray: + """Run inference, return logits""" + if hasattr(input_ids, 'numpy'): + input_ids = input_ids.numpy() + if hasattr(attention_mask, 'numpy'): + attention_mask = attention_mask.numpy() + if token_type_ids is not None and hasattr(token_type_ids, 'numpy'): + token_type_ids = token_type_ids.numpy() + + ort_inputs = { + "input_ids": input_ids.astype(np.int64), + "attention_mask": attention_mask.astype(np.int64), + } + # FinBERT requires token_type_ids + if "token_type_ids" in self._input_names: + if token_type_ids is None: + token_type_ids = np.zeros_like(input_ids) + ort_inputs["token_type_ids"] = token_type_ids.astype(np.int64) + + # Run inference + outputs = self.session.run(self._output_names, ort_inputs) + logits = outputs[0] # First output is typically logits + + return logits + + +class ONNXEmotionModel: + """ONNX Runtime wrapper for DistilRoBERTa emotion model (NO token_type_ids)""" + + def __init__(self, model_path: str, tokenizer_path: str, label_map_path: str = None): + self.model_path = model_path + self.tokenizer_path = tokenizer_path + self.label_map_path = label_map_path + + if TRANSFORMERS_AVAILABLE: + self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) + else: + self.tokenizer = MockTokenizer() + + # Load ONNX model with limited threads to avoid contention + session_options = ort.SessionOptions() + session_options.intra_op_num_threads = 2 + session_options.inter_op_num_threads = 2 + self.session = ort.InferenceSession(model_path, sess_options=session_options, providers=self._get_providers()) + + self.labels = ["anger", "disgust", "fear", "joy", "neutral", "sadness", "surprise"] + if label_map_path and Path(label_map_path).exists(): + import json + with open(label_map_path) as f: + self.labels = [v for k, v in sorted(json.load(f).items(), key=lambda x: int(x[0]))] + + self._input_names = [i.name for i in self.session.get_inputs()] + self._output_names = [o.name for o in self.session.get_outputs()] + + def _get_providers(self): + providers = ['CPUExecutionProvider'] + if ort.get_device() == 'GPU': + providers.insert(0, 'CUDAExecutionProvider') + return providers + + def __call__(self, input_ids, attention_mask, token_type_ids=None) -> np.ndarray: + """Run inference, return logits - DistilRoBERTa does NOT use token_type_ids""" + if hasattr(input_ids, 'numpy'): + input_ids = input_ids.numpy() + if hasattr(attention_mask, 'numpy'): + attention_mask = attention_mask.numpy() + + ort_inputs = { + "input_ids": input_ids.astype(np.int64), + "attention_mask": attention_mask.astype(np.int64), + } + # DistilRoBERTa does NOT have token_type_ids input + if "token_type_ids" in self._input_names and token_type_ids is not none: + if hasattr(token_type_ids, 'numpy'): + token_type_ids = token_type_ids.numpy() + ort_inputs["token_type_ids"] = token_type_ids.astype(np.int64) + + outputs = self.session.run(self._output_names, ort_inputs) + return outputs[0] + + +class ONNXEventModel: + """ONNX Runtime wrapper for BERT event classification model (requires token_type_ids)""" + + def __init__(self, model_path: str, tokenizer_path: str, label_map_path: str = None): + self.model_path = model_path + self.tokenizer_path = tokenizer_path + self.label_map_path = label_map_path + + if TRANSFORMERS_AVAILABLE: + self.tokenizer = AutoTokenizer.from_pretrained(tokenizer_path) + else: + self.tokenizer = MockTokenizer() + + # Load ONNX model with limited threads to avoid contention + session_options = ort.SessionOptions() + session_options.intra_op_num_threads = 2 + session_options.inter_op_num_threads = 2 + self.session = ort.InferenceSession(model_path, sess_options=session_options, providers=self._get_providers()) + + from sentiment_engine.schemas.processed import EventType + self.labels = [e.value for e in EventType if e != EventType.UNKNOWN] + if label_map_path and Path(label_map_path).exists(): + import json + with open(label_map_path) as f: + self.labels = [v for k, v in sorted(json.load(f).items(), key=lambda x: int(x[0]))] + + self._input_names = [i.name for i in self.session.get_inputs()] + self._output_names = [o.name for o in self.session.get_outputs()] + + def _get_providers(self): + providers = ['CPUExecutionProvider'] + if ort.get_device() == 'GPU': + providers.insert(0, 'CUDAExecutionProvider') + return providers + + def predict(self, input_ids, attention_mask, token_type_ids=None) -> np.ndarray: + """Run inference, return probabilities""" + if hasattr(input_ids, 'numpy'): + input_ids = input_ids.numpy() + if hasattr(attention_mask, 'numpy'): + attention_mask = attention_mask.numpy() + if token_type_ids is not none and hasattr(token_type_ids, 'numpy'): + token_type_ids = token_type_ids.numpy() + + ort_inputs = { + "input_ids": input_ids.astype(np.int64), + "attention_mask": attention_mask.astype(np.int64), + } + # BERT event model requires token_type_ids + if "token_type_ids" in self._input_names: + if token_type_ids is none: + token_type_ids = np.zeros_like(input_ids) + ort_inputs["token_type_ids"] = token_type_ids.astype(np.int64) + + outputs = self.session.run(self._output_names, ort_inputs) + logits = outputs[0] + + # Softmax + e_x = np.exp(logits - np.max(logits, axis=-1, keepdims=True)) + probs = e_x / e_x.sum(axis=-1, keepdims=True) + + return probs[0] + + +class CryptoSentimentCalibrator: + """ + Calibrates FinBERT outputs for crypto semantics. + + FinBERT (traditional finance): + - "surge/rally/pump" = risky/bubble = negative (index 0) + - "crash/drop/dump" = value/opportunity = positive (index 2) + - Native: [negative, neutral, positive] = [Bearish, Neutral, Bullish] + + Crypto semantics: + - "surge/pump/moon/rally" = bullish = Bullish (index 2) + - "crash/dump/rug/hack" = bearish = Bearish (index 0) + + This calibrator flips FinBERT's positive/negative ONLY when there's a semantic mismatch. + Uses word-boundary keyword matching for reliable crypto signal detection. + """ + + # Crypto-bullish keywords (should map to index 2 = Bullish) + # Comprehensive vocabulary covering price action, institutional, technical, on-chain, and news catalysts + CRYPTO_BULLISH_KEYWORDS = [ + # Price action + "surge", "surges", "surged", "surgeing", "pump", "pumps", "pumped", "pumping", + "moon", "mooning", "mooned", "to.the.moon", "to the moon", "to the moon", "rally", "rallies", "rallied", "rallying", + "breakout", "breakouts", "break.out", "breaking.out", "bullish", "bull.run", "bullrun", "bull run", "bull run", + "ath", "all.time.high", "all.time.highs", "all time high", "all time highs", "record.high", "record.highs", "record high", "record highs", "new.high", "new.highs", "new high", "new highs", + "higher.high", "higher.highs", "higher.low", "higher.lows", "higher high", "higher highs", "higher low", "higher lows", "uptrend", "uptrending", + "green", "green.candle", "green.candles", "pumping", "ripping", "rips", "ripped", + "parabolic", "parabolic.move", "explosive", "explosive.move", "violent.move.up", "violent.move", "violent.up", "violent.rally", "violent.move", "violent.up", "violent.rally", + + # Inflows & accumulation (in crypto: exchange OUTFLOWS = bullish, INFLOWS = bearish) + "outflow", "outflows", "net.outflow", "exchange.outflow", "exchange.outflows", + "capital.outflow", "institutional.outflow", "outflow.spike", "outflows.spike", + "balance.decreasing", "off.exchange", "cold.storage", + "whale.withdrawal", "whale.withdrawals", "large.withdrawal", "large.withdrawals", "exchange.withdrawal.to.cold", "whale.withdrawal.from.exchange", "whale.withdrawal.to.cold", "whale.withdraws", "whale.withdrawing", "break.out", "breaks.out", "breaking.out", "break out", "breaks out", "breaking out", "all.time.high", "all time high", "golden cross", "golden.cross", "violent.move.up", "violent move up", "short.squeeze", "short squeeze", "squeeze.higher", "squeeze higher", "short.cover", "short cover", "short.covering", "short covering", "violent.move.up", "violent move up", "violent.up", "violent up", "violent move", "violent up move", "violent move", "violent up move", "wipes.out.bearish", "wipes out bearish", "wiped.out.bearish", "wiped out bearish", "leverage.flush", "leverage flush", "leverage flush clears", "flush clears weak hands", "leverage flush clears", "flush clears weak hands", "short.liquidation", "short liquidation", "short.squeeze", "short squeeze", "short liquidation fuels", "short liquidation fuels squeeze", "short squeeze higher", "squeeze higher", "short liquidation fuels", "short liquidation fuels squeeze", "short squeeze higher", "squeeze higher", "squeeze.higher", "squeeze higher", "capitulation.over", "capitulation over", "capitulation over per", "capitulation ended", "capitulation over per", "capitulation ended", "miner.surrender", "miner surrender", "miner surrender evident", "surrender evident in", "evident in outflows", "miner surrender evident", "miners surrender", "miner capitulation", "miner surrender evident", "surrender evident in", "evident in outflows", "miner surrender evident", "miners surrender", "miner capitulation", "bank.run", "bank run", "bank run on", "run on staking", "run on exchange", "bank run on", "run on exchange", "run on staking", "bank run on", "run on staking", "run on exchange", "bank run on", "run on exchange", "run on staking", "supply.squeeze", "supply squeeze", "supply squeeze as", "coins lock up", "supply squeeze as", "coins lock up", "inflation.hedge", "inflation hedge", "store of value", "digital gold", "store of value", "digital gold", "balance.decreasing", "balance decreasing", "balance decreasing rapidly", "exchange balance decreasing", "balance decreasing rapidly", "exchange balance decreasing", "off.exchange", "off exchange", "moving off exchange", "coins moving off exchange", "moving off exchange", "coins moving off exchange", "cold.storage", "cold storage", "moving to cold storage", "whales moving to cold storage", "moving to cold storage", "whales moving to cold storage", "whale.withdrawal", "whale withdrawal", "whale.withdrawals", "whale withdrawals", "large whale withdrawal", "whale withdrawal from", "whale withdraws from", "large whale withdrawal", "whale withdrawal from", "whale withdraws from", "large.withdrawal", "large withdrawal", "large.withdrawals", "large withdrawals", "large withdrawal suggests", "withdrawal suggests accumulation", "large withdrawal suggests", "withdrawal suggests accumulation", "rsi.recovery", "rsi recovery", "rsi recovery from", "recovery from oversold", "rsi recovery from", "recovery from oversold", "rsi.oversold", "rsi oversold", "oversold territory", "rsi oversold at", "oversold territory", "rsi oversold at", "support.held", "support held", "support.holds", "support holds", "key support held", "support held on retest", "key support held", "support held on retest", "moving.average.support", "moving average support", "ma.support", "ma support", "200.day", "200 day", "200 day ma", "200-day ma", "200.day", "200 day", "200 day ma", "200-day ma", "golden.cross", "golden cross", "golden cross forms", "golden cross on daily", "golden cross on weekly", "golden cross forms", "golden cross on daily", "golden cross on weekly", "break.out", "break out", "breaks.out", "breaks out", "breaking.out", "breaking out", "break out of", "breaks out of", "breaking out of", "breakout confirmed", "breakout traps", "break out of", "breaks out of", "breaking out of", "breakout confirmed", "breakout traps", "all.time.high", "all time high", "all.time.highs", "all time highs", "hits all time high", "above all time high", "new all time high", "record high", "record highs", "hits all time high", "above all time high", "new all time high", "record high", "record highs", "leverage.flush", "leverage flush", "leverage flush clears", "flush clears weak hands", "leverage flush clears", "flush clears weak hands", "short.liquidation", "short liquidation", "short.squeeze", "short squeeze", "short liquidation fuels", "short liquidation fuels squeeze", "short squeeze higher", "squeeze higher", "short liquidation fuels", "short liquidation fuels squeeze", "short squeeze higher", "squeeze higher", "squeeze.higher", "squeeze higher", "capitulation.over", "capitulation over", "capitulation over per", "capitulation ended", "capitulation over per", "capitulation ended", "miner.surrender", "miner surrender", "miner surrender evident", "surrender evident in", "evident in outflows", "miner surrender evident", "miners surrender", "miner capitulation", "miner surrender evident", "surrender evident in", "evident in outflows", "miner surrender evident", "miners surrender", "miner capitulation", "bank.run", "bank run", "bank run on", "run on staking", "run on exchange", "bank run on", "run on exchange", "run on staking", "bank run on", "run on staking", "run on exchange", "bank run on", "run on exchange", "run on staking", "supply.squeeze", "supply squeeze", "supply squeeze as", "coins lock up", "supply squeeze as", "coins lock up", "inflation.hedge", "inflation hedge", "store of value", "digital gold", "store of value", "digital gold", "balance.decreasing", "balance decreasing", "balance decreasing rapidly", "exchange balance decreasing", "balance decreasing rapidly", "exchange balance decreasing", "off.exchange", "off exchange", "moving off exchange", "coins moving off exchange", "moving off exchange", "coins moving off exchange", "cold.storage", "cold storage", "moving to cold storage", "whales moving to cold storage", "moving to cold storage", "whales moving to cold storage", "whale.withdrawal", "whale withdrawal", "whale.withdrawals", "whale withdrawals", "large whale withdrawal", "whale withdrawal from", "whale withdraws from", "large whale withdrawal", "whale withdrawal from", "whale withdraws from", "large.withdrawal", "large withdrawal", "large.withdrawals", "large withdrawals", "large withdrawal suggests", "withdrawal suggests accumulation", "large withdrawal suggests", "withdrawal suggests accumulation", "rsi.recovery", "rsi recovery", "rsi recovery from", "recovery from oversold", "rsi recovery from", "recovery from oversold", "rsi.oversold", "rsi oversold", "oversold territory", "rsi oversold at", "oversold territory", "rsi oversold at", "support.held", "support held", "support.holds", "support holds", "key support held", "support held on retest", "key support held", "support held on retest", "moving.average.support", "moving average support", "ma.support", "ma support", "200.day", "200 day", "200 day ma", "200-day ma", "200.day", "200 day", "200 day ma", "200-day ma", "golden.cross", "golden cross", "golden cross forms", "golden cross on daily", "golden cross on weekly", "golden cross forms", "golden cross on daily", "golden cross on weekly", "break.out", "break out", "breaks.out", "breaks out", "breaking.out", "breaking out", "break out of", "breaks out of", "breaking out of", "breakout confirmed", "breakout traps", "break out of", "breaks out of", "breaking out of", "breakout confirmed", "breakout traps", "all.time.high", "all time high", "all.time.highs", "all time highs", "hits all time high", "above all time high", "new all time high", "record high", "record highs", "hits all time high", "above all time high", "new all time high", "record high", "record highs", "short.cover", "short cover", "short.covering", "short covering", "violent.move.up", "violent move up", "violent.up", "violent up", "violent move", "violent up move", "violent move", "violent up move", "wipes.out.bearish", "wipes out bearish", "wiped.out.bearish", "wiped out bearish", "leverage.flush", "leverage flush", "leverage flush clears", "flush clears weak hands", "leverage flush clears", "flush clears weak hands", "short.liquidation", "short liquidation", "short.squeeze", "short squeeze", "short liquidation fuels", "short liquidation fuels squeeze", "short squeeze higher", "squeeze higher", "short liquidation fuels", "short liquidation fuels squeeze", "short squeeze higher", "squeeze higher", "squeeze.higher", "squeeze higher", "capitulation.over", "capitulation over", "capitulation over per", "capitulation ended", "capitulation over per", "capitulation ended", "miner.surrender", "miner surrender", "miner surrender evident", "surrender evident in", "evident in outflows", "miner surrender evident", "miners surrender", "miner capitulation", "miner surrender evident", "surrender evident in", "evident in outflows", "miner surrender evident", "miners surrender", "miner capitulation", "bank.run", "bank run", "bank run on", "run on staking", "run on exchange", "bank run on", "run on exchange", "run on staking", "bank run on", "run on staking", "run on exchange", "bank run on", "run on exchange", "run on staking", "supply.squeeze", "supply squeeze", "supply squeeze as", "coins lock up", "supply squeeze as", "coins lock up", "inflation.hedge", "inflation hedge", "store of value", "digital gold", "store of value", "digital gold", "balance.decreasing", "balance decreasing", "balance decreasing rapidly", "exchange balance decreasing", "balance decreasing rapidly", "exchange balance decreasing", "off.exchange", "off exchange", "moving off exchange", "coins moving off exchange", "moving off exchange", "coins moving off exchange", "cold.storage", "cold storage", "moving to cold storage", "whales moving to cold storage", "moving to cold storage", "whales moving to cold storage", "whale.withdrawal", "whale withdrawal", "whale.withdrawals", "whale withdrawals", "large whale withdrawal", "whale withdrawal from", "whale withdraws from", "large whale withdrawal", "whale withdrawal from", "whale withdraws from", "large.withdrawal", "large withdrawal", "large.withdrawals", "large withdrawals", "large withdrawal suggests", "withdrawal suggests accumulation", "large withdrawal suggests", "withdrawal suggests accumulation", "rsi.recovery", "rsi recovery", "rsi recovery from", "recovery from oversold", "rsi recovery from", "recovery from oversold", "rsi.oversold", "rsi oversold", "oversold territory", "rsi oversold at", "oversold territory", "rsi oversold at", "support.held", "support held", "support.holds", "support holds", "key support held", "support held on retest", "key support held", "support held on retest", "moving.average.support", "moving average support", "ma.support", "ma support", "200.day", "200 day", "200 day ma", "200-day ma", "200.day", "200 day", "200 day ma", "200-day ma", "golden.cross", "golden cross", "golden cross forms", "golden cross on daily", "golden cross on weekly", "golden cross forms", "golden cross on daily", "golden cross on weekly", "break.out", "break out", "breaks.out", "breaks out", "breaking.out", "breaking out", "break out of", "breaks out of", "breaking out of", "breakout confirmed", "breakout traps", "break out of", "breaks out of", "breaking out of", "breakout confirmed", "breakout traps", "all.time.high", "all time high", "all.time.highs", "all time highs", "hits all time high", "above all time high", "new all time high", "record high", "record highs", "hits all time high", "above all time high", "new all time high", "record high", "record highs", "leverage.flush", "leverage flush", "leverage flush clears", "flush clears weak hands", "leverage flush clears", "flush clears weak hands", "short.liquidation", "short liquidation", "short.squeeze", "short squeeze", "short liquidation fuels", "short liquidation fuels squeeze", "short squeeze higher", "squeeze higher", "short liquidation fuels", "short liquidation fuels squeeze", "short squeeze higher", "squeeze higher", "squeeze.higher", "squeeze higher", "capitulation.over", "capitulation over", "capitulation over per", "capitulation ended", "capitulation over per", "capitulation ended", "miner.surrender", "miner surrender", "miner surrender evident", "surrender evident in", "evident in outflows", "miner surrender evident", "miners surrender", "miner capitulation", "miner surrender evident", "surrender evident in", "evident in outflows", "miner surrender evident", "miners surrender", "miner capitulation", "bank.run", "bank run", "bank run on", "run on staking", "run on exchange", "bank run on", "run on exchange", "run on staking", "bank run on", "run on staking", "run on exchange", "bank run on", "run on exchange", "run on staking", "supply.squeeze", "supply squeeze", "supply squeeze as", "coins lock up", "supply squeeze as", "coins lock up", "inflation.hedge", "inflation hedge", "store of value", "digital gold", "store of value", "digital gold", "balance.decreasing", "balance decreasing", "balance decreasing rapidly", "exchange balance decreasing", "balance decreasing rapidly", "exchange balance decreasing", "off.exchange", "off exchange", "moving off exchange", "coins moving off exchange", "moving off exchange", "coins moving off exchange", "cold.storage", "cold storage", "moving to cold storage", "whales moving to cold storage", "moving to cold storage", "whales moving to cold storage", "whale.withdrawal", "whale withdrawal", "whale.withdrawals", "whale withdrawals", "large whale withdrawal", "whale withdrawal from", "whale withdraws from", "large whale withdrawal", "whale withdrawal from", "whale withdraws from", "large.withdrawal", "large withdrawal", "large.withdrawals", "large withdrawals", "large withdrawal suggests", "withdrawal suggests accumulation", "large withdrawal suggests", "withdrawal suggests accumulation", "rsi.recovery", "rsi recovery", "rsi recovery from", "recovery from oversold", "rsi recovery from", "recovery from oversold", "rsi.oversold", "rsi oversold", "oversold territory", "rsi oversold at", "oversold territory", "rsi oversold at", "support.held", "support held", "support.holds", "support holds", "key support held", "support held on retest", "key support held", "support held on retest", "moving.average.support", "moving average support", "ma.support", "ma support", "200.day", "200 day", "200 day ma", "200-day ma", "200.day", "200 day", "200 day ma", "200-day ma", "golden.cross", "golden cross", "golden cross forms", "golden cross on daily", "golden cross on weekly", "golden cross forms", "golden cross on daily", "golden cross on weekly", "break.out", "break out", "breaks.out", "breaks out", "breaking.out", "breaking out", "break out of", "breaks out of", "breaking out of", "breakout confirmed", "breakout traps", "break out of", "breaks out of", "breaking out of", "breakout confirmed", "breakout traps", "all.time.high", "all time high", "all.time.highs", "all time highs", "hits all time high", "above all time high", "new all time high", "record high", "record highs", "hits all time high", "above all time high", "new all time high", "record high", "record highs", "capitulation.over", "capitulation over", "capitulation over per", "capitulation ended", "capitulation over per", "capitulation ended", "supply.squeeze", "supply squeeze", "supply squeeze as", "coins lock up", "supply squeeze as", "coins lock up", "inflation.hedge", "inflation hedge", "store of value", "digital gold", "store of value", "digital gold", "miner.surrender", "miner surrender", "miner surrender evident", "surrender evident in", "evident in outflows", "miner surrender evident", "miners surrender", "miner capitulation", "miner surrender evident", "surrender evident in", "evident in outflows", "miner surrender evident", "miners surrender", "miner capitulation", + "adoption", "adoptions", "mass.adoption", "mainstream.adoption", "accumulation", "accumulating", + "accumulate", "accumulates", "accumulated", "whale.accumulation", "smart.money.accumulation", + "hodl", "hodling", "diamond.hands", "strong.hands", "not.selling", "holding.strong", + "buy.the.dip", "buying.the.dip", "dip.bought", "buy the dip", "buying the dip", "dip bought", "dca", "dollar.cost.average", "dollar cost average", + + # Institutional & ETF + "etf", "etfs", "spot.etf", "spot.etfs", "bitcoin.etf", "ethereum.etf", "etf.approval", + "etf.approved", "etf.approves", "etf.launch", "etf.launches", "etf.listed", "etf.inflow", + "etf.inflows", "blackrock", "fidelity", "vanguard", "grayscale", "ark.invest", "institutional", + "institutions", "institutional.adoption", "corporate.treasury", "corporate.adoption", "institutional adoption", "corporate treasury", "corporate adoption", "institutional adoption", "corporate treasury", "corporate adoption", + "microstrategy", "tesla", "public.company", "balance.sheet", "treasury.reserve", + "approval", "approvals", "approved", "approves", "sec.approval", "regulatory.approval", + "regulatory.clarity", "clear.regulation", "friendly.regulation", "pro.crypto", + + # Exchange & listing + "listing", "listings", "listed", "lists", "list", "new.listing", "major.listing", + "tier1.listing", "tier1.exchange", "binance.listing", "coinbase.listing", "kraken.listing", + "bybit.listing", "okx.listing", "upbit.listing", "launch", "launches", "launched", + "mainnet.launch", "testnet.launch", "token.launch", "fair.launch", "mainnet launch", "testnet launch", "token launch", "fair launch", "mainnet launch", "testnet launch", "token launch", "fair launch", "ido", "ico", "ieo", + + # Partnerships & development + "partnership", "partnerships", "partner", "partners", "collaboration", "collaborations", + "integration", "integrations", "integrated", "integrates", "ecosystem", "ecosystem.growth", + "developer.activity", "dev.activity", "github.activity", "commits", "contributors", + "grant", "grants", "funding", "funded", "investment", "investments", "vc.funding", + "strategic.investment", "ecosystem.fund", "developer.fund", "bug.bounty", + + # Profit & gains + "profit", "profits", "profitable", "win", "wins", "winning", + "success", "successful", "breakthrough", "milestone", "milestones", "target.hit", + "price.target", "target.achieved", "price target", "target achieved", "allocation", "portfolio.allocation", "portfolio allocation", "diversification", + + # Technical indicators (bullish) + "golden.cross", "golden.cross", "macd.crossover", "macd.bullish", "rsi.oversold", + "rsi.recovery", "moving.average", "ma.support", "support.held", "support.holds", + "bounce", "bounces", "bounced", "bouncing", "reversal", "reverses", "reversed", + "bottom", "bottomed", "local.bottom", "macro.bottom", "capitulation.over", "capitulation.ended", "capitulation.done", "capitulation.finished", "capitulation.complete", "capitulation.ended", "capitulation.done", "capitulation.finished", "capitulation.complete", + "oversold", "oversold.rsi", "oversold rsi", "bullish.divergence", "hidden.bullish.divergence", "bullish divergence", "hidden bullish divergence", + + # On-chain bullish + "whale.accumulation", "whale.buying", "large.holder.accumulation", "exchange.outflow", + "exchange.outflows", "balance.decreasing", "off.exchange", "cold.storage", + "staking", "staked", "staking.rewards", "validator", "validators", "delegation", "staking rewards", "staking rewards", "deposit.to.vault", "deposit.to.protocol", "deposit.to.farm", "deposit.to.earn", "depositing.to.vault", "depositing.to.protocol", "yield.deposit", "farm.deposit", "deposit.to.vault", "deposit.to.protocol", "deposit.to.farm", "deposit.to.earn", "depositing.to.vault", "depositing.to.protocol", "yield.deposit", "farm.deposit", + "hashrate.up", "hashrate.rising", "difficulty.up", "difficulty up", "difficulty rising", "difficulty increases", "difficulty adjusts", "difficulty adjusts upward", "adjusts upward", "network.growth", "addresses.growing", + "active.addresses", "new.addresses", "transaction.count", "tx.count.rising", + + # DeFi & yield + "yield", "yield.farming", "liquidity.mining", "apy", "apr", "high.yield", + "tvl.growth", "tvl.rising", "total.value.locked", "tvl.all.time.high", + "protocol.revenue", "fees.growing", "revenue.growth", "buyback", "buybacks", + "token.burn", "burns", "burning", "deflationary", "supply.decreasing", + + # Macro & narrative + "halving", "bitcoin.halving", "supply.shock", "supply.squeeze", "scarcity", + "inflation.hedge", "store.of.value", "digital.gold", "monetary.policy", + "rate.cut", "rate.cuts", "pivot", "fed.pivot", "liquidity", "m2.growth", + "global.liquidity", "dollar.weakness", "dxy.down", "risk.on", "risk.on.environment", + + # Sentiment & social + "fomo", "fear.of.missing.out", "euphoria", "optimism", "optimistic", "bullish.sentiment", + "greed", "extreme.greed", "fear.and.greed", "greed.index", "social.dominance", + "mentions.rising", "trending", "trending.on.twitter", "trending.on.reddit", + "google.trends", "search.interest", "retail.fomo", "new.entrants", "onboarding", + # Additional compound phrases + "difficulty up", "difficulty rising", "difficulty increases", "hashrate up", "hashrate rising", + "addresses growing", "active addresses growing", "addresses growing exponentially", "active addresses", "new addresses", + "profit taking", "take profit", "taking profits", "taking gains", "realizing profits", "realizing gains", "locking profits", "locking gains", "crystalize gains", "crystalize profits", + "fake breakout", "fake breakout traps", "fake breakout confirmed", "fake out breakout", "fake out", "failed breakout", "failed breakout confirmed", + # Compound phrases (space-separated for regex matching) + "break out", "breaks out", "breaking out", "break out of", "breaks out of", "breaking out of", + "breakout confirmed", "breakout traps", "break out of consolidation", "breaks out of consolidation", + "all time high", "all time highs", "hits all time high", "above all time high", "new all time high", "record high", "record highs", + "golden cross", "golden cross forms", "golden cross on daily", "golden cross on weekly", + "violent move up", "violent move", "violent up move", "violent rally", + "short liquidation", "short liquidation fuels", "short liquidation fuels squeeze", "short squeeze higher", + "short squeeze", "squeeze higher", "short cover", "short covering", + "capitulation over", "capitulation over per", "capitulation ended", "capitulation complete", + "miner surrender", "miner surrender evident", "miners surrender", "miner capitulation", + "bank run", "bank run on", "run on exchange", "run on staking", + "leverage flush", "leverage flush clears", "flush clears weak hands", "clears weak hands", + "supply squeeze", "supply squeeze as", "coins lock up", "coins locking up", + "inflation hedge", "store of value", "digital gold", "bitcoin inflation hedge", + "balance decreasing", "balance decreasing rapidly", "balance decreasing rapidly", + "exchange balance decreasing", "exchange balance decreasing", + "off exchange", "moving off exchange", "coins moving off exchange", "moving to cold storage", + "cold storage", "moving to cold storage", "whales moving to cold storage", + "whale withdrawal", "whale withdrawal from", "whale withdraws from", "whale withdrawal to cold", + "large withdrawal", "large withdrawal suggests", "withdrawal suggests accumulation", "withdrawal suggests", + "rsi recovery", "rsi recovery from", "recovery from oversold", "oversold territory", "rsi oversold at", + "rsi oversold", "rsi oversold at", "oversold on daily", "oversold on weekly", + "support held", "support held on", "key support held", "support held on retest", "held on retest", + "support holds", "supports holds", "support holds on", "holds on retest", + "moving average support", "moving average support", "ma support", "ma support", + "200 day", "200 day ma", "200-day ma", "200 day moving average", + "break out", "break out of", "breaks out of", "breaking out of", "breakout confirmed", "breakout traps", + "all time high", "all time highs", "hits all time high", "above all time high", "new all time high", + "record high", "record highs", "new record high", "all time high broken", + "risk on", "risk on environment", "risk on sentiment", "bullish sentiment", + "dollar weakness", "dollar weakening", "dxy down", "dxy falling", "dollar falling", + "rate cut", "rate cuts", "fed pivot", "fed pivoting", "pivot to easing", "easing cycle", + "liquidity expansion", "liquidity expanding", "global liquidity", "m2 growth", "m2 growing", + "fomo retail", "retail fomo", "retail fomo drives", "fomo drives parabolic", + "euphoria phase", "euphoria phase per", "optimism high", "optimism at cycle highs", + "bullish sentiment", "extreme bullish", "greed extreme", "extreme greed", + "social dominance", "social dominance rising", "dominance rising", + "mentions rising", "mentions rising on", "trending on twitter", "trending on reddit", + "google trends", "google trends up", "search interest", "search interest rising", + "retail fomo", "new entrants", "new entrants joining", "entrants joining", "onboarding surge", "onboarding surging", + + # === ASSET-SPECIFIC BULLISH KEYWORDS FOR MISSED ASSETS === + # ZIL (Zilliqa) - stronger positive signals + "zilliqa", "zil", "zilliqa sharding", "zilliqa scaling", "zilliqa partnership", "zil adoption", "zil integration", "zilliqa mainnet", "zil staking", "zil governance", + # ONG (Ontology Gas) - positive signals + "ontology", "ong", "ontology gas", "ontology partnership", "ontology mainnet", "ong staking", "ong utility", "ontology dapp", "ong defi", "ontology bridge", + # ALGO (Algorand) - stronger positive signals + "algorand", "algo", "algorand partnership", "algo foundation", "algo staking", "algorand defi", "algo governance", "algorand defi", "algo staking rewards", "algorand mainnet", + # STX (Stacks) - positive signals + "stacks", "stx", "stacks bitcoin", "stacks nakamoto", "stx bitcoin", "stacks sbtc", "stx nakamoto release", "stacks defi", "stx defi", "stacks nft", + # FET (Fetch.ai) - positive signals + "fetch.ai", "fetch ai", "fetchai", "fet", "fetch.ai agent", "fetchai agent", "asi alliance", "asi merger", "fet staking", "fetch.ai staking", "asi staking", "fetch.ai agent", "asi agent", + # ONE (Harmony) - positive signals + "harmony", "one", "harmony one", "harmony sharding", "one staking", "harmony mainnet", "harmony defi", "one defi", "harmony bridge", "one bridge", + # XTZ (Tezos) - governance/rewards = NEUTRAL not bullish (reduce false positive) + "tezos", "xtz", "tezos governance", "xtz governance", "tezos baking", "xtz baking", "tezos rewards", "xtz rewards", "tezos delegation", "xtz delegation", "tezos upgrade", "xtz upgrade", + # ENJ (Enjin) - governance/ecosystem = NEUTRAL not bullish + "enjin", "enj", "enjin coin", "enjin platform", "enj staking", "enjin nft", "enjin gaming", "enjin beam", "enjin sdk", + # DASH (Dash) - positive signals + "dash", "dash instant send", "dash masternode", "dash governance", "dash treasury", "dash instant", "dash payment", "dash chain", + # LTC (Litecoin) - positive signals + "litecoin", "ltc", "litecoin halving", "ltc halving", "ltc mimblewimble", "ltc lightning", "ltc payment", "ltc merchant", + # DOGE (Dogecoin) - positive signals + "dogecoin", "doge", "doge elon", "doge twitter", "doge payment", "doge merchant", "doge tip", "doge community", + # TRX (TRON) - positive signals + "tron", "trx", "tron network", "trx staking", "tron defi", "tron sun", "justin sun", "tron chain", "trx chain", + # XLM (Stellar) - positive signals + "stellar", "xlm", "stellar lumen", "xlm anchor", "stellar anchor", "stellar payment", "xlm payment", "stellar bridge", "xlm bridge", + # ETC (Ethereum Classic) - positive signals + "ethereum classic", "etc", "etc mining", "etc pow", "etc immutability", "etc hashrate", "etc hashrate rising", + # TRX (TRON) - more specific + "tron network", "tron foundation", "tron mainnet", "tron defi ecosystem", + # XRP (Ripple) - positive signals + "xrp", "ripple", "xrp ledger", "xrpl", "xrp settlement", "xrp bridge", "xrp payment", "xrp sec", "xrp sec win", + # ADA (Cardano) - positive signals + "cardano", "ada", "cardano hydra", "ada staking", "cardano smart contract", "cardano defi", "cardano governance", + # AVAX (Avalanche) - positive signals + "avalanche", "avax", "avax subnet", "avax subnet", "avalanche subnet", "avax staking", "avax staking rewards", + # DOT (Polkadot) - positive signals + "polkadot", "dot", "dot parachain", "dot parachain auction", "dot staking", "polkadot parachain", "polkadot governance", + # MATIC (Polygon) - positive signals + "polygon", "matic", "polygon zkevm", "polygon zkevm", "matic staking", "polygon staking", "polygon bridge", + # KSM (Kusama) - positive signals + "kusama", "ksm", "kusama parachain", "ksm staking", "kusama governance", + # ATOM (Cosmos) - positive signals + "cosmos", "atom", "cosmos hub", "atom staking", "cosmos ibc", "cosmos interchain", "atom governance", + # APT (Aptos) - positive signals + "aptos", "apt", "aptos move", "aptos move language", "aptos mainnet", "apt staking", + # SUI (Sui) - positive signals + "sui", "sui move", "sui move language", "sui mainnet", "sui staking", "sui defi", + # NEAR (NEAR Protocol) - positive signals + "near protocol", "near", "near staking", "near sharding", "near defi", "near bridge", + # ICP (Internet Computer) - positive signals + "internet computer", "icp", "icp canister", "icp subnet", "icp staking", "icp governance", + ] + + # Crypto-bearish keywords (should map to index 0 = Bearish) + # Comprehensive vocabulary covering price action, institutional, technical, on-chain, regulatory, and news catalysts + CRYPTO_BEARISH_KEYWORDS = [ + # Price action - crashes & dumps + "crash", "crashes", "crashing", "crashed", "crash.down", "flash.crash", "market.crash", "crash down", "flash crash", "market crash", "crash down", "flash crash", "market crash", + "dump", "dumps", "dumped", "dumping", "violent.dump", "capitulation", "capitulating", + "panic", "panic.sell", "panic.selling", "panic.sold", "panic sell", "panic selling", "panic sold", "fear", "fearful", "bearish", + "bear.market", "bear market", "bearish.trend", "bearish trend", "downtrend", "downtrending", "lower.high", "lower.highs", + "lower.low", "lower.lows", "red", "red.candle", "red.candles", "bleeding", "bleeds", + "free.fall", "free fall", "freefall", "nose.dive", "nose dive", "nosedive", "plunge", "plunges", "plunged", + "collapse", "collapses", "collapsed", "collapsing", "implosion", "implodes", + "wipe", "wipes", "wiped", + + # Liquidations & leverage + "liquidation", "liquidations", "liquidated", "liquidating", "mass.liquidation", + "cascade.liquidation", "liquidation.cascade", "long.liquidation", "long.liquidations", + "short.liquidation", "short.liquidations", "forced.liquidation", "margin.call", + "margin.calls", "margin calls", "leverage.flush.out", "overleveraged", "over.leverage", "margin.call", "margin call", "blown.up", "blown up", "account.blown", "account blown", "blown up by", "unexpected move", "account blown up", "blown up by", "unexpected move", "account blown up", "rekt", "getting.rekt", "getting rekt", "getting rekt on", "rekt on leverage", "getting rekt on", "rekt on leverage", + "liquidation.price", "liq.price", "rekt", "getting.rekt", "blown.up", "account.blown", + + # Hacks, exploits, security + "hack", "hacks", "hacked", "hacking", "exploit", "exploits", "exploited", "exploiting", + "rug", "rugpull", "rug.pull", "rugged", "rug.pulled", "exit.scam", "exit scam", "exit scam suspected", "scam suspected as", "team disappears", "suspected as team", "exit scam suspected", "scam suspected as", "team disappears", "suspected as team", "exit.scams", + "stolen", "theft", "thefts", "stolen.funds", "drain", "drains", "drained", "draining", + "vulnerability", "vulnerabilities", "vuln", "bug", "bugs", "breach", "breaches", + "compromise", "compromised", "unauthorized", "unauthorized.access", "private.key.leak", + "seed.phrase", "phishing", "phished", "social.engineering", "insider.attack", + "smart.contract.bug", "reentrancy", "oracle.manipulation", "price.manipulation", + "governance.attack", "governance attack", "governance attack takes", "takes control of", "control of dao", "takes control of dao", "governance attack takes", "takes control of", "control of dao", "takes control of dao", "flash.loan.attack", "flash loan attack", "flash loan attack on", "attack on vulnerable", "on vulnerable pool", "flash loan attack on", "attack on vulnerable", "on vulnerable pool", "mev.attack", "mev attack", "sandwich.attack", "sandwich attack", + + # Depeg & stablecoin + "depeg", "depegs", "depegged", "depegging", "depeg.risk", "depeg.fear", + "stablecoin.depeg", "usdc.depeg", "usdt.depeg", "dai.depeg", "frax.depeg", + "peg.broken", "peg.lost", "lost.peg", "below.peg", "trading.below.peg", + "reserve.shortfall", "reserves.insufficient", "collateral.ratio", "undercollateralized", + + # Outflows & selling (in crypto: exchange INFLOWS = bearish) + "inflow", "inflows", "net.inflow", "exchange.inflow", "exchange.inflows", + "capital.inflow", "institutional.inflow", "inflow.spike", "inflows.spike", + "balance.increasing", "on.exchange", "hot.wallet", "deposit", "deposits", + "deposit.spike", "whale.deposit", "large.deposit", "exchange.deposit", + "sell", "sells", "selling", "sold", "sell.pressure", "selling.pressure", "heavy.selling", + "distribute", "distributes", "distributed", "distributing", "distribution", + "whale.selling", "whale.distribution", "smart.money.selling", "insider.selling", + "profit.taking", "take.profit", "taking.profit", "profit.takes", "profits.taken", "realized.gains", "realize.gains", "realizing.gains", "realizing.profits", "crystalize.gains", "crystalize.profits", "locking.gains", "locking.profits", + "paper.hands", "weak.hands", "capitulation.selling", "tax.loss.harvesting", + + # Losses & downside + "loss", "losses", "lost", "losing", "unrealized.loss", "drawdown", "max.drawdown", + "down", "downward", "declining", "decline", "declines", "dropping", "drops", "dropped", + "correction", "corrections", "pullback", "pullbacks", "retrace", "retraces", "retraced", + "bear.trap", "fake.out", "fakeout", "bull.trap", "failed.breakout", "failed.rally", + + # Regulatory & legal + "ban", "bans", "banned", "banning", "prohibition", "prohibited", "illegal", + "lawsuit", "lawsuits", "sued", "sues", "sue", "litigation", "legal.action", + "enforcement", "enforcement.action", "sec.enforcement", "cftc.enforcement", + "crackdown", "crackdowns", "cracking.down", "regulatory.crackdown", + "delist", "delists", "delisted", "delisting", "exchange.delisting", "delisting.announcement", + "wells.notice", "wells.notices", "subpoena", "subpoenas", "investigation", "investigations", + "sec.investigation", "doj.investigation", "fraud.charges", "securities.fraud", + "unregistered.securities", "securities.violation", "compliance", "non.compliant", + "license.revoked", "license.denied", "cease.and.desist", "cease.desist", + + # Bankruptcy & insolvency + "bankruptcy", "bankrupt", "insolvency", "insolvent", "chapter.11", "chapter.7", + "bank.run", "bank.run.on.exchange", "run.on.exchange", "run.on.bank", + "solvency.concerns", "solvency.fears", "solvency.issues", "proof.of.reserves", + "reserves.questioned", "reserves.doubted", "fractional.reserves", "insolvent.exchange", + "withdrawal.spike", "withdrawals.spike", "mass.withdrawal", "mass.withdrawals", + "users.fleeing", "users.flee", "exodus", "user.exodus", "platform.exodus", + "bank.run", "bank.run.on.exchange", "run.on.exchange", "run.on.bank", + "solvency.concerns", "solvency.fears", "solvency.issues", "proof.of.reserves", + "reserves.questioned", "reserves.doubted", "fractional.reserves", "insolvent.exchange", + "withdrawal.spike", "withdrawals.spike", "mass.withdrawal", "mass.withdrawals", + "users.fleeing", "users.flee", "exodus", "user.exodus", "platform.exodus", + "liquidation", "wind.down", "winding.down", "cease.operations", "shut.down", + "creditors", "creditor", "claims", "recovery", "recovery.rate", "haircut", + "ftx", "celcius", "voyager", "blockfi", "genesis", "three.arrows", "3ac", + "terra", "luna", "ust", "algorithmic.stablecoin", "ponzi", "pyramid.scheme", + + # Technical indicators (bearish) + "death.cross", "death.cross", "macd.bearish", "macd.crossunder", "rsi.overbought", + "rsi.rejection", "moving.average.resistance", "resistance.held", "resistance.holds", + "rejection", "rejects", "rejected", "rejecting", "failed.breakout", "fake.breakout", + "top", "topped", "local.top", "macro.top", "distribution.phase", "wyckoff.distribution", + "overbought", "overbought.rsi", "bearish.divergence", "hidden.bearish.divergence", + "head.and.shoulders", "double.top", "triple.top", "rising.wedge", "bearish.pattern", + + # On-chain bearish + "whale.selling", "whale.sells", "whale.dump", "whale.dumps", "large.holder.selling", + "exchange.inflow", "exchange.inflows", "balance.increasing", "on.exchange", + "hot.wallet", "exchange.deposit", "deposit.spike", "whale.deposit", "large.deposit", + "unstaking", "unstaked", "unstake", "unstakes", "withdrawal", "withdrawals", + "mass.withdrawal", "bank.run", "run.on.exchange", "solvency.concerns", "proof.of.reserves", + "hashrate.down", "hashrate.dropping", "difficulty.down", "miners.capitulation", "miner.capitulation", "miner.surrender", "miners.surrender", "miner.capitulation", "miner.surrender", "miners.surrender", + "miner.surrender", "miner.outflow", "miner.outflows", "miner.selling", "miner.sells", "miner.distributing", "hash.ribbons", + + # DeFi & protocol issues + "tvl.drop", "tvl.dropping", "tvl.crash", "total.value.locked.drop", "tvl.outflow", + "protocol.exploit", "protocol.hack", "smart.contract.exploit", "oracle.failure", + "short.liquidation", "short.squeeze", "short.cover", "squeeze.higher", "short.covering", "short.liquidation", "short.squeeze", "short.cover", "squeeze.higher", "short.covering", "liquidation.crisis", "bad.debt", "insolvent.protocol", "undercollateralized.vault", + "liquidation.bot", "mev.bot", "sandwich.attack", "front.run", "front.running", + "inflationary", "supply.increasing", "token.inflation", "emission", "emissions.high", + "unlock", "unlocks", "token.unlock", "vesting.unlock", "team.unlock", "investor.unlock", + "cliff.unlock", "linear.unlock", "supply.shock", "dilution", "dilutive", + + # Macro & risk-off + "rate.hike", "rate.hikes", "fed.hawkish", "hawkish.fed", "tightening", "qt", + "quantitative.tightening", "liquidity.drain", "liquidity.crunch", "credit.crunch", + "recession", "recession.fears", "recession.risk", "economic.slowdown", "gdp.slowing", + "inflation.high", "cpi.high", "ppi.high", "sticky.inflation", "core.inflation", + "dollar.strength", "dxy.up", "dxy.rising", "risk.off", "risk.off.environment", + "flight.to.safety", "safe.haven", "treasury.yields", "yields.rising", "bond.yields", + "correlation", "correlated.selloff", "broad.selloff", "market.wide.selloff", + "contagion", "contagion.risk", "systemic.risk", "domino.effect", "spillover", + + # Sentiment & social + "fud", "fear.uncertainty.doubt", "fear", "extreme.fear", "fear.and.greed", + "fear.index", "capitulation", "despair", "depression", "anger", "rage", + "selling.panic", "blind.panic", "herd.mentality", "herd.selling", "lemmings", + "social.dominance.drop", "mentions.falling", "interest.fading", "google.trends.down", + "search.interest.declining", "retail.exodus", "users.leaving", "offboarding", + "narrative.broken", "narrative.shift", "thesis.broken", "thesis.invalidated", + "disappointment", "missed.expectations", "guidance.lowered", "warn", "warning", + # Compound phrases (space-separated for regex matching) + "wipes out", "wiped out", "wiping out", "wipes out gains", "wiped out gains", "wiping out gains", + "peg broken", "peg lost", "lost peg", "below peg", "trading below peg", "peg breaks", "peg has broken", + "reserve shortfall", "reserves insufficient", "reserve shortfall confirmed", "shortfall confirmed by audit", + "algorithmic stablecoin", "algorithmic stablecoin loses", "stablecoin loses peg", "loses peg entirely", + "institutional inflow", "inflow to exchange", "inflow to exchange wallets", "to exchange wallets", + "exchange balance increasing", "balance increasing rapidly", "balance increasing", "exchange balance", + "supply on exchange", "on exchange supply", "supply on exchange at", "on exchange at", "yearly highs", "6 month highs", + "deposit spike", "deposit spike suggests", "spike suggests profit", "suggests profit taking", "profit taking", + "take profit", "take profit at", "profit at resistance", "at resistance", "traders take profit", "traders take", + "realized gains", "realizing gains", "holders realizing", "long term holders", "holders realizing gains", + "paper hands", "paper hands shaken", "hands shaken out", "shaken out at", "out at support", "weak hands shaken", + "selling pressure", "heavy selling pressure", "selling pressure evident", "pressure evident", + "institutional selling pressure", "evident selling", "evident selling pressure", + "smart money selling", "smart money selling the", "money selling the", "selling the rally", "the rally", + "smart money sells", "smart money distributes", "smart money distributes", + "institutional selling", "institutional sells", "institutional distributes", "institutional exits", + "institutional selling pressure", "selling pressure evident", + "mev bot sandwich", "mev bot sandwich attacks", "sandwich attack", "sandwich attacks", + "flash loan attack", "flash loan attack on", "attack on vulnerable", "on vulnerable pool", + "governance attack", "governance attack takes", "takes control of", "control of dao", "takes control of dao", + "margin call", "margin calls", "margin calls forcing", "forcing deleveraging", "deleveraging across board", + "blown up", "blown up by", "unexpected move", "account blown up", "account blown", + "rekt", "getting rekt", "getting rekt on", "rekt on leverage", + "exit scam", "exit scam suspected", "scam suspected as", "team disappears", "suspected as team", + "wells notice", "wells notice received", "notice received by", "received by crypto", "by crypto firm", + "fraud charges", "fraud charges filed", "charges filed against", "filed against founder", "against founder", + "unregistered securities", "tokens deemed unregistered", "deemed unregistered securities", + "cease and desist", "cease and desist order", "desist order issued", "order issued", + "bankruptcy filing", "crypto lender files", "lender files for", "files for chapter", "chapter 11", "filing for chapter", + "cease operations", "cease operations immediately", "operations immediately", "exchange ceases", "ceases operations", + "death cross", "death cross confirmed", "cross confirmed on", "confirmed on weekly", "on weekly", + "rsi rejection", "rsi rejection at", "rejection at overbought", "at overbought levels", + "resistance held", "resistance held for", "held for third", "for third time", "third time", + "fake breakout", "fake breakout traps", "breakout traps", "traps breakout traders", "breakout traders", + "failed rally", "failed rally confirms", "rally confirms distribution", "confirms distribution", + "head and shoulders", "head and shoulders pattern", "and shoulders pattern", "pattern completes", "shoulders pattern completes", + "double top", "double top confirmed", "top confirmed", "confirmed on break", "break of neckline", + "rising wedge", "rising wedge breakdown", "wedge breakdown", "breakdown imminent", "imminent breakdown", + "bank run", "bank run on", "run on staking", "run on exchange", + "solvency concerns", "solvency concerns for", "concerns for lending", "for lending protocol", "lending protocol", + "proof of reserves", "proof of reserves audit", "audit incomplete", "incomplete audit", "reserves audit", + "miner surrender", "miner surrender evident", "surrender evident in", "evident in outflows", + "hash ribbons", "hash ribbons signal", "ribbons signal miner", "signal miner distress", "miner distress", + "oracle failure", "oracle failure causes", "failure causes bad", "causes bad debt", "bad debt", + "bad debt", "protocol accumulates", "accumulates bad debt", "accumulates bad", + "emissions high", "emissions schedule", "schedule too aggressive", "too aggressive", "high emissions", + + # === ASSET-SPECIFIC BEARISH KEYWORDS FOR FALSE POSITIVES === + # XTZ (Tezos) - governance/rewards news often misread as bullish, but not price bullish + "tezos governance", "xtz governance", "tezos proposal", "xtz proposal", "tezos vote", "xtz vote", + "tezos baking", "xtz baking", "tezos baker", "xtz baker", "tezos rewards", "xtz rewards", + "tezos delegation", "xtz delegation", "tezos rewards distribution", "xtz rewards distribution", + "tezos inflation", "xtz inflation", "tezos staking rewards", "xtz staking rewards", + "tezos protocol upgrade", "xtz protocol upgrade", "tezos proposal approved", "xtz proposal approved", + + # ENJ (Enjin) - gaming/ecosystem news often misread as bullish + "enjin governance", "enj governance", "enjin proposal", "enj proposal", "enjin vote", "enj vote", + "enjin sdk", "enjin beam", "enjin nft", "enjin gaming", "enjin platform", + + # ZIL (Zilliqa) - competition, bridge issues, low volume + "zilliqa competition", "zil competition", "zilliqa bridge", "zil bridge", "zilliqa bridge hack", + "zil bridge hack", "zilliqa bridge exploit", "zil bridge exploit", "zilliqa low volume", + "zil low volume", "zilliqa centralization", "zil centralization", "zilliqa validator", + + # ENJ (Enjin) - governance issues + "enjin governance attack", "enj governance attack", "enjin governance exploit", + + # XTZ (Tezos) - governance attacks + "tezos governance attack", "xtz governance attack", "tezos proposal attack", + + # FET (Fetch.ai) - merger/ASI uncertainty + "fetch.ai merger", "fetch ai merger", "asi merger", "asi merger uncertainty", "asi token merger", + "fetch.ai token swap", "fet token swap", "asi token swap", "ocean protocol merger", "ocean fet merger", + "asi alliance uncertainty", "asi alliance risk", + + # ALGO (Algorand) - centralization concerns + "algorand centralization", "algo centralization", "algorand foundation control", + "algorand relay node", "algorand relay node centralization", + + # STX (Stacks) - bitcoin dependency risk + "stacks bitcoin dependency", "stx bitcoin dependency", "stacks btc anchor", "stx btc anchor", + + # DASH (Dash) - masternode centralization + "dash masternode centralization", "dash masternode control", "dash governance centralization", + + # LTC (Litecoin) - declining relevance + "litecoin declining", "ltc declining", "litecoin abandoned", "ltc abandoned", + "litecoin hashrate drop", "ltc hashrate drop", "litecoin miner exodus", + + # DOGE (Dogecoin) - meme volatility, elon dependency + "dogecoin meme", "doge meme", "doge elon dependency", "doge elon tweet", + "dogecoin pump dump", "doge pump dump", "dogecoin speculation", + + # TRX (TRON) - centralization, justin sun risk + "tron centralization", "trx centralization", "justin sun risk", "justin sun exit", + "tron sun network", "tron sun", "tron acquisition risk", + + # XLM (Stellar) - anchor dependency, inflation + "stellar inflation", "xlm inflation", "stellar anchor failure", "xlm anchor failure", + "stellar anchor down", "xlm anchor down", + + # ETC (Ethereum Classic) - 51% attack risk, hashrate drop + "ethereum classic 51 attack", "etc 51 attack", "etc hashrate drop", "etc hashrate declining", + "ethereum classic hashrate drop", "etc miner exodus", + + # XRP (Ripple) - SEC risk + "xrp sec", "xrp sec lawsuit", "xrp sec appeal", "ripple sec", "ripple lawsuit", + "xrp security", "xrp security risk", "xrp delisting risk", + + # ADA (Cardano) - slow development + "cardano slow", "ada slow development", "cardano delayed", "ada delayed", + "cardano hydra delayed", "cardano hydra delay", + + # XTZ (Tezos) - governance = NEUTRAL not bearish, but reduce false positive + "tezos governance", "xtz governance", "tezos proposal", "xtz proposal", + + # ENJ (Enjin) - governance = NEUTRAL not bearish, but reduce false positive + "enjin governance", "enj governance", "enjin proposal", "enj proposal", + + # XTZ (Tezos) - compound phrases + "tezos governance proposal", "xtz governance proposal", "tezos baking rewards", + "xtz baking rewards", "tezos staking rewards", "xtz staking rewards", + "tezos delegation rewards", "xtz delegation rewards", + + # ENJ (Enjin) - compound phrases + "enjin governance proposal", "enj governance proposal", "enjin nft gaming", + "enjin beam airdrop", "enjin beam qr", + ] + + # Whale action phrases - context-dependent sentiment (bullish) + WHALE_BULLISH_PHRASES = [ + "whale buys", "whale buys", "whale accumulates", "whale accumulates", "whale accumulating", + "whale loads", "whale loading", "whale adds", "whale buying", "whale accumulation", + "whale entry", "whale enters", "whale positions", "whale positions", "whale positioned", + "whale accumulation", "whale buying", "whale buys more", "whale adds more", + "whale buy", "whale buy", "whale bought", "whale has bought", "whale purchased", + "whale purchases", "whale acquiring", "whale acquires", "whale acquired", + "whale stacking", "whale stacks", "whale stacked", "whale building", "whale builds", + "whale building.position", "whale building.position", "whale increasing.position", + "whale adds.to.position", "whale adding.to.position", "whale scaled.in", "whale scaling.in", + "smart.money.accumulating", "smart.money.buys", "smart.money.accumulates", + "institutional.accumulation", "institutional.buying", "institutional.accumulates", + "large.holder.accumulating", "large.holder.buys", "large.holder.accumulates", + "large.holder.accumulation", "large.holder.buying", "large.holder.accumulates", + "large.holders.accumulating", "large.holders.buys", "large.holders.accumulate", + "smart.money.accumulation", "smart.money.buying", "institutional.buys", + "whale.absorbing", "whale.absorbs", "whale.absorbed", "absorption", "absorbing.supply", + "supply.absorption", "whale.absorbing.supply", "bid.absorption", "strong.bids", + "whale.bid", "whale.bids", "large.bid", "large.bids", "buy.wall", "buy.walls", + "massive.buy", "massive.buys", "huge.buy", "huge.buys", "large.buy", "large.buys", + ] + + WHALE_BEARISH_PHRASES = [ + "whale sells", "whale sells", "whale dumps", "whale dumping", "whale dumps", + "whale distributes", "whale distributing", "whale distributes", "whale exits", + "whale exits", "whale exits", "whale liquidates", "whale liquidating", + "whale takes profit", "whale takes profit", "whale profit taking", "whale takes profit", + "whale exits", "whale exits", "whale distributes", "whale distributing", + "whale sells", "whale sells", "whale sells", "whale unloads", "whale unloads", + "whale sell", "whale sell", "whale sold", "whale has sold", "whale selling", + "whale offloading", "whale offloads", "whale offloaded", "whale unloading", + "whale distributes", "whale distributing", "whale distribution", "whale.distributing", + "whale taking.profit", "whale.take.profit", "whale.profit.taking", "whale.realized", + "whale.realizing", "whale.realizes", "whale.realize", "profit.realized", + "whale.exits", "whale.exiting", "whale.exited", "whale.exit", "whale.exits.market", + "whale.reducing", "whale.reduces", "whale.reduced", "whale.trimmed", "whale.trims", + "whale.trimming", "whale.scaling.out", "whale.scales.out", "whale.scaled.out", + "smart.money.selling", "smart.money.sells", "smart.money.distributes", + "institutional.selling", "institutional.distributes", "institutional.exits", + "large.holder.selling", "large.holder.distributes", "large.holder.exits", + "large.holder.sells", "large.holder.selling", "large.holder.distributes", + "large.holders.selling", "large.holders.sells", "large.holders.distribute", + "smart.money.sells", "smart.money.distributing", "institutional.sells", + "whale.unloading", "whale.unload", "whale.unloaded", "supply.unloading", + "supply.flooding", "whale.flooding", "massive.sell", "massive.sells", "huge.sell", + "huge.sells", "large.sell", "large.sells", "sell.wall", "sell.walls", "massive.sell.wall", + "whale.ask", "whale.asks", "large.ask", "large.asks", "heavy.asks", "ask.side.heavy", + "whale.market.sell", "whale.market.sells", "market.sell", "market.sells", + "aggressive.selling", "aggressive.seller", "aggressive.sells", "market.order.sell", + # Compound bearish phrases with high priority + "profit taking", "take profit", "taking profits", "taking gains", "realizing profits", "realizing gains", + "locking profits", "locking gains", "crystalize gains", "crystalize profits", "profit taking after rally", "profit taking after pump", + "profit taking at top", "profit taking at highs", "profit taking at resistance", "profits taken", "gains taken", + "fake breakout", "fake breakout traps", "fake breakout confirmed", "fake out breakout", "fake out", + "failed breakout", "failed breakout confirmed", "failed breakout traps", "fakeout breakout", + "profit taking after 3x rally", "profit taking after pump", "profit taking at top", "profit taking at highs", + "profit taking at resistance", "take profit at resistance", "traders take profit", "traders taking profit", + ] + # Unified weighted sentiment lexicon (loaded from JSON) + _LEXICON: Dict[str, int] = {} + _LEXICON_LOADED = False + + @classmethod + def _load_lexicon(cls) -> None: + """Load the unified weighted lexicon from JSON file.""" + if cls._LEXICON_LOADED: + return + import json + from pathlib import Path + lexicon_path = Path("lexicon_weights.json") + if lexicon_path.exists(): + with open(lexicon_path) as f: + cls._LEXICON = json.load(f) + else: + # Fallback: build from class constants (legacy) + cls._build_legacy_lexicon() + cls._LEXICON_LOADED = True + + @classmethod + def _build_legacy_lexicon(cls) -> None: + """Build lexicon from legacy keyword lists (fallback).""" + cls._LEXICON = {} + # Bullish keywords + for kw in cls.CRYPTO_BULLISH_KEYWORDS: + canonical = kw.replace('.', ' ') + if ' ' in kw and '.' not in kw: + cls._LEXICON[canonical] = 30 + elif '.' in kw: + cls._LEXICON[canonical] = 20 + else: + cls._LEXICON[canonical] = 10 + # Bearish keywords + for kw in cls.CRYPTO_BEARISH_KEYWORDS: + canonical = kw.replace('.', ' ') + if ' ' in kw and '.' not in kw: + cls._LEXICON[canonical] = -30 + elif '.' in kw: + cls._LEXICON[canonical] = -20 + else: + cls._LEXICON[canonical] = -10 + # Whale phrases + for kw in cls.WHALE_BULLISH_PHRASES: + canonical = kw.replace('.', ' ') + cls._LEXICON[canonical] = 50 + for kw in cls.WHALE_BEARISH_PHRASES: + canonical = kw.replace('.', ' ') + cls._LEXICON[canonical] = -50 + + @classmethod + def _get_lexicon_score(cls, text: str) -> float: + """ + Compute weighted sentiment score from lexicon. + Returns a score in range [-100, 100] representing net sentiment. + Uses span-based matching with priority: longest matches first. + """ + cls._load_lexicon() + text_lower = text.lower() + + # Sort lexicon terms by length (longest first) for priority matching + sorted_terms = sorted(cls._LEXICON.items(), key=lambda x: -len(x[0])) + + matched_spans = [] + total_score = 0.0 + + for term, weight in sorted_terms: + # Skip zero-weight terms + if weight == 0: + continue + # Find all non-overlapping matches + for match in re.finditer(r'\b' + re.escape(term) + r'\b', text_lower): + span = (match.start(), match.end()) + # Check overlap + if not any(s[0] < span[1] and s[1] > span[0] for s in matched_spans): + matched_spans.append(span) + total_score += weight + + # Clamp to [-100, 100] + return max(-100.0, min(100.0, total_score)) + + @classmethod + def _lexicon_to_signal(cls, score: float) -> str: + """Convert lexicon score to signal direction.""" + if score > 10: + return "bullish" + elif score < -10: + return "bearish" + return "neutral" + + + + @classmethod + def _get_crypto_signal(cls, text: str) -> str: + """ + Determine crypto sentiment direction using unified weighted lexicon. + + The lexicon assigns each term a weight from -100 (extreme bearish) to +100 (extreme bullish). + Compound/whale terms have higher absolute weights. Matching uses longest-first priority + with span-based deduplication. + + This provides the EXPLICIT signal layer. The centroid/cosine layer in ScoringEngine + provides the SEMANTIC refinement layer. + """ + cls._load_lexicon() + score = cls._get_lexicon_score(text) + return cls._lexicon_to_signal(score) + + @classmethod + def get_lexicon_score(cls, text: str) -> float: + """Public method to get raw lexicon score for integration with centroid layer.""" + cls._load_lexicon() + return cls._get_lexicon_score(text) + + @classmethod + def _get_finbert_signal(cls, probs: np.ndarray) -> str: + """Determine FinBERT's predicted direction""" + # probs = [negative, neutral, positive] = [Bearish, Neutral, Bullish] + diff = probs[2] - probs[0] # positive - negative + if diff > 0.05: # clearly positive (Bullish) - lowered threshold from 0.15 + return "bullish" + elif diff < -0.05: # clearly negative (Bearish) + return "bearish" + return "neutral" + + @classmethod + def calibrate(cls, text: str, probs: np.ndarray) -> np.ndarray: + """ + Calibrate FinBERT probabilities using the weighted lexicon score. + + PRINCIPLE: Crypto-specific explicit lexicon WINS over general FinBERT + when they disagree. FinBERT is trained on traditional finance where + "surge/rally/pump" = risky/bubble = negative. In crypto, these are bullish. + + The lexicon provides a continuous score [-100, 100] representing + explicit keyword evidence with domain-specific weights. + + Blending logic: + - If lexicon and FinBERT AGREE on direction: amplify (trust both) + - If lexicon and FinBERT DISAGREE: lexicon wins (crypto-specific > general) + - If lexicon is NEUTRAL (|score| <= 10): trust FinBERT + - If FinBERT is NEUTRAL (|pos-neg| < 0.15): trust lexicon + + The centroid/cosine layer in ScoringEngine provides the SEMANTIC + refinement on top of this calibrated output. + """ + cls._load_lexicon() + + # Get continuous lexicon score [-100, 100] + lexicon_score = cls._get_lexicon_score(text) + + # FinBERT probabilities: [negative, neutral, positive] = [Bearish, Neutral, Bullish] + neg, neu, pos = probs[0], probs[1], probs[2] + finbert_polarity = pos - neg # [-1, 1] + finbert_confidence = max(probs) + + # Lexicon polarity: -1 (bearish) to +1 (bullish) + lexicon_polarity = np.clip(lexicon_score / 100.0, -1.0, 1.0) + lexicon_confidence = min(abs(lexicon_score) / 50.0, 1.0) # Full confidence at |50| + + # Determine agreement + finbert_dir = "bullish" if finbert_polarity > 0.1 else "bearish" if finbert_polarity < -0.1 else "neutral" + lexicon_dir = "bullish" if lexicon_polarity > 0.1 else "bearish" if lexicon_polarity < -0.1 else "neutral" + + # Case 1: Both agree on direction -> AMPLIFY + if finbert_dir == lexicon_dir and finbert_dir != "neutral": + # Weighted average favoring the stronger signal + total_conf = finbert_confidence + lexicon_confidence + if total_conf > 0: + w_finbert = finbert_confidence / total_conf + w_lexicon = lexicon_confidence / total_conf + else: + w_finbert = w_lexicon = 0.5 + blended_polarity = w_finbert * finbert_polarity + w_lexicon * lexicon_polarity + # Amplify slightly beyond both + blended_polarity = np.clip(blended_polarity * 1.2, -1.0, 1.0) + target_neu = neu * 0.7 # Reduce neutral when both agree + + # Case 2: Disagree -> LEXICON WINS (crypto-specific > general finance) + elif finbert_dir != "neutral" and lexicon_dir != "neutral" and finbert_dir != lexicon_dir: + # Lexicon dominates with high weight + blended_polarity = 0.85 * lexicon_polarity + 0.15 * finbert_polarity + target_neu = 0.15 # Low neutral when strong disagreement resolved + + # Case 3: Lexicon neutral -> trust FinBERT + elif lexicon_dir == "neutral": + blended_polarity = finbert_polarity + target_neu = neu + + # Case 4: FinBERT neutral -> trust lexicon + elif finbert_dir == "neutral": + blended_polarity = lexicon_polarity + target_neu = neu * (1 - lexicon_confidence * 0.5) + + # Case 5: One neutral, other directional + else: + blended_polarity = lexicon_polarity if lexicon_dir != "neutral" else finbert_polarity + target_neu = neu * 0.8 + + # Clamp polarity + blended_polarity = np.clip(blended_polarity, -0.98, 0.98) + target_neu = np.clip(target_neu, 0.05, 0.9) + + # Convert polarity back to probabilities + # pos - neg = blended_polarity + # pos + neg = 1 - target_neu + pos_prob = (blended_polarity + 1 - target_neu) / 2 + neg_prob = (1 - target_neu - blended_polarity) / 2 + + # Clamp + pos_prob = max(0.02, min(0.98, pos_prob)) + neg_prob = max(0.02, min(0.98, neg_prob)) + neu_prob = max(0.05, min(0.95, 1 - pos_prob - neg_prob)) + + # Renormalize + total = pos_prob + neg_prob + neu_prob + calibrated = np.array([neg_prob / total, neu_prob / total, pos_prob / total]) + + return calibrated + + +# ... rest of the file (all other classes remain the same) + + +class SentimentEmotionAnalyzer: + """Main sentiment and emotion analyzer orchestrating all NLP components""" + + def __init__(self): + self.settings = get_settings() + self._tokenizer = None + self._model = None + self._emotion_model = None + self._emotion_tokenizer = None + self._device = "cuda" if (TRANSFORMERS_AVAILABLE and torch.cuda.is_available()) else "cpu" + self._use_onnx = False + self._use_mock = True + self._labels = ["negative", "neutral", "positive"] + self._emotion_labels = ["joy", "fear", "anger", "greed", "sadness", "neutral"] + + async def initialize(self) -> None: + """Load models - priority: ONNX > PyTorch > Mock (with timeout handling)""" + settings = self.settings + + # Check for ONNX models first + onnx_finbert = Path("models/onnx/finbert/model.onnx") + onnx_emotion = Path("models/onnx/distilroberta-emotion/model.onnx") + + if ONNX_AVAILABLE and onnx_finbert.exists(): + print(f"DEBUG: Loading ONNX FinBERT from {onnx_finbert} ({onnx_finbert.stat().st_size / 1024 / 1024:.1f} MB)...") + import time + load_start = time.time() + try: + # Load tokenizer first (fast) + self._tokenizer = AutoTokenizer.from_pretrained("models/onnx/finbert") if TRANSFORMERS_AVAILABLE else MockTokenizer() + print(f"DEBUG: Tokenizer loaded in {time.time() - load_start:.1f}s") + + # Load ONNX model with timeout warning + model_start = time.time() + self._model = ONNXSentimentModel( + str(onnx_finbert), + "models/onnx/finbert", + "models/onnx/finbert/label_map.json" + ) + print(f"DEBUG: ONNX FinBERT loaded in {time.time() - model_start:.1f}s (total: {time.time() - load_start:.1f}s)") + + self._use_onnx = True + self._use_mock = False + logger.info("Loaded FinBERT via ONNX Runtime") + except Exception as e: + logger.warning(f"Failed to load ONNX FinBERT: {e}") + + # Skip emotion model to avoid memory contention with other models + # if ONNX_AVAILABLE and onnx_emotion.exists(): + # try: + # self._emotion_tokenizer = AutoTokenizer.from_pretrained("models/onnx/distilroberta-emotion") if TRANSFORMERS_AVAILABLE else MockTokenizer() + # self._emotion_model = ONNXEmotionModel( + # str(onnx_emotion), + # "models/onnx/distilroberta-emotion", + # "models/onnx/distilroberta-emotion/label_map.json" + # ) + # logger.info("Loaded DistilRoBERTa Emotion model via ONNX Runtime") + # except Exception as e: + # logger.warning(f"Failed to load ONNX Emotion model: {e}") + + # Fallback to PyTorch models + if self._use_mock and TRANSFORMERS_AVAILABLE: + print("DEBUG: Loading PyTorch FinBERT (fallback)...") + try: + self._tokenizer = AutoTokenizer.from_pretrained("ProsusAI/finbert") + self._model = AutoModelForSequenceClassification.from_pretrained("ProsusAI/finbert") + self._model.to(self._device) + self._model.eval() + + self._use_mock = False + self._use_onnx = False + logger.info(f"Loaded FinBERT via PyTorch on {self._device}") + except Exception as e: + logger.warning(f"Failed to load PyTorch FinBERT: {e}") + + # Final fallback to mock + if self._use_mock: + self._tokenizer = MockTokenizer() + self._model = MockSentimentModel() + self._emotion_model = None + self._emotion_tokenizer = None + logger.info("Using mock sentiment/emotion models") + + def _extract_context(self, text: str, span: Tuple[int, int], window: int = 200) -> str: + start, end = span + ctx_start = max(0, start - window) + ctx_end = min(len(text), end + window) + return text[ctx_start:ctx_end] + + async def analyze( + self, + text: str, + asset_mentions: List[Dict] + ) -> Tuple[Dict[str, SentimentScores], Dict[str, EmotionScores]]: + """Analyze sentiment and emotion for each asset mention""" + sentiment_results = {} + emotion_results = {} + + for mention in asset_mentions: + asset_id = mention.get("asset_id") + span = mention.get("span", (0, 0)) + + context = self._extract_context(text, span) + + sentiment = await self._analyze_sentiment(context) + emotions = await self._analyze_emotions(context) + + sentiment_results[asset_id] = sentiment + emotion_results[asset_id] = emotions + + return sentiment_results, emotion_results + + async def _analyze_sentiment(self, text: str) -> SentimentScores: + """Run sentiment classification""" + if self._use_mock or not self._model: + return self._heuristic_sentiment(text) + + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._run_sentiment, text) + + def _run_sentiment(self, text: str) -> SentimentScores: + """Synchronous sentiment inference""" + if self._use_onnx: + return self._run_sentiment_onnx(text) + else: + return self._run_sentiment_pytorch(text) + + def _run_sentiment_onnx(self, text: str) -> SentimentScores: + """Run ONNX sentiment inference with crypto calibration""" + inputs = self._tokenizer( + text, + return_tensors="np", + truncation=True, + max_length=512, + padding=True + ) + + token_type_ids = inputs.get("token_type_ids") + logits = self._model(inputs["input_ids"], inputs["attention_mask"], token_type_ids) + probs = self._softmax(logits)[0] + + # Apply crypto sentiment calibration + probs = CryptoSentimentCalibrator.calibrate(text, probs) + + neg, neu, pos = probs[0], probs[1], probs[2] + polarity = pos - neg + + return SentimentScores( + polarity=float(polarity), + confidence=float(max(probs)), + positive_prob=float(pos), + negative_prob=float(neg), + neutral_prob=float(neu) + ) + + def _run_sentiment_pytorch(self, text: str) -> SentimentScores: + """Run PyTorch sentiment inference""" + inputs = self._tokenizer( + text, + return_tensors="pt", + truncation=True, + max_length=512, + padding=True + ).to(self._device) + + with torch.no_grad(): + outputs = self._model(**inputs) + probs = torch.softmax(outputs.logits, dim=-1).cpu().numpy()[0] + + neg, neu, pos = probs[0], probs[1], probs[2] + polarity = pos - neg + + return SentimentScores( + polarity=float(polarity), + confidence=float(max(probs)), + positive_prob=float(pos), + negative_prob=float(neg), + neutral_prob=float(neu) + ) + + def _softmax(self, x: np.ndarray) -> np.ndarray: + """Numerically stable softmax""" + e_x = np.exp(x - np.max(x, axis=-1, keepdims=True)) + return e_x / e_x.sum(axis=-1, keepdims=True) + + def _heuristic_sentiment(self, text: str) -> SentimentScores: + """Fallback heuristic sentiment analysis""" + text_lower = text.lower() + + positive_words = ["rally", "surge", "pump", "moon", "bullish", "profit", "gain", "win", "success", "breakthrough"] + negative_words = ["crash", "dump", "panic", "fear", "scared", "worried", "risk", "danger", "collapse", "liquidation"] + + pos_count = sum(1 for kw in positive_words if kw in text_lower) + neg_count = sum(1 for kw in negative_words if kw in text_lower) + + polarity = (pos_count - neg_count) * 0.3 + polarity = max(-1.0, min(1.0, polarity)) + confidence = min(0.9, 0.3 + abs(polarity) * 0.5) + + return SentimentScores( + polarity=float(polarity), + confidence=confidence, + positive_prob=float(max(0, polarity)), + negative_prob=float(max(0, -polarity)), + neutral_prob=float(1 - abs(polarity)) + ) + + async def _analyze_emotions(self, text: str) -> EmotionScores: + """Run emotion classification""" + if self._use_mock or not self._emotion_model: + return self._heuristic_emotions(text) + + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, self._run_emotions, text) + + def _run_emotions(self, text: str) -> EmotionScores: + """Run emotion inference (ONNX or PyTorch)""" + if self._use_onnx and self._emotion_model: + return self._run_emotions_onnx(text) + else: + return self._heuristic_emotions(text) + + def _run_emotions_onnx(self, text: str) -> EmotionScores: + """Run ONNX emotion inference""" + inputs = self._emotion_tokenizer( + text, + return_tensors="np", + truncation=True, + max_length=512, + padding=True + ) + + logits = self._emotion_model(inputs["input_ids"], inputs["attention_mask"], None) + probs = self._softmax(logits)[0] + + # Label map: {"0": "joy", "1": "fear", "2": "anger", "3": "greed", "4": "sadness", "5": "neutral"} + return EmotionScores( + joy=float(probs[0]), + fear=float(probs[1]), + anger=float(probs[2]), + greed=float(probs[3]), + sadness=float(probs[4]), + intensity=float(max(probs)) + ) + + def _heuristic_emotions(self, text: str) -> EmotionScores: + """Heuristic emotion detection from keywords""" + text_lower = text.lower() + + emotion_keywords = { + "joy": ["rally", "surge", "pump", "moon", "bullish", "profit", "gain", "win", "success", "breakthrough"], + "fear": ["crash", "dump", "panic", "fear", "scared", "worried", "risk", "danger", "collapse", "liquidation"], + "anger": ["scam", "fraud", "hack", "exploit", "rug", "manipulation", "unfair", "angry", "furious"], + "greed": ["buy", "accumulate", "load", "bag", "greed", "fomo", "yolo", "ape", "lever", "margin"], + "sadness": ["loss", "lost", "down", "bear", "rekt", "pain", "hurt", "disappointed", "sad"], + } + + scores = {} + for emotion, keywords in emotion_keywords.items(): + count = sum(1 for kw in keywords if kw in text_lower) + scores[emotion] = min(1.0, count * 0.2) + + intensity = min(1.0, ( + text.count("!") * 0.1 + + sum(1 for w in text.split() if w.isupper() and len(w) > 2) * 0.05 + + text.lower().count("very") * 0.05 + + text.lower().count("extreme") * 0.1 + )) + + return EmotionScores( + joy=scores.get("joy", 0.0), + fear=scores.get("fear", 0.0), + anger=scores.get("anger", 0.0), + greed=scores.get("greed", 0.0), + sadness=scores.get("sadness", 0.0), + intensity=intensity + ) + + def compute_intensity(self, text: str) -> float: + """Compute overall emotional intensity""" + return self._heuristic_emotions(text).intensity