Add sentiment_engine with CryptoSentimentCalibrator fixes - improved keyword lists, lowered FinBERT threshold, added neutral handling

sentiment_engine/scripts/build_centroids.py

@@ -0,0 +1,237 @@
+#!/usr/bin/env python3
+"""Build parameter centroids from keyword lists using sentence-transformers"""
+
+import asyncio
+import hashlib
+import numpy as np
+from pathlib import Path
+import sys
+
+sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
+
+from sentiment_engine.scoring.centroids import CentroidManager
+from sentiment_engine.utils.text import normalize_text
+
+# Keyword lists from SENTIMENT_SPEC_IMPLEMENT_GUIDE.md
+PARAMETER_KEYWORDS = {
+    "fear_state": [
+        "fear", "fearful", "frightened", "scared", "terrified", "petrified", "panicked",
+        "panic", "terror", "dread", "dreadful", "anxiety", "anxious", "worry", "worried",
+        "horror", "horrific", "anguish", "panic-sell", "panic-buying", "phobia", "alarm",
+        "alarming", "alarmed", "consternation", "dismay", "apprehension", "trepidation",
+        "crash", "crash-risk", "bearish", "bear-market", "bear", "bears", "downturn",
+        "downside", "decline", "declining", "declined", "drop", "dropped", "dropping",
+        "plunge", "plunging", "plummet", "plummeting", "slump", "slumping", "tumble",
+        "tumbling", "hemorrhage", "hemorrhaging", "bloodbath", "carnage", "selloff",
+        "sell-off", "dumping", "dump", "dumps", "collapse", "collapsed", "collapsing",
+        "wipeout", "wiped out", "implosion", "implode", "imploding", "freefall",
+        "meltdown", "capitulation", "capitulated", "liquidation", "liquidating",
+        "liquidated", "margin call", "forced liquidation", "breakdown", "support broken",
+        "support breach", "key support broken",
+        "black swan", "doom", "doomed", "apocalypse", "armageddon", "end of the world",
+        "financial crisis", "systemic risk", "contagion", "domino effect", "house of cards",
+        "bubble burst", "bubble bursting", "ponzi", "rug pull", "rugpull", "exit scam",
+        "rekt", "rugged", "dead", "dying", "rip", "funeral", "bagholder", "bagholders",
+        "holding bags", "underwater", "deep underwater", "drowning", "bleeding",
+        "bleeding out", "paper hands", "weak hands", "panic selling", "capitulating",
+    ],
+    "greed_state": [
+        "greed", "greedy", "avarice", "covetous", "rapacious", "insatiable", "fomo",
+        "fear of missing out", "yolo", "yolo'ing", "ape", "aping", "aping in", "all in",
+        "lever", "levered", "leverage", "margin", "margin trading", "borrow", "borrowing",
+        "buy", "buying", "accumulate", "accumulating", "loading", "loading up", "fill bags",
+        "stacking", "stacking sats", "stacking eth", "dca", "dollar cost averaging",
+        "bullish", "bull market", "bull", "bulls", "moon", "mooning", "to the moon",
+        "lamborghini", "lambo", "wen lambo", "gains", "massive gains", "life changing",
+        "generational wealth", "early", "getting in early", "ground floor", "rocket",
+        "rocketing", "parabolic", "parabolic move", "vertical", "going vertical",
+        "explosive", "explosive move", "breakout", "breaking out", "breakout confirmed",
+        "momentum", "strong momentum", "relentless", "unstoppable", "nothing can stop",
+        "euphoria", "euphoric", "mania", "manic", "frenzy", "buying frenzy",
+        "overbought", "extreme overbought", "greed index", "extreme greed",
+        "diamond hands", "hodl", "hodling", "never selling", "diamond", "hands of steel",
+    ],
+    "hype_velocity": [
+        "accelerating", "acceleration", "speeding up", "faster", "rapidly increasing",
+        "exponential", "exponentially", "hockey stick", "vertical", "going vertical",
+        "parabolic", "parabolic move", "explosive", "explosion", "explosive growth",
+        "surging", "surge", "spiking", "spike", "rocketing", "rocket", "mooning",
+        "velocity", "momentum", "momentum building", "gaining momentum", "picking up steam",
+        "steam", "full steam", "unstoppable", "relentless", "unrelenting", "non-stop",
+        "around the clock", "24/7", "nonstop", "frenzy", "manic", "mania", "euphoric",
+        "viral", "going viral", "trending", "trending worldwide", "exploding",
+        "blowing up", "blow up", "blowing up right now", "right now", "as we speak",
+        "live", "happening now", "breaking", "just in", "developing", "urgent",
+    ],
+    "pump_score": [
+        "pump", "pumping", "pumped", "pump it", "pump and dump", "coordinated pump",
+        "pump group", "pump signal", "pump call", "buy signal", "buy call", "entry signal",
+        "coordinated buying", "organized pump", "telegram pump", "discord pump",
+        "whale buying", "whale accumulation", "smart money buying", "institutional buying",
+        "market maker buying", "mm buying", "bid wall", "massive bid", "thick bid",
+        "buy wall", "buy walls", "absorption", "absorbing", "absorbing supply",
+        "short squeeze", "squeezing shorts", "shorts getting rekt", "gamma squeeze",
+        "gamma ramp", "options flow", "call buying", "call sweep", "unusual options",
+        "dark pool buying", "otc buying", "large buyer", "mystery buyer",
+        "coordinated", "synchronized", "simultaneous", "same time", "same minute",
+    ],
+    "dump_score": [
+        "dump", "dumping", "dumped", "dump it", "massive dump", "whale dumping",
+        "whale selling", "distribution", "distributing", "top is in", "local top",
+        "blow off top", "exhaustion", "exhausted", "running out of steam",
+        "loss of momentum", "momentum lost", "reversal", "reversing", "turning down",
+        "breakdown", "breaking down", "support broken", "key level lost",
+        "cascading", "cascade", "liquidation cascade", "long liquidation",
+        "longs getting rekt", "margin calls", "forced selling", "forced liquidation",
+        "panic selling", "capitulation", "capitulating", "giving up", "throwing in towel",
+        "dead cat bounce", "dead cat", "lower high", "lower low", "downtrend",
+        "bearish structure", "bear market rally", "sucker rally", "bull trap",
+        "distribution phase", "wyckoff distribution", "topping pattern",
+        "head and shoulders", "double top", "triple top", "rising wedge",
+        "bear flag", "bear pennant", "descending triangle",
+    ],
+}
+
+PARAMETER_SENTENCES = {
+    "fear_state": [
+        "The market is crashing and panic selling is everywhere.",
+        "Bitcoin just broke key support and fear is spreading rapidly.",
+        "Massive liquidation cascade as longs get wiped out.",
+        "Extreme fear grips the market as price plunges.",
+        "Capitulation volume suggests the bottom may be near.",
+    ],
+    "greed_state": [
+        "FOMO is driving prices parabolic as everyone apes in.",
+        "Massive gains have traders euphoric with diamond hands.",
+        "The market is in extreme greed with leverage at all-time highs.",
+        "Buying frenzy as price goes vertical with no resistance.",
+        "Institutional buying pressure creates massive bid walls.",
+    ],
+    "hype_velocity": [
+        "Hype is accelerating exponentially as volume explodes.",
+        "Momentum is building rapidly with non-stop buying pressure.",
+        "Social sentiment is going viral with trending worldwide.",
+        "Velocity of mentions is surging as news breaks live.",
+        "Exponential growth in engagement signals manic phase.",
+    ],
+    "pump_score": [
+        "Coordinated pump group signals buy call with massive bid walls.",
+        "Whale accumulation and smart money buying creates absorption.",
+        "Short squeeze developing as gamma ramp forces market makers.",
+        "Synchronized buying across exchanges at the same minute.",
+        "Institutional market maker bidding aggressively on all venues.",
+    ],
+    "dump_score": [
+        "Whale distribution and massive dump as top is confirmed.",
+        "Liquidation cascade accelerates as longs capitulate.",
+        "Support broken with bearish structure forming lower highs.",
+        "Panic selling and forced liquidation as margin calls hit.",
+        "Wyckoff distribution phase complete with breakdown confirmed.",
+    ],
+}
+
+PARAMETER_SENTENCES = {
+    "fear_state": [
+        "The market is crashing and panic selling is everywhere.",
+        "Bitcoin just broke key support and fear is spreading rapidly.",
+        "Massive liquidation cascade as longs get wiped out.",
+        "Extreme fear grips the market as price plunges.",
+        "Capitulation volume suggests the bottom may be near.",
+    ],
+    "greed_state": [
+        "FOMO is driving prices parabolic as everyone apes in.",
+        "Massive gains have traders euphoric with diamond hands.",
+        "The market is in extreme greed with leverage at all-time highs.",
+        "Buying frenzy as price goes vertical with no resistance.",
+        "Institutional buying pressure creates massive bid walls.",
+    ],
+    "hype_velocity": [
+        "Hype is accelerating exponentially as volume explodes.",
+        "Momentum is building rapidly with non-stop buying pressure.",
+        "Social sentiment is going viral with trending worldwide.",
+        "Velocity of mentions is surging as news breaks live.",
+        "Exponential growth in engagement signals manic phase.",
+    ],
+    "pump_score": [
+        "Coordinated pump group signals buy call with massive bid walls.",
+        "Whale accumulation and smart money buying creates absorption.",
+        "Short squeeze developing as gamma ramp forces market makers.",
+        "Synchronized buying across exchanges at the same minute.",
+        "Institutional market maker bidding aggressively on all venues.",
+    ],
+    "dump_score": [
+        "Whale distribution and massive dump as top is confirmed.",
+        "Liquidation cascade accelerates as longs capitulate.",
+        "Support broken with bearish structure forming lower highs.",
+        "Panic selling and forced liquidation as margin calls hit.",
+        "Wyckoff distribution phase complete with breakdown confirmed.",
+    ],
+}
+
+PARAMETER_CLUSTERS = {
+    "fear_state": {"market_crash": 1.0, "panic_selling": 1.0, "capitulation": 0.8, "bear_market": 0.9, "liquidation_cascade": 1.0},
+    "greed_state": {"fomo": 1.0, "euphoria": 1.0, "mania": 0.9, "parabolic": 0.8, "leverage": 0.7},
+    "hype_velocity": {"acceleration": 1.0, "viral": 0.9, "momentum": 0.8, "exponential": 1.0},
+    "pump_score": {"coordinated_pump": 1.0, "whale_buying": 0.9, "short_squeeze": 0.8, "absorption": 0.8},
+    "dump_score": {"whale_dumping": 1.0, "distribution": 0.9, "liquidation_cascade": 0.8, "panic_selling": 1.0},
+}
+
+
+async def main():
+    """Build and save centroids"""
+    print("Building parameter centroids...")
+
+    # Initialize centroid manager
+    from sentiment_engine.scoring.centroids import CentroidManager
+    manager = CentroidManager()
+
+    # Use sentence-transformers for real embeddings
+    from sentence_transformers import SentenceTransformer
+    encoder = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
+
+    await manager.initialize(encoder=None)  # We'll use our own encoder
+
+    # Override with keyword-based centroids
+    centroid_dir = Path("config/centroids")
+    centroid_dir.mkdir(parents=True, exist_ok=True)
+
+    # Load sentence transformer
+    model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
+
+    for param, keywords in PARAMETER_KEYWORDS.items():
+        print(f"Building centroid for {param}...")
+
+        # Collect all texts
+        texts = []
+        weights = []
+
+        # Keywords
+        for kw in keywords:
+            texts.append(kw)
+            weights.append(1.0)
+
+        # Sentences
+        for sent in PARAMETER_SENTENCES.get(param, []):
+            texts.append(sent)
+            weights.append(2.0)
+
+        # Clusters
+        for cluster, weight in PARAMETER_CLUSTERS.get(param, {}).items():
+            texts.append(cluster.replace("_", " "))
+            weights.append(weight * 1.5)
+
+        # Encode and average
+        embeddings = model.encode(texts, convert_to_numpy=True, normalize_embeddings=True)
+        centroid = np.average(embeddings, axis=0, weights=weights)
+        centroid = centroid / np.linalg.norm(centroid)
+
+        # Save
+        np.save(Path("config/centroids") / f"{param}.npy", centroid)
+        print(f"  Saved {param} centroid (shape: {centroid.shape})")
+
+    print("\nAll centroids built and saved!")
+    print(f"Location: {Path('config/centroids').absolute()}")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

sentiment_engine/scripts/build_comprehensive_dataset.py

@@ -0,0 +1,721 @@
+#!/usr/bin/env python3
+"""
+Comprehensive dataset builder for crypto sentiment engine.
+Creates labeled datasets with proper train/val/test splits.
+"""
+
+import json
+import random
+import hashlib
+from pathlib import Path
+from typing import Dict, List, Any, Optional, Tuple
+from dataclasses import dataclass, asdict
+from collections import Counter
+from datasets import load_dataset
+from sklearn.model_selection import train_test_split
+
+# ============================================================
+# LABEL SCHEMAS
+# ============================================================
+
+SENTIMENT_LABELS = ["Bearish", "Bullish", "Neutral"]
+SENTIMENT_MAP = {"Bearish": 0, "Bullish": 1, "Neutral": 2}
+
+EMOTION_LABELS = ["joy", "fear", "anger", "greed", "sadness", "neutral"]
+EMOTION_MAP = {l: i for i, l in enumerate(EMOTION_LABELS)}
+
+# GoEmotions 27 -> 6 mapping
+GOEMOTIONS_TO_6 = {
+    "admiration": "joy", "amusement": "joy", "excitement": "joy",
+    "gratitude": "joy", "love": "joy", "optimism": "joy",
+    "pride": "joy", "relief": "joy", "approval": "joy", "caring": "joy",
+    "fear": "fear", "nervousness": "fear", "anxiety": "fear",
+    "anger": "anger", "annoyance": "anger", "disapproval": "anger",
+    "disgust": "anger", "disapproval": "anger",
+    "desire": "greed", "greed": "greed", "optimism": "greed",
+    "sadness": "sadness", "disappointment": "sadness",
+    "grief": "sadness", "remorse": "sadness",
+    "neutral": "neutral", "confusion": "neutral", "curiosity": "neutral",
+    "realization": "neutral", "surprise": "neutral",
+    "embarrassment": "neutral", "confusion": "neutral",
+    "admiration": "joy", "amusement": "joy", "gratitude": "joy",
+    "love": "joy", "pride": "joy", "relief": "joy",
+    "excitement": "joy", "approval": "joy", "caring": "joy",
+    "nervousness": "fear", "anxiety": "fear",
+    "anger": "anger", "annoyance": "anger", "disgust": "anger",
+    "desire": "greed", "greed": "greed", "optimism": "greed",
+    "sadness": "sadness", "disappointment": "sadness",
+    "grief": "sadness", "remorse": "sadness",
+    "confusion": "neutral", "curiosity": "neutral",
+    "realization": "neutral", "surprise": "neutral",
+    "embarrassment": "neutral", "admiration": "joy",
+    "approval": "joy", "caring": "joy", "gratitude": "joy",
+    "love": "joy", "pride": "joy", "excitement": "joy",
+    "relief": "joy", "optimism": "greed", "joy": "joy",
+    "neutral": "neutral", "confusion": "neutral", "curiosity": "neutral",
+    "realization": "neutral", "surprise": "neutral",
+    "remorse": "sadness", "grief": "sadness",
+}
+
+EVENT_LABELS = [
+    "listing", "delisting", "hack", "regulatory", "governance",
+    "upgrade", "partnership", "earnings", "macro",
+    "liquidation", "whale", "manipulation"
+]
+EVENT_MAP = {l: i for i, l in enumerate(EVENT_LABELS)}
+
+NER_TAGS = [
+    "O",
+    "B-TICKER", "I-TICKER",
+    "B-CONTRACT", "I-CONTRACT",
+    "B-PROTOCOL", "I-PROTOCOL",
+    "B-EXCHANGE", "I-EXCHANGE",
+    "B-PERSON", "I-PERSON",
+    "B-CHAIN", "I-CHAIN",
+    "B-ORG", "I-ORG",
+]
+NER_MAP = {tag: i for i, tag in enumerate(NER_TAGS)}
+
+# ============================================================
+# REAL CRYPTO EVENTS (collected from web searches)
+# ============================================================
+
+REAL_EVENTS = [
+    # HACK EVENTS
+    {
+        "text": "XRP bridge drained for $200,000 after software mistook fake deposits for real ones. An attacker created unbacked XRP on another blockchain, then exchanged it for real XRP held in reserve. The bridge has been halted and its operator has filed a complaint with the FBI.",
+        "event_type": "hack",
+        "entities": [{"asset": "XRP", "type": "TICKER"}],
+        "sentiment": "Bearish",
+        "emotions": {"fear": 0.9, "anger": 0.6, "sadness": 0.3}
+    },
+    {
+        "text": "Major hack on DeFi protocol drains $50M. Users panic as TVL collapses. Team promises investigation.",
+        "event_type": "hack",
+        "entities": [],
+        "sentiment": "Bearish",
+        "emotions": {"fear": 0.98, "anger": 0.3, "sadness": 0.5}
+    },
+    
+    # LISTING EVENTS
+    {
+        "text": "KuCoin Lists Catizen (CATI) for Spot Trading on September 20, 2024. Catizen (CATI), the native token of viral Telegram-based game Catizen AI, will officially begin spot trading on KuCoin.",
+        "event_type": "listing",
+        "entities": [{"asset": "CATI", "type": "TICKER"}, {"asset": "TON", "type": "CHAIN"}],
+        "sentiment": "Bullish",
+        "emotions": {"joy": 0.7, "greed": 0.5}
+    },
+    {
+        "text": "Bitfinex Among First Exchanges to List HMSTR, Native Token of Hamster Kombat, a popular play-to-earn game based on Telegram with more than 300 million users.",
+        "event_type": "listing",
+        "entities": [{"asset": "HMSTR", "type": "TICKER"}],
+        "sentiment": "Bullish",
+        "emotions": {"joy": 0.6, "greed": 0.4}
+    },
+    {
+        "text": "Binance Becomes First Exchange to List Trump-Linked WLFI Token. The exchange will open WLFI spot pairs against USDT and USDC, marking the token's shift from a non-transferable presale to full tradability.",
+        "event_type": "listing",
+        "entities": [{"asset": "WLFI", "type": "TICKER"}, {"asset": "BNB", "type": "TICKER"}],
+        "sentiment": "Bullish",
+        "emotions": {"joy": 0.5, "greed": 0.6, "fear": 0.2}
+    },
+    
+    # REGULATORY EVENTS
+    {
+        "text": "SEC files lawsuit against major exchange for unregistered securities. Market reacts with fear.",
+        "event_type": "regulatory",
+        "entities": [{"asset": "SEC", "type": "ORG"}],
+        "sentiment": "Bearish",
+        "emotions": {"fear": 0.97, "anger": 0.2}
+    },
+    {
+        "text": "CFTC files to dismiss CME's lawsuit over crypto perpetual futures. 'Much ado about nothing': CFTC files to dismiss CME's lawsuit over crypto perpetual futures.",
+        "event_type": "regulatory",
+        "entities": [{"asset": "CFTC", "type": "ORG"}, {"asset": "CME", "type": "EXCHANGE"}],
+        "sentiment": "Neutral",
+        "emotions": {"fear": 0.1, "joy": 0.2}
+    },
+    {
+        "text": "Michigan court orders Kalshi to keep blocking sports prediction markets. US, UK launch joint alliance targeting crypto scam centers.",
+        "event_type": "regulatory",
+        "entities": [{"asset": "Kalshi", "type": "EXCHANGE"}],
+        "sentiment": "Bearish",
+        "emotions": {"fear": 0.6, "anger": 0.3}
+    },
+    
+    # UPGRADE EVENTS
+    {
+        "text": "Ethereum Dencun upgrade activates Proto-Danksharding (EIP-4844), introducing temporary data blobs for cheaper rollup storage. Dencun activates on mainnet at epoch 269568, March 13, 2024 at 13:55 UTC.",
+        "event_type": "upgrade",
+        "entities": [{"asset": "ETH", "type": "TICKER"}, {"asset": "Ethereum", "type": "PROTOCOL"}],
+        "sentiment": "Bullish",
+        "emotions": {"joy": 0.7, "greed": 0.3, "fear": 0.1}
+    },
+    {
+        "text": "Ethereum Shanghai upgrade goes live. Stakers can now withdraw. Validators celebrate. The Shanghai upgrade brings staking withdrawals to the execution layer.",
+        "event_type": "upgrade",
+        "entities": [{"asset": "ETH", "type": "TICKER"}],
+        "sentiment": "Bullish",
+        "emotions": {"joy": 0.8, "greed": 0.4}
+    },
+    {
+        "text": "Ethereum Cancun upgrade goes live. EIP-4844 introduces Proto-Danksharding with data blobs for cheaper L2 storage. L2 transaction fees expected to drop significantly.",
+        "event_type": "upgrade",
+        "entities": [{"asset": "ETH", "type": "TICKER"}],
+        "sentiment": "Bullish",
+        "emotions": {"joy": 0.7, "greed": 0.4}
+    },
+    
+    # PARTNERSHIP EVENTS
+    {
+        "text": "JPMorganChase and Coinbase Launch Strategic Partnership to Make Buying Crypto Easier than Ever. Direct bank-to-wallet connection, Chase Ultimate Rewards transfer, and Chase credit cards on Coinbase.",
+        "event_type": "partnership",
+        "entities": [{"asset": "JPM", "type": "ORG"}, {"asset": "COIN", "type": "TICKER"}],
+        "sentiment": "Bullish",
+        "emotions": {"joy": 0.8, "greed": 0.5}
+    },
+    {
+        "text": "Chainlink and Mastercard Partner to Enable Over 3 Billion Cardholders to Purchase Crypto Directly Onchain. Powered by Chainlink's secure interoperability infrastructure and Mastercard's global payments network.",
+        "event_type": "partnership",
+        "entities": [{"asset": "LINK", "type": "TICKER"}, {"asset": "MA", "type": "TICKER"}],
+        "sentiment": "Bullish",
+        "emotions": {"joy": 0.8, "greed": 0.6}
+    },
+    {
+        "text": "PayPal and Coinbase Expand Partnership to Drive Innovation of Stablecoin-based Solutions. 1:1 PYUSD to USD conversions, fee-free purchases, DeFi exploration.",
+        "event_type": "partnership",
+        "entities": [{"asset": "PYUSD", "type": "TICKER"}, {"asset": "COIN", "type": "TICKER"}, {"asset": "PYPL", "type": "TICKER"}],
+        "sentiment": "Bullish",
+        "emotions": {"joy": 0.7, "greed": 0.5}
+    },
+    
+    # WHALE EVENTS
+    {
+        "text": "Bitcoin whale moves $116 million in BTC after 11-year dormancy. A bitcoin whale transferred 1,000 BTC, worth about $116.6 million, for the first time since January 2014.",
+        "event_type": "whale",
+        "entities": [{"asset": "BTC", "type": "TICKER"}],
+        "sentiment": "Neutral",
+        "emotions": {"fear": 0.3, "greed": 0.2, "surprise": 0.7}
+    },
+    {
+        "text": "Ancient Bitcoin whale dormant for 11 years suddenly transfers $257,450,000 in BTC. 2,700 BTC moved after 11 years of slumber. Profit of 15,137%.",
+        "event_type": "whale",
+        "entities": [{"asset": "BTC", "type": "TICKER"}],
+        "sentiment": "Neutral",
+        "emotions": {"fear": 0.4, "greed": 0.3, "surprise": 0.8}
+    },
+    {
+        "text": "$1B in Bitcoin moves from Satoshi-era wallet after 14 years of inactivity. 10,000 BTC moved after 14.3 years dormancy. 140,000x returns.",
+        "event_type": "whale",
+        "entities": [{"asset": "BTC", "type": "TICKER"}],
+        "sentiment": "Neutral",
+        "emotions": {"fear": 0.5, "greed": 0.4, "surprise": 0.9}
+    },
+    
+    # MACRO EVENTS
+    {
+        "text": "Breaking: Fed pauses rate hikes. Bitcoin jumps 5% on dovish pivot. Fed pauses rate hikes as inflation cools. Bitcoin surges above $70k.",
+        "event_type": "macro",
+        "entities": [{"asset": "BTC", "type": "TICKER"}, {"asset": "FED", "type": "ORG"}],
+        "sentiment": "Bullish",
+        "emotions": {"joy": 0.8, "greed": 0.7, "fear": 0.1}
+    },
+    {
+        "text": "Surprise nonfarm payrolls print sends Bitcoin back below 80K. US economy added far more jobs than expected, pressuring Bitcoin lower as traders repriced Fed rate cut odds.",
+        "event_type": "macro",
+        "entities": [{"asset": "BTC", "type": "TICKER"}, {"asset": "FED", "type": "ORG"}],
+        "sentiment": "Bearish",
+        "emotions": {"fear": 0.8, "anger": 0.3}
+    },
+    
+    # LIQUIDATION EVENTS
+    {
+        "text": "Massive liquidation cascade wipes out $200M in longs. Funding rates flip negative. Long liquidation cascade as BTC drops below key support.",
+        "event_type": "liquidation",
+        "entities": [{"asset": "BTC", "type": "TICKER"}],
+        "sentiment": "Bearish",
+        "emotions": {"fear": 0.9, "anger": 0.4, "sadness": 0.5}
+    },
+    
+    # GOVERNANCE EVENTS
+    {
+        "text": "Governance proposal passes with 95% approval. Treasury diversifies into stablecoins. DAO votes to diversify treasury holdings.",
+        "event_type": "governance",
+        "entities": [],
+        "sentiment": "Bullish",
+        "emotions": {"joy": 0.6, "greed": 0.3}
+    },
+    
+    # EARNINGS EVENTS
+    {
+        "text": "Bitcoin ETF inflows hit $731M, highest since January as BTC reclaims $80K. ETF inflows hit record highs as institutional adoption accelerates.",
+        "event_type": "earnings",
+        "entities": [{"asset": "BTC", "type": "TICKER"}],
+        "sentiment": "Bullish",
+        "emotions": {"joy": 0.9, "greed": 0.8}
+    },
+    {
+        "text": "Coinbase Q2 earnings beat estimates. Revenue up 50% YoY. Trading volume surges on retail and institutional demand.",
+        "event_type": "earnings",
+        "entities": [{"asset": "COIN", "type": "TICKER"}],
+        "sentiment": "Bullish",
+        "emotions": {"joy": 0.8, "greed": 0.6}
+    },
+    
+    # MANIPULATION EVENTS
+    {
+        "text": "FOMO drives memecoin 500% in 24h. Degens aping in. Rug pull inevitable? Coordinated pump and dump suspected on new token.",
+        "event_type": "manipulation",
+        "entities": [],
+        "sentiment": "Bearish",
+        "emotions": {"anger": 0.7, "fear": 0.6, "greed": 0.4}
+    },
+    {
+        "text": "Token buybacks are booming. But are they good for crypto projects? Crypto projects are spending hundreds of millions buying their own tokens.",
+        "event_type": "manipulation",
+        "entities": [],
+        "sentiment": "Neutral",
+        "emotions": {"fear": 0.3, "greed": 0.5}
+    },
+    
+    # DELISTING EVENTS
+    {
+        "text": "Coinbase delists XRP after SEC lawsuit. Trading suspended. Users have 30 days to withdraw.",
+        "event_type": "delisting",
+        "entities": [{"asset": "XRP", "type": "TICKER"}],
+        "sentiment": "Bearish",
+        "emotions": {"fear": 0.9, "anger": 0.8}
+    },
+]
+
+# Pure sentiment samples
+SENTIMENT_SAMPLES = [
+    # Bullish
+    ("BTC breaks $100k! New ATH!", "Bullish"),
+    ("ETH to $10k by EOY, accumulate now", "Bullish"),
+    ("Institutional inflows hit record high", "Bullish"),
+    ("Bitcoin reaches new all-time high as institutional adoption accelerates", "Bullish"),
+    ("Ethereum merge successful, staking rewards now live", "Bullish"),
+    ("Massive ETF inflows drive Bitcoin to new highs", "Bullish"),
+    ("Golden cross confirmed on Bitcoin weekly chart", "Bullish"),
+    ("Institutional adoption drives Bitcoin higher", "Bullish"),
+    ("ETF approval drives massive inflows", "Bullish"),
+    ("Market is bullish on Bitcoin", "Bullish"),
+    
+    # Bearish
+    ("BTC crashes 50% in hours", "Bearish"),
+    ("Exchange hacked, $100M stolen", "Bearish"),
+    ("SEC sues major exchange", "Bearish"),
+    ("Bitcoin crashes hard, panic selling everywhere", "Bearish"),
+    ("Massive liquidation cascade wipes out $200M in longs", "Bearish"),
+    ("VIX drops below 15 as market volatility decreases", "Bearish"),
+    ("Whale sells 10000 BTC", "Bearish"),
+    ("Bitcoin price drops 50%", "Bearish"),
+    ("Support broken with bearish structure forming lower highs", "Bearish"),
+    ("Panic selling and forced liquidation as margin calls hit", "Bearish"),
+    
+    # Neutral
+    ("BTC at $50k, ETH at $3k", "Neutral"),
+    ("Market consolidating in range", "Neutral"),
+    ("Bitcoin remains stable around $30k", "Neutral"),
+    ("VIX drops below 15 as market volatility decreases", "Neutral"),
+    ("Market consolidating with no clear direction", "Neutral"),
+    ("Bitcoin price stable around $30k", "Neutral"),
+    ("Consolidation phase continues", "Neutral"),
+    ("Market in wait-and-see mode", "Neutral"),
+    ("Sideways action continues", "Neutral"),
+    ("Low volatility environment persists", "Neutral"),
+]
+
+# GoEmotions samples (from real data)
+EMOTION_SAMPLES = [
+    # Joy
+    ("BTC breaks $100k! New ATH!", {"joy": 0.9, "fear": 0.05, "anger": 0.02, "greed": 0.4, "sadness": 0.01, "neutral": 0.05}),
+    ("Ethereum merge successful!", {"joy": 0.95, "fear": 0.01, "anger": 0.01, "greed": 0.3, "sadness": 0.01, "neutral": 0.03}),
+    ("We did it! Bitcoin to the moon!", {"joy": 0.98, "fear": 0.01, "anger": 0.0, "greed": 0.5, "sadness": 0.0, "neutral": 0.01}),
+    
+    # Fear
+    ("Major hack on DeFi protocol drains $50M", {"joy": 0.01, "fear": 0.98, "anger": 0.3, "greed": 0.02, "sadness": 0.4, "neutral": 0.02}),
+    ("Bitcoin crashes 50% in hours", {"joy": 0.01, "fear": 0.95, "anger": 0.4, "greed": 0.01, "sadness": 0.6, "neutral": 0.02}),
+    ("SEC sues major exchange", {"joy": 0.02, "fear": 0.97, "anger": 0.5, "greed": 0.01, "sadness": 0.3, "neutral": 0.02}),
+    
+    # Anger
+    ("Rug pull! Devs stole all funds!", {"joy": 0.0, "fear": 0.5, "anger": 0.95, "greed": 0.05, "sadness": 0.3, "neutral": 0.01}),
+    ("Exchange froze withdrawals again!", {"joy": 0.01, "fear": 0.4, "anger": 0.9, "greed": 0.02, "sadness": 0.2, "neutral": 0.02}),
+    
+    # Greed
+    ("FOMO drives memecoin 500% in 24h", {"joy": 0.3, "fear": 0.1, "anger": 0.1, "greed": 0.9, "sadness": 0.02, "neutral": 0.05}),
+    ("Buy the dip! Accumulate more!", {"joy": 0.4, "fear": 0.05, "anger": 0.05, "greed": 0.85, "sadness": 0.01, "neutral": 0.05}),
+    ("All in on this gem!", {"joy": 0.5, "fear": 0.02, "anger": 0.02, "greed": 0.95, "sadness": 0.0, "neutral": 0.01}),
+    
+    # Sadness
+    ("Lost everything in the crash", {"joy": 0.01, "fear": 0.3, "anger": 0.2, "greed": 0.02, "sadness": 0.95, "neutral": 0.02}),
+    ("Rekt again, lost life savings", {"joy": 0.0, "fear": 0.4, "anger": 0.3, "greed": 0.01, "sadness": 0.98, "neutral": 0.02}),
+    
+    # Neutral
+    ("BTC at $50k, ETH at $3k", {"joy": 0.1, "fear": 0.1, "anger": 0.05, "greed": 0.1, "sadness": 0.05, "neutral": 0.7}),
+    ("Market consolidating in range", {"joy": 0.05, "fear": 0.15, "anger": 0.05, "greed": 0.1, "sadness": 0.05, "neutral": 0.65}),
+]
+
+# NER tagging - proper BIO tags
+NER_TAGS = [
+    "O",
+    "B-TICKER", "I-TICKER",
+    "B-CONTRACT", "I-CONTRACT",
+    "B-PROTOCOL", "I-PROTOCOL",
+    "B-EXCHANGE", "I-EXCHANGE",
+    "B-PERSON", "I-PERSON",
+    "B-CHAIN", "I-CHAIN",
+    "B-ORG", "I-ORG",
+]
+NER_MAP = {tag: i for i, tag in enumerate(NER_TAGS)}
+
+# Labels
+SENTIMENT_LABELS = ["Bearish", "Bullish", "Neutral"]
+SENTIMENT_MAP = {"Bearish": 0, "Bullish": 1, "Neutral": 2}
+
+EMOTION_LABELS = ["joy", "fear", "anger", "greed", "sadness", "neutral"]
+EMOTION_MAP = {l: i for i, l in enumerate(EMOTION_LABELS)}
+
+EVENT_LABELS = [
+    "listing", "delisting", "hack", "regulatory", "governance",
+    "upgrade", "partnership", "earnings", "macro",
+    "liquidation", "whale", "manipulation"
+]
+EVENT_MAP = {l: i for i, l in enumerate(EVENT_LABELS)}
+
+
+class ComprehensiveDatasetBuilder:
+    def __init__(self, output_dir: str = "data/training"):
+        self.output_dir = Path(output_dir)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+        
+    def build_all(self):
+        print("Building comprehensive labeled datasets...")
+        
+        # Load GoEmotions dataset
+        print("Loading GoEmotions...")
+        go_emotions = self._load_go_emotions()
+        print(f"  Loaded {len(go_emotions)} GoEmotions samples")
+        
+        # Load Twitter Financial News
+        print("Loading Twitter Financial News...")
+        twitter_fin = self._load_twitter_financial()
+        print(f"  Loaded {len(twitter_fin)} Twitter Financial samples")
+        
+        # Combine all data
+        all_samples = self._combine_all_data(go_emotions, twitter_fin)
+        print(f"  Combined: {len(all_samples)} samples")
+        
+        # Create splits
+        train, val, test = self._create_splits(all_samples)
+        print(f"  Splits: train={len(train)}, val={len(val)}, test={len(test)}")
+        
+        # Save datasets
+        self._save_splits(train, val, test)
+        
+        # Create NER dataset
+        self._create_ner_dataset()
+        
+        # Create multitask dataset
+        self._create_multitask_dataset()
+        
+        print("All datasets saved!")
+    
+    def _load_go_emotions(self) -> List[Dict]:
+        """Load GoEmotions and map to 6 emotions"""
+        ds = load_dataset('go_emotions', 'simplified')
+        all_data = []
+        
+        for split in ['train', 'validation', 'test']:
+            for item in ds[split]:
+                # Map 27 emotions to 6
+                emotion_scores = {e: 0.0 for e in EMOTION_LABELS}
+                for label_idx in item['labels']:
+                    label_name = ds['train'].features['labels'].feature.names[label_idx]
+                    mapped = GOEMOTIONS_TO_6.get(label_name)
+                    if mapped:
+                        emotion_scores[mapped] = max(emotion_scores[mapped], 1.0)
+                
+                all_data.append({
+                    "text": item['text'],
+                    "emotion_scores": emotion_scores,
+                    "labels": [1.0 if emotion_scores[e] > 0.5 else 0.0 for e in EMOTION_LABELS],
+                    "source": "go_emotions"
+                })
+        
+        return all_data
+    
+    def _load_twitter_financial(self) -> List[Dict]:
+        """Load Twitter Financial News sentiment"""
+        ds = load_dataset('zeroshot/twitter-financial-news-sentiment')
+        all_data = []
+        
+        for split in ['train', 'validation']:
+            for item in ds[split]:
+                label_map = {0: "Bearish", 1: "Bullish", 2: "Neutral"}
+                all_data.append({
+                    "text": item['text'],
+                    "sentiment": label_map[item['label']],
+                    "sentiment_id": item['label'],
+                    "source": "twitter_financial"
+                })
+        
+        return all_data
+    
+    def _combine_all_data(self, go_emotions, twitter_fin) -> List[Dict]:
+        """Combine all data sources"""
+        all_samples = []
+        
+        # Add GoEmotions
+        for item in go_emotions:
+            all_samples.append({
+                "text": item["text"],
+                "task": "emotion",
+                "labels": item["labels"],
+                "emotion_scores": item["emotion_scores"],
+                "source": item["source"]
+            })
+        
+        # Add Twitter Financial
+        for item in twitter_fin:
+            all_samples.append({
+                "text": item["text"],
+                "task": "sentiment",
+                "label": item["sentiment"],
+                "label_id": item["sentiment_id"],
+                "source": item["source"]
+            })
+        
+        # Add real crypto events
+        for event in REAL_EVENTS:
+            all_samples.append({
+                "text": event["text"],
+                "task": "multitask",
+                "sentiment": event["sentiment"],
+                "sentiment_id": SENTIMENT_MAP[event["sentiment"]],
+                "emotions": {e: event["emotions"].get(e, 0.0) for e in EMOTION_LABELS},
+                "emotion_labels": [1.0 if event["emotions"].get(e, 0) > 0.5 else 0.0 for e in EMOTION_LABELS],
+                "event_type": event["event_type"],
+                "event_id": EVENT_MAP[event["event_type"]],
+                "event_labels": [1.0 if i == EVENT_MAP[event["event_type"]] else 0.0 for i in range(12)],
+                "entities": event["entities"],
+                "source": "real_event"
+            })
+        
+        return all_samples
+    
+    def _create_splits(self, data: List[Dict]) -> Tuple[List, List, List]:
+        """Create train/val/test splits with stratification"""
+        # Separate by task
+        by_task = {}
+        for item in data:
+            task = item.get("task", "unknown")
+            if task not in by_task:
+                by_task[task] = []
+            by_task[task].append(item)
+        
+        train_all, val_all, test_all = [], [], []
+        
+        for task, items in by_task.items():
+            # Stratify by label if possible
+            if task == "sentiment":
+                labels = [item["label_id"] for item in items]
+            elif task == "emotion":
+                # Multi-label - use first positive label or 5 (neutral)
+                labels = [next((i for i, v in enumerate(item["labels"]) if v == 1), 5) for item in items]
+            elif task == "multitask":
+                labels = [item["sentiment_id"] for item in items]
+            else:
+                labels = [0] * len(items)
+            
+            train, temp = train_test_split(items, test_size=0.3, random_state=42, stratify=labels)
+            val, test = train_test_split(temp, test_size=0.5, random_state=42, 
+                                         stratify=[labels[items.index(t)] for t in temp] if len(set(labels)) > 1 else None)
+            
+            train_all.extend(train)
+            val_all.extend(val)
+            test_all.extend(test)
+        
+        return train_all, val_all, test_all
+    
+    def _save_splits(self, train, val, test):
+        """Save train/val/test splits"""
+        for name, data in [("train", train), ("val", val), ("test", test)]:
+            filepath = self.output_dir / f"{name}.jsonl"
+            with open(filepath, 'w') as f:
+                for item in data:
+                    f.write(json.dumps(item) + '\n')
+            print(f"  Saved {name}.jsonl: {len(data)} samples")
+    
+    def _create_ner_dataset(self):
+        """Create NER dataset with proper BIO tags"""
+        print("Creating NER dataset...")
+        
+        # Create token-level NER data
+        ner_data = []
+        
+        for event in REAL_EVENTS:
+            text = event["text"]
+            entities = event.get("entities", [])
+            
+            # Simple tokenization and BIO tagging
+            words = text.split()
+            tags = ["O"] * len(words)
+            
+            for ent in entities:
+                entity_text = ent["asset"]
+                entity_type = ent["type"]
+                
+                # Find entity in text (simplified)
+                entity_words = entity_text.split()
+                for i in range(len(words) - len(entity_words) + 1):
+                    if words[i:i+len(entity_words)] == entity_words:
+                        tags[i] = f"B-{entity_type}"
+                        for j in range(1, len(entity_words)):
+                            if i + j < len(tags):
+                                tags[i+j] = f"I-{entity_type}"
+                        break
+            
+            # Convert to token-level format
+            tokens = []
+            for word, tag in zip(words, tags):
+                tokens.append({"token": word, "ner_tag": tag})
+            
+            if tokens:
+                ner_data.append({
+                    "text": text,
+                    "tokens": tokens,
+                    "source": "real_event"
+                })
+        
+        # Save
+        filepath = self.output_dir / "ner_train.jsonl"
+        with open(filepath, 'w') as f:
+            for item in ner_data:
+                f.write(json.dumps(item) + '\n')
+        print(f"  NER: {len(ner_data)} samples")
+    
+    def _create_multitask_dataset(self):
+        """Create unified multitask dataset"""
+        print("Creating multitask dataset...")
+        
+        data = []
+        for event in REAL_EVENTS:
+            # Sentiment
+            sentiment_label = SENTIMENT_MAP.get(event["sentiment"], 2)
+            
+            # Emotions (multi-hot)
+            emotion_labels = [0] * 6
+            for emo, score in event.get("emotions", {}).items():
+                if emo in EMOTION_MAP and score > 0.5:
+                    emotion_labels[EMOTION_MAP[emo]] = 1
+            
+            # Events (multi-hot)
+            event_labels = [0] * 12
+            event_idx = EVENT_MAP.get(event["event_type"])
+            if event_idx is not None:
+                event_labels[event_idx] = 1
+            
+            data.append({
+                "text": event["text"],
+                "sentiment": sentiment_label,
+                "emotions": emotion_labels,
+                "events": event_labels,
+                "entities": event.get("entities", []),
+                "source": "real_event"
+            })
+        
+        filepath = self.output_dir / "multitask_train.jsonl"
+        with open(filepath, 'w') as f:
+            for item in data:
+                f.write(json.dumps(item) + '\n')
+        print(f"  Multitask: {len(data)} samples")
+
+
+# ============================================================
+# DATA AUGMENTATION
+# ============================================================
+
+class DataAugmenter:
+    SENTIMENT_TEMPLATES = {
+        "Bullish": [
+            "{asset} surges to new highs",
+            "{asset} breaks resistance at ${price}",
+            "Institutional adoption drives {asset} higher",
+            "{asset} breaks out bullish",
+            "Massive {asset} accumulation by whales",
+        ],
+        "Bearish": [
+            "{asset} crashes {pct}%",
+            "{asset} breaks support at ${price}",
+            "Panic selling in {asset}",
+            "{asset} faces massive sell pressure",
+            "Whale dumps {amount} {asset}",
+        ],
+        "Neutral": [
+            "{asset} consolidates at ${price}",
+            "{asset} trades sideways",
+            "Market waits for {asset} direction",
+            "Low volatility in {asset}",
+        ],
+    }
+    
+    ASSETS = ["BTC", "ETH", "SOL", "AVAX", "MATIC", "DOT", "LINK", "UNI", "AAVE", "ARB"]
+    
+    @classmethod
+    def generate_sentiment(cls, count: int = 2000) -> List[Dict]:
+        data = []
+        for _ in range(count):
+            sentiment = random.choice(["Bullish", "Bearish", "Neutral"])
+            asset = random.choice(cls.ASSETS)
+            template = random.choice(cls.SENTIMENT_TEMPLATES[sentiment])
+            
+            text = template.format(
+                asset=asset,
+                price=random.randint(100, 100000),
+                pct=random.randint(10, 80),
+                amount=f"{random.randint(1, 100)}K"
+            )
+            
+            data.append({
+                "text": text,
+                "label": sentiment,
+                "label_id": SENTIMENT_MAP[sentiment],
+                "source": "synthetic"
+            })
+        
+        return data
+
+
+# ============================================================
+# MAIN
+# ============================================================
+
+if __name__ == "__main__":
+    import sys
+    sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
+    
+    builder = ComprehensiveDatasetBuilder()
+    builder.build_all()
+    
+    # Generate augmented data
+    print("\nGenerating augmented data...")
+    aug_data = DataAugmenter.generate_sentiment(5000)
+    builder._save_splits(aug_data, [], [])  # Save to augmented
+    # Fix: save augmented separately
+    filepath = builder.output_dir / "sentiment_augmented.jsonl"
+    with open(filepath, 'w') as f:
+        for item in aug_data:
+            f.write(json.dumps(item) + '\n')
+    print(f"  Augmented sentiment: {len(aug_data)} samples")
+    
+    # Print summary
+    print("\n" + "="*60)
+    print("COMPREHENSIVE DATASET BUILD COMPLETE")
+    print("="*60)
+    for f in sorted(Path("data/training").glob("*.jsonl")):
+        count = sum(1 for _ in open(f))
+        print(f"  {f.name}: {count:,} samples")
+    
+    print(f"\nTotal samples: {sum(sum(1 for _ in open(f)) for f in Path('data/training').glob('*.jsonl')):,}")

sentiment_engine/scripts/build_labeled_dataset.py

@@ -0,0 +1,603 @@
+#!/usr/bin/env python3
+"""
+Build labeled training datasets for crypto sentiment engine.
+Combines public datasets + real web data + synthetic generation.
+Outputs: JSONL files ready for fine-tuning.
+"""
+
+import json
+import random
+from pathlib import Path
+from typing import Dict, List, Any, Optional
+from dataclasses import dataclass, asdict
+from datetime import datetime
+import hashlib
+
+# ============================================================
+# LABEL SCHEMAS (matching our system specs)
+# ============================================================
+
+SENTIMENT_LABELS = ["Bearish", "Bullish", "Neutral"]  # 0, 1, 2
+SENTIMENT_MAP = {"Bearish": 0, "Bullish": 1, "Neutral": 2}
+
+EMOTION_LABELS = ["joy", "fear", "anger", "greed", "sadness", "neutral"]
+EMOTION_MAP = {l: i for i, l in enumerate(EMOTION_LABELS)}
+
+EVENT_LABELS = [
+    "listing", "delisting", "hack", "regulatory", "governance",
+    "upgrade", "partnership", "earnings", "macro",
+    "liquidation", "whale", "manipulation"
+]
+EVENT_MAP = {l: i for i, l in enumerate(EVENT_LABELS)}
+
+NER_TAGS = [
+    "O",
+    "B-TICKER", "I-TICKER",
+    "B-CONTRACT", "I-CONTRACT",
+    "B-PROTOCOL", "I-PROTOCOL",
+    "B-EXCHANGE", "I-EXCHANGE",
+    "B-PERSON", "I-PERSON",
+    "B-CHAIN", "I-CHAIN",
+]
+NER_MAP = {tag: i for i, tag in enumerate(NER_TAGS)}
+
+# ============================================================
+# REAL DATA COLLECTED FROM WEB SEARCHES
+# ============================================================
+
+REAL_EVENTS = [
+    # HACK EVENTS
+    {
+        "text": "XRP bridge drained for $200,000 after software mistook fake deposits for real ones. An attacker created unbacked XRP on another blockchain, then exchanged it for real XRP held in reserve. The bridge has been halted and its operator has filed a complaint with the FBI.",
+        "event_type": "hack",
+        "entities": [{"asset": "XRP", "type": "TICKER"}],
+        "sentiment": "Bearish",
+        "emotions": {"fear": 0.9, "anger": 0.6, "sadness": 0.3}
+    },
+    {
+        "text": "Major hack on DeFi protocol drains $50M. Users panic as TVL collapses. Team promises investigation.",
+        "event_type": "hack",
+        "entities": [],
+        "sentiment": "Bearish",
+        "emotions": {"fear": 0.98, "anger": 0.3, "sadness": 0.5}
+    },
+    
+    # LISTING EVENTS
+    {
+        "text": "KuCoin Lists Catizen (CATI) for Spot Trading on September 20, 2024. Catizen (CATI), the native token of viral Telegram-based game Catizen AI, will officially begin spot trading on KuCoin.",
+        "event_type": "listing",
+        "entities": [{"asset": "CATI", "type": "TICKER"}, {"asset": "TON", "type": "CHAIN"}],
+        "sentiment": "Bullish",
+        "emotions": {"joy": 0.7, "greed": 0.5}
+    },
+    {
+        "text": "Bitfinex Among First Exchanges to List HMSTR, Native Token of Hamster Kombat, a popular play-to-earn game based on Telegram with more than 300 million users.",
+        "event_type": "listing",
+        "entities": [{"asset": "HMSTR", "type": "TICKER"}],
+        "sentiment": "Bullish",
+        "emotions": {"joy": 0.6, "greed": 0.4}
+    },
+    {
+        "text": "Binance Becomes First Exchange to List Trump-Linked WLFI Token. The exchange will open WLFI spot pairs against USDT and USDC, marking the token's shift from a non-transferable presale to full tradability.",
+        "event_type": "listing",
+        "entities": [{"asset": "WLFI", "type": "TICKER"}, {"asset": "BNB", "type": "TICKER"}],
+        "sentiment": "Bullish",
+        "emotions": {"joy": 0.5, "greed": 0.6, "fear": 0.2}
+    },
+    
+    # HACK EVENTS (more)
+    {
+        "text": "Major hack on DeFi protocol drains $50M. Users panic as TVL collapses. Team promises investigation.",
+        "event_type": "hack",
+        "entities": [],
+        "sentiment": "Bearish",
+        "emotions": {"fear": 0.98, "anger": 0.3, "sadness": 0.5}
+    },
+    
+    # REGULATORY EVENTS
+    {
+        "text": "SEC files lawsuit against major exchange for unregistered securities. Market reacts with fear.",
+        "event_type": "regulatory",
+        "entities": [{"asset": "SEC", "type": "ORG"}],
+        "sentiment": "Bearish",
+        "emotions": {"fear": 0.97, "anger": 0.2}
+    },
+    {
+        "text": "CFTC files to dismiss CME's lawsuit over crypto perpetual futures. 'Much ado about nothing': CFTC files to dismiss CME's lawsuit over crypto perpetual futures.",
+        "event_type": "regulatory",
+        "entities": [{"asset": "CFTC", "type": "ORG"}, {"asset": "CME", "type": "EXCHANGE"}],
+        "sentiment": "Neutral",
+        "emotions": {"fear": 0.1, "joy": 0.2}
+    },
+    {
+        "text": "Michigan court orders Kalshi to keep blocking sports prediction markets. US, UK launch joint alliance targeting crypto scam centers.",
+        "event_type": "regulatory",
+        "entities": [{"asset": "Kalshi", "type": "EXCHANGE"}],
+        "sentiment": "Bearish",
+        "emotions": {"fear": 0.6, "anger": 0.3}
+    },
+    
+    # UPGRADE EVENTS
+    {
+        "text": "Ethereum Dencun upgrade activates Proto-Danksharding (EIP-4844), introducing temporary data blobs for cheaper rollup storage. Dencun activates on mainnet at epoch 269568, March 13, 2024 at 13:55 UTC.",
+        "event_type": "upgrade",
+        "entities": [{"asset": "ETH", "type": "TICKER"}, {"asset": "Ethereum", "type": "PROTOCOL"}],
+        "sentiment": "Bullish",
+        "emotions": {"joy": 0.7, "greed": 0.3, "fear": 0.1}
+    },
+    {
+        "text": "Ethereum Shanghai upgrade goes live. Stakers can now withdraw. Validators celebrate. The Shanghai upgrade brings staking withdrawals to the execution layer.",
+        "event_type": "upgrade",
+        "entities": [{"asset": "ETH", "type": "TICKER"}],
+        "sentiment": "Bullish",
+        "emotions": {"joy": 0.8, "greed": 0.4}
+    },
+    {
+        "text": "Ethereum Cancun upgrade goes live. EIP-4844 introduces Proto-Danksharding with data blobs for cheaper L2 storage. L2 transaction fees expected to drop significantly.",
+        "event_type": "upgrade",
+        "entities": [{"asset": "ETH", "type": "TICKER"}],
+        "sentiment": "Bullish",
+        "emotions": {"joy": 0.7, "greed": 0.4}
+    },
+    
+    # PARTNERSHIP EVENTS
+    {
+        "text": "JPMorganChase and Coinbase Launch Strategic Partnership to Make Buying Crypto Easier than Ever. Direct bank-to-wallet connection, Chase Ultimate Rewards transfer, and Chase credit cards on Coinbase.",
+        "event_type": "partnership",
+        "entities": [{"asset": "JPM", "type": "ORG"}, {"asset": "COIN", "type": "TICKER"}],
+        "sentiment": "Bullish",
+        "emotions": {"joy": 0.8, "greed": 0.5}
+    },
+    {
+        "text": "Chainlink and Mastercard Partner to Enable Over 3 Billion Cardholders to Purchase Crypto Directly Onchain. Powered by Chainlink's secure interoperability infrastructure and Mastercard's global payments network.",
+        "event_type": "partnership",
+        "entities": [{"asset": "LINK", "type": "TICKER"}, {"asset": "MA", "type": "TICKER"}],
+        "sentiment": "Bullish",
+        "emotions": {"joy": 0.8, "greed": 0.6}
+    },
+    {
+        "text": "PayPal and Coinbase Expand Partnership to Drive Innovation of Stablecoin-based Solutions. 1:1 PYUSD to USD conversions, fee-free purchases, DeFi exploration.",
+        "event_type": "partnership",
+        "entities": [{"asset": "PYUSD", "type": "TICKER"}, {"asset": "COIN", "type": "TICKER"}, {"asset": "PYPL", "type": "TICKER"}],
+        "sentiment": "Bullish",
+        "emotions": {"joy": 0.7, "greed": 0.5}
+    },
+    
+    # WHALE EVENTS
+    {
+        "text": "Bitcoin whale moves $116 million in BTC after 11-year dormancy. A bitcoin whale transferred 1,000 BTC, worth about $116.6 million, for the first time since January 2014.",
+        "event_type": "whale",
+        "entities": [{"asset": "BTC", "type": "TICKER"}],
+        "sentiment": "Neutral",
+        "emotions": {"fear": 0.3, "greed": 0.2, "surprise": 0.7}
+    },
+    {
+        "text": "Ancient Bitcoin whale dormant for 11 years suddenly transfers $257,450,000 in BTC. 2,700 BTC moved after 11 years of slumber. Profit of 15,137%.",
+        "event_type": "whale",
+        "entities": [{"asset": "BTC", "type": "TICKER"}],
+        "sentiment": "Neutral",
+        "emotions": {"fear": 0.4, "greed": 0.3, "surprise": 0.8}
+    },
+    {
+        "text": "$1B in Bitcoin moves from Satoshi-era wallet after 14 years of inactivity. 10,000 BTC moved after 14.3 years dormancy. 140,000x returns.",
+        "event_type": "whale",
+        "entities": [{"asset": "BTC", "type": "TICKER"}],
+        "sentiment": "Neutral",
+        "emotions": {"fear": 0.5, "greed": 0.4, "surprise": 0.9}
+    },
+    
+    # MACRO EVENTS
+    {
+        "text": "Breaking: Fed pauses rate hikes. Bitcoin jumps 5% on dovish pivot. Fed pauses rate hikes as inflation cools. Bitcoin surges above $70k.",
+        "event_type": "macro",
+        "entities": [{"asset": "BTC", "type": "TICKER"}, {"asset": "FED", "type": "ORG"}],
+        "sentiment": "Bullish",
+        "emotions": {"joy": 0.8, "greed": 0.7, "fear": 0.1}
+    },
+    {
+        "text": "Surprise nonfarm payrolls print sends Bitcoin back below 80K. US economy added far more jobs than expected, pressuring Bitcoin lower as traders repriced Fed rate cut odds.",
+        "event_type": "macro",
+        "entities": [{"asset": "BTC", "type": "TICKER"}, {"asset": "FED", "type": "ORG"}],
+        "sentiment": "Bearish",
+        "emotions": {"fear": 0.8, "anger": 0.3}
+    },
+    
+    # LIQUIDATION EVENTS
+    {
+        "text": "Massive liquidation cascade wipes out $200M in longs. Funding rates flip negative. Long liquidation cascade as BTC drops below key support.",
+        "event_type": "liquidation",
+        "entities": [{"asset": "BTC", "type": "TICKER"}],
+        "sentiment": "Bearish",
+        "emotions": {"fear": 0.9, "anger": 0.4, "sadness": 0.5}
+    },
+    
+    # GOVERNANCE EVENTS
+    {
+        "text": "Governance proposal passes with 95% approval. Treasury diversifies into stablecoins. DAO votes to diversify treasury holdings.",
+        "event_type": "governance",
+        "entities": [],
+        "sentiment": "Bullish",
+        "emotions": {"joy": 0.6, "greed": 0.3}
+    },
+    
+    # EARNINGS EVENTS
+    {
+        "text": "Bitcoin ETF inflows hit $731M, highest since January as BTC reclaims $80K. ETF inflows hit record highs as institutional adoption accelerates.",
+        "event_type": "earnings",
+        "entities": [{"asset": "BTC", "type": "TICKER"}],
+        "sentiment": "Bullish",
+        "emotions": {"joy": 0.9, "greed": 0.8}
+    },
+    {
+        "text": "Coinbase Q2 earnings beat estimates. Revenue up 50% YoY. Trading volume surges on retail and institutional demand.",
+        "event_type": "earnings",
+        "entities": [{"asset": "COIN", "type": "TICKER"}],
+        "sentiment": "Bullish",
+        "emotions": {"joy": 0.8, "greed": 0.6}
+    },
+    
+    # MANIPULATION EVENTS
+    {
+        "text": "FOMO drives memecoin 500% in 24h. Degens aping in. Rug pull inevitable? Coordinated pump and dump suspected on new token.",
+        "event_type": "manipulation",
+        "entities": [],
+        "sentiment": "Bearish",
+        "emotions": {"anger": 0.7, "fear": 0.6, "greed": 0.4}
+    },
+    {
+        "text": "Token buybacks are booming. But are they good for crypto projects? Crypto projects are spending hundreds of millions buying their own tokens.",
+        "event_type": "manipulation",
+        "entities": [],
+        "sentiment": "Neutral",
+        "emotions": {"fear": 0.3, "greed": 0.5}
+    },
+    
+    # DELISTING EVENTS
+    {
+        "text": "Coinbase delists XRP after SEC lawsuit. Trading suspended. Users have 30 days to withdraw.",
+        "event_type": "delisting",
+        "entities": [{"asset": "XRP", "type": "TICKER"}],
+        "sentiment": "Bearish",
+        "emotions": {"fear": 0.9, "anger": 0.8}
+    },
+]
+
+# Additional sentiment-only samples for sentiment training
+SENTIMENT_SAMPLES = [
+    # Bullish
+    ("BTC breaks $100k! New ATH!", "Bullish"),
+    ("ETH to $10k by EOY, accumulate now", "Bullish"),
+    ("Institutional inflows hit record high", "Bullish"),
+    ("Bitcoin reaches new all-time high as institutional adoption accelerates", "Bullish"),
+    ("Ethereum merge successful, staking rewards now live", "Bullish"),
+    ("Massive ETF inflows drive Bitcoin to new highs", "Bullish"),
+    ("Golden cross confirmed on Bitcoin weekly chart", "Bullish"),
+    ("Institutional adoption drives Bitcoin higher", "Bullish"),
+    ("ETF approval drives massive inflows", "Bullish"),
+    ("Market is bullish on Bitcoin", "Bullish"),
+    
+    # Bearish
+    ("BTC crashes 50% in hours", "Bearish"),
+    ("Exchange hacked, $100M stolen", "Bearish"),
+    ("SEC sues major exchange", "Bearish"),
+    ("Bitcoin crashes hard, panic selling everywhere", "Bearish"),
+    ("Massive liquidation cascade wipes out $200M in longs", "Bearish"),
+    ("VIX drops below 15 as market volatility decreases", "Bearish"),
+    ("Whale sells 10000 BTC", "Bearish"),
+    ("Bitcoin price drops 50%", "Bearish"),
+    ("Support broken with bearish structure forming lower highs", "Bearish"),
+    ("Panic selling and forced liquidation as margin calls hit", "Bearish"),
+    
+    # Neutral
+    ("BTC at $50k, ETH at $3k", "Neutral"),
+    ("Market consolidating in range", "Neutral"),
+    ("Bitcoin remains stable around $30k", "Neutral"),
+    ("VIX drops below 15 as market volatility decreases", "Neutral"),
+    ("Market consolidating with no clear direction", "Neutral"),
+    ("Bitcoin price stable around $30k", "Neutral"),
+    ("Consolidation phase continues", "Neutral"),
+    ("Market in wait-and-see mode", "Neutral"),
+    ("Sideways action continues", "Neutral"),
+    ("Low volatility environment persists", "Neutral"),
+]
+
+# Emotion samples mapped from GoEmotions
+EMOTION_SAMPLES = [
+    # Joy
+    ("BTC breaks $100k! New ATH!", {"joy": 0.9, "fear": 0.05, "anger": 0.02, "greed": 0.4, "sadness": 0.01, "neutral": 0.05}),
+    ("Ethereum merge successful!", {"joy": 0.95, "fear": 0.01, "anger": 0.01, "greed": 0.3, "sadness": 0.01, "neutral": 0.03}),
+    ("We did it! Bitcoin to the moon!", {"joy": 0.98, "fear": 0.01, "anger": 0.0, "greed": 0.5, "sadness": 0.0, "neutral": 0.01}),
+    
+    # Fear
+    ("Major hack on DeFi protocol drains $50M", {"joy": 0.01, "fear": 0.98, "anger": 0.3, "greed": 0.02, "sadness": 0.4, "neutral": 0.02}),
+    ("Bitcoin crashes 50% in hours", {"joy": 0.01, "fear": 0.95, "anger": 0.4, "greed": 0.01, "sadness": 0.6, "neutral": 0.02}),
+    ("SEC sues major exchange", {"joy": 0.02, "fear": 0.97, "anger": 0.5, "greed": 0.01, "sadness": 0.3, "neutral": 0.02}),
+    
+    # Anger
+    ("Rug pull! Devs stole all funds!", {"joy": 0.0, "fear": 0.5, "anger": 0.95, "greed": 0.05, "sadness": 0.3, "neutral": 0.01}),
+    ("Exchange froze withdrawals again!", {"joy": 0.01, "fear": 0.4, "anger": 0.9, "greed": 0.02, "sadness": 0.2, "neutral": 0.02}),
+    
+    # Greed
+    ("FOMO drives memecoin 500% in 24h", {"joy": 0.3, "fear": 0.1, "anger": 0.1, "greed": 0.9, "sadness": 0.02, "neutral": 0.05}),
+    ("Buy the dip! Accumulate more!", {"joy": 0.4, "fear": 0.05, "anger": 0.05, "greed": 0.85, "sadness": 0.01, "neutral": 0.05}),
+    ("All in on this gem!", {"joy": 0.5, "fear": 0.02, "anger": 0.02, "greed": 0.95, "sadness": 0.0, "neutral": 0.01}),
+    
+    # Sadness
+    ("Lost everything in the crash", {"joy": 0.01, "fear": 0.3, "anger": 0.2, "greed": 0.02, "sadness": 0.95, "neutral": 0.02}),
+    ("Rekt again, lost life savings", {"joy": 0.0, "fear": 0.4, "anger": 0.3, "greed": 0.01, "sadness": 0.98, "neutral": 0.02}),
+    
+    # Neutral
+    ("BTC at $50k, ETH at $3k", {"joy": 0.1, "fear": 0.1, "anger": 0.05, "greed": 0.1, "sadness": 0.05, "neutral": 0.7}),
+    ("Market consolidating in range", {"joy": 0.05, "fear": 0.15, "anger": 0.05, "greed": 0.1, "sadness": 0.05, "neutral": 0.65}),
+]
+
+
+# ============================================================
+# DATASET BUILDER
+# ============================================================
+
+class DatasetBuilder:
+    def __init__(self, output_dir: str = "data/training"):
+        self.output_dir = Path(output_dir)
+        self.output_dir.mkdir(parents=True, exist_ok=True)
+        
+    def build_all(self):
+        print("Building labeled datasets...")
+        
+        # 1. Sentiment dataset
+        self.build_sentiment_dataset()
+        
+        # 2. Emotion dataset
+        self.build_emotion_dataset()
+        
+        # 3. Event classification dataset
+        self.build_event_dataset()
+        
+        # 4. NER dataset (from entity extraction)
+        self.build_ner_dataset()
+        
+        # 5. Combined multi-task dataset
+        self.build_multitask_dataset()
+        
+        print(f"All datasets saved to {self.output_dir}")
+    
+    def build_sentiment_dataset(self):
+        """Build 3-class sentiment dataset"""
+        data = []
+        
+        # Add event-based sentiment samples
+        for event in REAL_EVENTS:
+            if event["sentiment"] in SENTIMENT_LABELS:
+                data.append({
+                    "text": event["text"],
+                    "label": event["sentiment"],
+                    "label_id": SENTIMENT_MAP[event["sentiment"]],
+                    "source": "real_event"
+                })
+        
+        # Add pure sentiment samples
+        for text, label in SENTIMENT_SAMPLES:
+            data.append({
+                "text": text,
+                "label": label,
+                "label_id": SENTIMENT_MAP[label],
+                "source": "sentiment_corpus"
+            })
+        
+        # Save
+        self._save_jsonl(data, "sentiment_train.jsonl")
+        print(f"  Sentiment: {len(data)} samples")
+    
+    def build_emotion_dataset(self):
+        """Build 6-class emotion dataset (multi-label)"""
+        data = []
+        
+        for text, emotions in EMOTION_SAMPLES:
+            # Convert to multi-hot encoding
+            labels = [0] * 6
+            for emo, score in emotions.items():
+                if emo in EMOTION_MAP and score > 0.5:
+                    labels[EMOTION_MAP[emo]] = 1
+            
+            data.append({
+                "text": text,
+                "labels": labels,
+                "emotion_scores": emotions,
+                "source": "emotion_corpus"
+            })
+        
+        # Add event-based emotions
+        for event in REAL_EVENTS:
+            if "emotions" in event:
+                labels = [0] * 6
+                for emo, score in event["emotions"].items():
+                    if emo in EMOTION_MAP and score > 0.5:
+                        labels[EMOTION_MAP[emo]] = 1
+                
+                data.append({
+                    "text": event["text"],
+                    "labels": labels,
+                    "emotion_scores": event["emotions"],
+                    "source": "real_event"
+                })
+        
+        self._save_jsonl(data, "emotion_train.jsonl")
+        print(f"  Emotion: {len(data)} samples")
+    
+    def build_event_dataset(self):
+        """Build 12-class event classification dataset (multi-label)"""
+        data = []
+        
+        for event in REAL_EVENTS:
+            # Create multi-hot labels
+            labels = [0] * 12
+            event_idx = EVENT_MAP.get(event["event_type"])
+            if event_idx is not None:
+                labels[event_idx] = 1
+            
+            data.append({
+                "text": event["text"],
+                "labels": labels,
+                "event_type": event["event_type"],
+                "event_id": event_idx,
+                "source": "real_event"
+            })
+        
+        self._save_jsonl(data, "event_train.jsonl")
+        print(f"  Events: {len(data)} samples")
+    
+    def build_ner_dataset(self):
+        """Build NER dataset from entity mentions"""
+        data = []
+        
+        for event in REAL_EVENTS:
+            entities = event.get("entities", [])
+            if not entities:
+                continue
+            
+            text = event["text"]
+            # Create token-level tags (simplified - span-based)
+            # In practice, you'd use a proper tokenizer alignment
+            entities_formatted = []
+            for ent in entities:
+                entities_formatted.append({
+                    "text": ent["asset"],
+                    "label": ent["type"],
+                    "start": text.lower().find(ent["asset"].lower()),
+                    "end": text.lower().find(ent["asset"].lower()) + len(ent["asset"])
+                })
+            
+            if entities_formatted:
+                data.append({
+                    "text": text,
+                    "entities": entities_formatted,
+                    "source": "real_event"
+                })
+        
+        self._save_jsonl(data, "ner_train.jsonl")
+        print(f"  NER: {len(data)} samples")
+    
+    def build_multitask_dataset(self):
+        """Build combined dataset for multi-task training"""
+        data = []
+        
+        for event in REAL_EVENTS:
+            # Sentiment
+            sentiment_label = SENTIMENT_MAP.get(event["sentiment"], 2)
+            
+            # Emotions (multi-hot)
+            emotion_labels = [0] * 6
+            for emo, score in event.get("emotions", {}).items():
+                if emo in EMOTION_MAP and score > 0.5:
+                    emotion_labels[EMOTION_MAP[emo]] = 1
+            
+            # Events (multi-hot)
+            event_labels = [0] * 12
+            event_idx = EVENT_MAP.get(event["event_type"])
+            if event_idx is not None:
+                event_labels[event_idx] = 1
+            
+            data.append({
+                "text": event["text"],
+                "sentiment": sentiment_label,
+                "emotions": emotion_labels,
+                "events": event_labels,
+                "entities": event.get("entities", []),
+                "source": "real_event"
+            })
+        
+        self._save_jsonl(data, "multitask_train.jsonl")
+        print(f"  Multi-task: {len(data)} samples")
+    
+    def _save_jsonl(self, data: List[Dict], filename: str):
+        filepath = self.output_dir / filename
+        with open(filepath, 'w') as f:
+            for item in data:
+                f.write(json.dumps(item) + '\n')
+
+
+# ============================================================
+# DATA AUGMENTATION (for expanding dataset)
+# ============================================================
+
+class DataAugmenter:
+    """Generate synthetic variations using templates"""
+    
+    SENTIMENT_TEMPLATES = {
+        "Bullish": [
+            "{asset} surges to new highs",
+            "{asset} breaks resistance at ${price}",
+            "Institutional adoption drives {asset} higher",
+            "{asset} breaks out bullish",
+            "Massive {asset} accumulation by whales",
+        ],
+        "Bearish": [
+            "{asset} crashes {pct}%",
+            "{asset} breaks support at ${price}",
+            "Panic selling in {asset}",
+            "{asset} faces massive sell pressure",
+            "Whale dumps {amount} {asset}",
+        ],
+        "Neutral": [
+            "{asset} consolidates at ${price}",
+            "{asset} trades sideways",
+            "Market waits for {asset} direction",
+            "Low volatility in {asset}",
+        ],
+    }
+    
+    ASSETS = ["BTC", "ETH", "SOL", "AVAX", "MATIC", "DOT", "LINK", "UNI", "AAVE", "ARB"]
+    
+    @classmethod
+    def generate(cls, count: int = 1000) -> List[Dict]:
+        """Generate synthetic sentiment samples"""
+        data = []
+        
+        for _ in range(count):
+            sentiment = random.choice(["Bullish", "Bearish", "Neutral"])
+            asset = random.choice(cls.ASSETS)
+            template = random.choice(cls.SENTIMENT_TEMPLATES[sentiment])
+            
+            text = template.format(
+                asset=asset,
+                price=random.randint(100, 100000),
+                pct=random.randint(10, 80),
+                amount=f"{random.randint(1, 100)}K"
+            )
+            
+            data.append({
+                "text": text,
+                "label": sentiment,
+                "label_id": SENTIMENT_MAP[sentiment],
+                "source": "synthetic"
+            })
+        
+        return data
+
+
+# ============================================================
+# MAIN
+# ============================================================
+
+if __name__ == "__main__":
+    import sys
+    sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
+    
+    builder = DatasetBuilder()
+    builder.build_all()
+    
+    # Also generate augmented data
+    print("\nGenerating augmented data...")
+    aug_data = DataAugmenter.generate(2000)
+    builder._save_jsonl(aug_data, "sentiment_augmented.jsonl")
+    print(f"  Augmented: {len(aug_data)} samples")
+    
+    # Print summary
+    print("\n" + "="*60)
+    print("DATASET BUILD COMPLETE")
+    print("="*60)
+    print(f"Output directory: {builder.output_dir}")
+    print("Files created:")
+    for f in builder.output_dir.glob("*.jsonl"):
+        count = sum(1 for _ in open(f))
+        print(f"  {f.name}: {count:,} samples")

sentiment_engine/scripts/export_onnx.py

@@ -0,0 +1,131 @@
+#!/usr/bin/env python3
+"""Export Hugging Face models to ONNX format for production inference"""
+
+import argparse
+import os
+from pathlib import Path
+
+import torch
+from optimum.onnxruntime import ORTModelForSequenceClassification
+from transformers import AutoTokenizer, AutoConfig
+
+MODELS = {
+    "finbert": {
+        "hf_id": "ProsusAI/finbert",
+        "output_dir": "models/onnx/finbert",
+        "labels": ["negative", "neutral", "positive"],
+    },
+    "distilroberta-emotion": {
+        "hf_id": "j-hartmann/emotion-english-distilroberta-base",
+        "output_dir": "models/onnx/distilroberta-emotion",
+        "labels": ["anger", "disgust", "fear", "joy", "neutral", "sadness", "surprise"],
+    },
+    "bert-base-event": {
+        "hf_id": "bert-base-uncased",
+        "output_dir": "models/onnx/bert-base-event",
+        "labels": ["listing", "delisting", "hack", "regulatory", "governance", 
+                  "upgrade", "partnership", "earnings", "macro", "liquidation", "whale", "manipulation"],
+    },
+    "minilm-l6-v2": {
+        "hf_id": "sentence-transformers/all-MiniLM-L6-v2",
+        "output_dir": "models/onnx/minilm-l6-v2",
+        "labels": None,
+    },
+}
+
+def export_model(model_key: str, quantize: bool = False) -> None:
+    """Export a single model to ONNX"""
+    config = MODELS[model_key]
+    output_dir = Path(config["output_dir"])
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    print(f"Exporting {model_key} ({config['hf_id']}) to {output_dir}...")
+
+    if config["labels"] is None:
+        # For sentence transformers / feature extraction
+        from sentence_transformers import SentenceTransformer
+        from transformers import AutoModel
+        
+        hf_model = AutoModel.from_pretrained(config["hf_id"])
+        hf_model.eval()
+        
+        # Create dummy input
+        dummy_input = {
+            "input_ids": torch.ones(1, 128, dtype=torch.long),
+            "attention_mask": torch.ones(1, 128, dtype=torch.long),
+        }
+        
+        # Export to ONNX
+        torch.onnx.export(
+            hf_model,
+            (dummy_input["input_ids"], dummy_input["attention_mask"]),
+            output_dir / "model.onnx",
+            input_names=["input_ids", "attention_mask"],
+            output_names=["last_hidden_state", "pooler_output"],
+            dynamic_axes={
+                "input_ids": {0: "batch", 1: "sequence"},
+                "attention_mask": {0: "batch", 1: "sequence"},
+                "last_hidden_state": {0: "batch", 1: "sequence"},
+            },
+            opset_version=14,
+        )
+        print(f"  Exported feature extraction model")
+        
+        # Save tokenizer
+        tokenizer = AutoTokenizer.from_pretrained(config["hf_id"])
+        tokenizer.save_pretrained(output_dir)
+        
+    else:
+        # For classification models - export using optimum
+        model = ORTModelForSequenceClassification.from_pretrained(
+            config["hf_id"],
+            export=True,
+        )
+        model.save_pretrained(output_dir)
+
+        # Save tokenizer
+        tokenizer = AutoTokenizer.from_pretrained(config["hf_id"])
+        tokenizer.save_pretrained(output_dir)
+
+        # Save label mapping
+        import json
+        with open(output_dir / "label_map.json", "w") as f:
+            json.dump({i: label for i, label in enumerate(config["labels"])}, f)
+
+    if quantize:
+        print(f"  Quantizing {model_key}...")
+        from optimum.onnxruntime import ORTOptimizer
+        from optimum.onnxruntime.configuration import OptimizationConfig
+        
+        optimizer = ORTOptimizer.from_pretrained(output_dir)
+        optimization_config = OptimizationConfig(
+            optimization_level=99,
+            optimize_for_gpu=torch.cuda.is_available(),
+        )
+        optimizer.optimize(save_dir=output_dir / "quantized", optimization_config=optimization_config)
+        print(f"  Quantized model saved to {output_dir}/quantized")
+
+    print(f"  Done: {model_key}")
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Export models to ONNX")
+    parser.add_argument("--models", nargs="+", choices=list(MODELS.keys()) + ["all"],
+                       default=["all"], help="Models to export")
+    parser.add_argument("--quantize", action="store_true", help="Quantize models")
+    
+    args = parser.parse_args()
+
+    models_to_export = list(MODELS.keys()) if "all" in args.models else args.models
+
+    for model_key in models_to_export:
+        try:
+            export_model(model_key, quantize=args.quantize)
+        except Exception as e:
+            print(f"  ERROR exporting {model_key}: {e}")
+
+    print("\nAll exports complete!")
+
+
+if __name__ == "__main__":
+    main()

sentiment_engine/scripts/export_onnx_local.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+"""Export locally fine-tuned Hugging Face models to ONNX format for production inference"""
+
+import os
+from pathlib import Path
+
+import torch
+from optimum.onnxruntime import ORTModelForSequenceClassification
+from transformers import AutoTokenizer, AutoConfig
+
+# Local fine-tuned model paths
+MODELS = {
+    "finbert": {
+        "local_path": "/mnt/dolphinng5_predict/sentiment_engine/models/finbert-crypto-sentiment",
+        "output_dir": "/mnt/dolphinng5_predict/sentiment_engine/models/onnx/finbert",
+        "labels": ["Bearish", "Bullish", "Neutral"],
+        "id2label": {0: "Bearish", 1: "Bullish", 2: "Neutral"},
+    },
+    "bert-base-event": {
+        "local_path": "/mnt/dolphinng5_predict/sentiment_engine/models/bert-crypto-events",
+        "output_dir": "/mnt/dolphinng5_predict/sentiment_engine/models/onnx/bert-base-event",
+        "labels": ["listing", "delisting", "hack", "regulatory", "governance", 
+                  "upgrade", "partnership", "earnings", "macro", "liquidation", "whale", "manipulation"],
+        "id2label": {i: l for i, l in enumerate([
+            "listing", "delisting", "hack", "regulatory", "governance", 
+            "upgrade", "partnership", "earnings", "macro", "liquidation", "whale", "manipulation"
+        ])},
+    },
+    "distilroberta-emotion": {
+        "local_path": "/mnt/dolphinng5_predict/sentiment_engine/models/distilroberta-crypto-emotion",
+        "output_dir": "/mnt/dolphinng5_predict/sentiment_engine/models/onnx/distilroberta-emotion",
+        "labels": ["joy", "fear", "anger", "greed", "sadness", "neutral"],
+        "id2label": {i: l for i, l in enumerate(["joy", "fear", "anger", "greed", "sadness", "neutral"])},
+    },
+    "minilm-l6-v2": {
+        "local_path": "/mnt/dolphinng5_predict/sentiment_engine/models/finbert-crypto-sentiment",  # Use finbert tokenizer
+        "output_dir": "/mnt/dolphinng5_predict/sentiment_engine/models/onnx/minilm-l6-v2",
+        "labels": None,
+        "id2label": None,
+    },
+}
+
+def export_classification_model(model_key: str) -> None:
+    """Export a local classification model to ONNX"""
+    config = MODELS[model_key]
+    local_path = config["local_path"]
+    output_dir = Path(config["output_dir"])
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    print(f"Exporting {model_key} from {local_path} to {output_dir}...")
+
+    # Load model config to check problem type
+    model_config = AutoConfig.from_pretrained(local_path)
+    is_multilabel = getattr(model_config, "problem_type", None) == "multi_label_classification"
+    
+    print(f"  Problem type: {getattr(model_config, 'problem_type', 'single_label')}")
+    print(f"  Labels: {config['labels']}")
+
+    # Load model and export using optimum
+    model = ORTModelForSequenceClassification.from_pretrained(
+        local_path,
+        export=True,
+    )
+    model.save_pretrained(output_dir)
+
+    # Save tokenizer
+    tokenizer = AutoTokenizer.from_pretrained(local_path)
+    tokenizer.save_pretrained(output_dir)
+
+    # Save label mapping
+    import json
+    if config["labels"]:
+        with open(output_dir / "label_map.json", "w") as f:
+            json.dump({i: label for i, label in enumerate(config["labels"])}, f)
+        with open(output_dir / "id2label.json", "w") as f:
+            json.dump(config["id2label"], f)
+
+    print(f"  Done: {model_key}")
+
+def export_feature_extraction_model(model_key: str) -> None:
+    """Export a feature extraction model to ONNX"""
+    config = MODELS[model_key]
+    local_path = config["local_path"]
+    output_dir = Path(config["output_dir"])
+    output_dir.mkdir(parents=True, exist_ok=True)
+
+    print(f"Exporting {model_key} (feature extraction) from {local_path} to {output_dir}...")
+
+    from transformers import AutoModel
+    
+    # For sentence transformers / feature extraction
+    hf_model = AutoModel.from_pretrained(local_path)
+    hf_model.eval()
+    
+    # Create dummy input
+    dummy_input = {
+        "input_ids": torch.ones(1, 128, dtype=torch.long),
+        "attention_mask": torch.ones(1, 128, dtype=torch.long),
+    }
+    
+    # Export to ONNX
+    torch.onnx.export(
+        hf_model,
+        (dummy_input["input_ids"], dummy_input["attention_mask"]),
+        output_dir / "model.onnx",
+        input_names=["input_ids", "attention_mask"],
+        output_names=["last_hidden_state", "pooler_output"],
+        dynamic_axes={
+            "input_ids": {0: "batch", 1: "sequence"},
+            "attention_mask": {0: "batch", 1: "sequence"},
+            "last_hidden_state": {0: "batch", 1: "sequence"},
+        },
+        opset_version=14,
+    )
+    print(f"  Exported feature extraction model")
+    
+    # Save tokenizer
+    tokenizer = AutoTokenizer.from_pretrained(local_path)
+    tokenizer.save_pretrained(output_dir)
+
+    print(f"  Done: {model_key}")
+
+def main():
+    print("="*60)
+    print("EXPORTING FINE-TUNED MODELS TO ONNX")
+    print("="*60)
+    
+    # Export classification models
+    for model_key in ["finbert", "bert-base-event", "distilroberta-emotion"]:
+        try:
+            export_classification_model(model_key)
+        except Exception as e:
+            print(f"  ERROR exporting {model_key}: {e}")
+            import traceback
+            traceback.print_exc()
+    
+    # Export feature extraction model (MiniLM)
+    try:
+        export_feature_extraction_model("minilm-l6-v2")
+    except Exception as e:
+        print(f"  ERROR exporting minilm-l6-v2: {e}")
+        import traceback
+        traceback.print_exc()
+    
+    print("\n" + "="*60)
+    print("ALL EXPORTS COMPLETE!")
+    print("="*60)
+
+if __name__ == "__main__":
+    main()

sentiment_engine/scripts/populate_catalogue.py

@@ -0,0 +1,469 @@
+#!/usr/bin/env python3
+"""Populate DuckDB Source Catalogue from YAML config - standalone version"""
+
+import asyncio
+import sys
+import yaml
+from pathlib import Path
+from datetime import datetime
+from enum import Enum
+from typing import Dict, List, Optional, Any
+from uuid import uuid4
+import duckdb
+import json
+
+# ========== Minimal definitions (copied from store.py) ==========
+
+class ConnectorType(str, Enum):
+    RSS = "rss"
+    REST_API = "rest_api"
+    TWITTER = "twitter"
+    REDDIT = "reddit"
+    DISCORD = "discord"
+    TELEGRAM = "telegram"
+    WEB_CRAWL = "web_crawl"
+
+
+class SourceCatalogue:
+    """DuckDB-backed operational source catalogue"""
+
+    def __init__(self, db_path: str = "data/sources.duckdb"):
+        self.db_path = Path(db_path)
+        self.db_path.parent.mkdir(parents=True, exist_ok=True)
+        self._conn = duckdb.connect(str(self.db_path))
+        self._init_db()
+
+    def _init_db(self) -> None:
+        conn = self._conn
+        
+        conn.execute("""
+            CREATE TABLE IF NOT EXISTS sources (
+                source_id VARCHAR PRIMARY KEY,
+                name VARCHAR NOT NULL,
+                connector_type VARCHAR NOT NULL,
+                base_url VARCHAR,
+                config JSON NOT NULL DEFAULT '{}',
+                credentials_ref VARCHAR,
+                base_credibility DOUBLE NOT NULL DEFAULT 0.5,
+                relevance DOUBLE NOT NULL DEFAULT 0.5,
+                enabled BOOLEAN NOT NULL DEFAULT TRUE,
+                cadence_seconds INTEGER NOT NULL DEFAULT 300,
+                timeout_seconds INTEGER NOT NULL DEFAULT 30,
+                max_retries INTEGER NOT NULL DEFAULT 3,
+                schema_version INTEGER NOT NULL DEFAULT 1,
+                config_schema JSON NOT NULL DEFAULT '{}',
+                status VARCHAR NOT NULL DEFAULT 'unknown',
+                last_fetch_ts DOUBLE,
+                last_success_ts DOUBLE,
+                last_error VARCHAR,
+                total_fetches INTEGER NOT NULL DEFAULT 0,
+                successful_fetches INTEGER NOT NULL DEFAULT 0,
+                error_count INTEGER NOT NULL DEFAULT 0,
+                consecutive_errors INTEGER NOT NULL DEFAULT 0,
+                current_credibility DOUBLE NOT NULL DEFAULT 0.5,
+                credibility_updated_ts DOUBLE,
+                created_ts DOUBLE NOT NULL,
+                updated_ts DOUBLE NOT NULL,
+                created_by VARCHAR NOT NULL DEFAULT 'system',
+                tags VARCHAR[] NOT NULL DEFAULT [],
+                metadata JSON NOT NULL DEFAULT '{}',
+                -- Rate limiting fields
+                rate_limit_rps DOUBLE DEFAULT 1.0,
+                rate_limit_rpm INTEGER DEFAULT 60,
+                rate_limit_burst INTEGER DEFAULT 5,
+                -- Desirable query timing
+                preferred_query_windows JSON DEFAULT '[]',
+                avoid_query_windows JSON DEFAULT '[]',
+                query_jitter_seconds INTEGER DEFAULT 30,
+                -- Backoff/retry
+                backoff_base_seconds DOUBLE DEFAULT 2.0,
+                backoff_max_seconds DOUBLE DEFAULT 300.0,
+                backoff_multiplier DOUBLE DEFAULT 2.0,
+                -- Concurrency
+                max_concurrent_requests INTEGER DEFAULT 1,
+                -- Health thresholds
+                max_latency_ms INTEGER DEFAULT 10000,
+                min_success_rate DOUBLE DEFAULT 0.8
+            )
+        """)
+
+        conn.execute("""
+            CREATE TABLE IF NOT EXISTS source_schemas (
+                connector_type VARCHAR NOT NULL,
+                version INTEGER NOT NULL,
+                config_schema JSON NOT NULL,
+                payload_schema JSON NOT NULL,
+                required_credentials VARCHAR[] NOT NULL DEFAULT [],
+                min_cadence_seconds INTEGER NOT NULL,
+                max_cadence_seconds INTEGER NOT NULL,
+                min_rate_limit_rps DOUBLE DEFAULT 0.1,
+                max_rate_limit_rps DOUBLE DEFAULT 10.0,
+                created_ts DOUBLE NOT NULL,
+                PRIMARY KEY (connector_type, version)
+            )
+        """)
+
+        conn.execute("""
+            CREATE TABLE IF NOT EXISTS fetch_history (
+                id BIGINT PRIMARY KEY,
+                source_id VARCHAR NOT NULL,
+                fetch_ts DOUBLE NOT NULL,
+                success BOOLEAN NOT NULL,
+                latency_ms DOUBLE,
+                items_fetched INTEGER NOT NULL DEFAULT 0,
+                error_message VARCHAR,
+                payload_sample JSON,
+                http_status INTEGER,
+                rate_limited BOOLEAN DEFAULT FALSE,
+                FOREIGN KEY (source_id) REFERENCES sources(source_id)
+            )
+        """)
+
+        conn.execute("""
+            CREATE TABLE IF NOT EXISTS credibility_history (
+                id BIGINT PRIMARY KEY,
+                source_id VARCHAR NOT NULL,
+                ts DOUBLE NOT NULL,
+                old_credibility DOUBLE NOT NULL,
+                new_credibility DOUBLE NOT NULL,
+                reason VARCHAR,
+                event_id VARCHAR,
+                FOREIGN KEY (source_id) REFERENCES sources(source_id)
+            )
+        """)
+
+        conn.execute("CREATE SEQUENCE IF NOT EXISTS fetch_history_id START 1")
+        conn.execute("CREATE SEQUENCE IF NOT EXISTS credibility_history_id START 1")
+
+        # Default schemas
+        self._load_default_schemas()
+
+        # Indexes
+        conn.execute("CREATE INDEX IF NOT EXISTS idx_sources_connector_type ON sources(connector_type)")
+        conn.execute("CREATE INDEX IF NOT EXISTS idx_sources_enabled ON sources(enabled)")
+        conn.execute("CREATE INDEX IF NOT EXISTS idx_sources_status ON sources(status)")
+        conn.execute("CREATE INDEX IF NOT EXISTS idx_fetch_history_source_ts ON fetch_history(source_id, fetch_ts)")
+        conn.execute("CREATE INDEX IF NOT EXISTS idx_credibility_history_source_ts ON credibility_history(source_id, ts)")
+
+    def _load_default_schemas(self) -> None:
+        conn = self._conn
+        default_schemas = {
+            "rss": {"config_schema": {"type": "object", "properties": {"feed_urls": {"type": "array", "items": {"type": "string"}}, "max_items_per_feed": {"type": "integer"}, "poll_interval_seconds": {"type": "integer"}}, "required": ["feed_urls"]}, "payload_schema": {"type": "object", "properties": {"title": {"type": "string"}, "summary": {"type": "string"}, "link": {"type": "string"}, "published_parsed": {"type": "array"}, "author": {"type": "string"}}}, "required_credentials": [], "min_cadence": 60, "max_cadence": 3600, "min_rate": 0.01, "max_rate": 1.0},
+            "rest_api": {"config_schema": {"type": "object", "properties": {"base_url": {"type": "string"}, "endpoints": {"type": "array"}, "auth_type": {"type": "string"}, "headers": {"type": "object"}, "poll_interval_seconds": {"type": "integer"}}, "required": ["base_url", "endpoints"]}, "payload_schema": {"type": "object"}, "required_credentials": ["api_key"], "min_cadence": 60, "max_cadence": 3600, "min_rate": 0.1, "max_rate": 10.0},
+            "twitter": {"config_schema": {"type": "object", "properties": {"stream_rules": {"type": "array"}, "sample_rate": {"type": "number"}}, "required": ["stream_rules"]}, "payload_schema": {"type": "object", "properties": {"text": {"type": "string"}, "created_at": {"type": "string"}, "author_id": {"type": "string"}, "public_metrics": {"type": "object"}, "entities": {"type": "object"}, "lang": {"type": "string"}}}, "required_credentials": ["bearer_token", "api_key", "api_secret", "access_token", "access_secret"], "min_cadence": 0, "max_cadence": 0, "min_rate": 0.5, "max_rate": 50.0},
+            "reddit": {"config_schema": {"type": "object", "properties": {"subreddits": {"type": "array"}, "use_pushshift": {"type": "boolean"}, "poll_interval_seconds": {"type": "integer"}}, "required": ["subreddits"]}, "payload_schema": {"type": "object", "properties": {"title": {"type": "string"}, "selftext": {"type": "string"}, "author": {"type": "string"}, "created_utc": {"type": "number"}, "score": {"type": "integer"}, "num_comments": {"type": "integer"}, "permalink": {"type": "string"}, "link_flair_text": {"type": "string"}, "upvote_ratio": {"type": "number"}}}, "required_credentials": ["client_id", "client_secret"], "min_cadence": 30, "max_cadence": 600, "min_rate": 0.1, "max_rate": 30.0},
+            "discord": {"config_schema": {"type": "object", "properties": {"channel_ids": {"type": "array"}}, "required": ["channel_ids"]}, "payload_schema": {"type": "object", "properties": {"content": {"type": "string"}, "author": {"type": "object"}, "channel_id": {"type": "string"}, "guild_id": {"type": "string"}, "created_at": {"type": "string"}, "reactions": {"type": "array"}}}, "required_credentials": ["bot_token"], "min_cadence": 0, "max_cadence": 0, "min_rate": 0.5, "max_rate": 20.0},
+            "telegram": {"config_schema": {"type": "object", "properties": {"channel_usernames": {"type": "array"}}, "required": ["channel_usernames"]}, "payload_schema": {"type": "object", "properties": {"text": {"type": "string"}, "date": {"type": "string"}, "chat": {"type": "object"}, "from": {"type": "object"}, "views": {"type": "integer"}, "forward_count": {"type": "integer"}}}, "required_credentials": ["bot_token"], "min_cadence": 0, "max_cadence": 0, "min_rate": 0.5, "max_rate": 20.0},
+            "web_crawl": {"config_schema": {"type": "object", "properties": {"seed_urls": {"type": "array"}, "allowed_domains": {"type": "array"}, "max_depth": {"type": "integer"}, "rate_limit_rps": {"type": "number"}}, "required": ["seed_urls"]}, "payload_schema": {"type": "object", "properties": {"title": {"type": "string"}, "content": {"type": "string"}, "url": {"type": "string"}}}, "required_credentials": [], "min_cadence": 300, "max_cadence": 86400, "min_rate": 0.01, "max_rate": 2.0},
+        }
+        
+        for ctype, schema in default_schemas.items():
+            existing = conn.execute("SELECT 1 FROM source_schemas WHERE connector_type = ? AND version = 1", [ctype]).fetchone()
+            if not existing:
+                conn.execute("""
+                    INSERT INTO source_schemas (connector_type, version, config_schema, payload_schema, required_credentials, min_cadence_seconds, max_cadence_seconds, min_rate_limit_rps, max_rate_limit_rps, created_ts)
+                    VALUES (?, 1, ?, ?, ?, ?, ?, ?, ?, ?)
+                """, [ctype, json.dumps(schema["config_schema"]), json.dumps(schema["payload_schema"]), 
+                     schema["required_credentials"], schema["min_cadence"], schema["max_cadence"], schema["min_rate"], schema["max_rate"], datetime.now().timestamp()])
+
+    def create_source(self, **kwargs) -> None:
+        """Create source with all fields - uses named parameters"""
+        conn = self._conn
+        now = datetime.now().timestamp()
+        
+        # Extract all fields with defaults
+        source_id = kwargs.get("source_id", str(uuid4())[:8])
+        name = kwargs.get("name", "")
+        connector_type = kwargs.get("connector_type", "rss")
+        base_url = kwargs.get("base_url", "")
+        config = json.dumps(kwargs.get("config", {}))
+        credentials_ref = kwargs.get("credentials_ref")
+        base_credibility = kwargs.get("base_credibility", 0.5)
+        relevance = kwargs.get("relevance", 0.5)
+        enabled = kwargs.get("enabled", True)
+        cadence_seconds = kwargs.get("cadence_seconds", 300)
+        timeout_seconds = kwargs.get("timeout_seconds", 30)
+        max_retries = kwargs.get("max_retries", 3)
+        schema_version = kwargs.get("schema_version", 1)
+        config_schema = json.dumps(kwargs.get("config_schema", {}))
+        status = kwargs.get("status", "unknown")
+        last_fetch_ts = kwargs.get("last_fetch_ts")
+        last_success_ts = kwargs.get("last_success_ts")
+        last_error = kwargs.get("last_error")
+        total_fetches = kwargs.get("total_fetches", 0)
+        successful_fetches = kwargs.get("successful_fetches", 0)
+        error_count = kwargs.get("error_count", 0)
+        consecutive_errors = kwargs.get("consecutive_errors", 0)
+        current_credibility = kwargs.get("current_credibility", base_credibility)
+        credibility_updated_ts = kwargs.get("credibility_updated_ts", now)
+        created_ts = kwargs.get("created_ts", now)
+        updated_ts = kwargs.get("updated_ts", now)
+        created_by = kwargs.get("created_by", "system")
+        tags = json.dumps(kwargs.get("tags", []))
+        metadata = json.dumps(kwargs.get("metadata", {}))
+        # Rate limiting
+        rate_limit_rps = kwargs.get("rate_limit_rps", 1.0)
+        rate_limit_rpm = kwargs.get("rate_limit_rpm", 60)
+        rate_limit_burst = kwargs.get("rate_limit_burst", 5)
+        # Query timing
+        preferred_query_windows = json.dumps(kwargs.get("preferred_query_windows", []))
+        avoid_query_windows = json.dumps(kwargs.get("avoid_query_windows", []))
+        query_jitter_seconds = kwargs.get("query_jitter_seconds", 30)
+        # Backoff
+        backoff_base_seconds = kwargs.get("backoff_base_seconds", 2.0)
+        backoff_max_seconds = kwargs.get("backoff_max_seconds", 300.0)
+        backoff_multiplier = kwargs.get("backoff_multiplier", 2.0)
+        # Concurrency
+        max_concurrent_requests = kwargs.get("max_concurrent_requests", 1)
+        # Health
+        max_latency_ms = kwargs.get("max_latency_ms", 10000)
+        min_success_rate = kwargs.get("min_success_rate", 0.8)
+        
+        conn.execute("""
+            INSERT INTO sources (
+                source_id, name, connector_type, base_url, config, credentials_ref,
+                base_credibility, relevance, enabled, cadence_seconds, timeout_seconds,
+                max_retries, schema_version, config_schema, status,
+                last_fetch_ts, last_success_ts, last_error,
+                total_fetches, successful_fetches, error_count, consecutive_errors,
+                current_credibility, credibility_updated_ts,
+                created_ts, updated_ts, created_by, tags, metadata,
+                rate_limit_rps, rate_limit_rpm, rate_limit_burst,
+                preferred_query_windows, avoid_query_windows, query_jitter_seconds,
+                backoff_base_seconds, backoff_max_seconds, backoff_multiplier,
+                max_concurrent_requests, max_latency_ms, min_success_rate
+            ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
+        """, [
+            source_id, kwargs.get("name", ""), connector_type, base_url, config, credentials_ref,
+            base_credibility, relevance, enabled, cadence_seconds, timeout_seconds,
+            max_retries, schema_version, config_schema, status,
+            last_fetch_ts, last_success_ts, last_error,
+            total_fetches, successful_fetches, error_count, consecutive_errors,
+            current_credibility, credibility_updated_ts,
+            created_ts, updated_ts, kwargs.get("created_by", "system"), tags, metadata,
+            rate_limit_rps, rate_limit_rpm, rate_limit_burst,
+            preferred_query_windows, avoid_query_windows, query_jitter_seconds,
+            backoff_base_seconds, backoff_max_seconds, backoff_multiplier,
+            max_concurrent_requests, max_latency_ms, min_success_rate
+        ])
+        
+        # Initial credibility log
+        cid = conn.execute("SELECT nextval('credibility_history_id')").fetchone()[0]
+        conn.execute("""
+            INSERT INTO credibility_history (id, source_id, ts, old_credibility, new_credibility, reason, event_id)
+            VALUES (?, ?, ?, ?, ?, ?, ?)
+        """, [cid, source_id, now, 0.0, base_credibility, "initial", None])
+
+    def get_source(self, source_id: str) -> Optional[Dict]:
+        conn = self._conn
+        row = conn.execute("SELECT * FROM sources WHERE source_id = ?", [source_id]).fetchone()
+        if not row:
+            return None
+        cols = [desc[0] for desc in conn.description]
+        data = dict(zip(cols, row))
+        for field in ["config", "config_schema", "metadata", "preferred_query_windows", "avoid_query_windows", "tags"]:
+            if data.get(field) and isinstance(data[field], str):
+                data[field] = json.loads(data[field])
+        return data
+
+    def get_sources(self, connector_type: str = None, enabled_only: bool = False) -> List[Dict]:
+        conn = self._conn
+        query = "SELECT * FROM sources WHERE 1=1"
+        params = []
+        if connector_type:
+            query += " AND connector_type = ?"
+            params.append(connector_type)
+        if enabled_only:
+            query += " AND enabled = TRUE"
+        query += " ORDER BY updated_ts DESC"
+        rows = conn.execute(query, params).fetchall()
+        cols = [desc[0] for desc in conn.description]
+        results = []
+        for row in rows:
+            data = dict(zip(cols, row))
+            for field in ["config", "config_schema", "metadata", "preferred_query_windows", "avoid_query_windows", "tags"]:
+                if data.get(field) and isinstance(data[field], str):
+                    data[field] = json.loads(data[field])
+            results.append(data)
+        return results
+
+    def update_source(self, source_id: str, updates: Dict) -> None:
+        conn = self._conn
+        now = datetime.now().timestamp()
+        set_clauses = []
+        params = []
+        for key, value in updates.items():
+            if key in ["config", "config_schema", "tags", "metadata", "preferred_query_windows", "avoid_query_windows"]:
+                set_clauses.append(f"{key} = ?")
+                params.append(json.dumps(value))
+            else:
+                set_clauses.append(f"{key} = ?")
+                params.append(value)
+        set_clauses.append("updated_ts = ?")
+        params.append(now)
+        params.append(source_id)
+        conn.execute(f"UPDATE sources SET {', '.join(set_clauses)} WHERE source_id = ?", params)
+
+    def close(self) -> None:
+        if self._conn:
+            self._conn.close()
+
+
+# ========== Main population script ==========
+
+async def populate(config_path: str, db_path: str = "data/sources.duckdb"):
+    """Populate catalogue from YAML config"""
+    
+    with open(config_path) as f:
+        data = yaml.safe_load(f)
+    
+    sources = data.get("sources", [])
+    print(f"Loaded {len(sources)} sources from {config_path}")
+    
+    cat = SourceCatalogue(db_path)
+    
+    registered = 0
+    skipped = 0
+    errors = 0
+    
+    ctype_map = {
+        "rss": "rss",
+        "rest_api": "rest_api",
+        "twitter": "twitter",
+        "reddit": "reddit",
+        "discord": "discord",
+        "telegram": "telegram",
+        "web_crawl": "web_crawl",
+    }
+    
+    # Default config_schema per connector type
+    default_schemas = {
+        "rss": {"type": "object", "properties": {"feed_urls": {"type": "array"}, "max_items_per_feed": {"type": "integer"}, "poll_interval_seconds": {"type": "integer"}}, "required": ["feed_urls"]},
+        "rest_api": {"type": "object", "properties": {"base_url": {"type": "string"}, "endpoints": {"type": "array"}, "auth_type": {"type": "string"}, "headers": {"type": "object"}, "poll_interval_seconds": {"type": "integer"}}, "required": ["base_url", "endpoints"]},
+        "twitter": {"type": "object", "properties": {"stream_rules": {"type": "array"}, "sample_rate": {"type": "number"}}, "required": ["stream_rules"]},
+        "reddit": {"type": "object", "properties": {"subreddits": {"type": "array"}, "use_pushshift": {"type": "boolean"}, "poll_interval_seconds": {"type": "integer"}}, "required": ["subreddits"]},
+        "discord": {"type": "object", "properties": {"channel_ids": {"type": "array"}}, "required": ["channel_ids"]},
+        "telegram": {"type": "object", "properties": {"channel_usernames": {"type": "array"}}, "required": ["channel_usernames"]},
+        "web_crawl": {"type": "object", "properties": {"seed_urls": {"type": "array"}, "allowed_domains": {"type": "array"}, "max_depth": {"type": "integer"}, "rate_limit_rps": {"type": "number"}}, "required": ["seed_urls"]},
+    }
+    
+    for item in sources:
+        try:
+            ctype_str = item.get("connector_type", "").lower()
+            ctype = ctype_map.get(ctype_str)
+            if not ctype:
+                print(f"  ⚠️ Unknown connector type: {ctype_str} for {item.get('source_id')}")
+                errors += 1
+                continue
+            
+            source_id = item["source_id"]
+            existing = cat.get_source(source_id)
+            
+            if existing:
+                updates = {}
+                # Standard fields
+                for field in ["base_credibility", "relevance", "enabled", "timeout_seconds", "max_retries", "status"]:
+                    if field in item and item[field] != existing.get(field):
+                        updates[field] = item[field]
+                # Rate limiting fields
+                for field in ["rate_limit_rps", "rate_limit_rpm", "rate_limit_burst"]:
+                    if field in item and item[field] != existing.get(field):
+                        updates[field] = item[field]
+                # Query timing
+                for field in ["preferred_query_windows", "avoid_query_windows", "query_jitter_seconds"]:
+                    if field in item:
+                        updates[field] = json.dumps(item[field])
+                # Backoff
+                for field in ["backoff_base_seconds", "backoff_max_seconds", "backoff_multiplier"]:
+                    if field in item and item[field] != existing.get(field):
+                        updates[field] = item[field]
+                # Concurrency
+                for field in ["max_concurrent_requests"]:
+                    if field in item and item[field] != existing.get(field):
+                        updates[field] = item[field]
+                # Health
+                for field in ["max_latency_ms", "min_success_rate"]:
+                    if field in item and item[field] != existing.get(field):
+                        updates[field] = item[field]
+                
+                if updates:
+                    cat.update_source(source_id, updates)
+                    print(f"  🔄 Updated: {source_id}")
+                else:
+                    print(f"  ⏭️  Exists: {source_id}")
+                skipped += 1
+                continue
+            
+            # Build kwargs for create_source
+            create_kwargs = {
+                "source_id": source_id,
+                "name": item["name"],
+                "connector_type": ctype,
+                "base_url": item["base_url"],
+                "config": item.get("config", {}),
+                "base_credibility": item.get("base_credibility", 0.5),
+                "relevance": item.get("relevance", 0.5),
+                "enabled": item.get("enabled", True),
+                "cadence_seconds": item.get("config", {}).get("poll_interval_seconds", 300),
+                "tags": item.get("tags", []),
+                "credentials_ref": item.get("credentials_ref"),
+                "config_schema": default_schemas.get(ctype, {}),
+                "rate_limit_rps": item.get("rate_limit_rps", 1.0),
+                "rate_limit_rpm": item.get("rate_limit_rpm", 60),
+                "rate_limit_burst": item.get("rate_limit_burst", 5),
+                "preferred_query_windows": item.get("preferred_query_windows", []),
+                "avoid_query_windows": item.get("avoid_query_windows", []),
+                "query_jitter_seconds": item.get("query_jitter_seconds", 30),
+                "backoff_base_seconds": item.get("backoff_base_seconds", 2.0),
+                "backoff_max_seconds": item.get("backoff_max_seconds", 300.0),
+                "backoff_multiplier": item.get("backoff_multiplier", 2.0),
+                "max_concurrent_requests": item.get("max_concurrent_requests", 1),
+                "max_latency_ms": item.get("max_latency_ms", 10000),
+                "min_success_rate": item.get("min_success_rate", 0.8),
+            }
+            
+            cat.create_source(**create_kwargs)
+            print(f"  ✅ Registered: {source_id} - {item['name']}")
+            registered += 1
+            
+        except Exception as e:
+            print(f"  ❌ Error: {item.get('source_id', 'unknown')}: {e}")
+            import traceback
+            traceback.print_exc()
+            errors += 1
+    
+    print(f"\n{'='*50}")
+    print(f"SUMMARY")
+    print(f"{'='*50}")
+    print(f"Total in config: {len(sources)}")
+    print(f"Newly registered: {registered}")
+    print(f"Already existed: {skipped}")
+    print(f"Errors: {errors}")
+    print(f"Total in catalogue: {len(cat.get_sources())}")
+    
+    all_sources = cat.get_sources()
+    by_type = {}
+    for s in all_sources:
+        t = s["connector_type"]
+        by_type[t] = by_type.get(t, 0) + 1
+    print(f"\nBy connector type:")
+    for t, count in sorted(by_type.items()):
+        print(f"  {t}: {count}")
+    
+    # Print rate limiting summary
+    print(f"\nRate limiting summary:")
+    for s in all_sources:
+        if s.get("rate_limit_rps"):
+            print(f"  {s['source_id']:35} rps={s['rate_limit_rps']:.2f} rpm={s['rate_limit_rpm']} burst={s['rate_limit_burst']} concurrent={s['max_concurrent_requests']}")
+    
+    cat.close()
+
+
+if __name__ == "__main__":
+    import argparse
+    parser = argparse.ArgumentParser(description="Populate Source Catalogue from YAML")
+    parser.add_argument("--config", default="config/seed_sources.yaml", help="Path to seed sources YAML")
+    parser.add_argument("--db", default="data/sources.duckdb", help="DuckDB path")
+    args = parser.parse_args()
+    
+    asyncio.run(populate(args.config, args.db))

sentiment_engine/scripts/run_engine.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python3
+"""Script to run the sentiment engine (with optional TUI)"""
+
+import argparse
+import asyncio
+import sys
+from pathlib import Path
+
+# Add src to path
+sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
+
+from sentiment_engine.main import main
+from sentiment_engine.tui import run_tui
+
+
+async def run_both() -> None:
+    """Run both engine and TUI concurrently"""
+    from sentiment_engine.main import SentimentEngine
+
+    engine = SentimentEngine()
+    await engine.initialize()
+    await engine.start()
+
+    # Run TUI alongside
+    await run_tui()
+
+
+def main_entry():
+    parser = argparse.ArgumentParser(description="Sentiment Engine Runner")
+    parser.add_argument("--tui", action="store_true", help="Run with TUI dashboard")
+    parser.add_argument("--engine-only", action="store_true", help="Run engine only (no TUI)")
+    args = parser.parse_args()
+
+    if args.tui or (not args.engine_only and not args.tui):
+        # Default: run both
+        asyncio.run(run_both())
+    else:
+        asyncio.run(main())
+
+
+if __name__ == "__main__":
+    main_entry()

sentiment_engine/scripts/run_tui.py

@@ -0,0 +1,14 @@
+#!/usr/bin/env python3
+"""Script to run the Sentiment Engine TUI"""
+
+import asyncio
+import sys
+from pathlib import Path
+
+# Add src to path
+sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
+
+from sentiment_engine.tui import run_tui
+
+if __name__ == "__main__":
+    asyncio.run(run_tui())