#!/usr/bin/env python3 """ Add hard negative/positive examples that the current model gets wrong. """ import json from pathlib import Path HARD_EXAMPLES = [ # Current model gets these WRONG - needs correction { "text": "BTC breaks 100k! New ATH, institutional buying surging", "sentiment": "Bullish", "entities": ["BTC"], "source": "hard_positive", }, { "text": "Major hack on DeFi protocol, 50M drained from liquidity pools", "sentiment": "Bearish", "entities": ["DeFi"], "source": "hard_negative", }, { "text": "Rug pull suspected, dev wallet drained liquidity", "sentiment": "Bearish", "entities": ["token"], "source": "hard_negative", }, { "text": "SEC sues exchange for unregistered securities", "sentiment": "Bearish", "entities": ["SEC", "exchange"], "source": "hard_negative", }, { "text": "Major hack on DeFi protocol drains $50M from liquidity pools", "sentiment": "Bearish", "entities": ["DeFi"], "source": "hard_negative", }, { "text": "Panic selling BTC at 50k, liquidation cascade", "sentiment": "Bearish", "entities": ["BTC"], "source": "hard_negative", }, { "text": "HODL strong hands, diamond hands win", "sentiment": "Bullish", "entities": ["BTC"], "source": "hard_positive", }, { "text": "ETF approval sends Bitcoin to new highs", "sentiment": "Bullish", "entities": ["BTC"], "source": "hard_positive", }, { "text": "Whale accumulation pushes ETH above 3k", "sentiment": "Bullish", "entities": ["ETH"], "source": "hard_positive", }, { "text": "SEC sues exchange for unregistered securities, regulatory crackdown", "sentiment": "Bearish", "entities": ["SEC", "exchange"], "source": "hard_negative", }, { "text": "Rug pull suspected on new memecoin, dev wallet drains liquidity", "sentiment": "Bearish", "entities": ["memecoin"], "source": "hard_negative", }, { "text": "Panic selling as Bitcoin drops below $50K support", "sentiment": "Bearish", "entities": ["BTC"], "source": "hard_negative", }, { "text": "FOMO buying drives PEPE to new ATH, experts warn of correction", "sentiment": "Bullish", "entities": ["PEPE"], "source": "hard_positive", }, { "text": "New ETF approved for Solana, price surges 20%", "sentiment": "Bullish", "entities": ["SOL"], "source": "hard_positive", }, { "text": "Rug pull suspected on new memecoin, dev wallet drained liquidity", "sentiment": "Bearish", "entities": ["memecoin"], "source": "hard_negative", }, { "text": "HODL strategy pays off as long-term holders profit", "sentiment": "Bullish", "entities": ["BTC"], "source": "hard_positive", }, # More hard examples for boundary cases { "text": "Major exchange hack suspected, $100M in BTC moved to unknown wallets", "sentiment": "Bearish", "entities": ["BTC"], "source": "hard_negative", }, { "text": "Coinbase to delist 5 tokens including REN, BAND, MANA, CVC, ALGO", "sentiment": "Bearish", "entities": ["REN", "BAND", "MANA", "CVC", "ALGO"], "source": "hard_negative", }, { "text": "Ethereum ETF outflows hit $280M as Grayscale ETHE bleeds", "sentiment": "Bearish", "entities": ["ETH", "Grayscale"], "source": "hard_negative", }, { "text": "MicroStrategy adds 12,000 BTC, total holdings exceed 252,000 BTC", "sentiment": "Bullish", "entities": ["BTC", "MicroStrategy"], "source": "hard_positive", }, { "text": "Binance deal gives Circle a boost in stablecoin race with Tether", "sentiment": "Bullish", "entities": ["Circle", "Tether", "Binance"], "source": "hard_positive", }, { "text": "Major DeFi hack drains $50M from liquidity pools, users panic", "sentiment": "Bearish", "entities": ["DeFi"], "source": "hard_negative", }, { "text": "Ethereum Layer 2 adoption hits record high, Arbitrum and Optimism lead", "sentiment": "Bullish", "entities": ["ETH", "Arbitrum", "Optimism"], "source": "hard_positive", }, { "text": "SEC sues Binance for unregistered securities, BNB drops 15%", "sentiment": "Bearish", "entities": ["BNB", "Binance", "SEC"], "source": "hard_negative", }, { "text": "Solana outage halts network for 5 hours, SOL drops 10%", "sentiment": "Bearish", "entities": ["SOL", "Solana"], "source": "hard_negative", }, { "text": "New ETF approved for Solana, price surges 20% on launch", "sentiment": "Bullish", "entities": ["SOL"], "source": "hard_positive", }, { "text": "Rug pull suspected on new memecoin, dev wallet drains liquidity", "sentiment": "Bearish", "entities": ["memecoin"], "source": "hard_negative", }, ] # Load existing augmented with open("/mnt/dolphinng5_predict/sentiment_engine/data/final_augmented_set.jsonl") as f: data = [json.loads(line) for line in open("/mnt/dolphinng5_predict/sentiment_engine/data/final_augmented_set.jsonl")] # Add hard examples existing_texts = set(d["text"] for d in data) added = 0 for ex in HARD_EXAMPLES: if ex["text"] not in [d["text"] for d in data]: data.append({ "text": ex["text"], "sentiment": ex["sentiment"], "event_type": "price_action", "entities": ex["entities"], "source": ex["source"], }) added += 1 print(f"Added {added} hard examples") print(f"Total: {len(data)} samples") # Save with open("/mnt/dolphinng5_predict/sentiment_engine/data/final_labeled_complete.jsonl", "w") as f: for d in data: json.dump(d, f) f.write("\n") # Stats from collections import Counter dist = Counter(d["sentiment"] for d in data) print(f"Final distribution: {dict(dist)}")