#!/usr/bin/env python3 """ CPU-optimized FinBERT fine-tuning for crypto sentiment. """ import json import random from pathlib import Path from typing import List, Dict, Any from dataclasses import dataclass import torch import torch.nn as nn from torch.utils.data import DataLoader, Dataset from transformers import ( AutoTokenizer, AutoModelForSequenceClassification, TrainingArguments, Trainer, EarlyStoppingCallback ) from datasets import load_dataset from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score, f1_score from sklearn.utils.class_weight import compute_class_weight import numpy as np # ============================================================ # CPU-OPTIMIZED SETTINGS # ============================================================ CPU_CONFIG = { "batch_size": 8, "grad_accum": 8, "epochs": 3, "lr": 1.5e-5, "warmup_ratio": 0.1, "max_length": 128, "weight_decay": 0.01, "eval_strategy": "epoch", "save_strategy": "epoch", "dataloader_workers": 2, } SENTIMENT_LABELS = ["Bearish", "Bullish", "Neutral"] SENTIMENT_MAP = {"Bearish": 0, "Bullish": 1, "Neutral": 2} # ============================================================ # SENTIMENT TEMPLATES (module-level for augmentation) # ============================================================ SENTIMENT_TEMPLATES = { "Bullish": [ "{asset} surges to new highs", "{asset} breaks resistance at ${price}", "Institutional adoption drives {asset} higher", "{asset} breaks out bullish", "Massive {asset} accumulation by whales", "{asset} ETF approval drives massive inflows", "Golden cross confirmed on {asset} chart", ], "Bearish": [ "{asset} crashes {pct}%", "{asset} breaks support at ${price}", "Panic selling in {asset}", "{asset} faces massive sell pressure", "Whale dumps {amount} {asset}", "{asset} price drops {pct}% on bad news", "Support broken on {asset} chart", ], "Neutral": [ "{asset} consolidates at ${price}", "{asset} trades sideways", "Market waits for {asset} direction", "Low volatility in {asset}", "{asset} trades in tight range", ], } ASSETS = ["BTC", "ETH", "SOL", "AVAX", "MATIC", "DOT", "LINK", "UNI", "AAVE", "ARB"] SENTIMENT_LABELS = ["Bearish", "Bullish", "Neutral"] SENTIMENT_MAP = {"Bearish": 0, "Bullish": 1, "Neutral": 2} # ============================================================ # REAL CRYPTO EVENTS # ============================================================ REAL_EVENTS = [ {"text": "XRP bridge drained for $200,000 after software mistook fake deposits for real ones. An attacker created unbacked XRP on another blockchain, then exchanged it for real XRP held in reserve. The bridge has been halted and its operator has filed a complaint with the FBI.", "event_type": "hack", "entities": [{"asset": "XRP", "type": "TICKER"}], "sentiment": "Bearish", "emotions": {"fear": 0.9, "anger": 0.6, "sadness": 0.3}}, {"text": "Major hack on DeFi protocol drains $50M. Users panic as TVL collapses. Team promises investigation.", "event_type": "hack", "entities": [], "sentiment": "Bearish", "emotions": {"fear": 0.98, "anger": 0.3, "sadness": 0.5}}, {"text": "KuCoin Lists Catizen (CATI) for Spot Trading on September 20, 2024. Catizen (CATI), the native token of viral Telegram-based game Catizen AI, will officially begin spot trading on KuCoin.", "event_type": "listing", "entities": [{"asset": "CATI", "type": "TICKER"}, {"asset": "TON", "type": "CHAIN"}], "sentiment": "Bullish", "emotions": {"joy": 0.7, "greed": 0.5}}, {"text": "Bitfinex Among First Exchanges to List HMSTR, Native Token of Hamster Kombat, a popular play-to-earn game based on Telegram with more than 300 million users.", "event_type": "listing", "entities": [{"asset": "HMSTR", "type": "TICKER"}], "sentiment": "Bullish", "emotions": {"joy": 0.6, "greed": 0.4}}, {"text": "Binance Becomes First Exchange to List Trump-Linked WLFI Token. The exchange will open WLFI spot pairs against USDT and USDC, marking the token's shift from a non-transferable presale to full tradability.", "event_type": "listing", "entities": [{"asset": "WLFI", "type": "TICKER"}, {"asset": "BNB", "type": "TICKER"}], "sentiment": "Bullish", "emotions": {"joy": 0.5, "greed": 0.6, "fear": 0.2}}, {"text": "SEC files lawsuit against major exchange for unregistered securities. Market reacts with fear.", "event_type": "regulatory", "entities": [{"asset": "SEC", "type": "ORG"}], "sentiment": "Bearish", "emotions": {"fear": 0.97, "anger": 0.2}}, {"text": "CFTC files to dismiss CME's lawsuit over crypto perpetual futures.", "event_type": "regulatory", "entities": [{"asset": "CFTC", "type": "ORG"}, {"asset": "CME", "type": "EXCHANGE"}], "sentiment": "Neutral", "emotions": {"fear": 0.1, "joy": 0.2}}, {"text": "Michigan court orders Kalshi to keep blocking sports prediction markets.", "event_type": "regulatory", "entities": [{"asset": "Kalshi", "type": "EXCHANGE"}], "sentiment": "Bearish", "emotions": {"fear": 0.6, "anger": 0.3}}, {"text": "Ethereum Dencun upgrade activates Proto-Danksharding (EIP-4844), introducing temporary data blobs for cheaper rollup storage.", "event_type": "upgrade", "entities": [{"asset": "ETH", "type": "TICKER"}, {"asset": "Ethereum", "type": "PROTOCOL"}], "sentiment": "Bullish", "emotions": {"joy": 0.7, "greed": 0.3, "fear": 0.1}}, {"text": "Ethereum Shanghai upgrade goes live. Stakers can now withdraw. Validators celebrate.", "event_type": "upgrade", "entities": [{"asset": "ETH", "type": "TICKER"}], "sentiment": "Bullish", "emotions": {"joy": 0.8, "greed": 0.4}}, {"text": "Ethereum Cancun upgrade goes live. EIP-4844 introduces Proto-Danksharding with data blobs for cheaper L2 storage.", "event_type": "upgrade", "entities": [{"asset": "ETH", "type": "TICKER"}], "sentiment": "Bullish", "emotions": {"joy": 0.7, "greed": 0.4}}, {"text": "JPMorganChase and Coinbase Launch Strategic Partnership to Make Buying Crypto Easier than Ever.", "event_type": "partnership", "entities": [{"asset": "JPM", "type": "ORG"}, {"asset": "COIN", "type": "TICKER"}], "sentiment": "Bullish", "emotions": {"joy": 0.8, "greed": 0.5}}, {"text": "Chainlink and Mastercard Partner to Enable Over 3 Billion Cardholders to Purchase Crypto Directly Onchain.", "event_type": "partnership", "entities": [{"asset": "LINK", "type": "TICKER"}, {"asset": "MA", "type": "TICKER"}], "sentiment": "Bullish", "emotions": {"joy": 0.8, "greed": 0.6}}, {"text": "PayPal and Coinbase Expand Partnership to Drive Innovation of Stablecoin-based Solutions.", "event_type": "partnership", "entities": [{"asset": "PYUSD", "type": "TICKER"}, {"asset": "COIN", "type": "TICKER"}, {"asset": "PYPL", "type": "TICKER"}], "sentiment": "Bullish", "emotions": {"joy": 0.7, "greed": 0.5}}, {"text": "Bitcoin whale moves $116 million in BTC after 11-year dormancy.", "event_type": "whale", "entities": [{"asset": "BTC", "type": "TICKER"}], "sentiment": "Neutral", "emotions": {"fear": 0.3, "greed": 0.2}}, {"text": "Ancient Bitcoin whale dormant for 11 years suddenly transfers $257,450,000 in BTC.", "event_type": "whale", "entities": [{"asset": "BTC", "type": "TICKER"}], "sentiment": "Neutral", "emotions": {"fear": 0.4, "greed": 0.3, "surprise": 0.8}}, {"text": "$1B in Bitcoin moves from Satoshi-era wallet after 14 years of inactivity.", "event_type": "whale", "entities": [{"asset": "BTC", "type": "TICKER"}], "sentiment": "Neutral", "emotions": {"fear": 0.5, "greed": 0.4, "surprise": 0.9}}, {"text": "Breaking: Fed pauses rate hikes. Bitcoin jumps 5% on dovish pivot.", "event_type": "macro", "entities": [{"asset": "BTC", "type": "TICKER"}, {"asset": "FED", "type": "ORG"}], "sentiment": "Bullish", "emotions": {"joy": 0.8, "greed": 0.7, "fear": 0.1}}, {"text": "Surprise nonfarm payrolls print sends Bitcoin back below 80K.", "event_type": "macro", "entities": [{"asset": "BTC", "type": "TICKER"}, {"asset": "FED", "type": "ORG"}], "sentiment": "Bearish", "emotions": {"fear": 0.8, "anger": 0.3}}, {"text": "Massive liquidation cascade wipes out $200M in longs. Funding rates flip negative.", "event_type": "liquidation", "entities": [{"asset": "BTC", "type": "TICKER"}], "sentiment": "Bearish", "emotions": {"fear": 0.9, "anger": 0.4, "sadness": 0.5}}, {"text": "Governance proposal passes with 95% approval. Treasury diversifies into stablecoins.", "event_type": "governance", "entities": [], "sentiment": "Bullish", "emotions": {"joy": 0.6, "greed": 0.3}}, {"text": "Bitcoin ETF inflows hit $731M, highest since January as BTC reclaims $80K.", "event_type": "earnings", "entities": [{"asset": "BTC", "type": "TICKER"}], "sentiment": "Bullish", "emotions": {"joy": 0.9, "greed": 0.8}}, {"text": "Coinbase Q2 earnings beat estimates. Revenue up 50% YoY.", "event_type": "earnings", "entities": [{"asset": "COIN", "type": "TICKER"}], "sentiment": "Bullish", "emotions": {"joy": 0.8, "greed": 0.6}}, {"text": "FOMO drives memecoin 500% in 24h. Degens aping in. Rug pull inevitable?", "event_type": "manipulation", "entities": [], "sentiment": "Bearish", "emotions": {"anger": 0.7, "fear": 0.6, "greed": 0.4}}, {"text": "Token buybacks are booming. But are they good for crypto projects?", "event_type": "manipulation", "entities": [], "sentiment": "Neutral", "emotions": {"fear": 0.3, "greed": 0.5}}, {"text": "Coinbase delists XRP after SEC lawsuit. Trading suspended.", "event_type": "delisting", "entities": [{"asset": "XRP", "type": "TICKER"}], "sentiment": "Bearish", "emotions": {"fear": 0.9, "anger": 0.8}}, ] SENTIMENT_LABELS = ["Bearish", "Bullish", "Neutral"] SENTIMENT_MAP = {"Bearish": 0, "Bullish": 1, "Neutral": 2} SENTIMENT_SAMPLES = [ ("BTC breaks $100k! New ATH!", "Bullish"), ("ETH to $10k by EOY, accumulate now", "Bullish"), ("Institutional inflows hit record high", "Bullish"), ("Bitcoin reaches new all-time high as institutional adoption accelerates", "Bullish"), ("Ethereum merge successful, staking rewards now live", "Bullish"), ("Massive ETF inflows drive Bitcoin to new highs", "Bullish"), ("Golden cross confirmed on Bitcoin weekly chart", "Bullish"), ("Institutional adoption drives Bitcoin higher", "Bullish"), ("ETF approval drives massive inflows", "Bullish"), ("Market is bullish on Bitcoin", "Bullish"), ("BTC crashes 50% in hours", "Bearish"), ("Exchange hacked, $100M stolen", "Bearish"), ("SEC sues major exchange", "Bearish"), ("Bitcoin crashes hard, panic selling everywhere", "Bearish"), ("Massive liquidation cascade wipes out $200M in longs", "Bearish"), ("VIX drops below 15 as market volatility decreases", "Bearish"), ("Whale sells 10000 BTC", "Bearish"), ("Bitcoin price drops 50%", "Bearish"), ("Support broken with bearish structure forming lower highs", "Bearish"), ("Panic selling and forced liquidation as margin calls hit", "Bearish"), ("BTC at $50k, ETH at $3k", "Neutral"), ("Market consolidating in range", "Neutral"), ("Bitcoin remains stable around $30k", "Neutral"), ("VIX drops below 15 as market volatility decreases", "Neutral"), ("Market consolidating with no clear direction", "Neutral"), ("Bitcoin price stable around $30k", "Neutral"), ("Consolidation phase continues", "Neutral"), ("Market in wait-and-see mode", "Neutral"), ("Sideways action continues", "Neutral"), ("Low volatility environment persists", "Neutral"), ] # ============================================================ # CPU-OPTIMIZED SETTINGS # ============================================================ CPU_CONFIG = { "batch_size": 8, "grad_accum": 8, "epochs": 3, "lr": 1.5e-5, "warmup_ratio": 0.1, "max_length": 128, "weight_decay": 0.01, "eval_strategy": "epoch", "save_strategy": "epoch", "dataloader_workers": 2, } SENTIMENT_LABELS = ["Bearish", "Bullish", "Neutral"] SENTIMENT_MAP = {"Bearish": 0, "Bullish": 1, "Neutral": 2} SENTIMENT_TEMPLATES = { "Bullish": [ "{asset} surges to new highs", "{asset} breaks resistance at ${price}", "Institutional adoption drives {asset} higher", "{asset} breaks out bullish", "Massive {asset} accumulation by whales", "{asset} ETF approval drives massive inflows", "Golden cross confirmed on {asset} chart", ], "Bearish": [ "{asset} crashes {pct}%", "{asset} breaks support at ${price}", "Panic selling in {asset}", "{asset} faces massive sell pressure", "Whale dumps {amount} {asset}", "{asset} price drops {pct}% on bad news", "Support broken on {asset} chart", ], "Neutral": [ "{asset} consolidates at ${price}", "{asset} trades sideways", "Market waits for {asset} direction", "Low volatility in {asset}", "{asset} trades in tight range", ], } ASSETS = ["BTC", "ETH", "SOL", "AVAX", "MATIC", "DOT", "LINK", "UNI", "AAVE", "ARB"] SENTIMENT_LABELS = ["Bearish", "Bullish", "Neutral"] SENTIMENT_MAP = {"Bearish": 0, "Bullish": 1, "Neutral": 2} class CryptoSentimentDataset(Dataset): def __init__(self, samples: List[Dict], tokenizer, max_length: int = 128): self.samples = samples self.tokenizer = tokenizer self.max_length = max_length def __len__(self): return len(self.samples) def __getitem__(self, idx): item = self.samples[idx] text = item["text"] label = item.get("label_id", item.get("label", 2)) if isinstance(label, str): label = SENTIMENT_MAP.get(label, 2) encoding = self.tokenizer( text, truncation=True, max_length=self.max_length, padding="max_length", return_tensors="pt" ) return { "input_ids": encoding["input_ids"].squeeze(0), "attention_mask": encoding["attention_mask"].squeeze(0), "labels": torch.tensor(label, dtype=torch.long) } def load_all_sentiment_data() -> List[Dict]: all_samples = [] # 1. Twitter Financial News print("Loading Twitter Financial News...") try: ds = load_dataset("zeroshot/twitter-financial-news-sentiment") label_map = {0: "Bearish", 1: "Bullish", 2: "Neutral"} for split in ["train", "validation"]: for item in load_dataset("zeroshot/twitter-financial-news-sentiment", split=split): all_samples.append({ "text": item["text"], "label": label_map[item["label"]], "label_id": item["label"], "source": "twitter_financial" }) print(f" Loaded {len([s for s in all_samples if s['source']=='twitter_financial'])} Twitter Financial samples") except Exception as e: print(f" Error loading Twitter Financial: {e}") # 2. FiQA print("Loading FiQA...") try: ds = load_dataset("explodinggradients/fiqa", "main") for split in ["train", "validation", "test"]: for item in load_dataset("explodinggradients/fiqa", "main", split=split): all_samples.append({ "text": item.get("question", "") + " " + item.get("answer", ""), "label": "Neutral", "label_id": 2, "source": "fiqa" }) print(f" Loaded FiQA samples") except Exception as e: print(f" Error loading FiQA: {e}") # 3. Add real crypto events for event in REAL_EVENTS: if event["sentiment"] in SENTIMENT_LABELS: all_samples.append({ "text": event["text"], "label": event["sentiment"], "label_id": SENTIMENT_MAP[event["sentiment"]], "source": "real_crypto_event" }) # 4. Add manual sentiment samples for text, label in SENTIMENT_SAMPLES: all_samples.append({ "text": text, "label": label, "label_id": SENTIMENT_MAP[label], "source": "manual_corpus" }) print(f"Total real samples: {len(all_samples)}") return all_samples def create_augmented_data(count: int = 3000) -> List[Dict]: data = [] for _ in range(count): sentiment = random.choice(["Bullish", "Bearish", "Neutral"]) asset = random.choice(ASSETS) template = random.choice(SENTIMENT_TEMPLATES[sentiment]) text = template.format( asset=asset, price=random.randint(100, 100000), pct=random.randint(10, 80), amount=f"{random.randint(1, 100)}K" ) data.append({ "text": text, "label": sentiment, "label_id": SENTIMENT_MAP[sentiment], "source": "synthetic" }) return data # ============================================================ # MAIN # ============================================================ if __name__ == "__main__": import torch import torch.nn as nn from torch.utils.data import Dataset from transformers import ( AutoTokenizer, AutoModelForSequenceClassification, TrainingArguments, Trainer, EarlyStoppingCallback ) from datasets import load_dataset from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score, f1_score from sklearn.utils.class_weight import compute_class_weight import numpy as np # ============================================================ # LOCAL CLASSES FOR MAIN # ============================================================ class CryptoSentimentDataset(Dataset): def __init__(self, samples: List[Dict], tokenizer, max_length: int = 128): self.samples = samples self.tokenizer = tokenizer self.max_length = max_length def __len__(self): return len(self.samples) def __getitem__(self, idx): item = self.samples[idx] text = item["text"] label = item.get("label_id", item.get("label", 2)) if isinstance(label, str): label = SENTIMENT_MAP.get(label, 2) encoding = self.tokenizer( text, truncation=True, max_length=self.max_length, padding="max_length", return_tensors="pt" ) return { "input_ids": encoding["input_ids"].squeeze(0), "attention_mask": encoding["attention_mask"].squeeze(0), "labels": torch.tensor(label, dtype=torch.long) } # ============================================================ # MAIN # ============================================================ print("=" * 60) print("FinBERT Crypto Sentiment Fine-Tuning (CPU Optimized)") print("=" * 60) print("\n1. Loading all sentiment data...") all_samples = load_all_sentiment_data() # Add augmented data print("\n2. Generating augmented data...") augmented = [] for item in create_augmented_data(3000): all_samples.append(item) print(f"Total samples: {len(all_samples)}") # Split train/val/test print("\n3. Creating train/val/test splits...") labels = [s["label_id"] for s in all_samples] train_samples, temp_samples = train_test_split( all_samples, test_size=0.3, random_state=42, stratify=labels ) temp_labels = [s["label_id"] for s in temp_samples] val_samples, test_samples = train_test_split( temp_samples, test_size=0.5, random_state=42, stratify=temp_labels ) print(f" Train: {len(train_samples)}, Val: {len(val_samples)}, Test: {len(test_samples)}") # Class weights train_labels = [s["label_id"] for s in train_samples] class_weights = compute_class_weight("balanced", classes=np.unique(train_labels), y=train_labels) class_weights = torch.tensor(class_weights, dtype=torch.float) print(f" Class weights: {class_weights}") # Tokenizer & Model print("\n4. Loading FinBERT...") tokenizer = AutoTokenizer.from_pretrained("ProsusAI/finbert") model = AutoModelForSequenceClassification.from_pretrained( "ProsusAI/finbert", num_labels=3, id2label={0: "Bearish", 1: "Bullish", 2: "Neutral"}, label2id={"Bearish": 0, "Bullish": 1, "Neutral": 2} ) # Create datasets train_dataset = CryptoSentimentDataset(train_samples, tokenizer, max_length=128) val_dataset = CryptoSentimentDataset(val_samples, tokenizer, max_length=128) test_dataset = CryptoSentimentDataset(test_samples, tokenizer, max_length=128) # Training arguments training_args = TrainingArguments( output_dir="./models/finbert-crypto-sentiment", num_train_epochs=3, per_device_train_batch_size=8, per_device_eval_batch_size=16, gradient_accumulation_steps=8, warmup_ratio=0.1, weight_decay=0.01, learning_rate=1.5e-5, lr_scheduler_type="cosine", evaluation_strategy="epoch", save_strategy="epoch", load_best_model_at_end=True, metric_for_best_model="f1_macro", greater_is_better=True, fp16=False, dataloader_num_workers=2, logging_steps=50, save_total_limit=2, remove_unused_columns=False, report_to="none", ) # Class weights class_weights = compute_class_weight("balanced", classes=np.unique(train_labels), y=train_labels) class_weights_tensor = torch.tensor(class_weights, dtype=torch.float) # Trainer trainer = Trainer( model=model, args=TrainingArguments( output_dir="./models/finbert-crypto-sentiment", num_train_epochs=3, per_device_train_batch_size=8, per_device_eval_batch_size=16, gradient_accumulation_steps=8, warmup_ratio=0.1, weight_decay=0.01, learning_rate=1.5e-5, lr_scheduler_type="cosine", evaluation_strategy="epoch", save_strategy="epoch", load_best_model_at_end=True, metric_for_best_model="f1_macro", greater_is_better=True, fp16=False, dataloader_num_workers=2, logging_steps=50, save_total_limit=2, remove_unused_columns=False, report_to="none", ), train_dataset=CryptoSentimentDataset(train_samples, tokenizer, max_length=128), eval_dataset=CryptoSentimentDataset(val_samples, tokenizer, max_length=128), tokenizer=tokenizer, compute_metrics=lambda eval_pred: { "accuracy": accuracy_score(eval_pred.label_ids, np.argmax(eval_pred.predictions, axis=-1)), "f1_macro": f1_score(eval_pred.label_ids, np.argmax(eval_pred.predictions, axis=-1), average="macro"), "f1_per_class": f1_score(eval_pred.label_ids, np.argmax(eval_pred.predictions, axis=-1), average=None).tolist() }, callbacks=[EarlyStoppingCallback(early_stopping_patience=2)] ) print("\n5. Starting training (CPU optimized)...") print(f" Effective batch size: 64") print(f" Epochs: 3") print(f" Max length: 128") trainer.train() # Evaluate print("\n6. Evaluating on test set...") test_results = trainer.evaluate() print(f"Test Results: {test_results}") # Save print("\n7. Saving model...") trainer.save_model("./models/finbert-crypto-sentiment-final") AutoTokenizer.from_pretrained("ProsusAI/finbert").save_pretrained("./models/finbert-crypto-sentiment-final") print("Model saved!") # Quick test print("\nQuick inference test...") model.eval() test_texts = [ "BTC surges to new all-time high!", "Bitcoin crashes 50% in panic selling", "BTC consolidates around $50k", ] for text in test_texts: inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=128, padding=True) with torch.no_grad(): outputs = model(**inputs) probs = torch.softmax(outputs.logits, dim=-1)[0] pred = torch.argmax(probs).item() polarity = probs[1].item() - probs[0].item() print(f" '{text[:50]}...' -> {SENTIMENT_LABELS[pred]} (polarity: {polarity:.3f})") print("\n✅ FinBERT fine-tuning complete!") if __name__ == "__main__": import torch import torch.nn as nn from torch.utils.data import Dataset from transformers import ( AutoTokenizer, AutoModelForSequenceClassification, TrainingArguments, Trainer, EarlyStoppingCallback ) from datasets import load_dataset from sklearn.model_selection import train_test_split from sklearn.metrics import accuracy_score, f1_score from sklearn.utils.class_weight import compute_class_weight import numpy as np # Dataset class (needs to be at module level for pickling) class CryptoSentimentDataset(Dataset): def __init__(self, samples: List[Dict], tokenizer, max_length: int = 128): self.samples = samples self.tokenizer = tokenizer self.max_length = max_length def __len__(self): return len(self.samples) def __getitem__(self, idx): item = self.samples[idx] text = item["text"] label = item.get("label_id", item.get("label", 2)) if isinstance(label, str): label = SENTIMENT_MAP.get(label, 2) encoding = AutoTokenizer.from_pretrained("ProsusAI/finbert")( text, truncation=True, max_length=128, padding="max_length", return_tensors="pt" ) return { "input_ids": encoding["input_ids"].squeeze(0), "attention_mask": encoding["attention_mask"].squeeze(0), "labels": torch.tensor(label, dtype=torch.long) } # Load data all_samples = [] def load_all_sentiment_data() -> List[Dict]: all_samples = [] # 1. Twitter Financial News print("Loading Twitter Financial News...") try: ds = load_dataset("zeroshot/twitter-financial-news-sentiment") label_map = {0: "Bearish", 1: "Bullish", 2: "Neutral"} for split in ["train", "validation"]: for item in load_dataset("zeroshot/twitter-financial-news-sentiment", split=split): all_samples.append({ "text": item["text"], "label": label_map[item["label"]], "label_id": item["label"], "source": "twitter_financial" }) print(f" Loaded {len([s for s in all_samples if s['source']=='twitter_financial'])} Twitter Financial samples") except Exception as e: print(f" Error loading Twitter Financial: {e}") # 2. FiQA print("Loading FiQA...") try: ds = load_dataset("explodinggradients/fiqa", "main") for split in ["train", "validation", "test"]: for item in load_dataset("explodinggradients/fiqa", "main", split=split): all_samples.append({ "text": item.get("question", "") + " " + item.get("answer", ""), "label": "Neutral", "label_id": 2, "source": "fiqa" }) print(f" Loaded FiQA samples") except Exception as e: print(f" Error loading FiQA: {e}") # 3. Real crypto events for event in REAL_EVENTS: if event["sentiment"] in SENTIMENT_LABELS: all_samples.append({ "text": event["text"], "label": event["sentiment"], "label_id": SENTIMENT_MAP[event["sentiment"]], "source": "real_crypto_event" }) # Manual samples for text, label in SENTIMENT_SAMPLES: all_samples.append({ "text": text, "label": label, "label_id": SENTIMENT_MAP[label], "source": "manual_corpus" }) print(f"Total real samples: {len(all_samples)}") return all_samples # Create augmented data def create_augmented_data(count: int = 3000) -> List[Dict]: data = [] for _ in range(count): sentiment = random.choice(["Bullish", "Bearish", "Neutral"]) asset = random.choice(ASSETS) template = random.choice(SENTIMENT_TEMPLATES[sentiment]) text = template.format( asset=asset, price=random.randint(100, 100000), pct=random.randint(10, 80), amount=f"{random.randint(1, 100)}K" ) data.append({ "text": text, "label": sentiment, "label_id": SENTIMENT_MAP[sentiment], "source": "synthetic" }) return data # Load all data all_samples = [] all_samples = load_all_sentiment_data() # Add augmented augmented = [] for item in create_augmented_data(3000): all_samples.append(item) print(f"Total samples: {len(all_samples)}") # Split labels = [s["label_id"] for s in all_samples] train_samples, temp_samples = train_test_split(all_samples, test_size=0.3, random_state=42, stratify=labels) temp_labels = [s["label_id"] for s in temp_samples] val_samples, test_samples = train_test_split(temp_samples, test_size=0.5, random_state=42, stratify=temp_labels) print(f"Train: {len(train_samples)}, Val: {len(val_samples)}, Test: {len(test_samples)}") # Class weights train_labels = [s["label_id"] for s in train_samples] class_weights = compute_class_weight("balanced", classes=np.unique(train_labels), y=train_labels) class_weights = torch.tensor(class_weights, dtype=torch.float) print(f" Class weights: {class_weights}") # Tokenizer & Model tokenizer = AutoTokenizer.from_pretrained("ProsusAI/finbert") model = AutoModelForSequenceClassification.from_pretrained( "ProsusAI/finbert", num_labels=3, id2label={0: "Bearish", 1: "Bullish", 2: "Neutral"}, label2id={"Bearish": 0, "Bullish": 1, "Neutral": 2} ) # Tokenizer for dataset class tokenizer = AutoTokenizer.from_pretrained("ProsusAI/finbert") # Datasets train_dataset = CryptoSentimentDataset(train_samples, tokenizer, max_length=128) val_dataset = CryptoSentimentDataset(val_samples, tokenizer, max_length=128) test_dataset = CryptoSentimentDataset(test_samples, tokenizer, max_length=128) # Training args training_args = TrainingArguments( output_dir="./models/finbert-crypto-sentiment", num_train_epochs=3, per_device_train_batch_size=8, per_device_eval_batch_size=16, gradient_accumulation_steps=8, warmup_ratio=0.1, weight_decay=0.01, learning_rate=1.5e-5, lr_scheduler_type="cosine", evaluation_strategy="epoch", save_strategy="epoch", load_best_model_at_end=True, metric_for_best_model="f1_macro", greater_is_better=True, fp16=False, dataloader_num_workers=2, logging_steps=50, save_total_limit=2, remove_unused_columns=False, report_to="none", ) # Trainer trainer = Trainer( model=model, args=TrainingArguments( output_dir="./models/finbert-crypto-sentiment", num_train_epochs=3, per_device_train_batch_size=8, per_device_eval_batch_size=16, gradient_accumulation_steps=8, warmup_ratio=0.1, weight_decay=0.01, learning_rate=1.5e-5, lr_scheduler_type="cosine", evaluation_strategy="epoch", save_strategy="epoch", load_best_model_at_end=True, metric_for_best_model="f1_macro", greater_is_better=True, fp16=False, dataloader_num_workers=2, logging_steps=50, save_total_limit=2, remove_unused_columns=False, report_to="none", ), train_dataset=CryptoSentimentDataset(train_samples, AutoTokenizer.from_pretrained("ProsusAI/finbert"), max_length=128), eval_dataset=CryptoSentimentDataset(val_samples, AutoTokenizer.from_pretrained("ProsusAI/finbert"), max_length=128), tokenizer=AutoTokenizer.from_pretrained("ProsusAI/finbert"), compute_metrics=lambda eval_pred: { "accuracy": accuracy_score(eval_pred.label_ids, np.argmax(eval_pred.predictions, axis=-1)), "f1_macro": f1_score(eval_pred.label_ids, np.argmax(eval_pred.predictions, axis=-1), average="macro"), "f1_per_class": f1_score(eval_pred.label_ids, np.argmax(eval_pred.predictions, axis=-1), average=None).tolist() }, callbacks=[EarlyStoppingCallback(early_stopping_patience=2)] ) print("\n5. Starting training (CPU optimized)...") print(f" Effective batch size: 64") print(f" Epochs: 3") print(f" Max length: 128") trainer.train() # Evaluate print("\nEvaluating on test set...") test_results = trainer.evaluate() print(f"Test Results: {test_results}") # Save trainer.save_model("./models/finbert-crypto-sentiment-final") AutoTokenizer.from_pretrained("ProsusAI/finbert").save_pretrained("./models/finbert-crypto-sentiment-final") print("Model saved!") # Quick test print("\nQuick inference test...") model.eval() test_texts = [ "BTC surges to new all-time high!", "Bitcoin crashes 50% in panic selling", "BTC consolidates around $50k", ] for text in test_texts: inputs = tokenizer(text, return_tensors="pt", truncation=True, max_length=128, padding=True) with torch.no_grad(): outputs = model(**inputs) probs = torch.softmax(outputs.logits, dim=-1)[0] pred = torch.argmax(probs).item() polarity = probs[1].item() - probs[0].item() print(f" '{text[:50]}...' -> {SENTIMENT_LABELS[pred]} (polarity: {polarity:.3f})") print("\n✅ FinBERT fine-tuning complete!")