621 lines
30 KiB
Python
621 lines
30 KiB
Python
|
#!/usr/bin/env python3
|
||
|
|
"""
|
||
|
|
Fine-tune all 3 models using the labeled data from labeling pipeline.
|
||
|
|
"""
|
||
|
|
|
||
|
|
import json
|
||
|
|
import random
|
||
|
|
import torch
|
||
|
|
import numpy as np
|
||
|
|
from pathlib import Path
|
||
|
|
from typing import List, Dict
|
||
|
|
from torch.utils.data import Dataset
|
||
|
|
from transformers import (
|
||
|
|
AutoTokenizer, AutoModelForSequenceClassification,
|
||
|
|
TrainingArguments, Trainer, EarlyStoppingCallback
|
||
|
|
)
|
||
|
|
from datasets import load_dataset
|
||
|
|
from sklearn.model_selection import train_test_split
|
||
|
|
from sklearn.metrics import accuracy_score, f1_score
|
||
|
|
from sklearn.utils.class_weight import compute_class_weight
|
||
|
|
import torch.nn as nn
|
||
|
|
|
||
|
|
# ============================================================
|
||
|
|
# LABELS & CONSTANTS
|
||
|
|
# ============================================================
|
||
|
|
|
||
|
|
SENTIMENT_LABELS = ["Bearish", "Bullish", "Neutral"]
|
||
|
|
SENTIMENT_MAP = {"Bearish": 0, "Bullish": 1, "Neutral": 2}
|
||
|
|
|
||
|
|
EMOTION_LABELS = ["joy", "fear", "anger", "greed", "sadness", "neutral"]
|
||
|
|
EMOTION_MAP = {l: i for i, l in enumerate(EMOTION_LABELS)}
|
||
|
|
|
||
|
|
EVENT_LABELS = [
|
||
|
|
"listing", "delisting", "hack", "regulatory", "governance",
|
||
|
|
"upgrade", "partnership", "earnings", "macro",
|
||
|
|
"liquidation", "whale", "manipulation"
|
||
|
|
]
|
||
|
|
EVENT_MAP = {l: i for i, l in enumerate(EVENT_LABELS)}
|
||
|
|
|
||
|
|
REAL_EVENTS = [
|
||
|
|
{"text": "XRP bridge drained for $200,000 after software mistook fake deposits for real ones. An attacker created unbacked XRP on another blockchain, then exchanged it for real XRP held in reserve. The bridge has been halted and its operator has filed a complaint with the FBI.", "label_id": 0, "event_type": "hack"},
|
||
|
|
{"text": "Major hack on DeFi protocol drains $50M. Users panic as TVL collapses. Team promises investigation.", "label_id": 0, "event_type": "hack"},
|
||
|
|
{"text": "KuCoin Lists Catizen (CATI) for Spot Trading on September 20, 2024.", "label_id": 1, "event_type": "listing"},
|
||
|
|
{"text": "Bitfinex Among First Exchanges to List HMSTR, Native Token of Hamster Kombat.", "label_id": 1, "event_type": "listing"},
|
||
|
|
{"text": "Binance Becomes First Exchange to List Trump-Linked WLFI Token.", "label_id": 1, "event_type": "listing"},
|
||
|
|
{"text": "SEC files lawsuit against major exchange for unregistered securities.", "label_id": 0, "event_type": "regulatory"},
|
||
|
|
{"text": "CFTC files to dismiss CME's lawsuit over crypto perpetual futures.", "label_id": 2, "event_type": "regulatory"},
|
||
|
|
{"text": "Michigan court orders Kalshi to keep blocking sports prediction markets.", "label_id": 0, "event_type": "regulatory"},
|
||
|
|
{"text": "Ethereum Dencun upgrade activates Proto-Danksharding (EIP-4844).", "label_id": 1, "event_type": "upgrade"},
|
||
|
|
{"text": "Ethereum Shanghai upgrade goes live. Stakers can now withdraw.", "label_id": 1, "event_type": "upgrade"},
|
||
|
|
{"text": "JPMorganChase and Coinbase Launch Strategic Partnership.", "label_id": 1, "event_type": "partnership"},
|
||
|
|
{"text": "Chainlink and Mastercard Partner to Enable Over 3 Billion Cardholders.", "label_id": 1, "event_type": "partnership"},
|
||
|
|
{"text": "PayPal and Coinbase Expand Partnership to Drive Innovation.", "label_id": 1, "event_type": "partnership"},
|
||
|
|
{"text": "Bitcoin whale moves $116 million in BTC after 11-year dormancy.", "label_id": 2, "event_type": "whale"},
|
||
|
|
{"text": "Ancient Bitcoin whale dormant for 11 years suddenly transfers $257,450,000 in BTC.", "label_id": 2, "event_type": "whale"},
|
||
|
|
{"text": "$1B in Bitcoin moves from Satoshi-era wallet after 14 years of inactivity.", "label_id": 2, "event_type": "whale"},
|
||
|
|
{"text": "Breaking: Fed pauses rate hikes. Bitcoin jumps 5% on dovish pivot.", "label_id": 1, "event_type": "macro"},
|
||
|
|
{"text": "Surprise nonfarm payrolls print sends Bitcoin back below 80K.", "label_id": 0, "event_type": "macro"},
|
||
|
|
{"text": "Massive liquidation cascade wipes out $200M in longs.", "label_id": 0, "event_type": "liquidation"},
|
||
|
|
{"text": "Governance proposal passes with 95% approval.", "label_id": 1, "event_type": "governance"},
|
||
|
|
{"text": "Bitcoin ETF inflows hit $731M, highest since January.", "label_id": 1, "event_type": "earnings"},
|
||
|
|
{"text": "Coinbase Q2 earnings beat estimates. Revenue up 50% YoY.", "label_id": 1, "event_type": "earnings"},
|
||
|
|
{"text": "FOMO drives memecoin 500% in 24h. Degens aping in. Rug pull inevitable?", "label_id": 0, "event_type": "manipulation"},
|
||
|
|
{"text": "Token buybacks are booming. But are they good for crypto projects?", "label_id": 2, "event_type": "manipulation"},
|
||
|
|
{"text": "Coinbase delists XRP after SEC lawsuit. Trading suspended.", "label_id": 0, "event_type": "delisting"},
|
||
|
|
]
|
||
|
|
|
||
|
|
SENTIMENT_LABELS = ["Bearish", "Bullish", "Neutral"]
|
||
|
|
SENTIMENT_MAP = {"Bearish": 0, "Bullish": 1, "Neutral": 2}
|
||
|
|
|
||
|
|
EMOTION_LABELS = ["joy", "fear", "anger", "greed", "sadness", "neutral"]
|
||
|
|
EMOTION_MAP = {l: i for i, l in enumerate(EMOTION_LABELS)}
|
||
|
|
|
||
|
|
EVENT_LABELS = [
|
||
|
|
"listing", "delisting", "hack", "regulatory", "governance",
|
||
|
|
"upgrade", "partnership", "earnings", "macro",
|
||
|
|
"liquidation", "whale", "manipulation"
|
||
|
|
]
|
||
|
|
EVENT_MAP = {l: i for i, l in enumerate(EVENT_LABELS)}
|
||
|
|
|
||
|
|
EMOTION_LABELS = ["joy", "fear", "anger", "greed", "sadness", "neutral"]
|
||
|
|
EMOTION_MAP = {l: i for i, l in enumerate(EMOTION_LABELS)}
|
||
|
|
|
||
|
|
EVENT_LABELS = [
|
||
|
|
"listing", "delisting", "hack", "regulatory", "governance",
|
||
|
|
"upgrade", "partnership", "earnings", "macro",
|
||
|
|
"liquidation", "whale", "manipulation"
|
||
|
|
]
|
||
|
|
EVENT_MAP = {l: i for i, l in enumerate(EVENT_LABELS)}
|
||
|
|
|
||
|
|
EMOTION_SAMPLES = [
|
||
|
|
("BTC breaks $100k! New ATH!", [1,0,0,1,0,0]),
|
||
|
|
("Ethereum merge successful!", [1,0,0,1,0,0]),
|
||
|
|
("Major hack on DeFi protocol drains $50M", [0,1,1,0,1,0]),
|
||
|
|
("Bitcoin crashes 50% in hours", [0,1,1,0,1,0]),
|
||
|
|
("SEC sues major exchange", [0,1,1,0,1,0]),
|
||
|
|
("Rug pull! Devs stole all funds!", [0,1,1,0,0,0]),
|
||
|
|
("FOMO drives memecoin 500% in 24h", [0,0,0,1,0,0]),
|
||
|
|
("Buy the dip! Accumulate more!", [0,0,0,1,0,0]),
|
||
|
|
("Lost everything in the crash", [0,0,0,0,1,0]),
|
||
|
|
("BTC at $50k, ETH at $3k", [0,0,0,0,0,1]),
|
||
|
|
]
|
||
|
|
|
||
|
|
EVENT_LABELS = [
|
||
|
|
"listing", "delisting", "hack", "regulatory", "governance",
|
||
|
|
"upgrade", "partnership", "earnings", "macro",
|
||
|
|
"liquidation", "whale", "manipulation"
|
||
|
|
]
|
||
|
|
EVENT_MAP = {l: i for i, l in enumerate(EVENT_LABELS)}
|
||
|
|
|
||
|
|
EMOTION_SAMPLES = [
|
||
|
|
("BTC breaks $100k! New ATH!", [1,0,0,1,0,0]),
|
||
|
|
("Ethereum merge successful!", [1,0,0,1,0,0]),
|
||
|
|
("Major hack on DeFi protocol drains $50M", [0,1,1,0,1,0]),
|
||
|
|
("Bitcoin crashes 50% in hours", [0,1,1,0,1,0]),
|
||
|
|
("SEC sues major exchange", [0,1,1,0,1,0]),
|
||
|
|
("Rug pull! Devs stole all funds!", [0,1,1,0,0,0]),
|
||
|
|
("FOMO drives memecoin 500% in 24h", [0,0,0,1,0,0]),
|
||
|
|
("Buy the dip! Accumulate more!", [0,0,0,1,0,0]),
|
||
|
|
("Lost everything in the crash", [0,0,0,0,1,0]),
|
||
|
|
("BTC at $50k, ETH at $3k", [0,0,0,0,0,1]),
|
||
|
|
]
|
||
|
|
|
||
|
|
EVENT_LABELS = [
|
||
|
|
"listing", "delisting", "hack", "regulatory", "governance",
|
||
|
|
"upgrade", "partnership", "earnings", "macro",
|
||
|
|
"liquidation", "whale", "manipulation"
|
||
|
|
]
|
||
|
|
EVENT_MAP = {l: i for i, l in enumerate(EVENT_LABELS)}
|
||
|
|
|
||
|
|
EMOTION_SAMPLES = [
|
||
|
|
("BTC breaks $100k! New ATH!", [1,0,0,1,0,0]),
|
||
|
|
("Ethereum merge successful!", [1,0,0,1,0,0]),
|
||
|
|
("Major hack on DeFi protocol drains $50M", [0,1,1,0,1,0]),
|
||
|
|
("Bitcoin crashes 50% in hours", [0,1,1,0,1,0]),
|
||
|
|
("SEC sues major exchange", [0,1,1,0,1,0]),
|
||
|
|
("Rug pull! Devs stole all funds!", [0,1,1,0,0,0]),
|
||
|
|
("FOMO drives memecoin 500% in 24h", [0,0,0,1,0,0]),
|
||
|
|
("Buy the dip! Accumulate more!", [0,0,0,1,0,0]),
|
||
|
|
("Lost everything in the crash", [0,0,0,0,1,0]),
|
||
|
|
("BTC at $50k, ETH at $3k", [0,0,0,0,0,1]),
|
||
|
|
]
|
||
|
|
|
||
|
|
EVENT_LABELS = [
|
||
|
|
"listing", "delisting", "hack", "regulatory", "governance",
|
||
|
|
"upgrade", "partnership", "earnings", "macro",
|
||
|
|
"liquidation", "whale", "manipulation"
|
||
|
|
]
|
||
|
|
EVENT_MAP = {l: i for i, l in enumerate(EVENT_LABELS)}
|
||
|
|
|
||
|
|
# ============================================================
|
||
|
|
# LOAD LABELED DATA
|
||
|
|
# ============================================================
|
||
|
|
|
||
|
|
def load_labeled_data(label_file):
|
||
|
|
"""Load verified labeled data from JSONL file"""
|
||
|
|
texts, labels = [], []
|
||
|
|
with open(label_file) as f:
|
||
|
|
for line in open(label_file):
|
||
|
|
r = json.loads(line)
|
||
|
|
if r.get('verified', False):
|
||
|
|
texts.append(r['text'])
|
||
|
|
labels.append(r['labels']['sentiment'])
|
||
|
|
return texts, labels
|
||
|
|
|
||
|
|
def get_sentiment_data():
|
||
|
|
texts, labels = [], []
|
||
|
|
# Manual samples
|
||
|
|
for text, label in [
|
||
|
|
("BTC breaks $100k! New ATH!", 1), ("ETH to $10k by EOY, accumulate now", 1),
|
||
|
|
("Institutional inflows hit record high", 1), ("Bitcoin reaches new all-time high", 1),
|
||
|
|
("Ethereum merge successful, staking rewards now live", 1),
|
||
|
|
("Massive ETF inflows drive Bitcoin to new highs", 1),
|
||
|
|
("Golden cross confirmed on Bitcoin weekly chart", 1),
|
||
|
|
("Institutional adoption drives Bitcoin higher", 1),
|
||
|
|
("ETF approval drives massive inflows", 1), ("Market is bullish on Bitcoin", 1),
|
||
|
|
("BTC crashes 50% in hours", 0), ("Exchange hacked, $100M stolen", 0),
|
||
|
|
("SEC sues major exchange", 0), ("Bitcoin crashes hard, panic selling everywhere", 0),
|
||
|
|
("Massive liquidation cascade wipes out $200M in longs", 0),
|
||
|
|
("VIX drops below 15 as market volatility decreases", 0),
|
||
|
|
("Whale sells 10000 BTC", 0), ("Bitcoin price drops 50%", 0),
|
||
|
|
("Support broken with bearish structure", 0), ("Panic selling and forced liquidation", 0),
|
||
|
|
("BTC at $50k, ETH at $3k", 2), ("Market consolidating in range", 2),
|
||
|
|
("Bitcoin remains stable around $30k", 2), ("VIX drops below 15", 2),
|
||
|
|
("Market consolidating with no clear direction", 2),
|
||
|
|
("Bitcoin price stable around $30k", 2), ("Consolidation phase continues", 2),
|
||
|
|
("Market in wait-and-see mode", 2), ("Sideways action continues", 2),
|
||
|
|
("Low volatility environment persists", 2),
|
||
|
|
]:
|
||
|
|
yield t, l
|
||
|
|
for event in REAL_EVENTS:
|
||
|
|
yield event["text"], event["label_id"]
|
||
|
|
|
||
|
|
def get_event_data():
|
||
|
|
texts, labels = [], []
|
||
|
|
for event in REAL_EVENTS:
|
||
|
|
lbls = [0]*12
|
||
|
|
lbls[EVENT_MAP[event["event_type"]]] = 1
|
||
|
|
yield event["text"], lbls
|
||
|
|
|
||
|
|
def get_emotion_data():
|
||
|
|
texts, labels = [], []
|
||
|
|
for text, labels in [
|
||
|
|
("BTC breaks $100k! New ATH!", [1,0,0,1,0,0]),
|
||
|
|
("Ethereum merge successful!", [1,0,0,1,0,0]),
|
||
|
|
("Major hack on DeFi protocol drains $50M", [0,1,1,0,1,0]),
|
||
|
|
("Bitcoin crashes 50% in hours", [0,1,1,0,1,0]),
|
||
|
|
("SEC sues major exchange", [0,1,1,0,1,0]),
|
||
|
|
("Rug pull! Devs stole all funds!", [0,1,1,0,0,0]),
|
||
|
|
("FOMO drives memecoin 500% in 24h", [0,0,0,1,0,0]),
|
||
|
|
("Buy the dip! Accumulate more!", [0,0,0,1,0,0]),
|
||
|
|
("Lost everything in the crash", [0,0,0,0,1,0]),
|
||
|
|
("BTC at $50k, ETH at $3k", [0,0,0,0,0,1]),
|
||
|
|
]:
|
||
|
|
yield text, labels
|
||
|
|
|
||
|
|
def get_event_data():
|
||
|
|
texts, labels = [], []
|
||
|
|
for event in REAL_EVENTS:
|
||
|
|
lbls = [0]*12
|
||
|
|
lbls[EVENT_MAP[event["event_type"]]] = 1
|
||
|
|
yield event["text"], lbls
|
||
|
|
|
||
|
|
def get_emotion_data():
|
||
|
|
texts, labels = [], []
|
||
|
|
for text, labels in [
|
||
|
|
("BTC breaks $100k! New ATH!", [1,0,0,1,0,0]),
|
||
|
|
("Ethereum merge successful!", [1,0,0,1,0,0]),
|
||
|
|
("Major hack on DeFi protocol drains $50M", [0,1,1,0,1,0]),
|
||
|
|
("Bitcoin crashes 50% in hours", [0,1,1,0,1,0]),
|
||
|
|
("SEC sues major exchange", [0,1,1,0,1,0]),
|
||
|
|
("Rug pull! Devs stole all funds!", [0,1,1,0,0,0]),
|
||
|
|
("FOMO drives memecoin 500% in 24h", [0,0,0,1,0,0]),
|
||
|
|
("Buy the dip! Accumulate more!", [0,0,0,1,0,0]),
|
||
|
|
("Lost everything in the crash", [0,0,0,0,1,0]),
|
||
|
|
("BTC at $50k, ETH at $3k", [0,0,0,0,0,1]),
|
||
|
|
]:
|
||
|
|
yield text, labels
|
||
|
|
|
||
|
|
def get_event_data():
|
||
|
|
texts, labels = [], []
|
||
|
|
for event in REAL_EVENTS:
|
||
|
|
lbls = [0]*12
|
||
|
|
lbls[EVENT_MAP[event["event_type"]]] = 1
|
||
|
|
yield event["text"], lbls
|
||
|
|
|
||
|
|
def get_emotion_data():
|
||
|
|
texts, labels = [], []
|
||
|
|
for text, labels in [
|
||
|
|
("BTC breaks $100k! New ATH!", [1,0,0,1,0,0]),
|
||
|
|
("Ethereum merge successful!", [1,0,0,1,0,0]),
|
||
|
|
("Major hack on DeFi protocol drains $50M", [0,1,1,0,1,0]),
|
||
|
|
("Bitcoin crashes 50% in hours", [0,1,1,0,1,0]),
|
||
|
|
("SEC sues major exchange", [0,1,1,0,1,0]),
|
||
|
|
("Rug pull! Devs stole all funds!", [0,1,1,0,0,0]),
|
||
|
|
("FOMO drives memecoin 500% in 24h", [0,0,0,1,0,0]),
|
||
|
|
("Buy the dip! Accumulate more!", [0,0,0,1,0,0]),
|
||
|
|
("Lost everything in the crash", [0,0,0,0,1,0]),
|
||
|
|
("BTC at $50k, ETH at $3k", [0,0,0,0,0,1]),
|
||
|
|
]:
|
||
|
|
yield text, labels
|
||
|
|
|
||
|
|
# ============================================================
|
||
|
|
# LOAD LABELED DATA FROM LABELING PIPELINE
|
||
|
|
# ============================================================
|
||
|
|
|
||
|
|
def load_labeled_data():
|
||
|
|
"""Load all verified labeled data from labeling pipeline outputs"""
|
||
|
|
sentiment_texts, sentiment_labels = [], []
|
||
|
|
event_texts, event_labels = [], []
|
||
|
|
emotion_texts, emotion_labels = [], []
|
||
|
|
|
||
|
|
# Load from labeled_output.jsonl
|
||
|
|
for label_file in ['data/labeled_output.jsonl', 'data/labeled_large.jsonl', 'data/labeled_large.jsonl']:
|
||
|
|
try:
|
||
|
|
with open(label_file) as f:
|
||
|
|
for line in open(label_file):
|
||
|
|
r = json.loads(line)
|
||
|
|
if r.get('verified', False):
|
||
|
|
# Sentiment
|
||
|
|
texts.append(r['text'])
|
||
|
|
labels.append(r['labels']['sentiment'])
|
||
|
|
except:
|
||
|
|
pass
|
||
|
|
|
||
|
|
return texts, labels
|
||
|
|
|
||
|
|
# ============================================================
|
||
|
|
# DATASET CLASS
|
||
|
|
# ============================================================
|
||
|
|
|
||
|
|
class QuickDataset(torch.utils.data.Dataset):
|
||
|
|
def __init__(self, texts, labels, tokenizer, max_len=64):
|
||
|
|
self.texts = texts; self.labels = labels
|
||
|
|
self.tokenizer = tokenizer; self.max_len = 64
|
||
|
|
def __len__(self): return len(self.texts)
|
||
|
|
def __getitem__(self, i):
|
||
|
|
enc = self.tokenizer(self.texts[i], truncation=True, max_length=self.max_len, padding="max_length", return_tensors="pt")
|
||
|
|
lbl = self.labels[i]
|
||
|
|
if isinstance(lbl, list):
|
||
|
|
lbl = torch.tensor(lbl, dtype=torch.float)
|
||
|
|
else:
|
||
|
|
lbl = torch.tensor(lbl, dtype=torch.long)
|
||
|
|
enc = self.tokenizer(self.texts[i], truncation=True, max_length=self.max_len, padding="max_length", return_tensors="pt")
|
||
|
|
return {"input_ids": enc["input_ids"].squeeze(0), "attention_mask": enc["attention_mask"].squeeze(0), "labels": lbl}
|
||
|
|
|
||
|
|
|
||
|
|
def train_sentiment():
|
||
|
|
print("\n" + "="*50)
|
||
|
|
print("1. TRAINING SENTIMENT (FinBERT)")
|
||
|
|
print("="*50)
|
||
|
|
|
||
|
|
# Collect all sentiment data
|
||
|
|
texts, labels = [], []
|
||
|
|
for text, label in [
|
||
|
|
("BTC breaks $100k! New ATH!", 1), ("ETH to $10k by EOY, accumulate now", 1),
|
||
|
|
("Institutional inflows hit record high", 1), ("Bitcoin reaches new all-time high", 1),
|
||
|
|
("Ethereum merge successful, staking rewards now live", 1),
|
||
|
|
("Massive ETF inflows drive Bitcoin to new highs", 1),
|
||
|
|
("Golden cross confirmed on Bitcoin weekly chart", 1),
|
||
|
|
("Institutional adoption drives Bitcoin higher", 1),
|
||
|
|
("ETF approval drives massive inflows", 1), ("Market is bullish on Bitcoin", 1),
|
||
|
|
("BTC crashes 50% in hours", 0), ("Exchange hacked, $100M stolen", 0),
|
||
|
|
("SEC sues major exchange", 0), ("Bitcoin crashes hard, panic selling everywhere", 0),
|
||
|
|
("Massive liquidation cascade wipes out $200M in longs", 0),
|
||
|
|
("VIX drops below 15 as market volatility decreases", 0),
|
||
|
|
("Whale sells 10000 BTC", 0), ("Bitcoin price drops 50%", 0),
|
||
|
|
("Support broken with bearish structure", 0), ("Panic selling and forced liquidation", 0),
|
||
|
|
("BTC at $50k, ETH at $3k", 2), ("Market consolidating in range", 2),
|
||
|
|
("Bitcoin remains stable around $30k", 2), ("VIX drops below 15", 2),
|
||
|
|
("Market consolidating with no clear direction", 2),
|
||
|
|
("Bitcoin price stable around $30k", 2), ("Consolidation phase continues", 2),
|
||
|
|
("Market in wait-and-see mode", 2), ("Sideways action continues", 2),
|
||
|
|
("Low volatility environment persists", 2),
|
||
|
|
]:
|
||
|
|
texts.append(text); labels.append(label)
|
||
|
|
for event in REAL_EVENTS:
|
||
|
|
texts.append(event["text"]); labels.append(event["label_id"])
|
||
|
|
|
||
|
|
train_t, temp_t, train_l, temp_l = train_test_split(texts, labels, test_size=0.3, random_state=42, stratify=labels)
|
||
|
|
temp_t, test_t, temp_l, test_l = train_test_split(temp_t, temp_l, test_size=0.5, random_state=42, stratify=temp_l)
|
||
|
|
|
||
|
|
print(f"Train: {len(train_t)}, Val: {len(temp_t)}, Test: {len(test_t)}")
|
||
|
|
|
||
|
|
tokenizer = AutoTokenizer.from_pretrained("ProsusAI/finbert")
|
||
|
|
model = AutoModelForSequenceClassification.from_pretrained(
|
||
|
|
"ProsusAI/finbert", num_labels=3,
|
||
|
|
id2label={0:"Bearish",1:"Bullish",2:"Neutral"},
|
||
|
|
label2id={"Bearish":0,"Bullish":1,"Neutral":2})
|
||
|
|
|
||
|
|
class QuickDataset(torch.utils.data.Dataset):
|
||
|
|
def __init__(self, texts, labels, tokenizer, max_len=64):
|
||
|
|
self.texts = texts; self.labels = labels
|
||
|
|
self.tokenizer = AutoTokenizer.from_pretrained("ProsusAI/finbert"); self.max_len = 64
|
||
|
|
def __len__(self): return len(self.texts)
|
||
|
|
def __getitem__(self, i):
|
||
|
|
enc = self.tokenizer(self.texts[i], truncation=True, max_length=self.max_len, padding="max_length", return_tensors="pt")
|
||
|
|
return {"input_ids": enc["input_ids"].squeeze(0), "attention_mask": enc["attention_mask"].squeeze(0), "labels": torch.tensor(self.labels[i], dtype=torch.long)}
|
||
|
|
|
||
|
|
train_ds = QuickDataset(train_t, train_l, AutoTokenizer.from_pretrained("ProsusAI/finbert"))
|
||
|
|
val_ds = QuickDataset(temp_t, temp_l, AutoTokenizer.from_pretrained("ProsusAI/finbert"))
|
||
|
|
|
||
|
|
model = AutoModelForSequenceClassification.from_pretrained(
|
||
|
|
"ProsusAI/finbert", num_labels=3,
|
||
|
|
id2label={0:"Bearish",1:"Bullish",2:"Neutral"},
|
||
|
|
label2id={"Bearish":0,"Bullish":1,"Neutral":2})
|
||
|
|
|
||
|
|
trainer = Trainer(
|
||
|
|
model=model,
|
||
|
|
args=TrainingArguments(
|
||
|
|
output_dir="./models/finbert-crypto-sentiment",
|
||
|
|
num_train_epochs=2, per_device_train_batch_size=16,
|
||
|
|
per_device_eval_batch_size=32, gradient_accumulation_steps=2,
|
||
|
|
warmup_ratio=0.1, learning_rate=2e-5, lr_scheduler_type="cosine",
|
||
|
|
eval_strategy="epoch", save_strategy="epoch",
|
||
|
|
load_best_model_at_end=True, metric_for_best_model="f1_macro",
|
||
|
|
greater_is_better=True, fp16=False, dataloader_num_workers=0,
|
||
|
|
logging_steps=10, save_total_limit=1, remove_unused_columns=False,
|
||
|
|
report_to="none",
|
||
|
|
),
|
||
|
|
train_dataset=QuickDataset(train_t, train_l, AutoTokenizer.from_pretrained("ProsusAI/finbert")),
|
||
|
|
eval_dataset=QuickDataset(temp_t, temp_l, AutoTokenizer.from_pretrained("ProsusAI/finbert")),
|
||
|
|
tokenizer=AutoTokenizer.from_pretrained("ProsusAI/finbert"),
|
||
|
|
compute_metrics=lambda ep: {"f1_macro": f1_score(ep.label_ids, np.argmax(ep.predictions, axis=-1), average="macro")},
|
||
|
|
callbacks=[EarlyStoppingCallback(early_stopping_patience=1)]
|
||
|
|
)
|
||
|
|
|
||
|
|
print("\n1. TRAINING SENTIMENT (FinBERT)")
|
||
|
|
print("="*50)
|
||
|
|
print(f"Train: {len(train_t)}, Val: {len(temp_t)}")
|
||
|
|
print("Training Sentiment (2 epochs, ~3 min)...")
|
||
|
|
trainer.train()
|
||
|
|
|
||
|
|
model.save_pretrained("./models/finbert-crypto-sentiment")
|
||
|
|
AutoTokenizer.from_pretrained("ProsusAI/finbert").save_pretrained("./models/finbert-crypto-sentiment")
|
||
|
|
print("✅ Sentiment model saved!")
|
||
|
|
return model
|
||
|
|
|
||
|
|
|
||
|
|
def train_events():
|
||
|
|
print("\n" + "="*50)
|
||
|
|
print("2. TRAINING EVENT CLASSIFIER (BERT)")
|
||
|
|
print("="*50)
|
||
|
|
|
||
|
|
texts, labels = [], []
|
||
|
|
for event in REAL_EVENTS:
|
||
|
|
lbls = [0]*12
|
||
|
|
lbls[EVENT_MAP[event["event_type"]]] = 1
|
||
|
|
texts.append(event["text"])
|
||
|
|
labels.append(lbls)
|
||
|
|
|
||
|
|
train_t, temp_t, train_l, temp_l = train_test_split(texts, labels, test_size=0.3, random_state=42)
|
||
|
|
temp_t, test_t, temp_l, test_l = train_test_split(temp_t, temp_l, test_size=0.5, random_state=42)
|
||
|
|
|
||
|
|
print(f"Train: {len(train_t)}, Val: {len(temp_t)}, Test: {len(test_t)}")
|
||
|
|
|
||
|
|
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
|
||
|
|
model = AutoModelForSequenceClassification.from_pretrained(
|
||
|
|
"bert-base-uncased", num_labels=12,
|
||
|
|
id2label={i:l for i,l in enumerate(EVENT_LABELS)}, label2id=EVENT_MAP,
|
||
|
|
problem_type="multi_label_classification")
|
||
|
|
|
||
|
|
class MultiLabelDataset(torch.utils.data.Dataset):
|
||
|
|
def __init__(self, texts, labels, tokenizer, max_len=64):
|
||
|
|
self.texts = texts; self.labels = labels
|
||
|
|
self.tokenizer = tokenizer; self.max_len = 64
|
||
|
|
def __len__(self): return len(self.texts)
|
||
|
|
def __getitem__(self, i):
|
||
|
|
enc = self.tokenizer(self.texts[i], truncation=True, max_length=self.max_len, padding="max_length", return_tensors="pt")
|
||
|
|
return {"input_ids": enc["input_ids"].squeeze(0), "attention_mask": enc["attention_mask"].squeeze(0), "labels": torch.tensor(self.labels[i], dtype=torch.float)}
|
||
|
|
|
||
|
|
texts, labels = [], []
|
||
|
|
for event in REAL_EVENTS:
|
||
|
|
lbls = [0]*12
|
||
|
|
lbls[EVENT_MAP[event["event_type"]]] = 1
|
||
|
|
texts.append(event["text"])
|
||
|
|
labels.append(lbls)
|
||
|
|
|
||
|
|
train_t, temp_t, train_l, temp_l = train_test_split(texts, labels, test_size=0.3, random_state=42)
|
||
|
|
temp_t, test_t, temp_l, test_l = train_test_split(temp_t, temp_l, test_size=0.5, random_state=42)
|
||
|
|
|
||
|
|
print(f"Train: {len(train_t)}, Val: {len(temp_t)}, Test: {len(test_t)}")
|
||
|
|
|
||
|
|
tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
|
||
|
|
model = AutoModelForSequenceClassification.from_pretrained(
|
||
|
|
"bert-base-uncased", num_labels=12,
|
||
|
|
id2label={i:l for i,l in enumerate(EVENT_LABELS)}, label2id=EVENT_MAP,
|
||
|
|
problem_type="multi_label_classification")
|
||
|
|
|
||
|
|
class MultiLabelDataset(torch.utils.data.Dataset):
|
||
|
|
def __init__(self, texts, labels, tokenizer, max_len=64):
|
||
|
|
self.texts = texts; self.labels = labels
|
||
|
|
self.tokenizer = tokenizer; self.max_len = 64
|
||
|
|
def __len__(self): return len(self.texts)
|
||
|
|
def __getitem__(self, i):
|
||
|
|
enc = self.tokenizer(self.texts[i], truncation=True, max_length=self.max_len, padding="max_length", return_tensors="pt")
|
||
|
|
return {"input_ids": enc["input_ids"].squeeze(0), "attention_mask": enc["attention_mask"].squeeze(0), "labels": torch.tensor(self.labels[i], dtype=torch.float)}
|
||
|
|
|
||
|
|
train_ds = MultiLabelDataset([t for t in texts if t in train_t], [l for t,l in zip(texts, labels) if t in train_t], AutoTokenizer.from_pretrained("bert-base-uncased"))
|
||
|
|
val_ds = MultiLabelDataset([t for t in texts if t in temp_t], [l for t,l in zip(texts, labels) if t in temp_t], AutoTokenizer.from_pretrained("bert-base-uncased"))
|
||
|
|
|
||
|
|
model = AutoModelForSequenceClassification.from_pretrained(
|
||
|
|
"bert-base-uncased", num_labels=12,
|
||
|
|
id2label={i:l for i,l in enumerate(EVENT_LABELS)}, label2id=EVENT_MAP,
|
||
|
|
problem_type="multi_label_classification")
|
||
|
|
|
||
|
|
trainer = Trainer(
|
||
|
|
model=model,
|
||
|
|
args=TrainingArguments(
|
||
|
|
output_dir="./models/bert-crypto-events",
|
||
|
|
num_train_epochs=2, per_device_train_batch_size=8,
|
||
|
|
per_device_eval_batch_size=16, gradient_accumulation_steps=4,
|
||
|
|
warmup_ratio=0.1, learning_rate=2e-5, lr_scheduler_type="cosine",
|
||
|
|
eval_strategy="epoch", save_strategy="epoch",
|
||
|
|
load_best_model_at_end=True, metric_for_best_model="f1_macro",
|
||
|
|
greater_is_better=True, fp16=False, dataloader_num_workers=0,
|
||
|
|
logging_steps=10, save_total_limit=1, remove_unused_columns=False,
|
||
|
|
report_to="none",
|
||
|
|
),
|
||
|
|
train_dataset=MultiLabelDataset([t for t in texts if t in train_t], [l for t,l in zip(texts, labels) if t in train_t], AutoTokenizer.from_pretrained("bert-base-uncased")),
|
||
|
|
eval_dataset=MultiLabelDataset([t for t in texts if t in temp_t], [l for t,l in zip(texts, labels) if t in temp_t], AutoTokenizer.from_pretrained("bert-base-uncased")),
|
||
|
|
tokenizer=AutoTokenizer.from_pretrained("bert-base-uncased"),
|
||
|
|
compute_metrics=lambda ep: {"f1_macro": f1_score(ep.label_ids, (np.array(ep.predictions) > 0.5).astype(int), average="macro")},
|
||
|
|
callbacks=[EarlyStoppingCallback(early_stopping_patience=1)]
|
||
|
|
)
|
||
|
|
|
||
|
|
print("\n2. TRAINING EVENT CLASSIFIER (BERT)")
|
||
|
|
print("="*50)
|
||
|
|
print(f"Train: {len(train_t)}, Val: {len(temp_t)}, Test: {len(test_t)}")
|
||
|
|
print("Training Events (2 epochs, ~5 min)...")
|
||
|
|
trainer.train()
|
||
|
|
|
||
|
|
model.save_pretrained("./models/bert-crypto-events")
|
||
|
|
AutoTokenizer.from_pretrained("bert-base-uncased").save_pretrained("./models/bert-crypto-events")
|
||
|
|
print("✅ Event model saved!")
|
||
|
|
return model
|
||
|
|
|
||
|
|
|
||
|
|
def train_emotion():
|
||
|
|
print("\n" + "="*50)
|
||
|
|
print("3. TRAINING EMOTION (DistilRoBERTa)")
|
||
|
|
print("="*50)
|
||
|
|
|
||
|
|
texts, labels = [], []
|
||
|
|
for text, lbls in [
|
||
|
|
("BTC breaks $100k! New ATH!", [1,0,0,1,0,0]),
|
||
|
|
("Ethereum merge successful!", [1,0,0,1,0,0]),
|
||
|
|
("Major hack on DeFi protocol drains $50M", [0,1,1,0,1,0]),
|
||
|
|
("Bitcoin crashes 50% in hours", [0,1,1,0,1,0]),
|
||
|
|
("SEC sues major exchange", [0,1,1,0,1,0]),
|
||
|
|
("Rug pull! Devs stole all funds!", [0,1,1,0,0,0]),
|
||
|
|
("FOMO drives memecoin 500% in 24h", [0,0,0,1,0,0]),
|
||
|
|
("Buy the dip! Accumulate more!", [0,0,0,1,0,0]),
|
||
|
|
("Lost everything in the crash", [0,0,0,0,1,0]),
|
||
|
|
("BTC at $50k, ETH at $3k", [0,0,0,0,0,1]),
|
||
|
|
]:
|
||
|
|
texts.append(text); labels.append(lbls)
|
||
|
|
|
||
|
|
train_t, temp_t, train_l, temp_l = train_test_split(texts, labels, test_size=0.3, random_state=42)
|
||
|
|
temp_t, test_t, temp_l, test_l = train_test_split(temp_t, temp_l, test_size=0.5, random_state=42)
|
||
|
|
|
||
|
|
print(f"Train: {len(train_t)}, Val: {len(temp_t)}, Test: {len(test_t)}")
|
||
|
|
|
||
|
|
tokenizer = AutoTokenizer.from_pretrained("j-hartmann/emotion-english-distilroberta-base")
|
||
|
|
model = AutoModelForSequenceClassification.from_pretrained(
|
||
|
|
"j-hartmann/emotion-english-distilroberta-base", num_labels=6,
|
||
|
|
id2label={i:l for i,l in enumerate(EMOTION_LABELS)}, label2id=EMOTION_MAP,
|
||
|
|
problem_type="multi_label_classification", ignore_mismatched_sizes=True)
|
||
|
|
|
||
|
|
class MultiLabelDataset(torch.utils.data.Dataset):
|
||
|
|
def __init__(self, texts, labels, tokenizer, max_len=64):
|
||
|
|
self.texts = texts; self.labels = labels
|
||
|
|
self.tokenizer = tokenizer; self.max_len = 64
|
||
|
|
def __len__(self): return len(self.texts)
|
||
|
|
def __getitem__(self, i):
|
||
|
|
enc = self.tokenizer(self.texts[i], truncation=True, max_length=self.max_len, padding="max_length", return_tensors="pt")
|
||
|
|
return {"input_ids": enc["input_ids"].squeeze(0), "attention_mask": enc["attention_mask"].squeeze(0), "labels": torch.tensor(self.labels[i], dtype=torch.float)}
|
||
|
|
|
||
|
|
train_ds = MultiLabelDataset([t for t in texts if t in train_t], [l for t,l in zip(texts, labels) if t in train_t], AutoTokenizer.from_pretrained("j-hartmann/emotion-english-distilroberta-base"))
|
||
|
|
val_ds = MultiLabelDataset([t for t in texts if t in temp_t], [l for t,l in zip(texts, labels) if t in temp_t], AutoTokenizer.from_pretrained("j-hartmann/emotion-english-distilroberta-base"))
|
||
|
|
|
||
|
|
model = AutoModelForSequenceClassification.from_pretrained(
|
||
|
|
"j-hartmann/emotion-english-distilroberta-base", num_labels=6,
|
||
|
|
id2label={i:l for i,l in enumerate(EMOTION_LABELS)}, label2id=EMOTION_MAP,
|
||
|
|
problem_type="multi_label_classification", ignore_mismatched_sizes=True)
|
||
|
|
|
||
|
|
trainer = Trainer(
|
||
|
|
model=model,
|
||
|
|
args=TrainingArguments(
|
||
|
|
output_dir="./models/distilroberta-crypto-emotion",
|
||
|
|
num_train_epochs=2, per_device_train_batch_size=8,
|
||
|
|
per_device_eval_batch_size=16, gradient_accumulation_steps=4,
|
||
|
|
warmup_ratio=0.1, learning_rate=2e-5, lr_scheduler_type="cosine",
|
||
|
|
eval_strategy="epoch", save_strategy="epoch",
|
||
|
|
load_best_model_at_end=True, metric_for_best_model="f1_macro",
|
||
|
|
greater_is_better=True, fp16=False, dataloader_num_workers=0,
|
||
|
|
logging_steps=10, save_total_limit=1, remove_unused_columns=False,
|
||
|
|
report_to="none",
|
||
|
|
),
|
||
|
|
train_dataset=MultiLabelDataset([t for t in texts if t in train_t], [l for t,l in zip(texts, labels) if t in train_t], AutoTokenizer.from_pretrained("j-hartmann/emotion-english-distilroberta-base")),
|
||
|
|
eval_dataset=MultiLabelDataset([t for t in texts if t in temp_t], [l for t,l in zip(texts, labels) if t in temp_t], AutoTokenizer.from_pretrained("j-hartmann/emotion-english-distilroberta-base")),
|
||
|
|
tokenizer=AutoTokenizer.from_pretrained("j-hartmann/emotion-english-distilroberta-base"),
|
||
|
|
compute_metrics=lambda ep: {"f1_macro": f1_score(ep.label_ids, (np.array(ep.predictions) > 0.5).astype(int), average="macro")},
|
||
|
|
callbacks=[EarlyStoppingCallback(early_stopping_patience=1)]
|
||
|
|
)
|
||
|
|
|
||
|
|
print("\n3. TRAINING EMOTION (DistilRoBERTa)")
|
||
|
|
print("="*50)
|
||
|
|
print(f"Train: {len(train_t)}, Val: {len(temp_t)}, Test: {len(test_t)}")
|
||
|
|
print("Training Emotion (2 epochs, ~3 min)...")
|
||
|
|
trainer.train()
|
||
|
|
|
||
|
|
model.save_pretrained("./models/distilroberta-crypto-emotion")
|
||
|
|
AutoTokenizer.from_pretrained("j-hartmann/emotion-english-distilroberta-base").save_pretrained("./models/distilroberta-crypto-emotion")
|
||
|
|
print("✅ Emotion model saved!")
|
||
|
|
return model
|
||
|
|
|
||
|
|
|
||
|
|
def main():
|
||
|
|
print("="*60)
|
||
|
|
print("DOMAIN ADAPTATION: FINE-TUNING ALL MODELS")
|
||
|
|
print("="*60)
|
||
|
|
|
||
|
|
import torch
|
||
|
|
from transformers import AutoTokenizer, AutoModelForSequenceClassification, TrainingArguments, Trainer, EarlyStoppingCallback
|
||
|
|
from datasets import load_dataset
|
||
|
|
from sklearn.model_selection import train_test_split
|
||
|
|
from sklearn.metrics import accuracy_score, f1_score
|
||
|
|
from sklearn.utils.class_weight import compute_class_weight
|
||
|
|
import numpy as np
|
||
|
|
import random
|
||
|
|
|
||
|
|
# 1. SENTIMENT
|
||
|
|
train_sentiment()
|
||
|
|
|
||
|
|
# 2. EVENTS
|
||
|
|
train_events()
|
||
|
|
|
||
|
|
# 3. EMOTION
|
||
|
|
train_emotion()
|
||
|
|
|
||
|
|
print("\n" + "="*60)
|
||
|
|
print("✅ ALL MODELS TRAINED AND SAVED!")
|
||
|
|
print("="*60)
|
||
|
|
print("Models saved to ./models/")
|
||
|
|
print(" - finbert-crypto-sentiment/")
|
||
|
|
print(" - bert-crypto-events/")
|
||
|
|
print(" - distilroberta-crypto-emotion/")
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
import torch
|
||
|
|
from transformers import AutoTokenizer, AutoModelForSequenceClassification, TrainingArguments, Trainer, EarlyStoppingCallback
|
||
|
|
from datasets import load_dataset
|
||
|
|
from sklearn.model_selection import train_test_split
|
||
|
|
from sklearn.metrics import accuracy_score, f1_score
|
||
|
|
from sklearn.utils.class_weight import compute_class_weight
|
||
|
|
import numpy as np
|
||
|
|
import random
|
||
|
|
|
||
|
|
main()
|