hjnormey/sentiment-engine
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add sentiment_engine with CryptoSentimentCalibrator fixes - improved keyword lists, lowered FinBERT threshold, added neutral handling

Browse Source
This commit is contained in:
Codex
2026-09-14 13:30:05 +02:00
parent 19a7812094
commit a276aeaded
149 changed files with 35226 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

0
sentiment_engine/config/FinancialPhraseBank.csv Normal file
View File

72
sentiment_engine/config/asset_aliases.yaml Normal file
View File

@@ -0,0 +1,72 @@
# Asset alias mappings - maps common aliases to canonical tickers
aliases:
# Major crypto
"BTC": "BTC"
"BITCOIN": "BTC"
"XBT": "BTC"
"ETH": "ETH"
"ETHEREUM": "ETH"
"ETHER": "ETH"
"SOL": "SOL"
"SOLANA": "SOL"
"BNB": "BNB"
"BINANCE": "BNB"
"ADA": "ADA"
"CARDANO": "ADA"
"XRP": "XRP"
"RIPPLE": "XRP"
"DOGE": "DOGE"
"DOGECOIN": "DOGE"
"MATIC": "MATIC"
"POLYGON": "MATIC"
"AVAX": "AVAX"
"AVALANCHE": "AVAX"
"DOT": "DOT"
"POLKADOT": "DOT"
"LINK": "LINK"
"CHAINLINK": "LINK"
"UNI": "UNI"
"UNISWAP": "UNI"
"AAVE": "AAVE"
"ARB": "ARB"
"ARBITRUM": "ARB"
"OP": "OP"
"OPTIMISM": "OP"
# People aliases
"VITALIK": "ETH"
"VITALIK BUTERIN": "ETH"
"CZ": "BNB"
"CHANGPENG ZHAO": "BNB"
"ELON": "DOGE"
"ELON MUSK": "DOGE"
"SAYLOR": "BTC"
"MICHAEL SAYLOR": "BTC"
"SBF": "SOL" # Historical
# Stablecoins
"USDT": "USDT"
"TETHER": "USDT"
"USDC": "USDC"
"CIRCLE": "USDC"
"DAI": "DAI"
"MAKER": "MKR"
# Meme/Other
"SHIB": "SHIB"
"SHIBA": "SHIB"
"PEPE": "PEPE"
"WIF": "WIF"
"BONK": "BONK"

89
sentiment_engine/config/asset_industry_map.yaml Normal file
View File

@@ -0,0 +1,89 @@
# Asset to industry/class mapping for hierarchical aggregation
mapping:
# Layer 1: Base protocols
BTC: "Store of Value"
ETH: "Smart Contract Platform"
SOL: "Smart Contract Platform"
BNB: "Smart Contract Platform"
ADA: "Smart Contract Platform"
AVAX: "Smart Contract Platform"
DOT: "Smart Contract Platform"
MATIC: "Smart Contract Platform"
ARB: "Smart Contract Platform"
OP: "Smart Contract Platform"
# Layer 2: DeFi
UNI: "DeFi - DEX"
AAVE: "DeFi - Lending"
LINK: "DeFi - Oracle"
MKR: "DeFi - Stablecoin"
CRV: "DeFi - DEX"
SUSHI: "DeFi - DEX"
BAL: "DeFi - DEX"
YFI: "DeFi - Yield"
COMP: "DeFi - Lending"
# Stablecoins
USDT: "Stablecoin"
USDC: "Stablecoin"
DAI: "Stablecoin"
BUSD: "Stablecoin"
TUSD: "Stablecoin"
FRAX: "Stablecoin"
# Meme
DOGE: "Meme"
SHIB: "Meme"
PEPE: "Meme"
WIF: "Meme"
BONK: "Meme"
FLOKI: "Meme"
# Gaming/Metaverse
AXS: "Gaming"
SAND: "Gaming"
MANA: "Gaming"
GALA: "Gaming"
ILV: "Gaming"
APE: "Gaming"
# Infrastructure
LINK: "Infrastructure - Oracle"
GRT: "Infrastructure - Indexing"
BAND: "Infrastructure - Oracle"
API3: "Infrastructure - Oracle"
# Privacy
XMR: "Privacy"
ZEC: "Privacy"
DASH: "Privacy"
# Exchange tokens
FTT: "Exchange Token" # Historical
OKB: "Exchange Token"
CRO: "Exchange Token"
KCS: "Exchange Token"
HT: "Exchange Token"
# NFT/Collectibles
APE: "NFT"
BLUR: "NFT"
LOOKS: "NFT"
weights:
"Store of Value": 1.0
"Smart Contract Platform": 1.0
"DeFi - DEX": 0.8
"DeFi - Lending": 0.8
"DeFi - Oracle": 0.7
"DeFi - Stablecoin": 0.7
"DeFi - Yield": 0.6
"Stablecoin": 0.5
"Meme": 0.4
"Gaming": 0.6
"Infrastructure - Oracle": 0.7
"Infrastructure - Indexing": 0.6
"Privacy": 0.5
"Exchange Token": 0.6
"NFT": 0.5
"UNKNOWN": 0.3

85
sentiment_engine/config/known_entities.yaml Normal file
View File

@@ -0,0 +1,85 @@
# Known entities with contract addresses and metadata
entities:
BTC:
name: "Bitcoin"
type: "crypto"
chain: "bitcoin"
contracts: []
market_cap_rank: 1
ETH:
name: "Ethereum"
type: "crypto"
chain: "ethereum"
contracts: ["0xC02aaA39b223FE8D0A0e5C4F27eAD9083C756Cc2"] # WETH
market_cap_rank: 2
SOL:
name: "Solana"
type: "crypto"
chain: "solana"
contracts: ["So11111111111111111111111111111111111111112"]
market_cap_rank: 5
BNB:
name: "BNB"
type: "crypto"
chain: "bsc"
contracts: ["0xbb4CdB9CBd36B01bD1cBaEBF2De08d9173bc095c"] # WBNB
market_cap_rank: 4
USDT:
name: "Tether USD"
type: "stablecoin"
chain: "ethereum"
contracts: ["0xdAC17F958D2ee523a2206206994597C13D831ec7"]
market_cap_rank: 3
USDC:
name: "USD Coin"
type: "stablecoin"
chain: "ethereum"
contracts: ["0xA0b86a33E6441b8C4C8C8C8C8C8C8C8C8C8C8C8C8"] # placeholder
market_cap_rank: 6
MATIC:
name: "Polygon"
type: "crypto"
chain: "polygon"
contracts: ["0x0000000000000000000000000000000000001010"]
market_cap_rank: 15
ARB:
name: "Arbitrum"
type: "crypto"
chain: "arbitrum"
contracts: []
market_cap_rank: 35
OP:
name: "Optimism"
type: "crypto"
chain: "optimism"
contracts: []
market_cap_rank: 40
UNI:
name: "Uniswap"
type: "defi"
chain: "ethereum"
contracts: ["0x1f9840a85d5aF5bf1D1762F925BDADdC4201F984"]
market_cap_rank: 20
AAVE:
name: "Aave"
type: "defi"
chain: "ethereum"
contracts: ["0x7Fc66500c84A76Ad7e9c93437bFc5Ac33E2DDaE9"]
market_cap_rank: 50
LINK:
name: "Chainlink"
type: "oracle"
chain: "ethereum"
contracts: ["0x514910771AF9Ca656af840dff83E8264EcF986CA"]
market_cap_rank: 18

1147
sentiment_engine/config/seed_sources.yaml Normal file
View File

File diff suppressed because it is too large Load Diff

257
sentiment_engine/config/settings.yaml Normal file
View File

@@ -0,0 +1,257 @@
# Sentiment Engine Configuration v2.0.0
# =============================================================================
# NATS JetStream Configuration
# =============================================================================
nats:
servers: ["nats://localhost:4222"]
stream_ingestion: "sentiment_ingestion"
stream_processed: "sentiment_processed"
subjects:
news: "sentiment.ingest.news"
social: "sentiment.ingest.social"
regulatory: "sentiment.ingest.regulatory"
exchange: "sentiment.ingest.exchange"
consumer_durable: "sentiment-engine"
ack_wait_seconds: 30
max_deliver: 3
# =============================================================================
# ClickHouse Configuration
# =============================================================================
clickhouse:
host: "localhost"
port: 8123
database: "dolphin"
user: "default"
password: "${CLICKHOUSE_PASSWORD}"
tables:
sentiment_events: "sentiment_events"
sentiment_scores: "sentiment_scores"
sentiment_raw_items: "sentiment_raw_items"
sentiment_otel: "sentiment_otel"
# =============================================================================
# Hazelcast Configuration
# =============================================================================
hazelcast:
cluster_name: "dolphin"
cluster_members: ["localhost:5701"]
maps:
sentiment_scores: "sentiment_scores_*"
sentiment_streams: "sentiment_streams"
# =============================================================================
# LatticeDB (Graph Layer) Configuration
# =============================================================================
latticedb:
enabled: true
host: "localhost"
port: 7878
# For source credibility propagation, entity co-occurrence graph
# =============================================================================
# NLP Model Configuration
# =============================================================================
nlp:
models:
entity_extraction:
model_name: "ProsusAI/finbert"
device: "cuda"
batch_size: 32
max_length: 512
sentiment_emotion:
model_name: "google/gemma-3-4b"
device: "cuda"
batch_size: 8
max_length: 2048
quantization: "4bit"
event_classification:
model_name: "custom/finbert-event-classifier"
device: "cuda"
batch_size: 16
max_length: 512
embeddings:
model_name: "intfloat/e5-large-v2"
device: "cuda"
batch_size: 64
max_length: 4096
multilingual_embeddings:
model_name: "intfloat/multilingual-e5-large"
device: "cuda"
batch_size: 32
language_detection:
model: "fasttext"
supported_languages: ["en"]
translate_non_english: false
asset_mapping:
ticker_regex: "\\$?[A-Z]{2,10}\\b"
contract_address_regex: "0x[a-fA-F0-9]{40}|[1-9A-HJ-NP-Za-km-z]{32,44}"
alias_file: "config/asset_aliases.yaml"
known_entities_file: "config/known_entities.yaml"
# =============================================================================
# Scoring Engine Configuration
# =============================================================================
scoring:
parameters:
fear_state:
halflife_minutes: 180
confidence_floor: 0.15
proximity_boost: 0.5
centroid_weight_keywords: 1.0
centroid_weight_sentences: 2.0
centroid_weight_clusters: 1.5
greed_state:
halflife_minutes: 180
confidence_floor: 0.15
proximity_boost: 0.5
hype_velocity:
halflife_minutes: 60
confidence_floor: 0.20
proximity_boost: 0.3
velocity_window_minutes: 15
pub_velocity:
halflife_minutes: 120
window_minutes: 60
min_sources: 3
pump_score:
halflife_minutes: 240
confidence_floor: 0.25
multi_source_threshold: 3
coordination_window_minutes: 30
dump_score:
halflife_minutes: 240
confidence_floor: 0.25
event_flags:
halflife_minutes: 480
event_types:
- "listing"
- "delisting"
- "hack"
- "regulatory"
- "governance"
- "upgrade"
- "partnership"
- "earnings"
- "macro"
- "liquidation"
- "whale"
- "manipulation"
aggregation:
asset_to_industry_map: "config/asset_industry_map.yaml"
industry_weights: "equal" # or "market_cap"
market_weights: "equal"
decay:
asset_halflife_minutes: 30
industry_halflife_minutes: 60
market_halflife_minutes: 120
# =============================================================================
# Source Credibility Registry
# =============================================================================
credibility:
registry_file: "config/source_credibility.yaml"
default_credibility: 0.5
decay:
half_life_days: 30
min_credibility: 0.1
feedback_loop:
enabled: true
lookback_days: 90
impact_threshold: 0.02 # 2% price move attributed to event
# =============================================================================
# Source Connector Configuration
# =============================================================================
connectors:
rss:
poll_interval_seconds: 120 # 2 minutes
max_feeds_per_poll: 500
timeout_seconds: 30
user_agent: "DOLPHIN-SentimentEngine/2.0"
api:
poll_interval_seconds: 300 # 5 minutes
rate_limit_rpm: 100
timeout_seconds: 30
twitter:
bearer_token: "${TWITTER_BEARER_TOKEN}"
api_key: "${TWITTER_API_KEY}"
api_secret: "${TWITTER_API_SECRET}"
access_token: "${TWITTER_ACCESS_TOKEN}"
access_secret: "${TWITTER_ACCESS_SECRET}"
stream_rules: ["crypto", "bitcoin", "ethereum", "defi", "web3"]
sample_rate: 0.1
reddit:
client_id: "${REDDIT_CLIENT_ID}"
client_secret: "${REDDIT_CLIENT_SECRET}"
user_agent: "DOLPHIN-SentimentEngine/2.0"
subreddits: ["CryptoCurrency", "Bitcoin", "EthTrader", "CryptoMoon", "SatoshiStreetBets"]
poll_interval_seconds: 300
use_pushshift: true
discord:
bot_token: "${DISCORD_BOT_TOKEN}"
channels: [] # channel IDs to monitor
telegram:
bot_token: "${TELEGRAM_BOT_TOKEN}"
channels: [] # channel usernames/IDs
web_crawl:
enabled: true
tool: "hister" # or "scrapy"
job_timeout_seconds: 3600
max_depth: 2
allowed_domains: []
rate_limit_rps: 1
# =============================================================================
# Prefect Configuration
# =============================================================================
prefect:
api_url: "http://localhost:4200/api"
work_pool: "sentiment-engine"
deployment_tags: ["sentiment", "production"]
flows:
rss_ingest:
schedule: "*/2 * * * *" # every 2 minutes
timeout_seconds: 300
api_ingest:
schedule: "*/5 * * * *" # every 5 minutes
timeout_seconds: 300
web_crawl:
schedule: "0 */30 * * *" # every 30 minutes
timeout_seconds: 7200
# =============================================================================
# Trading Engine Integration
# =============================================================================
trading_integration:
exf_map_key: "exf_latest"
acb_keys:
- "market_sentiment_state"
- "aggregate_pump_risk"
- "fear_state"
- "greed_state"
- "hype_velocity"
book_health_gate:
pump_score_veto_threshold: 75
alpha_exit_v7:
dump_score_threshold: 70
fear_state_threshold: 80
# =============================================================================
# Observability
# =============================================================================
observability:
otel:
endpoint: "http://localhost:4317"
service_name: "sentiment-engine"
resource_attributes:
deployment.environment: "production"
prometheus:
port: 9090
path: "/metrics"
logging:
level: "INFO"
format: "json"
output: "stdout"

123
sentiment_engine/config/source_credibility.yaml Normal file
View File

@@ -0,0 +1,123 @@
# Source credibility registry
# base_credibility: 0-1 static score
# relevance: 0-1 how relevant to crypto markets
# enabled: whether to use this source
sources:
# Crypto-native news (high credibility)
- source_id: "rss:coindesk.com"
name: "CoinDesk"
url: "https://www.coindesk.com"
source_type: "news"
base_credibility: 0.85
relevance: 0.9
enabled: true
- source_id: "rss:cointelegraph.com"
name: "CoinTelegraph"
url: "https://cointelegraph.com"
source_type: "news"
base_credibility: 0.75
relevance: 0.85
enabled: true
- source_id: "rss:theblock.co"
name: "The Block"
url: "https://www.theblock.co"
source_type: "news"
base_credibility: 0.85
relevance: 0.9
enabled: true
- source_id: "rss:decrypt.co"
name: "Decrypt"
url: "https://decrypt.co"
source_type: "news"
base_credibility: 0.75
relevance: 0.8
enabled: true
- source_id: "rss:messari.io"
name: "Messari"
url: "https://messari.io"
source_type: "research"
base_credibility: 0.8
relevance: 0.85
enabled: true
# Traditional finance
- source_id: "api:fred_vix"
name: "FRED VIX"
url: "https://fred.stlouisfed.org"
source_type: "regulatory"
base_credibility: 0.95
relevance: 0.7
enabled: true
- source_id: "api:fred_dxy"
name: "FRED DXY"
url: "https://fred.stlouisfed.org"
source_type: "regulatory"
base_credibility: 0.95
relevance: 0.7
enabled: true
# Exchange announcements
- source_id: "rss:binance.com"
name: "Binance Announcements"
url: "https://www.binance.com"
source_type: "exchange_ann"
base_credibility: 0.9
relevance: 0.95
enabled: true
- source_id: "rss:blog.coinbase.com"
name: "Coinbase Blog"
url: "https://blog.coinbase.com"
source_type: "exchange_ann"
base_credibility: 0.85
relevance: 0.9
enabled: true
# Social - Twitter
- source_id: "twitter:stream"
name: "Twitter/X Stream"
url: "https://twitter.com"
source_type: "social"
base_credibility: 0.4
relevance: 0.8
enabled: true
# Social - Reddit
- source_id: "reddit:CryptoCurrency"
name: "r/CryptoCurrency"
url: "https://reddit.com/r/CryptoCurrency"
source_type: "social"
base_credibility: 0.35
relevance: 0.75
enabled: true
- source_id: "reddit:Bitcoin"
name: "r/Bitcoin"
url: "https://reddit.com/r/Bitcoin"
source_type: "social"
base_credibility: 0.35
relevance: 0.8
enabled: true
- source_id: "reddit:EthTrader"
name: "r/EthTrader"
url: "https://reddit.com/r/EthTrader"
source_type: "social"
base_credibility: 0.3
relevance: 0.7
enabled: true
# Web crawl (lower credibility)
- source_id: "web:coindesk.com"
name: "CoinDesk (crawl)"
url: "https://www.coindesk.com"
source_type: "news"
base_credibility: 0.6
relevance: 0.85
enabled: true