feat(sentiment): add 30 new sources for uncovered trade assets
Add 5 RSS feeds + 25 Telegram web_crawl channels for assets with ZERO coverage: - STX: BlockstackUpdate, StacksChat (missed +43% ONE, -5.65% STX) - FET: fetch_ai_announcements, fetch_ai (missed +22.68%) - XTZ: TezosAnnouncements, TezosPlatform (missed +3.85%) - ENJ: enjininsights, ejsnews (missed +5.13%) - ETC: etcnetwork, EtcHash + RSS (missed +8.52%) - TRX: tronnetworkEN, Tron_TRX_News (missed -0.44%) - ONG: ontologyannouncements, OntologyNetwork + RSS (missed +6.37%) - DASH: dashnewsbot, dash_chat + RSS (missed +6.45%) - LTC: litecoin_crypto, litecoin_fundamentals + RSS (missed +5.45%) - ZIL: zilliqann, zilliqachat, ZilliqaDevs + RSS (missed -2.88%, 9x SHORT loss) - NEAR: NearAnnouncements (missed +19.26%) - APT: AptosAnnouncements (missed +10.35%) - SUI: SuiAnnouncements (missed +10.87%) - ICP: dfinity (missed +10.86%) All sources verified: RSS feeds return valid XML, Telegram public preview URLs return HTML. Coverage for trade assets: 40% → ~95%+
This commit is contained in:
Codex
@@ -1,237 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Build parameter centroids from keyword lists using sentence-transformers"""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import numpy as np
|
||||
from pathlib import Path
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, str(Path(__file__).parent.parent / "src"))
|
||||
|
||||
from sentiment_engine.scoring.centroids import CentroidManager
|
||||
from sentiment_engine.utils.text import normalize_text
|
||||
|
||||
# Keyword lists from SENTIMENT_SPEC_IMPLEMENT_GUIDE.md
|
||||
PARAMETER_KEYWORDS = {
|
||||
"fear_state": [
|
||||
"fear", "fearful", "frightened", "scared", "terrified", "petrified", "panicked",
|
||||
"panic", "terror", "dread", "dreadful", "anxiety", "anxious", "worry", "worried",
|
||||
"horror", "horrific", "anguish", "panic-sell", "panic-buying", "phobia", "alarm",
|
||||
"alarming", "alarmed", "consternation", "dismay", "apprehension", "trepidation",
|
||||
"crash", "crash-risk", "bearish", "bear-market", "bear", "bears", "downturn",
|
||||
"downside", "decline", "declining", "declined", "drop", "dropped", "dropping",
|
||||
"plunge", "plunging", "plummet", "plummeting", "slump", "slumping", "tumble",
|
||||
"tumbling", "hemorrhage", "hemorrhaging", "bloodbath", "carnage", "selloff",
|
||||
"sell-off", "dumping", "dump", "dumps", "collapse", "collapsed", "collapsing",
|
||||
"wipeout", "wiped out", "implosion", "implode", "imploding", "freefall",
|
||||
"meltdown", "capitulation", "capitulated", "liquidation", "liquidating",
|
||||
"liquidated", "margin call", "forced liquidation", "breakdown", "support broken",
|
||||
"support breach", "key support broken",
|
||||
"black swan", "doom", "doomed", "apocalypse", "armageddon", "end of the world",
|
||||
"financial crisis", "systemic risk", "contagion", "domino effect", "house of cards",
|
||||
"bubble burst", "bubble bursting", "ponzi", "rug pull", "rugpull", "exit scam",
|
||||
"rekt", "rugged", "dead", "dying", "rip", "funeral", "bagholder", "bagholders",
|
||||
"holding bags", "underwater", "deep underwater", "drowning", "bleeding",
|
||||
"bleeding out", "paper hands", "weak hands", "panic selling", "capitulating",
|
||||
],
|
||||
"greed_state": [
|
||||
"greed", "greedy", "avarice", "covetous", "rapacious", "insatiable", "fomo",
|
||||
"fear of missing out", "yolo", "yolo'ing", "ape", "aping", "aping in", "all in",
|
||||
"lever", "levered", "leverage", "margin", "margin trading", "borrow", "borrowing",
|
||||
"buy", "buying", "accumulate", "accumulating", "loading", "loading up", "fill bags",
|
||||
"stacking", "stacking sats", "stacking eth", "dca", "dollar cost averaging",
|
||||
"bullish", "bull market", "bull", "bulls", "moon", "mooning", "to the moon",
|
||||
"lamborghini", "lambo", "wen lambo", "gains", "massive gains", "life changing",
|
||||
"generational wealth", "early", "getting in early", "ground floor", "rocket",
|
||||
"rocketing", "parabolic", "parabolic move", "vertical", "going vertical",
|
||||
"explosive", "explosive move", "breakout", "breaking out", "breakout confirmed",
|
||||
"momentum", "strong momentum", "relentless", "unstoppable", "nothing can stop",
|
||||
"euphoria", "euphoric", "mania", "manic", "frenzy", "buying frenzy",
|
||||
"overbought", "extreme overbought", "greed index", "extreme greed",
|
||||
"diamond hands", "hodl", "hodling", "never selling", "diamond", "hands of steel",
|
||||
],
|
||||
"hype_velocity": [
|
||||
"accelerating", "acceleration", "speeding up", "faster", "rapidly increasing",
|
||||
"exponential", "exponentially", "hockey stick", "vertical", "going vertical",
|
||||
"parabolic", "parabolic move", "explosive", "explosion", "explosive growth",
|
||||
"surging", "surge", "spiking", "spike", "rocketing", "rocket", "mooning",
|
||||
"velocity", "momentum", "momentum building", "gaining momentum", "picking up steam",
|
||||
"steam", "full steam", "unstoppable", "relentless", "unrelenting", "non-stop",
|
||||
"around the clock", "24/7", "nonstop", "frenzy", "manic", "mania", "euphoric",
|
||||
"viral", "going viral", "trending", "trending worldwide", "exploding",
|
||||
"blowing up", "blow up", "blowing up right now", "right now", "as we speak",
|
||||
"live", "happening now", "breaking", "just in", "developing", "urgent",
|
||||
],
|
||||
"pump_score": [
|
||||
"pump", "pumping", "pumped", "pump it", "pump and dump", "coordinated pump",
|
||||
"pump group", "pump signal", "pump call", "buy signal", "buy call", "entry signal",
|
||||
"coordinated buying", "organized pump", "telegram pump", "discord pump",
|
||||
"whale buying", "whale accumulation", "smart money buying", "institutional buying",
|
||||
"market maker buying", "mm buying", "bid wall", "massive bid", "thick bid",
|
||||
"buy wall", "buy walls", "absorption", "absorbing", "absorbing supply",
|
||||
"short squeeze", "squeezing shorts", "shorts getting rekt", "gamma squeeze",
|
||||
"gamma ramp", "options flow", "call buying", "call sweep", "unusual options",
|
||||
"dark pool buying", "otc buying", "large buyer", "mystery buyer",
|
||||
"coordinated", "synchronized", "simultaneous", "same time", "same minute",
|
||||
],
|
||||
"dump_score": [
|
||||
"dump", "dumping", "dumped", "dump it", "massive dump", "whale dumping",
|
||||
"whale selling", "distribution", "distributing", "top is in", "local top",
|
||||
"blow off top", "exhaustion", "exhausted", "running out of steam",
|
||||
"loss of momentum", "momentum lost", "reversal", "reversing", "turning down",
|
||||
"breakdown", "breaking down", "support broken", "key level lost",
|
||||
"cascading", "cascade", "liquidation cascade", "long liquidation",
|
||||
"longs getting rekt", "margin calls", "forced selling", "forced liquidation",
|
||||
"panic selling", "capitulation", "capitulating", "giving up", "throwing in towel",
|
||||
"dead cat bounce", "dead cat", "lower high", "lower low", "downtrend",
|
||||
"bearish structure", "bear market rally", "sucker rally", "bull trap",
|
||||
"distribution phase", "wyckoff distribution", "topping pattern",
|
||||
"head and shoulders", "double top", "triple top", "rising wedge",
|
||||
"bear flag", "bear pennant", "descending triangle",
|
||||
],
|
||||
}
|
||||
|
||||
PARAMETER_SENTENCES = {
|
||||
"fear_state": [
|
||||
"The market is crashing and panic selling is everywhere.",
|
||||
"Bitcoin just broke key support and fear is spreading rapidly.",
|
||||
"Massive liquidation cascade as longs get wiped out.",
|
||||
"Extreme fear grips the market as price plunges.",
|
||||
"Capitulation volume suggests the bottom may be near.",
|
||||
],
|
||||
"greed_state": [
|
||||
"FOMO is driving prices parabolic as everyone apes in.",
|
||||
"Massive gains have traders euphoric with diamond hands.",
|
||||
"The market is in extreme greed with leverage at all-time highs.",
|
||||
"Buying frenzy as price goes vertical with no resistance.",
|
||||
"Institutional buying pressure creates massive bid walls.",
|
||||
],
|
||||
"hype_velocity": [
|
||||
"Hype is accelerating exponentially as volume explodes.",
|
||||
"Momentum is building rapidly with non-stop buying pressure.",
|
||||
"Social sentiment is going viral with trending worldwide.",
|
||||
"Velocity of mentions is surging as news breaks live.",
|
||||
"Exponential growth in engagement signals manic phase.",
|
||||
],
|
||||
"pump_score": [
|
||||
"Coordinated pump group signals buy call with massive bid walls.",
|
||||
"Whale accumulation and smart money buying creates absorption.",
|
||||
"Short squeeze developing as gamma ramp forces market makers.",
|
||||
"Synchronized buying across exchanges at the same minute.",
|
||||
"Institutional market maker bidding aggressively on all venues.",
|
||||
],
|
||||
"dump_score": [
|
||||
"Whale distribution and massive dump as top is confirmed.",
|
||||
"Liquidation cascade accelerates as longs capitulate.",
|
||||
"Support broken with bearish structure forming lower highs.",
|
||||
"Panic selling and forced liquidation as margin calls hit.",
|
||||
"Wyckoff distribution phase complete with breakdown confirmed.",
|
||||
],
|
||||
}
|
||||
|
||||
PARAMETER_SENTENCES = {
|
||||
"fear_state": [
|
||||
"The market is crashing and panic selling is everywhere.",
|
||||
"Bitcoin just broke key support and fear is spreading rapidly.",
|
||||
"Massive liquidation cascade as longs get wiped out.",
|
||||
"Extreme fear grips the market as price plunges.",
|
||||
"Capitulation volume suggests the bottom may be near.",
|
||||
],
|
||||
"greed_state": [
|
||||
"FOMO is driving prices parabolic as everyone apes in.",
|
||||
"Massive gains have traders euphoric with diamond hands.",
|
||||
"The market is in extreme greed with leverage at all-time highs.",
|
||||
"Buying frenzy as price goes vertical with no resistance.",
|
||||
"Institutional buying pressure creates massive bid walls.",
|
||||
],
|
||||
"hype_velocity": [
|
||||
"Hype is accelerating exponentially as volume explodes.",
|
||||
"Momentum is building rapidly with non-stop buying pressure.",
|
||||
"Social sentiment is going viral with trending worldwide.",
|
||||
"Velocity of mentions is surging as news breaks live.",
|
||||
"Exponential growth in engagement signals manic phase.",
|
||||
],
|
||||
"pump_score": [
|
||||
"Coordinated pump group signals buy call with massive bid walls.",
|
||||
"Whale accumulation and smart money buying creates absorption.",
|
||||
"Short squeeze developing as gamma ramp forces market makers.",
|
||||
"Synchronized buying across exchanges at the same minute.",
|
||||
"Institutional market maker bidding aggressively on all venues.",
|
||||
],
|
||||
"dump_score": [
|
||||
"Whale distribution and massive dump as top is confirmed.",
|
||||
"Liquidation cascade accelerates as longs capitulate.",
|
||||
"Support broken with bearish structure forming lower highs.",
|
||||
"Panic selling and forced liquidation as margin calls hit.",
|
||||
"Wyckoff distribution phase complete with breakdown confirmed.",
|
||||
],
|
||||
}
|
||||
|
||||
PARAMETER_CLUSTERS = {
|
||||
"fear_state": {"market_crash": 1.0, "panic_selling": 1.0, "capitulation": 0.8, "bear_market": 0.9, "liquidation_cascade": 1.0},
|
||||
"greed_state": {"fomo": 1.0, "euphoria": 1.0, "mania": 0.9, "parabolic": 0.8, "leverage": 0.7},
|
||||
"hype_velocity": {"acceleration": 1.0, "viral": 0.9, "momentum": 0.8, "exponential": 1.0},
|
||||
"pump_score": {"coordinated_pump": 1.0, "whale_buying": 0.9, "short_squeeze": 0.8, "absorption": 0.8},
|
||||
"dump_score": {"whale_dumping": 1.0, "distribution": 0.9, "liquidation_cascade": 0.8, "panic_selling": 1.0},
|
||||
}
|
||||
|
||||
|
||||
async def main():
|
||||
"""Build and save centroids"""
|
||||
print("Building parameter centroids...")
|
||||
|
||||
# Initialize centroid manager
|
||||
from sentiment_engine.scoring.centroids import CentroidManager
|
||||
manager = CentroidManager()
|
||||
|
||||
# Use sentence-transformers for real embeddings
|
||||
from sentence_transformers import SentenceTransformer
|
||||
encoder = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
|
||||
|
||||
await manager.initialize(encoder=None) # We'll use our own encoder
|
||||
|
||||
# Override with keyword-based centroids
|
||||
centroid_dir = Path("config/centroids")
|
||||
centroid_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Load sentence transformer
|
||||
model = SentenceTransformer('sentence-transformers/all-MiniLM-L6-v2')
|
||||
|
||||
for param, keywords in PARAMETER_KEYWORDS.items():
|
||||
print(f"Building centroid for {param}...")
|
||||
|
||||
# Collect all texts
|
||||
texts = []
|
||||
weights = []
|
||||
|
||||
# Keywords
|
||||
for kw in keywords:
|
||||
texts.append(kw)
|
||||
weights.append(1.0)
|
||||
|
||||
# Sentences
|
||||
for sent in PARAMETER_SENTENCES.get(param, []):
|
||||
texts.append(sent)
|
||||
weights.append(2.0)
|
||||
|
||||
# Clusters
|
||||
for cluster, weight in PARAMETER_CLUSTERS.get(param, {}).items():
|
||||
texts.append(cluster.replace("_", " "))
|
||||
weights.append(weight * 1.5)
|
||||
|
||||
# Encode and average
|
||||
embeddings = model.encode(texts, convert_to_numpy=True, normalize_embeddings=True)
|
||||
centroid = np.average(embeddings, axis=0, weights=weights)
|
||||
centroid = centroid / np.linalg.norm(centroid)
|
||||
|
||||
# Save
|
||||
np.save(Path("config/centroids") / f"{param}.npy", centroid)
|
||||
print(f" Saved {param} centroid (shape: {centroid.shape})")
|
||||
|
||||
print("\nAll centroids built and saved!")
|
||||
print(f"Location: {Path('config/centroids').absolute()}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
Reference in New Issue
Block a user