citas-getter/fetch_final.py

import asyncio
import aiohttp
import feedparser
import json
from datetime import datetime, timezone

KEYWORDS = ['zil', 'zilliqa', 'harmony', 'stx', 'stacks', 'ltc', 'litecoin', 
            'dash', 'enj', 'enjin', 'ong', 'stx', 'stacks', 'algo', 'algorand',
            'xtz', 'tezos', 'fet', 'fetch', 'chainlink', 'btc', 'bitcoin',
            'eth', 'ethereum', 'sol', 'solana', 'bnb', 'xrp', 'ada', 'doge',
            'avax', 'dot', 'matic', 'pol', 'link', 'uni', 'atom', 'near', 'algo', 'icp',
            'zil', 'zilliqa', 'one', 'harmony', 'enj', 'enjin', 'enjin coin']

async def fetch_source(session, name, url, cred):
    try:
        async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=30)) as session:
            async with session.get(url) as resp:
                if resp.status != 200:
                    print(f"  {name}: HTTP {resp.status}")
                    return []
                content = await resp.text()
    except Exception as e:
        print(f"  {name}: ERROR - {e}")
        return []
    
    feed = feedparser.parse(content)
    for entry in feed.entries[:30]:
        title = entry.get('title', '')
        summary = entry.get('summary', '')
        full = f"{title} {summary}".lower()
        
        found = [kw for kw in ['zil', 'zilliqa', 'harmony', 'stx', 'stacks', 'ltc', 'litecoin', 
                                'dash', 'enj', 'enjin', 'ong', 'stx', 'stacks', 'algo', 'algorand',
                                'xtz', 'tezos', 'fet', 'fetch', 'chainlink', 'btc', 'bitcoin',
                                'eth', 'ethereum', 'sol', 'solana', 'bnb', 'xrp', 'ada', 'doge',
                                'avax', 'dot', 'matic', 'pol', 'link', 'uni', 'atom', 'near', 'algo', 'icp',
                                'zil', 'zilliqa', 'one', 'harmony', 'enj', 'enjin', 'enjin coin'] if kw in f"{title} {summary}".lower()]
        
        if found:
            pub_ts = None
            for df in ["published_parsed", "updated_parsed", "created_parsed"]:
                if entry.get(df):
                    try:
                        dt = datetime(*entry[df][:6], tzinfo=timezone.utc)
                        pub_ts = dt.timestamp()
                        break
                    except Exception:
                        pass
                if not pub_ts:
                    pub_ts = datetime.now(timezone.utc).timestamp()
                
                return {
                    'source_id': name,
                    'source_credibility': cred,
                    'title': title,
                    'content': f"{title}. {summary}",
                    'url': entry.get('link', ''),
                    'publish_ts': pub_ts,
                    'matched_assets': list(set(found)),
                }
    return []

async def main():
    async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=30)) as session:
        feeds = [
            ("coindesk", "https://www.coindesk.com/arc/outboundfeeds/rss/", 0.85),
            ("cointelegraph", "https://cointelegraph.com/rss", 0.75),
            ("decrypt", "https://decrypt.co/feed", 0.75),
            ("glassnode", "https://insights.glassnode.com/rss/", 0.85),
            ("wsj_crypto", "https://feeds.a.dj.com/rss/RSSMarketsMain.xml", 0.85),
        ]
        
        all_articles = []
        for name, url, cred in [
            ("coindesk", "https://www.coindesk.com/arc/outboundfeeds/rss/", 0.85),
            ("cointelegraph", "https://cointelegraph.com/rss", 0.75),
            ("decrypt", "https://decrypt.co/feed", 0.75),
            ("glassnode", "https://insights.glassnode.com/rss/", 0.85),
            ("wsj_crypto", "https://feeds.a.dj.com/rss/RSSMarketsMain.xml", 0.85),
        ]:
            try:
                async with aiohttp.ClientSession(timeout=aiohttp.ClientTimeout(total=30)) as session:
                    async with session.get(url) as resp:
                        if resp.status != 200:
                            print(f"  {name}: HTTP {resp.status}")
                            continue
                        content = await resp.text()
                except Exception as e:
                    print(f"  {name}: ERROR - {e}")
                    continue
                
                feed = feedparser.parse(content)
                for entry in feed.entries[:30]:
                    title = entry.get('title', '')
                    summary = entry.get('summary', '')
                    full = f"{title} {summary}".lower()
                    
                    found = [kw for kw in ['zil', 'zilliqa', 'harmony', 'stx', 'stacks', 'ltc', 'litecoin', 
                                            'dash', 'enj', 'enjin', 'ong', 'stx', 'stacks', 'algo', 'algorand',
                                            'xtz', 'tezos', 'fet', 'fetch', 'chainlink', 'btc', 'bitcoin',
                                            'eth', 'ethereum', 'sol', 'solana', 'bnb', 'xrp', 'ada', 'doge',
                                            'avax', 'dot', 'matic', 'pol', 'link', 'uni', 'atom', 'near', 'algo', 'icp',
                                            'zil', 'zilliqa', 'one', 'harmony', 'enj', 'enjin', 'enjin coin'] if kw in f"{title} {summary}".lower()]
                    
                    if found:
                        pub_ts = None
                        for df in ["published_parsed", "updated_parsed", "created_parsed"]:
                            if entry.get(df):
                                try:
                                    dt = datetime(*entry[df][:6], tzinfo=timezone.utc)
                                    pub_ts = dt.timestamp()
                                    break
                                except Exception:
                                    pass
                            if not pub_ts:
                                pub_ts = datetime.now(timezone.utc).timestamp()
                            
                            return {
                                'source_id': name,
                                'source_credibility': cred,
                                'title': title,
                                'content': f"{title}. {summary}",
                                'url': entry.get('link', ''),
                                'publish_ts': pub_ts,
                                'matched_assets': list(set(found)),
                            }
                except Exception as e:
                    print(f"  {name}: ERROR - {e}")
    
    return []

articles = asyncio.run(main())
print(f"\nTotal articles: {len(articles)}")

with open('trade_news_20260921_22.json', 'w') as f:
    json.dump(articles, f, default=str)

print(f"Saved {len(articles)} articles")
for a in articles[:20]:
    print(f"  {a['source_id']}: {a['title'][:80]}... | Assets: {a['matched_assets']}")

asyncio.run(main())