sentiment-engine/MALKHUT/malkhut/training/news_sources.py

"""
News Source Repository — industry-standard tracking for market regime sources.

Tracks:
  - Source metadata (name, URL, type, relevance)
  - Fetch history (timestamps, success/failure)
  - Source ranking (by regime relevance, freshness, reliability)
  - Source health (error rates, response times)
  - Auto-discovery of new sources

Format: JSON-based, compatible with standard news aggregation pipelines.
"""
from __future__ import annotations

import hashlib
import json
import os
import time
from dataclasses import dataclass, field
from typing import Any, Dict, List, Optional, Set, Tuple


@dataclass(frozen=True, slots=True)
class SourceMetadata:
    """Full metadata for a news/data source."""
    source_id: str
    name: str
    url: str
    source_type: str  # "news", "data", "research", "exchange", "social"
    regime_relevance: float  # 0-1
    reliability: float  # 0-1
    freshness_hours: float  # how often to fetch
    last_fetched_ns: int = 0
    fetch_count: int = 0
    error_count: int = 0
    avg_response_ms: float = 0.0
    tags: Tuple[str, ...] = ()
    enabled: bool = True
    created_ns: int = 0

    @property
    def health_score(self) -> float:
        if self.fetch_count == 0:
            return 0.5
        error_rate = self.error_count / self.fetch_count
        return max(0.0, 1.0 - error_rate) * self.reliability


class NewsSourceRepository:
    """
    Industry-standard repository for tracking news/data sources.

    Features:
      - Source registration and metadata
      - Fetch history tracking
      - Source ranking by relevance, freshness, reliability
      - Source health monitoring
      - Auto-discovery hooks
      - JSON persistence

    Compatible with standard news aggregation pipelines.
    """

    def __init__(self, repo_path: str = "news_sources.json") -> None:
        self._path = repo_path
        self._sources: Dict[str, SourceMetadata] = {}
        self._load()

    def _load(self) -> None:
        if os.path.exists(self._path):
            try:
                with open(self._path) as f:
                    data = json.load(f)
                for entry in data:
                    self._sources[entry["source_id"]] = SourceMetadata(**entry)
            except Exception:
                pass

    def _save(self) -> None:
        try:
            with open(self._path, "w") as f:
                json.dump([{
                    "source_id": s.source_id, "name": s.name, "url": s.url,
                    "source_type": s.source_type, "regime_relevance": s.regime_relevance,
                    "reliability": s.reliability, "freshness_hours": s.freshness_hours,
                    "last_fetched_ns": s.last_fetched_ns, "fetch_count": s.fetch_count,
                    "error_count": s.error_count, "avg_response_ms": s.avg_response_ms,
                    "tags": list(s.tags), "enabled": s.enabled, "created_ns": s.created_ns,
                } for s in self._sources.values()], f, indent=2)
        except OSError:
            pass

    def register(self, source_id: str, name: str, url: str,
                 source_type: str = "news", regime_relevance: float = 0.5,
                 reliability: float = 0.8, freshness_hours: float = 1.0,
                 tags: Tuple[str, ...] = ()) -> None:
        """Register a new source."""
        self._sources[source_id] = SourceMetadata(
            source_id=source_id, name=name, url=url,
            source_type=source_type, regime_relevance=regime_relevance,
            reliability=reliability, freshness_hours=freshness_hours,
            tags=tags, created_ns=time.time_ns(),
        )
        self._save()

    def record_fetch(self, source_id: str, success: bool, response_ms: float = 0.0) -> None:
        """Record a fetch attempt."""
        if source_id in self._sources:
            s = self._sources[source_id]
            new_count = s.fetch_count + 1
            new_errors = s.error_count + (0 if success else 1)
            new_avg = ((s.avg_response_ms * s.fetch_count) + response_ms) / new_count
            self._sources[source_id] = SourceMetadata(
                source_id=s.source_id, name=s.name, url=s.url,
                source_type=s.source_type, regime_relevance=s.regime_relevance,
                reliability=s.reliability, freshness_hours=s.freshness_hours,
                last_fetched_ns=time.time_ns(), fetch_count=new_count,
                error_count=new_errors, avg_response_ms=new_avg,
                tags=s.tags, enabled=s.enabled, created_ns=s.created_ns,
            )
            self._save()

    def rank_by_relevance(self) -> List[SourceMetadata]:
        """Rank sources by regime relevance."""
        return sorted(self._sources.values(), key=lambda s: s.regime_relevance, reverse=True)

    def rank_by_health(self) -> List[SourceMetadata]:
        """Rank sources by health score."""
        return sorted(self._sources.values(), key=lambda s: s.health_score, reverse=True)

    def rank_by_freshness(self) -> List[SourceMetadata]:
        """Rank sources by last fetch time (most recent first)."""
        return sorted(self._sources.values(), key=lambda s: s.last_fetched_ns, reverse=True)

    def get_enabled(self) -> List[SourceMetadata]:
        return [s for s in self._sources.values() if s.enabled]

    def get_by_type(self, source_type: str) -> List[SourceMetadata]:
        return [s for s in self._sources.values() if s.source_type == source_type]

    def get_by_tag(self, tag: str) -> List[SourceMetadata]:
        return [s for s in self._sources.values() if tag in s.tags]

    def disable(self, source_id: str) -> None:
        if source_id in self._sources:
            s = self._sources[source_id]
            self._sources[source_id] = SourceMetadata(
                source_id=s.source_id, name=s.name, url=s.url,
                source_type=s.source_type, regime_relevance=s.regime_relevance,
                reliability=s.reliability, freshness_hours=s.freshness_hours,
                last_fetched_ns=s.last_fetched_ns, fetch_count=s.fetch_count,
                error_count=s.error_count, avg_response_ms=s.avg_response_ms,
                tags=s.tags, enabled=False, created_ns=s.created_ns,
            )
            self._save()

    def enable(self, source_id: str) -> None:
        if source_id in self._sources:
            s = self._sources[source_id]
            self._sources[source_id] = SourceMetadata(
                source_id=s.source_id, name=s.name, url=s.url,
                source_type=s.source_type, regime_relevance=s.regime_relevance,
                reliability=s.reliability, freshness_hours=s.freshness_hours,
                last_fetched_ns=s.last_fetched_ns, fetch_count=s.fetch_count,
                error_count=s.error_count, avg_response_ms=s.avg_response_ms,
                tags=s.tags, enabled=True, created_ns=s.created_ns,
            )
            self._save()

    @property
    def source_count(self) -> int:
        return len(self._sources)

    @property
    def enabled_count(self) -> int:
        return len(self.get_enabled())

    def seed_defaults(self) -> None:
        """Seed with industry-standard crypto news sources."""
        defaults = [
            ("coindesk", "CoinDesk", "https://www.coindesk.com", "news", 0.8, 0.9, 1.0, ("crypto", "news")),
            ("cointelegraph", "Cointelegraph", "https://cointelegraph.com", "news", 0.7, 0.85, 1.0, ("crypto", "news")),
            ("the_block", "The Block", "https://www.theblock.co", "news", 0.8, 0.9, 0.5, ("crypto", "news", "research")),
            ("cryptoquant", "CryptoQuant", "https://cryptoquant.com", "data", 0.9, 0.95, 4.0, ("on_chain", "data")),
            ("glassnode", "Glassnode", "https://glassnode.com", "data", 0.9, 0.95, 4.0, ("on_chain", "data")),
            ("coinglass", "Coinglass", "https://www.coinglass.com", "data", 0.8, 0.9, 1.0, ("derivatives", "data")),
            ("binance_research", "Binance Research", "https://www.binance.com/en/research", "research", 0.7, 0.85, 24.0, ("exchange", "research")),
            ("messari", "Messari", "https://messari.io", "research", 0.7, 0.85, 24.0, ("research", "fundamentals")),
            ("defillama", "DefiLlama", "https://defillama.com", "data", 0.6, 0.9, 1.0, ("defi", "data")),
            ("dune", "Dune Analytics", "https://dune.com", "data", 0.7, 0.85, 4.0, ("on_chain", "data")),
            ("the_blockResearch", "The Block Research", "https://www.theblock.co/research", "research", 0.8, 0.9, 24.0, ("research", "institutional")),
            ("coingecko", "CoinGecko", "https://www.coingecko.com", "data", 0.6, 0.85, 1.0, ("market_data", "data")),
        ]
        for sid, name, url, stype, relevance, reliability, freshness, tags in defaults:
            self.register(sid, name, url, stype, relevance, reliability, freshness, tags)