129 lines
3.7 KiB
Python
129 lines
3.7 KiB
Python
"""Web crawl Prefect flow"""
|
|
|
|
import asyncio
|
|
import logging
|
|
import subprocess
|
|
import tempfile
|
|
from pathlib import Path
|
|
from typing import List
|
|
|
|
from prefect import flow, task
|
|
|
|
from sentiment_engine.utils.config import get_settings
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
@task(retries=1, retry_delay_seconds=300)
|
|
async def run_hister_crawl(
|
|
seed_urls: List[str],
|
|
allowed_domains: List[str],
|
|
max_depth: int = 2,
|
|
job_timeout: int = 3600
|
|
) -> List[dict]:
|
|
"""Run Hister crawl job"""
|
|
with tempfile.TemporaryDirectory() as tmpdir:
|
|
seed_file = Path(tmpdir) / "seeds.txt"
|
|
seed_file.write_text("\n".join(seed_urls))
|
|
output_file = Path(tmpdir) / "output.jsonl"
|
|
|
|
cmd = [
|
|
"hister", "crawl",
|
|
"--input", str(seed_file),
|
|
"--job-id", f"prefect-crawl-{asyncio.current_task().get_name()}",
|
|
"--depth", str(max_depth),
|
|
"--delay", "1.0",
|
|
"--output", str(output_file),
|
|
"--format", "jsonl"
|
|
]
|
|
|
|
if allowed_domains:
|
|
cmd.extend(["--allowed-domain", ",".join(allowed_domains)])
|
|
|
|
try:
|
|
proc = await asyncio.create_subprocess_exec(
|
|
*cmd,
|
|
stdout=asyncio.subprocess.PIPE,
|
|
stderr=asyncio.subprocess.PIPE
|
|
)
|
|
|
|
stdout, stderr = await asyncio.wait_for(
|
|
proc.communicate(), timeout=job_timeout
|
|
)
|
|
|
|
if proc.returncode != 0:
|
|
logger.error(f"Hister failed: {stderr.decode()}")
|
|
return []
|
|
|
|
# Parse output
|
|
items = []
|
|
if output_file.exists():
|
|
import json
|
|
with open(output_file) as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if not line:
|
|
continue
|
|
try:
|
|
data = json.loads(line)
|
|
items.append(data)
|
|
except json.JSONDecodeError:
|
|
continue
|
|
|
|
return items
|
|
|
|
except asyncio.TimeoutError:
|
|
logger.error(f"Hister job timed out after {job_timeout}s")
|
|
return []
|
|
except FileNotFoundError:
|
|
logger.error("Hister not installed")
|
|
return []
|
|
|
|
|
|
@flow(
|
|
name="web_crawl",
|
|
log_prints=True
|
|
)
|
|
async def web_crawl_flow(
|
|
seed_urls: List[str] = None,
|
|
allowed_domains: List[str] = None,
|
|
max_depth: int = 2
|
|
):
|
|
"""Web crawl flow for sites without RSS/API"""
|
|
if seed_urls is None:
|
|
seed_urls = [
|
|
"https://www.coindesk.com",
|
|
"https://cointelegraph.com",
|
|
"https://www.theblock.co",
|
|
"https://decrypt.co",
|
|
"https://cryptoslate.com",
|
|
]
|
|
|
|
if allowed_domains is None:
|
|
allowed_domains = [
|
|
"coindesk.com", "cointelegraph.com", "theblock.co",
|
|
"decrypt.co", "cryptoslate.com", "bitcoinmagazine.com"
|
|
]
|
|
|
|
items = await run_hister_crawl(seed_urls, allowed_domains, max_depth)
|
|
|
|
logger.info(f"Crawled {len(items)} pages")
|
|
|
|
# Convert to normalized items
|
|
normalized = []
|
|
for item in items:
|
|
normalized.append({
|
|
"source_id": f"web:{item.get('url', '').split('/')[2] if item.get('url') else 'unknown'}",
|
|
"source_type": "news",
|
|
"raw_text": f"{item.get('title', '')}\n\n{item.get('content', item.get('text', ''))}",
|
|
"title": item.get("title"),
|
|
"url": item.get("url"),
|
|
"metadata": {"crawler": "hister", "job": "prefect"}
|
|
})
|
|
|
|
return normalized
|
|
|
|
|
|
if __name__ == "__main__":
|
|
asyncio.run(web_crawl_flow())
|