DOLPHIN/prod/services_ctl.py

#!/usr/bin/env python3
"""
DOLPHIN Services Control
========================
Unified control for all Prefect-managed services.

Usage:
    python services_ctl.py status    # Check all services
    python services_ctl.py deploy    # Deploy to Prefect
    python services_ctl.py start     # Start worker
    python services_ctl.py stop      # Stop all services
    python services_ctl.py logs      # Show recent logs
"""

import argparse
import subprocess
import sys
from pathlib import Path

SERVICES = [
    ('scan-bridge-daemon', 'Arrow → Hz bridge', 'DOLPHIN_FEATURES:latest_eigen_scan'),
    ('exf-service', 'External factors', 'DOLPHIN_FEATURES:exf_latest'),
    ('obf-service', 'Order book features', 'DOLPHIN_FEATURES_SHARD_00:*'),
    ('acb-processor-daemon', 'ACB calculations', 'DOLPHIN_FEATURES:acb_boost'),
    ('watchdog-daemon', 'Survival stack', 'DOLPHIN_SAFETY:latest'),
]

POOL_NAME = "dolphin-services"


def run_cmd(cmd, capture=True):
    """Run shell command."""
    result = subprocess.run(cmd, shell=True, capture_output=capture, text=True)
    return result


def status():
    """Show status of all services."""
    print("🐬 DOLPHIN Services Status")
    print("=" * 70)
    
    # Check Prefect deployments
    print("\n📋 Prefect Deployments:")
    result = run_cmd("prefect deployment ls 2>/dev/null | grep -E 'dolphin|scan|exf|obf|acb|watchdog' || echo '  No deployments found'")
    print(result.stdout if result.stdout else "  (No deployments)")
    
    # Check running processes
    print("\n🔧 Running Processes:")
    result = run_cmd("ps aux | grep -E 'scan_bridge|exf_prefect|obf_prefect|acb_processor|system_watchdog' | grep -v grep")
    if result.stdout:
        for line in result.stdout.strip().split('\n')[:10]:
            parts = line.split()
            if len(parts) >= 11:
                pid = parts[1]
                cmd = ' '.join(parts[10:])[:50]
                print(f"  PID {pid}: {cmd}")
    else:
        print("  No services running")
    
    # Check Hz data
    print("\n📡 Hazelcast Data:")
    try:
        import sys
        sys.path.insert(0, '/mnt/dolphinng5_predict')
        import hazelcast
        
        client = hazelcast.HazelcastClient(
            cluster_name="dolphin",
            cluster_members=["127.0.0.1:5701"],
        )
        
        features = client.get_map('DOLPHIN_FEATURES').blocking()
        safety = client.get_map('DOLPHIN_SAFETY').blocking()
        
        checks = [
            ('Scan Data', 'latest_eigen_scan', features.get('latest_eigen_scan') is not None),
            ('EXTF', 'exf_latest', features.get('exf_latest') is not None),
            ('ACB', 'acb_boost', features.get('acb_boost') is not None),
            ('Safety', 'latest', safety.get('latest') is not None),
        ]
        
        for name, key, present in checks:
            icon = "✅" if present else "❌"
            print(f"  {icon} {name:15} ({key})")
        
        client.shutdown()
    except Exception as e:
        print(f"  Error: {e}")
    
    print("\n" + "=" * 70)


def deploy():
    """Deploy all services to Prefect."""
    print("🚀 Deploying DOLPHIN Services")
    print("=" * 70)
    
    # Check if pool exists
    result = run_cmd(f"prefect work-pool ls | grep {POOL_NAME}")
    if POOL_NAME not in (result.stdout or ""):
        print(f"\n📦 Creating work pool: {POOL_NAME}")
        run_cmd(f"prefect work-pool create {POOL_NAME} --type process", capture=False)
    
    # Deploy
    print("\n🔨 Deploying services...")
    run_cmd(f"prefect deploy prefect_all_services.yaml", capture=False)
    
    print("\n" + "=" * 70)
    print("✅ Deployment complete!")
    print(f"\nStart worker: prefect worker start --pool {POOL_NAME}")


def start():
    """Show start commands."""
    print("🚀 Start DOLPHIN Services")
    print("=" * 70)
    print("\n1. Start Prefect worker:")
    print(f"   prefect worker start --pool {POOL_NAME}")
    print("\n2. Or start services manually:")
    for name, desc, _ in SERVICES:
        print(f"   prefect deployment run '{name}/{name}'")


def stop():
    """Stop all services."""
    print("🛑 Stopping DOLPHIN Services")
    print("=" * 70)
    
    # Kill processes
    print("\nStopping processes...")
    patterns = ['scan_bridge', 'exf_prefect', 'obf_prefect', 'acb_processor', 'system_watchdog']
    for pattern in patterns:
        run_cmd(f"pkill -f '{pattern}' 2>/dev/null")
    
    print("✅ All services stopped")


def logs():
    """Show recent logs."""
    print("📜 Recent Service Logs")
    print("=" * 70)
    
    log_files = [
        ('Scan Bridge', '/tmp/scan_bridge.log'),
        ('ACB', '/tmp/acb_service.log'),
        ('Watchdog', '/tmp/watchdog_service.log'),
        ('EXF', '/tmp/exf_service.log'),
        ('OBF', '/tmp/obf_service.log'),
    ]
    
    for name, path in log_files:
        print(f"\n{name} ({path}):")
        result = run_cmd(f"tail -5 {path} 2>/dev/null || echo '  (no log)'")
        print(result.stdout)


def main():
    parser = argparse.ArgumentParser(description="Control DOLPHIN services")
    parser.add_argument('action', choices=['status', 'deploy', 'start', 'stop', 'logs'],
                       default='status', nargs='?')
    args = parser.parse_args()
    
    if args.action == 'status':
        status()
    elif args.action == 'deploy':
        deploy()
    elif args.action == 'start':
        start()
    elif args.action == 'stop':
        stop()
    elif args.action == 'logs':
        logs()


if __name__ == "__main__":
    main()