sentiment-engine/prod/ops/v20r1_monitor.sh

#!/usr/bin/env bash
# v20r1 monitor — sandbox ops helper (repo-tracked: prod/ops/).
# Run anytime to snapshot: process, lease, heartbeat, FORCE_ENGAGE cadence,
# and independent HL testnet truth (unsigned /info — the same host the kernel's
# HlVenueTruthSupplier reads: hl/venue_truth.py + hl/instrument_provider.py).
# Testnet-only; reads only. No system code/data is touched by this helper.
#
# Usage: /mnt/dolphinng5_predict/prod/ops/v20r1_monitor.sh   (or from anywhere)
#
# NOTE: v20r1-specific constants (signer, spool, PIDs, HL host) are hardcoded;
# when a new campaign boots, bump ADDR/LOG/PID_*. The HL host is the canonical
# testnet hostname from prod/hl/config.py resolve_urls() => 'api.hyperliquid-testnet.xyz'
# (liquid, NOT 'hyperlight'). A typo'd host resolves as NXDOMAIN and would
# silently block venue truth — see section [5] error handling below.
set -u
ADDR="0x6af790574E5E9EA067FB4Ce7458B429039cE3f96"
LOG="/root/dolphin_logs/uv_journal_runs/f13_terminal_slot_release_v20r1_20260826/runner.log"
LEASE="/tmp/flight13_hl_testnet_single_writer.lock"
PID_WRAPPER=790059   # `timeout --signal=TERM 8h launch_flight13_testnet.sh`
PID_KERNEL=790060    # `/root/uv-wt/f13-hl/.f13_boot.py`
HL_API="https://api.hyperliquid-testnet.xyz/info"   # canonical testnet host (liquid, not hyperlight)

echo "=== v20r1 @ $(date -u +%FT%TZ) ==="

echo "[1] PROCESS (wrapper=${PID_WRAPPER} kernel=${PID_KERNEL})"
ps -p "$PID_WRAPPER","$PID_KERNEL" -o pid,ppid,stat,etime,args 2>/dev/null || echo "  !! PIDs ${PID_WRAPPER}/${PID_KERNEL} NOT alive"

echo "[2] EXCLUSIVE-WRITER LEASE (${LEASE})"
ls -la "$LEASE" 2>/dev/null && cat "$LEASE" 2>/dev/null || echo "  !! lease file absent"

echo "[3] HEARTBEAT (runner.log tail — TPSL ticker + latest BAR; stale-metadata noise excluded, surfaced in [6])"
# Sample a wide window then strip the per-200ms stale-metadata spam so the real
# heartbeat/TBAR lines survive even during the current steady-state warning storm.
tail -n 80 "$LOG" 2>/dev/null | grep -vE "RuntimeWarning|stddev|warnings.warn|c /= stddev|numpy|stale.*metadata" | tail -n 12 || echo "  (log unreadable)"

echo "[4] FORCE_ENGAGE cadence (synthetic SHORT firehose)"
printf '  synthetic-enter log lines: '; grep -c "FORCE_ENGAGE: SYNTHETIC ENTER" "$LOG" 2>/dev/null || echo 0
grep "in_flight=" "$LOG" 2>/dev/null | tail -n 1 | sed 's/^/  last in_flight: /'

echo "[5] INDEPENDENT HL TESTNET TRUTH (unsigned /info) — signer ${ADDR} @ ${HL_API}"
# Capture curl exit + HTTP code SEPARATELY from the body so a DNS/TLS failure
# prints UNREACHABLE — not a misleading JSON "Expecting value" parse error
# (which previously masked a bad hostname as a parse quirk).
_hl_info() {  # $1 = HL /info type (openOrders | clearinghouseState | userFills | allMids)
  local ef bf code rc
  ef=$(mktemp); bf=$(mktemp)
  code=$(curl -sS --max-time 12 -o "$bf" -w "%{http_code}" \
          -X POST "$HL_API" \
          -H 'Content-Type: application/json' \
          -d "{\"type\":\"$1\",\"user\":\"$ADDR\"}" 2>"$ef"); rc=$?
  if [ "$rc" -ne 0 ] || [ "${code:-000}" = "000" ]; then
    printf '  %s: UNREACHABLE — venue truth blocked (curl rc=%s http=%s; DNS/venue outage)\n' "$1" "${rc:-0}" "${code:-000}"
    printf '    detail: %s\n' "$(head -c 200 "$ef" 2>/dev/null)"
    rm -f "$ef" "$bf"; return
  fi
  python3 - "$bf" "$1" <<'PY'
import sys,json
bf,typ=sys.argv[1],sys.argv[2]
try:
    d=json.load(open(bf))
    if typ=='openOrders':
        o=d; print('  openOrders:', len(o) if isinstance(o,list) else o)
        for x in (o if isinstance(o,list) else []): print('    -',x.get('coin'),x.get('side'),'px',x.get('px'),'oid',x.get('oid'))
    else:
        m=d['marginSummary']; print('  accountValue=%s totalNtlPos=%s withdrawable=%s'%(m['accountValue'],m['totalNtlPos'],d['withdrawable']))
        for p in d.get('assetPositions',[]):po=p['position']
        if float(po['szi'])!=0: print('  POSITION',po['coin'],'szi',po['szi'],'entry',po['entryPx'],'ROE%',po['returnOnEquity'])
except Exception as e: print('  '+typ+' parse failed:',e)
PY
  rm -f "$ef" "$bf"
}
_hl_info openOrders
_hl_info clearinghouseState

echo "[6] ALERTS (tail CRITICAL/ERROR/REFUSING/LEASE-stale-metadata)"
grep -E "CRITICAL|ERROR|REFUSING|LEASE REFUSED|stale.*metadata" "$LOG" 2>/dev/null | tail -n 6 | sed 's/^/  /'