70 lines
3.6 KiB
Bash
70 lines
3.6 KiB
Bash
#!/usr/bin/env bash
|
|
# v20r1 monitor — sandbox ops helper (now repo-tracked: prod/ops/).
|
|
# Run anytime to snapshot: process, lease, heartbeat, FORCE_ENGAGE cadence,
|
|
# and independent HL testnet truth (unsigned /info — same source of truth the
|
|
# kernel's HlVenueTruthSupplier uses). Testnet-only; reads only.
|
|
#
|
|
# Usage: /mnt/dolphinng5_predict/prod/ops/v20r1_monitor.sh (or from anywhere)
|
|
#
|
|
# NOTE: v20r1-specific constants (signer, spool, PIDs) are hardcoded; when a new
|
|
# campaign boots, bump ADDR/SPOOL and the PID pair in section [1].
|
|
set -u
|
|
ADDR="0x6af790574E5E9EA067FB4Ce7458B429039cE3f96"
|
|
LOG="/root/dolphin_logs/uv_journal_runs/f13_terminal_slot_release_v20r1_20260826/runner.log"
|
|
LEASE="/tmp/flight13_hl_testnet_single_writer.lock"
|
|
PID_WRAPPER=790059 # `timeout --signal=TERM 8h launch_flight13_testnet.sh`
|
|
PID_KERNEL=790060 # `/root/uv-wt/f13-hl/.f13_boot.py`
|
|
|
|
echo "=== v20r1 @ $(date -u +%FT%TZ) ==="
|
|
|
|
echo "[1] PROCESS (wrapper=${PID_WRAPPER} kernel=${PID_KERNEL})"
|
|
ps -p "$PID_WRAPPER","$PID_KERNEL" -o pid,ppid,stat,etime,args 2>/dev/null || echo " !! PIDs ${PID_WRAPPER}/${PID_KERNEL} NOT alive"
|
|
|
|
echo "[2] EXCLUSIVE-WRITER LEASE (${LEASE})"
|
|
ls -la "$LEASE" 2>/dev/null && cat "$LEASE" 2>/dev/null || echo " !! lease file absent"
|
|
|
|
echo "[3] HEARTBEAT (runner.log tail — TPSL ticker + latest BAR)"
|
|
tail -n 12 "$LOG" 2>/dev/null | grep -vE "RuntimeWarning|stddev|warnings.warn|c /= stddev|numpy" || echo " (log unreadable)"
|
|
|
|
echo "[4] FORCE_ENGAGE cadence (synthetic SHORT firehose)"
|
|
printf ' synthetic-enter log lines: '; grep -c "FORCE_ENGAGE: SYNTHETIC ENTER" "$LOG" 2>/dev/null || echo 0
|
|
grep "in_flight=" "$LOG" 2>/dev/null | tail -n 1 | sed 's/^/ last in_flight: /'
|
|
|
|
echo "[5] INDEPENDENT HL TESTNET TRUTH (unsigned /info) — signer ${ADDR}"
|
|
# Probe captures curl exit+http code separately from the body so a DNS/TLS
|
|
# outage (the current failure mode: api.hyperlight-testnet.xyz NXDOMAIN across
|
|
# resolvers) is reported as UNREACHABLE — not a misleading JSON parse error.
|
|
_hl_info() { # $1 = HL /info type (openOrders | clearinghouseState | userFills | allMids)
|
|
local ef bf code rc
|
|
ef=$(mktemp); bf=$(mktemp)
|
|
code=$(curl -sS --max-time 12 -o "$bf" -w "%{http_code}" \
|
|
-X POST https://api.hyperlight-testnet.xyz/info \
|
|
-H 'Content-Type: application/json' \
|
|
-d "{\"type\":\"$1\",\"user\":\"$ADDR\"}" 2>"$ef"); rc=$?
|
|
if [ "$rc" -ne 0 ] || [ "${code:-000}" = "000" ]; then
|
|
printf ' %s: UNREACHABLE — venue truth blocked (curl rc=%s http=%s; DNS/venue outage)\n' "$1" "${rc:-0}" "${code:-000}"
|
|
printf ' detail: %s\n' "$(head -c 200 "$ef" 2>/dev/null)"
|
|
rm -f "$ef" "$bf"; return
|
|
fi
|
|
python3 - "$bf" "$1" <<'PY'
|
|
import sys,json
|
|
bf,typ=sys.argv[1],sys.argv[2]
|
|
try:
|
|
d=json.load(open(bf))
|
|
if typ=='openOrders':
|
|
o=d; print(' openOrders:', len(o) if isinstance(o,list) else o)
|
|
for x in (o if isinstance(o,list) else []): print(' -',x.get('coin'),x.get('side'),'px',x.get('px'),'oid',x.get('oid'))
|
|
else:
|
|
m=d['marginSummary']; print(' accountValue=%s totalNtlPos=%s withdrawable=%s'%(m['accountValue'],m['totalNtlPos'],d['withdrawable']))
|
|
for p in d.get('assetPositions',[]):po=p['position']
|
|
if float(po['szi'])!=0: print(' POSITION',po['coin'],'szi',po['szi'],'entry',po['entryPx'],'ROE%',po['returnOnEquity'])
|
|
except Exception as e: print(' '+typ+' parse failed:',e)
|
|
PY
|
|
rm -f "$ef" "$bf"
|
|
}
|
|
_hl_info openOrders
|
|
_hl_info clearinghouseState
|
|
|
|
echo "[6] ALERTS (tail CRITICAL/ERROR/REFUSING/LEASE-stale-metadata)"
|
|
grep -E "CRITICAL|ERROR|REFUSING|LEASE REFUSED|stale.*metadata" "$LOG" 2>/dev/null | tail -n 6 | sed 's/^/ /'
|