pi_wake_agent: --pane targeting + crontab backend fallback + cronicle HCL fix + hermetic tests

- --pane/--pane-id: write-chars AND write(13) hit terminal_1 (bottom pane)
- cronicle_available()=shutil.which + daemon-live check; crontab/crnd auto-fallback
  (cronicle daemon down here -> live scheduler is system crnd)
- fix cronicle_install invalid HCL (command=[...] array, was bare comma-list)
- rewrite TestCrontabEntry hermetic (subprocess mocked; real crontab never touched)
- add --pane-id / no-pane omission / HCL-array mutation-litmus tests; .sh --pane
- cronicle.hcl: corrected pi_wake_pi_test_20m (msg + --pane terminal_1), valid HCL
- live 20m crontab doorbell installed for pi_test/terminal_1, bell verified delivered
- SIGQUIT rescued a locked pi (final_test @1318s); steer queued + bus-posted
This commit is contained in:
Codex
2026-09-14 21:31:37 +02:00
parent a276aeaded
commit 5e9168ac1d
4 changed files with 1000 additions and 93 deletions

View File

@@ -17,6 +17,7 @@ import argparse
import logging
import os
import re
import shutil
import shlex
import signal
import subprocess
@@ -36,8 +37,20 @@ AGENT_NICK = "pi_nvnemo"
H5I_AGENT = "pi_nvnemo"
H5I_BUS_ROOT = Path("/mnt/dolphinng5_predict")
DEFAULT_INTERVAL = "1h"
CRONICLE_CONFIG = Path(os.environ.get("PI_WAKE_CRONICLE_CONFIG", "/mnt/dolphinng5_predict/cronicle.hcl"))
BACKEND_ENV = "PI_WAKE_BACKEND"
CRON_PATH = "/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin"
PYTHON_BIN = "/usr/bin/python3"
# Bootstrap PATH at import: crnd fires this script under a minimal env, so
# make zjet/h5i/cronicle discoverable for the subprocess calls below. (The
# shebang resolves python3 via the caller PATH; the script is stdlib-only.)
os.environ["PATH"] = CRON_PATH + os.environ.get("PATH", "")
# ─── Logging Setup ───────────────────────────────────────────────────────
ZELIG_BIN = 'zellij'
def setup_logging(debug: bool = False) -> logging.Logger:
log_rotate()
logger = logging.getLogger("pi_wake_agent")
@@ -115,50 +128,234 @@ def interval_to_human(interval: str) -> str:
return interval
# ─── Cron Management ────────────────────────────────────────────────────
# ─── Cronicle Support ────────────────────────────────────────────────────
def cronicle_available() -> bool:
"""True if the cronicle binary is installed (backend present)."""
return shutil.which("cronicle") is not None
def cronicle_daemon_running() -> bool:
"""True if a cronicle daemon is actually serving (binary alone is not enough).
On this host cronicle has been flaky (the walker never ran) and its daemon
is NOT supervised (see AGENTS.md). Schedules written to cronicle.hcl only
fire while a daemon is serving it.
"""
try:
result = subprocess.run(["pgrep", "-f", "cronicle run --path"],
capture_output=True, text=True, timeout=5)
return result.returncode == 0
except Exception:
return False
def use_cronicle() -> bool:
"""Decide the live scheduler backend.
PI_WAKE_BACKEND override (cronicle|crontab) wins for tests/ops. Otherwise
cronicle is used only when the binary is installed AND its daemon is
running; otherwise crontab (system crnd is the live scheduler here).
"""
backend = os.environ.get(BACKEND_ENV, "").strip().lower()
if backend == "cronicle":
return True
if backend == "crontab":
return False
return cronicle_available() and cronicle_daemon_running()
def cron_comment(sessions: List[str], interval: str) -> str:
sessions_str = ",".join(sessions)
return f"{CRON_COMMENT_PREFIX}:{sessions_str}:{interval}"
def install_cron(sessions: List[str], interval: str, message: str, logger: logging.Logger) -> None:
cron_sched = interval_to_cron(interval)
comment = cron_comment(sessions, interval)
sessions_arg = f"'{','.join(sessions)}'"
cmd = f"cd {H5I_BUS_ROOT} && export H5I_AGENT={H5I_AGENT} && {SCRIPT_PATH} --run --sessions {sessions_arg} --msg {shlex.quote(message)}"
def cronicle_config_path() -> Path:
"""Return path to cronicle config file."""
return CRONICLE_CONFIG
def cronicle_schedule_name(sessions: List[str], interval: str) -> str:
"""Generate unique schedule name for cronicle."""
sessions_str = "_".join(sessions)
return f"pi_wake_{sessions_str}_{interval}"
def wake_command_args(sessions: List[str], interval: str, message: str, pane: Optional[str] = None) -> List[str]:
"""Build the argv list for the --run wake command (shared by both backends).
Always passes --interval so the cronicle/crnd entry is self-describing.
--pane targets a specific zjet pane (e.g. the bottom Soundcheck pane).
"""
args = [str(SCRIPT_PATH), "--run", "--interval", interval, "--sessions", ",".join(sessions)]
if pane:
args += ["--pane", pane]
args += ["--msg", message]
return args
def cronicle_install(sessions: List[str], interval: str, message: str, logger: logging.Logger, pane: Optional[str] = None) -> None:
"""Install wake schedule using cronicle HCL config."""
schedule_name = cronicle_schedule_name(sessions, interval)
result = subprocess.run(["crontab", "-l"], capture_output=True, text=True)
existing = result.stdout if result.returncode == 0 else ""
lines = [line for line in existing.splitlines() if comment not in line]
lines.append(f"{cron_sched} {cmd} # {comment}")
new_cron = "\n".join(lines) + "\n"
subprocess.run(["crontab", "-"], input=new_cron, text=True, check=True)
logger.info(f"Installed cron: {cron_sched} -> {sessions} every {interval}")
# Build task command
sessions_arg = f"'{','.join(sessions)}'"
cmd = f"cd {H5I_BUS_ROOT} && export H5I_AGENT={H5I_AGENT} && {SCRIPT_PATH} --run --interval {interval} --sessions {sessions_arg} --msg {shlex.quote(message)}"
# Read existing config
config_path = cronicle_config_path()
existing = ""
if config_path.exists():
existing = config_path.read_text()
# Remove existing schedule with same name
lines = existing.splitlines()
new_lines = []
i = 0
while i < len(lines):
if f'schedule "{schedule_name}"' in lines[i]:
# Skip this schedule block (find closing brace)
brace_count = 0
while i < len(lines):
brace_count += lines[i].count('{')
brace_count -= lines[i].count('}')
i += 1
if brace_count == 0:
break
continue
new_lines.append(lines[i])
i += 1
# Add new schedule
cron_sched = interval_to_cron(interval)
cron_sched = cron_sched.replace("*/", "@every ").replace("0 */", "@every ").replace(" * * *", "")
# Convert cron to cronicle format
match = re.match(r"^(\d+)([hm])$", interval)
if match:
value, unit = int(match.group(1)), match.group(2)
if unit == "h":
cronicle_cron = f"@every {value}h"
else:
cronicle_cron = f"@every {value}m"
else:
cronicle_cron = "@every 1h"
new_schedule = f'''
schedule "{schedule_name}" {{
cron = "{cronicle_cron}"
task "wake" {{
command = [{", ".join(chr(34) + a + chr(34) for a in wake_command_args(sessions, interval, message, pane))}]
}}
}}'''
new_config = "\n".join(new_lines) + "\n\n" + new_schedule + "\n"
config_path.write_text(new_config)
logger.info(f"Installed cronicle schedule: {schedule_name} -> {sessions} every {interval}")
def remove_cron(sessions: List[str], interval: str, logger: logging.Logger) -> None:
comment = cron_comment(sessions, interval)
result = subprocess.run(["crontab", "-l"], capture_output=True, text=True)
existing = result.stdout if result.returncode == 0 else ""
lines = [line for line in existing.splitlines() if comment not in line]
new_cron = "\n".join(lines) + ("\n" if lines else "")
subprocess.run(["crontab", "-"], input=new_cron, text=True, check=True)
logger.info(f"Removed cron for {sessions} ({interval})")
def cronicle_remove(sessions: List[str], interval: str, logger: logging.Logger) -> None:
"""Remove wake schedule from cronicle HCL config."""
schedule_name = cronicle_schedule_name(sessions, interval)
config_path = cronicle_config_path()
if not config_path.exists():
return
existing = config_path.read_text()
lines = existing.splitlines()
new_lines = []
i = 0
while i < len(lines):
if f'schedule "{schedule_name}"' in lines[i]:
brace_count = 0
while i < len(lines):
brace_count += lines[i].count('{')
brace_count -= lines[i].count('}')
i += 1
if brace_count == 0:
break
continue
new_lines.append(lines[i])
i += 1
config_path.write_text("\n".join(new_lines) + "\n")
logger.info(f"Removed cronicle schedule: {schedule_name}")
def list_cron(logger: logging.Logger) -> None:
print("=== pi_wake_agent cron entries ===")
result = subprocess.run(["crontab", "-l"], capture_output=True, text=True)
existing = result.stdout if result.returncode == 0 else ""
def cronicle_list(logger: logging.Logger) -> None:
"""List cronicle wake schedules."""
config_path = cronicle_config_path()
if not config_path.exists():
print(" (none)")
return
existing = config_path.read_text()
found = False
for line in existing.splitlines():
if CRON_COMMENT_PREFIX in line:
print(f" {line}")
if 'schedule "pi_wake_' in line:
print(f" {line.strip()}")
found = True
if not found:
print(" (none)")
def install_cron(sessions: List[str], interval: str, message: str, logger: logging.Logger, pane: Optional[str] = None) -> None:
if use_cronicle():
if not cronicle_daemon_running():
logger.warning(
"cronicle binary present but its daemon is NOT running "
"schedules written to cronicle.hcl will not fire; set "
"PI_WAKE_BACKEND=crontab to use system crnd."
)
cronicle_install(sessions, interval, message, logger, pane)
else:
cron_sched = interval_to_cron(interval)
comment = cron_comment(sessions, interval)
sessions_arg = f"'{','.join(sessions)}'"
run_cmd = f"{PYTHON_BIN} {SCRIPT_PATH} --run --interval {interval} --sessions {sessions_arg} --msg {shlex.quote(message)}"
if pane:
run_cmd += f" --pane {shlex.quote(pane)}"
cmd = f"cd {H5I_BUS_ROOT} && export PATH={CRON_PATH} && export H5I_AGENT={H5I_AGENT} && {run_cmd}"
result = subprocess.run(["crontab", "-l"], capture_output=True, text=True)
existing = result.stdout if result.returncode == 0 else ""
lines = [line for line in existing.splitlines() if comment not in line]
lines.append(f"{cron_sched} {cmd} # {comment}")
new_cron = "\n".join(lines) + "\n"
subprocess.run(["crontab", "-"], input=new_cron, text=True, check=True)
logger.info(f"Installed cron: {cron_sched} -> {sessions} every {interval}")
def remove_cron(sessions: List[str], interval: str, logger: logging.Logger) -> None:
if use_cronicle():
cronicle_remove(sessions, interval, logger)
else:
comment = cron_comment(sessions, interval)
result = subprocess.run(["crontab", "-l"], capture_output=True, text=True)
existing = result.stdout if result.returncode == 0 else ""
lines = [line for line in existing.splitlines() if comment not in line]
new_cron = "\n".join(lines) + ("\n" if lines else "")
subprocess.run(["crontab", "-"], input=new_cron, text=True, check=True)
logger.info(f"Removed cron for {sessions} ({interval})")
def list_cron(logger: logging.Logger) -> None:
if use_cronicle():
cronicle_list(logger)
else:
print("=== pi_wake_agent cron entries ===")
result = subprocess.run(["crontab", "-l"], capture_output=True, text=True)
existing = result.stdout if result.returncode == 0 else ""
found = False
for line in existing.splitlines():
if CRON_COMMENT_PREFIX in line:
print(f" {line}")
found = True
if not found:
print(" (none)")
# ─── Zellij Operations ──────────────────────────────────────────────────
def zellij_session_exists(session: str) -> bool:
try:
@@ -172,19 +369,26 @@ def zellij_session_exists(session: str) -> bool:
return False
def zellij_write_chars(session: str, text: str) -> bool:
def zellij_write_chars(session: str, text: str, pane: Optional[str] = None) -> bool:
"""Write text into a zjet pane (focused, or --pane-id when given)."""
cmd = [ZELIG_BIN, "--session", session, "action", "write-chars"]
if pane:
cmd += ["--pane-id", pane]
cmd.append(text)
try:
subprocess.run(["zellij", "--session", session, "action", "write-chars", text],
capture_output=True, timeout=5)
subprocess.run(cmd, capture_output=True, timeout=5)
return True
except Exception:
return False
def zellij_write_enter(session: str) -> bool:
def zellij_write_enter(session: str, pane: Optional[str] = None) -> bool:
"""Press ENTER (CR) in a zjet pane (focused, or --pane-id when given)."""
cmd = [ZELIG_BIN, "--session", session, "action", "write", "13"]
if pane:
cmd += ["--pane-id", pane]
try:
subprocess.run(["zellij", "--session", session, "action", "write", "13"],
capture_output=True, timeout=5)
subprocess.run(cmd, capture_output=True, timeout=5)
return True
except Exception:
return False
@@ -195,7 +399,7 @@ def zellij_write_enter(session: str) -> bool:
def _wake():
time.sleep(interval_seconds)
run_wake(sessions, message, logger)
run_wake(sessions, message, logger, pane)
thread = threading.Thread(target=_wake, daemon=True)
thread.start()
@@ -204,15 +408,21 @@ def zellij_write_enter(session: str) -> bool:
pid_file.write_text(str(os.getpid()))
def run_wake(sessions: List[str], message: str, logger: logging.Logger) -> None:
def run_wake(sessions: List[str], message: str, logger: logging.Logger, pane: Optional[str] = None) -> None:
sessions_str = ",".join(sessions)
logger.info(f"Waking {sessions_str} with message: {message}")
logger.info(f"Waking {sessions_str} with message: {message}" + (f" (pane={pane})" if pane else ""))
for session in sessions:
zellij_write_chars(session, f"[{AGENT_NICK} via zellij] {message} Run: h5i-bus msg inbox")
for _ in range(5):
zellij_write_enter(session)
time.sleep(1)
# Verify session exists before attempting to wake
if not zellij_session_exists(session):
logger.warning(f"Session {session} does not exist, skipping")
continue
zellij_write_chars(session, f"[{AGENT_NICK} via zellij] {message} Run: h5i-bus msg inbox", pane)
# Send more ENTER keypresses with longer delay for reliability
for _ in range(10):
zellij_write_enter(session, pane)
time.sleep(0.5)
# Fire-and-forget h5i bus message (non-blocking)
def _send_bus():
@@ -231,13 +441,13 @@ def run_wake(sessions: List[str], message: str, logger: logging.Logger) -> None:
logger.info(f"Wake sent to {sessions_str}")
def run_once(sessions: List[str], interval: str, message: str, logger: logging.Logger) -> None:
def run_once(sessions: List[str], interval: str, message: str, logger: logging.Logger, pane: Optional[str] = None) -> None:
interval_seconds = parse_interval(interval)
logger.info(f"One-shot timer set for {interval} ({interval_to_human(interval)})")
def _wake():
time.sleep(interval_seconds)
run_wake(sessions, message, logger)
run_wake(sessions, message, logger, pane)
thread = threading.Thread(target=_wake, daemon=True)
thread.start()
@@ -247,22 +457,22 @@ def run_once(sessions: List[str], interval: str, message: str, logger: logging.L
return 0
def run_daemon(sessions: List[str], interval: str, message: str, logger: logging.Logger) -> None:
def run_daemon(sessions: List[str], interval: str, message: str, logger: logging.Logger, pane: Optional[str] = None) -> None:
interval_seconds = parse_interval(interval)
logger.info("=== DAEMON START ===")
logger.info(f"Interval: {interval} ({interval_to_human(interval)})")
logger.info(f"Sessions: {sessions}")
logger.info(f"Message: {message}")
run_wake(sessions, message, logger)
run_wake(sessions, message, logger, pane)
while True:
logger.debug(f"Sleeping for {interval_seconds}s...")
time.sleep(interval_seconds)
run_wake(sessions, message, logger)
run_wake(sessions, message, logger, pane)
def run_succession(sessions: List[str], interval: str, count: int, message: str, logger: logging.Logger) -> None:
def run_succession(sessions: List[str], interval: str, count: int, message: str, logger: logging.Logger, pane: Optional[str] = None) -> None:
"""Run wake N times at interval, then self-clean (remove cron if installed)."""
interval_seconds = parse_interval(interval)
logger.info(f"=== SUCCESSION START === Count: {count}, Interval: {interval} ({interval_to_human(interval)})")
@@ -271,7 +481,7 @@ def run_succession(sessions: List[str], interval: str, count: int, message: str,
for i in range(1, count + 1):
logger.info(f"Succession {i}/{count}")
run_wake(sessions, message, logger)
run_wake(sessions, message, logger, pane)
if i < count:
logger.debug(f"Sleeping for {interval_seconds}s until next succession...")
time.sleep(interval_seconds)
@@ -292,6 +502,17 @@ def status(logger: logging.Logger) -> None:
print(f"Log: {LOG_FILE}")
print(f"Log size: {LOG_FILE.stat().st_size if LOG_FILE.exists() else 'N/A'} bytes")
print(f"Agent: {AGENT_NICK}")
backend = os.environ.get(BACKEND_ENV, "").strip().lower()
if backend == "cronicle":
backend_label = "cronicle (forced by PI_WAKE_BACKEND)"
elif backend == "crontab":
backend_label = "crontab (forced by PI_WAKE_BACKEND)"
elif use_cronicle():
backend_label = "cronicle"
else:
backend_label = "crontab (system crnd)"
print(f"Backend: {backend_label}")
print(f"cronicle binary={cronicle_available()} daemon_running={cronicle_daemon_running()}")
print()
list_cron(logger)
print()
@@ -363,10 +584,6 @@ EXAMPLES:
pi_wake_agent.py --validate --session cc_UV_dev0_Fb
"""
)
parser = argparse.ArgumentParser(
description="pi_wake_agent.py — Reusable multi-agent wake-up timer with self-cron/daemon/succession",
formatter_class=argparse.RawDescriptionHelpFormatter,
)
parser.add_argument("--install", action="store_const", const="install", dest="mode", help="Install recurring cron timer")
parser.add_argument("--once", action="store_const", const="once", dest="mode", help="One-shot wake (no cron)")
parser.add_argument("--daemon", action="store_const", const="daemon", dest="mode", help="Run as long-lived daemon (no cron)")
@@ -380,6 +597,7 @@ EXAMPLES:
parser.add_argument("--interval", default=DEFAULT_INTERVAL, help="Interval (default: 1h). Formats: 30m, 1h, 90m, 2h, etc.")
parser.add_argument("--session", action="append", dest="session_list", help="Zellij session name (can repeat)")
parser.add_argument("--sessions", help="Comma-separated list of sessions")
parser.add_argument("--pane", dest="pane", default=None, help="Target a specific zjet pane (focused pane if unset), e.g. terminal_1")
parser.add_argument("--msg", default=f"Operator says CONTINUE. {AGENT_NICK} here, saying hi!", help="Wake message")
parser.add_argument("--debug", action="store_true", help="Enable debug logging")
parser.add_argument("--dry-run", action="store_true", help="Show what would be done without executing")
@@ -415,6 +633,8 @@ def main() -> int:
logger.info(f"=== pi_wake_agent {args.mode} ===")
logger.info(f"Interval: {args.interval} ({interval_to_human(args.interval)})")
logger.info(f"Sessions: {sessions}")
if args.pane:
logger.info(f"Pane target: {args.pane}")
logger.info(f"Message: {args.msg}")
if args.mode == "succession":
logger.info(f"Count: {args.count}")
@@ -427,15 +647,15 @@ def main() -> int:
try:
if args.mode == "install":
install_cron(sessions, args.interval, args.msg, logger)
install_cron(sessions, args.interval, args.msg, logger, args.pane)
elif args.mode == "once":
run_once(sessions, args.interval, args.msg, logger)
run_once(sessions, args.interval, args.msg, logger, args.pane)
elif args.mode == "daemon":
run_daemon(sessions, args.interval, args.msg, logger)
run_daemon(sessions, args.interval, args.msg, logger, args.pane)
elif args.mode == "succession":
run_succession(sessions, args.interval, args.count, args.msg, logger)
run_succession(sessions, args.interval, args.count, args.msg, logger, args.pane)
elif args.mode == "run":
run_wake(sessions, args.msg, logger)
run_wake(sessions, args.msg, logger, args.pane)
elif args.mode == "remove":
remove_cron(sessions, args.interval, logger)
elif args.mode == "list":