Varys — Status
Checks HTTP reachability for all services in the NxM stack and monitors agent run status. Refreshes every 15 minutes.
Updated
Services
{service_cards}
Agents
{backup_section}
"""
def render_summary_md(services: list, agents: list) -> str:
now = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
up = sum(1 for s in services if s["status"] == "up")
down = [s["name"] for s in services if s["status"] in ("down", "degraded")]
lines = [f"# Varys — Status Summary\n\nUpdated: {now}\n"]
lines.append(f"## Services: {up}/{len(services)} up")
if down:
lines.append(f"Issues: {', '.join(down)}")
lines.append("")
lines.append("## Agents")
for a in agents:
ts = a["timestamp"][:16].replace("T", " ") + " UTC" if a["timestamp"] else "no data"
lines.append(f"- {a['name']}: {a['status']} ({ts}) — {a['result']}")
return "\n".join(lines)
def main():
config = yaml.safe_load(CONFIG_FILE.read_text())
services = []
for svc in config.get("services", []):
result = check_service(svc["name"], svc["url"])
logger.info(f"{svc['name']}: {result['status']} ({result['ms']} ms)")
services.append(result)
agents = []
for agent in config.get("agents", []):
result = read_agent_status(agent["name"])
logger.info(f"agent {agent['name']}: {result['status']}")
agents.append(result)
# Check Proxmox backups
backup_vmids = [str(v) for v in config.get("proxmox_backup_vmids", [])]
backups = check_proxmox_backups(backup_vmids) if backup_vmids else []
prev_states = _load_prev_states()
prev_services = prev_states.get("services", {})
prev_agents = prev_states.get("agents", {})
prev_backups = prev_states.get("backups", {})
for s in services:
prev = prev_services.get(s["name"])
if prev and prev != "down" and s["status"] == "down":
_notify_raven(f"{s['name']} is DOWN", "critical")
elif prev == "down" and s["status"] not in ("down", "degraded"):
_notify_raven(f"{s['name']} recovered (UP)", "info")
for a in agents:
prev = prev_agents.get(a["name"])
if a["status"] == "failure" and prev != "failure":
detail = a.get("result", "no details")
_notify_raven(f"{a['name']} failed: {detail}", "critical")
elif prev == "failure" and a["status"] == "success":
_notify_raven(f"{a['name']} recovered (success)", "info")
prev_backup_ctimes = prev_states.get("backup_ctimes", {})
for b in backups:
vid = b["vmid"]
prev_status = prev_backups.get(vid)
prev_ctime = prev_backup_ctimes.get(vid, 0)
curr_ctime = b.get("ctime", 0)
# New backup detected — ctime is newer than last recorded
if curr_ctime and curr_ctime > prev_ctime:
_notify_raven(
f"✅ Proxmox backup completed — VM {vid} ({b.get('size_gb', '?')} GB) at {b.get('last_backup', '?')}",
"info",
)
# Backup gone overdue or missing
if b["status"] in ("warning", "error") and prev_status not in ("warning", "error"):
age = f"{b['age_days']}d" if b.get("age_days") is not None else "unknown age"
reason = b.get("reason") or f"last backup {age} ago"
_notify_raven(f"⚠️ Proxmox backup VM {vid}: {b['status'].upper()} — {reason}", "critical")
elif prev_status in ("warning", "error") and b["status"] == "ok":
_notify_raven(f"✅ Proxmox backup VM {vid}: recovered (last backup {b.get('age_days', 0)}d ago)", "info")
_save_states(services, agents)
# Save backup states and ctimes (extend existing format)
state_path = AGENT_OS_DIR / "logs" / "varys-monitor" / "service-states.json"
if state_path.exists():
try:
existing = json.loads(state_path.read_text())
except Exception:
existing = {}
existing["backups"] = {b["vmid"]: b["status"] for b in backups}
existing["backup_ctimes"] = {b["vmid"]: b.get("ctime", 0) for b in backups}
state_path.write_text(json.dumps(existing))
out_dir = SITES_DIR / "varys"
out_dir.mkdir(parents=True, exist_ok=True)
(out_dir / "index.html").write_text(render_html(services, agents, backups))
(out_dir / "last-output.md").write_text(render_summary_md(services, agents))
down_count = sum(1 for s in services if s["status"] != "up")
backup_issues = sum(1 for b in backups if b["status"] != "ok")
status = {
"agent": "varys-monitor",
"timestamp": datetime.now(timezone.utc).isoformat(),
"status": "success",
"result": f"{len(services)} services checked, {down_count} not up; {len(backups)} backups checked, {backup_issues} issues",
}
log_dir = AGENT_OS_DIR / "logs" / "varys-monitor"
log_dir.mkdir(parents=True, exist_ok=True)
(log_dir / "last-run.json").write_text(json.dumps(status, indent=2))
logger.info(f"done — {len(services)} services, {down_count} not up; {len(backups)} backups, {backup_issues} issues")
if __name__ == "__main__":
main()