373 lines
15 KiB
Python
373 lines
15 KiB
Python
import json
|
|
import logging
|
|
import os
|
|
import smtplib
|
|
from contextlib import asynccontextmanager
|
|
from datetime import datetime, timezone
|
|
from email.mime.text import MIMEText
|
|
from pathlib import Path
|
|
|
|
import httpx
|
|
from fastapi import FastAPI
|
|
from fastapi.middleware.cors import CORSMiddleware
|
|
from pydantic import BaseModel
|
|
|
|
logging.basicConfig(level=logging.INFO, format="%(asctime)s [raven] %(message)s")
|
|
logger = logging.getLogger("raven")
|
|
|
|
DISCORD_WEBHOOK_URL = os.getenv("DISCORD_WEBHOOK_URL", "")
|
|
SMTP_HOST = os.getenv("SMTP_HOST", "smtp.gmail.com")
|
|
SMTP_PORT = int(os.getenv("SMTP_PORT", "587"))
|
|
SMTP_USER = os.getenv("SMTP_USER", "")
|
|
SMTP_PASSWORD = os.getenv("SMTP_PASSWORD", "")
|
|
SMTP_FROM = os.getenv("SMTP_FROM", "")
|
|
SMTP_TO = os.getenv("SMTP_TO", "")
|
|
SITES_DIR = Path(os.getenv("SITES_DIR", "/opt/sites"))
|
|
AGENT_OS_DIR = Path(os.getenv("AGENT_OS_DIR", "/opt/agent-os"))
|
|
PORT = int(os.getenv("PORT", "8400"))
|
|
|
|
MAX_HISTORY = 50
|
|
SEVERITY_COLORS = {"critical": 15158332, "warning": 16776960, "info": 3066993}
|
|
|
|
|
|
def _state_path() -> Path:
|
|
return SITES_DIR / "raven" / "state.json"
|
|
|
|
|
|
def _load_state() -> dict:
|
|
p = _state_path()
|
|
if not p.exists():
|
|
return {"email_enabled": True}
|
|
try:
|
|
return json.loads(p.read_text())
|
|
except Exception:
|
|
return {"email_enabled": True}
|
|
|
|
|
|
def _save_state(state: dict):
|
|
p = _state_path()
|
|
p.parent.mkdir(parents=True, exist_ok=True)
|
|
p.write_text(json.dumps(state, indent=2))
|
|
|
|
|
|
class NotifyRequest(BaseModel):
|
|
message: str
|
|
severity: str = "info"
|
|
source: str = "unknown"
|
|
|
|
|
|
def _history_path() -> Path:
|
|
return SITES_DIR / "raven" / "history.json"
|
|
|
|
|
|
def _load_history() -> list:
|
|
p = _history_path()
|
|
if not p.exists():
|
|
return []
|
|
try:
|
|
return json.loads(p.read_text())
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
def _save_history(history: list):
|
|
p = _history_path()
|
|
p.parent.mkdir(parents=True, exist_ok=True)
|
|
p.write_text(json.dumps(history[-MAX_HISTORY:], indent=2))
|
|
|
|
|
|
def _send_discord(message: str, severity: str, source: str, timestamp: str):
|
|
if not DISCORD_WEBHOOK_URL:
|
|
logger.warning("DISCORD_WEBHOOK_URL not set — skipping Discord")
|
|
return
|
|
color = SEVERITY_COLORS.get(severity.lower(), 3066993)
|
|
payload = {
|
|
"embeds": [{
|
|
"title": f"[{severity.upper()}] {source}",
|
|
"description": message,
|
|
"color": color,
|
|
"timestamp": timestamp,
|
|
"footer": {"text": "raven-notify · nxm.co.za"},
|
|
}]
|
|
}
|
|
try:
|
|
r = httpx.post(DISCORD_WEBHOOK_URL, json=payload, timeout=10)
|
|
r.raise_for_status()
|
|
logger.info(f"discord: sent [{severity}] {source}")
|
|
except Exception as e:
|
|
logger.error(f"discord send failed: {e}")
|
|
|
|
|
|
def _send_email(message: str, severity: str, source: str):
|
|
if not _load_state().get("email_enabled", True):
|
|
logger.info("email: disabled by toggle — skipping")
|
|
return
|
|
if not all([SMTP_USER, SMTP_PASSWORD, SMTP_TO]):
|
|
logger.warning("SMTP not configured — skipping email")
|
|
return
|
|
subject = f"[{severity.upper()}] {source} — {message[:80]}"
|
|
body = f"Severity: {severity.upper()}\nSource: {source}\n\n{message}"
|
|
msg = MIMEText(body)
|
|
msg["Subject"] = subject
|
|
msg["From"] = SMTP_FROM or SMTP_USER
|
|
msg["To"] = SMTP_TO
|
|
try:
|
|
with smtplib.SMTP(SMTP_HOST, SMTP_PORT) as smtp:
|
|
smtp.ehlo()
|
|
smtp.starttls()
|
|
smtp.login(SMTP_USER, SMTP_PASSWORD)
|
|
smtp.sendmail(SMTP_FROM or SMTP_USER, [SMTP_TO], msg.as_string())
|
|
logger.info(f"email: sent [{severity}] {source}")
|
|
except Exception as e:
|
|
logger.error(f"email send failed: {e}")
|
|
|
|
|
|
def _render_html(history: list, timestamp: str, email_enabled: bool = True) -> str:
|
|
rows = ""
|
|
for entry in reversed(history[-20:]):
|
|
sev = entry.get("severity", "info")
|
|
colors = {"critical": "#f85149", "warning": "#d29922", "info": "#3fb950"}
|
|
color = colors.get(sev, "#8b949e")
|
|
ts = entry.get("timestamp", "")
|
|
rows += f"""
|
|
<tr>
|
|
<td><span data-utc="{ts}"></span></td>
|
|
<td style="color:{color};font-weight:bold">{sev.upper()}</td>
|
|
<td class="src">{entry.get('source', '')}</td>
|
|
<td>{entry.get('message', '')}</td>
|
|
</tr>"""
|
|
|
|
total = len(history)
|
|
email_color = "#3fb950" if email_enabled else "#f85149"
|
|
email_label = "ENABLED" if email_enabled else "DISABLED"
|
|
btn_label = "Disable Email" if email_enabled else "Enable Email"
|
|
return f"""<!DOCTYPE html>
|
|
<html lang="en">
|
|
<head>
|
|
<meta charset="UTF-8">
|
|
<meta name="viewport" content="width=device-width, initial-scale=1">
|
|
<title>Raven — Notifications</title>
|
|
<link rel="preconnect" href="https://fonts.googleapis.com">
|
|
<link href="https://fonts.googleapis.com/css2?family=Inter:wght@400;500;600&display=swap" rel="stylesheet">
|
|
<style>
|
|
:root {{ --bg: #0d1117; --surface: #161b22; --border: #30363d; --dim: #21262d; --text: #e6edf3; --muted: #8b949e; }}
|
|
*, *::before, *::after {{ box-sizing: border-box; margin: 0; padding: 0; }}
|
|
body {{ font-family: 'Inter', -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif; background: var(--bg); color: var(--text); min-height: 100vh; font-size: 15px; line-height: 1.6; }}
|
|
.site-nav {{ background: var(--surface); border-bottom: 1px solid var(--border); padding: 0.7rem 2rem; display: flex; align-items: center; gap: 0.6rem; position: sticky; top: 0; z-index: 10; }}
|
|
.nav-brand {{ color: #58a6ff; font-weight: 600; font-size: 0.9rem; text-decoration: none; }}
|
|
.nav-brand:hover {{ color: var(--text); }}
|
|
.nav-sep {{ color: var(--border); }}
|
|
.nav-title {{ color: var(--text); font-weight: 500; font-size: 0.9rem; }}
|
|
.nav-right {{ margin-left: auto; }}
|
|
.nav-back {{ color: var(--muted); font-size: 0.85rem; text-decoration: none; }}
|
|
.nav-back:hover {{ color: var(--text); }}
|
|
.main {{ max-width: 1100px; margin: 0 auto; padding: 2.5rem 2rem 4rem; }}
|
|
h1 {{ font-size: 1.5rem; font-weight: 600; color: var(--text); margin-bottom: 0.2rem; }}
|
|
h2 {{ color: var(--muted); font-size: 0.7rem; font-weight: 600; text-transform: uppercase; letter-spacing: 0.1em; margin: 2rem 0 0.875rem; }}
|
|
.desc {{ color: var(--muted); font-size: 0.9rem; margin: 0.3rem 0 0.2rem; }}
|
|
.meta {{ color: var(--muted); font-size: 0.82rem; margin-top: 0.2rem; margin-bottom: 0; }}
|
|
.meta a {{ color: var(--muted); text-decoration: none; }}
|
|
.meta a:hover {{ color: var(--text); }}
|
|
.controls {{ display: flex; align-items: center; gap: 1rem; margin-top: 1.5rem; margin-bottom: 0; padding: 0.75rem 1rem; background: var(--surface); border: 1px solid var(--border); border-radius: 8px; }}
|
|
.controls span {{ font-size: 0.9rem; }}
|
|
.controls button {{ font-family: 'Inter', -apple-system, sans-serif; font-size: 0.82rem; padding: 0.35rem 0.85rem; border: 1px solid var(--border); border-radius: 6px; background: var(--dim); color: var(--text); cursor: pointer; transition: background 0.15s; }}
|
|
.controls button:hover {{ background: var(--border); }}
|
|
.controls button:disabled {{ opacity: 0.5; cursor: default; }}
|
|
table {{ width: 100%; border-collapse: collapse; }}
|
|
td {{ padding: 0.5rem 0.75rem; border-bottom: 1px solid var(--dim); font-size: 0.875rem; vertical-align: top; }}
|
|
td:first-child {{ white-space: nowrap; color: var(--muted); min-width: 9rem; }}
|
|
td:nth-child(2) {{ white-space: nowrap; min-width: 6rem; }}
|
|
.src {{ color: #79c0ff; white-space: nowrap; min-width: 8rem; }}
|
|
td:last-child {{ line-height: 1.5; }}
|
|
.empty {{ color: var(--muted); font-size: 0.9rem; padding: 1rem 0; }}
|
|
</style>
|
|
</head>
|
|
<body>
|
|
<nav class="site-nav">
|
|
<a class="nav-brand" href="/">◈ NxM</a>
|
|
<span class="nav-sep">·</span>
|
|
<span class="nav-title">Raven — Notifications</span>
|
|
<span class="nav-right"><a class="nav-back" href="/">← home</a></span>
|
|
</nav>
|
|
<main class="main">
|
|
<h1>Raven — Notifications</h1>
|
|
<p class="desc">Sends alerts and notifications via Discord and email. Triggered by Varys (service down/up) and Grafana (infrastructure alerts).</p>
|
|
<p class="meta">Updated <span data-utc="{timestamp}"></span> · {total} total</p>
|
|
<div class="controls">
|
|
<span>Email alerts: <strong style="color:{email_color}">{email_label}</strong></span>
|
|
<button id="toggle-btn" onclick="toggleEmail()">{btn_label}</button>
|
|
</div>
|
|
<h2>Recent Notifications (last 20)</h2>
|
|
{"<table><tbody>" + rows + "</tbody></table>" if rows else '<p class="empty">No notifications yet.</p>'}
|
|
<script>
|
|
document.querySelectorAll('[data-utc]').forEach(el => {{
|
|
el.textContent = new Date(el.dataset.utc).toLocaleString(undefined, {{year:'numeric',month:'2-digit',day:'2-digit',hour:'2-digit',minute:'2-digit'}});
|
|
}});
|
|
function toggleEmail() {{
|
|
const btn = document.getElementById('toggle-btn');
|
|
btn.disabled = true;
|
|
fetch('/raven/toggle', {{method: 'POST'}})
|
|
.then(() => location.reload())
|
|
.catch(e => {{ alert('Toggle failed: ' + e); btn.disabled = false; }});
|
|
}}
|
|
</script>
|
|
</main>
|
|
</body>
|
|
</html>"""
|
|
|
|
|
|
def _render_md(history: list, timestamp: str) -> str:
|
|
now = timestamp[:16].replace("T", " ") + " UTC"
|
|
lines = [f"# Raven — Notification Log\n\nUpdated: {now} | {len(history)} total\n"]
|
|
for e in reversed(history[-10:]):
|
|
ts = e.get("timestamp", "")[:16].replace("T", " ") + " UTC"
|
|
lines.append(f"- [{e.get('severity','?').upper()}] {ts} · **{e.get('source','')}** — {e.get('message','')}")
|
|
return "\n".join(lines)
|
|
|
|
|
|
def _write_status(history: list, last_event: dict = None):
|
|
timestamp = datetime.now(timezone.utc).isoformat()
|
|
email_enabled = _load_state().get("email_enabled", True)
|
|
out_dir = SITES_DIR / "raven"
|
|
out_dir.mkdir(parents=True, exist_ok=True)
|
|
(out_dir / "index.html").write_text(_render_html(history, timestamp, email_enabled))
|
|
(out_dir / "last-output.md").write_text(_render_md(history, timestamp))
|
|
|
|
log_dir = AGENT_OS_DIR / "logs" / "raven-notify"
|
|
log_dir.mkdir(parents=True, exist_ok=True)
|
|
last_msg = (
|
|
f"last: [{last_event['severity']}] {last_event['source']} — {last_event['message'][:50]}"
|
|
if last_event else "running, no notifications yet"
|
|
)
|
|
(log_dir / "last-run.json").write_text(json.dumps({
|
|
"agent": "raven-notify",
|
|
"timestamp": timestamp,
|
|
"status": "success",
|
|
"result": f"{len(history)} notifications total; {last_msg}",
|
|
}, indent=2))
|
|
|
|
|
|
def _do_notify(message: str, severity: str, source: str) -> dict:
|
|
timestamp = datetime.now(timezone.utc).isoformat()
|
|
severity = severity.lower()
|
|
entry = {"timestamp": timestamp, "severity": severity, "source": source, "message": message}
|
|
|
|
history = _load_history()
|
|
history.append(entry)
|
|
_save_history(history)
|
|
|
|
_send_discord(message, severity, source, timestamp)
|
|
_send_email(message, severity, source)
|
|
_write_status(history, entry)
|
|
|
|
logger.info(f"notified [{severity}] {source}: {message[:80]}")
|
|
return {"ok": True, "timestamp": timestamp}
|
|
|
|
|
|
@asynccontextmanager
|
|
async def lifespan(app: FastAPI):
|
|
history = _load_history()
|
|
_write_status(history)
|
|
logger.info(f"raven-notify started on port {PORT} — {len(history)} notifications in history")
|
|
yield
|
|
|
|
|
|
app = FastAPI(title="raven-notify", lifespan=lifespan)
|
|
|
|
app.add_middleware(
|
|
CORSMiddleware,
|
|
allow_origins=["*"],
|
|
allow_methods=["POST"],
|
|
allow_headers=["*"],
|
|
)
|
|
|
|
|
|
@app.get("/health")
|
|
def health():
|
|
return {"status": "ok", "agent": "raven-notify"}
|
|
|
|
|
|
@app.post("/notify")
|
|
def handle_notify(req: NotifyRequest):
|
|
return _do_notify(req.message, req.severity, req.source)
|
|
|
|
|
|
@app.post("/email/toggle")
|
|
def toggle_email():
|
|
state = _load_state()
|
|
state["email_enabled"] = not state.get("email_enabled", True)
|
|
_save_state(state)
|
|
_write_status(_load_history())
|
|
logger.info(f"email alerts {'enabled' if state['email_enabled'] else 'disabled'} via toggle")
|
|
return {"ok": True, "email_enabled": state["email_enabled"]}
|
|
|
|
|
|
@app.post("/webhook/grafana")
|
|
def handle_grafana(payload: dict):
|
|
status = payload.get("status", "firing")
|
|
alerts = payload.get("alerts", [])
|
|
if alerts:
|
|
for alert in alerts:
|
|
alert_status = alert.get("status", status)
|
|
name = alert.get("labels", {}).get("alertname", "unknown")
|
|
if alert_status == "resolved":
|
|
# Grafana always sends the firing annotation text in resolved payloads —
|
|
# override with a clear recovery message so Discord/email isn't confusing.
|
|
msg = f"✅ RESOLVED: {name} — alert has cleared, service is back online."
|
|
sev = "info"
|
|
else:
|
|
msg = alert.get("annotations", {}).get("description", name)
|
|
sev = "critical"
|
|
_do_notify(msg, sev, f"grafana/{name}")
|
|
return {"ok": True, "processed": len(alerts)}
|
|
msg = payload.get("message", "") or payload.get("title", "Grafana Alert")
|
|
sev = "critical" if status == "firing" else "info"
|
|
return _do_notify(msg, sev, "grafana")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
import uvicorn
|
|
uvicorn.run(app, host="0.0.0.0", port=PORT)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Jon Snow approval gate — action notification endpoint
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class ActionNotifyRequest(BaseModel):
|
|
description: str
|
|
approve_url: str
|
|
reject_url: str
|
|
action_type: str
|
|
source: str = "jon-snow"
|
|
|
|
|
|
@app.post("/notify-with-actions")
|
|
def handle_action_notify(req: ActionNotifyRequest):
|
|
"""Send a Discord embed with clickable approve/reject links for Jon Snow's approval gate."""
|
|
if not DISCORD_WEBHOOK_URL:
|
|
logger.warning("DISCORD_WEBHOOK_URL not set — skipping Discord")
|
|
return {"ok": False, "error": "no webhook configured"}
|
|
|
|
payload = {
|
|
"embeds": [{
|
|
"title": "⚡ Action Approval Required",
|
|
"description": f"**{req.source}** wants to: `{req.description}`",
|
|
"color": 16776960, # yellow
|
|
"fields": [
|
|
{"name": "Action Type", "value": f"`{req.action_type}`", "inline": True},
|
|
{"name": "Expires", "value": "15 minutes", "inline": True},
|
|
],
|
|
"footer": {"text": "jon-snow approval gate · nxm.co.za"},
|
|
}],
|
|
"content": f"✅ **Approve:** {req.approve_url}\n❌ **Reject:** {req.reject_url}",
|
|
}
|
|
try:
|
|
r = httpx.post(DISCORD_WEBHOOK_URL, json=payload, timeout=10)
|
|
r.raise_for_status()
|
|
logger.info(f"discord: action approval request sent for {req.action_type}")
|
|
return {"ok": True}
|
|
except Exception as e:
|
|
logger.error(f"discord action notify failed: {e}")
|
|
return {"ok": False, "error": str(e)}
|