diff --git a/CLAUDE.md b/CLAUDE.md index be55ad9..02eb7cf 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -32,11 +32,14 @@ Claude Code, then relays/acts on the agent's JSON result. re-probe. Kuma's root-owned `kuma.db` is read via a throwaway `keinos/sqlite3` container mounting the `uptime-kuma-data` volume read-only. - **Health loop** (5 min): down services confirmed over `down_confirmations` - cycles → invoke stack-support agent to remediate → report to Matrix. + cycles → invoke stack-support agent to remediate → report to Matrix. Also + reads Kuma's `important=1` heartbeat transitions since the last cycle (tracked + by heartbeat id) to catch services that blipped DOWN→UP on their own, and + hands those self-recovered windows to the agent for a log post-mortem. - **Upgrade loop** (7 days): resolve containers behind tracked services → agent researches + applies safe upgrades (test, pin, rollback); escalates major/breaking ones to Matrix. -- **Matrix**: bot `@maintainer:semprini.me` DMs `@paul:semprini.me`. Escalations +- **Matrix**: bot `@maintainer-bot:semprini.me` DMs `@paul:semprini.me`. Escalations are numbered questions; the reply is fed back to the agent to carry out. - **Autonomy** (`config.json` → `autonomy`): `full` (default) | `fix-only` | `notify`. @@ -56,7 +59,7 @@ Claude Code, then relays/acts on the agent's JSON result. |---|---| | `maintainer.py` | The daemon (stdlib only). | | `config.example.json` | Template — copy to `config.json`. | -| `register-matrix-bot.sh` | Creates `@maintainer` account + DM room; writes token to config. | +| `register-matrix-bot.sh` | Creates `@maintainer-bot` account + DM room; writes token to config. | | `install.sh` | Syntax-check, install unit, enable + start. | | `semprini-maintainer.service` | systemd unit (runs as `paul`). | | `docs/architecture.md` | Architecture decision record. | diff --git a/config.example.json b/config.example.json index a13b288..c38b4b0 100644 --- a/config.example.json +++ b/config.example.json @@ -4,7 +4,7 @@ "agent_file": "agents/stack-support.agent.md", "runbook_file": "docs/stack-support-runbook.md", "claude_bin": "/home/paul/.local/bin/claude", - "claude_model": "claude-opus-4-8", + "claude_model": "claude-sonnet-4-6", "claude_extra_args": [], "agent_timeout_seconds": 1800, @@ -22,7 +22,7 @@ "matrix": { "homeserver": "https://matrix.semprini.me", - "user_id": "@claude-code:semprini.me", + "user_id": "@maintainer-bot:semprini.me", "access_token": "FILLED_BY_register-matrix-bot.sh", "room_id": "FILLED_BY_register-matrix-bot.sh", "admin_user_id": "@paul:semprini.me" diff --git a/maintainer.py b/maintainer.py index 6a5adfd..65ebf3e 100755 --- a/maintainer.py +++ b/maintainer.py @@ -57,7 +57,7 @@ DEFAULTS = { # default, which may be a 1M-context ([1m]) model; once a cycle's context # crosses 200K that escalates to the 1M tier and fails on subscription # plans with "Usage credits are required for long context requests" (429). - "claude_model": "claude-opus-4-8", + "claude_model": "claude-sonnet-4-6", "claude_extra_args": [], "agent_timeout_seconds": 1800, "kuma_volume": "uptime-kuma-data", @@ -112,6 +112,8 @@ def load_state(cfg): "down_streak": {}, # monitor name -> consecutive-down count "matrix_since": None, # /sync pagination token "pending": None, # outstanding question awaiting admin reply + "last_event_check": None, # max kuma heartbeat id seen (outage history) + "open_outages": {}, # monitor_id -> open DOWN window, cross-cycle } @@ -168,6 +170,92 @@ def read_kuma_monitors(cfg): return json.loads(data) +# Kuma flags every up/down transition with important=1. This reads those +# transitions since a given heartbeat id, plus the current max id (watermark), +# in one shot — so we can spot outages that have already self-recovered. +KUMA_EVENTS_QUERY = ( + "select json_object(" + "'watermark', (select max(id) from heartbeat), " + "'events', (select json_group_array(json_object(" + "'monitor_id', h.monitor_id, 'name', m.name, 'status', h.status, " + "'time', h.time, 'msg', h.msg)) " + "from heartbeat h join monitor m on m.id = h.monitor_id " + "where h.important = 1 {clause} order by h.id));" +) + + +def read_kuma_events(cfg, since_id): + """Return (watermark, events) of monitor up/down transitions. + + `events` are the important (transition) heartbeats with id > since_id, in + chronological order; `watermark` is the current max heartbeat id to pass + back next time. since_id=None means first call: returns the watermark with + NO events (a baseline, so we don't replay weeks of history). Uses the same + throwaway sqlite container as read_kuma_monitors. + """ + clause = "and 1=0" if since_id is None else f"and h.id > {int(since_id)}" + query = KUMA_EVENTS_QUERY.format(clause=clause) + script = f"cp /src/kuma.db /tmp/k.db && sqlite3 /tmp/k.db \"{query}\"" + out = subprocess.run( + [ + "docker", "run", "--rm", + "-v", f"{cfg['kuma_volume']}:/src:ro", + cfg["sqlite_image"], "sh", "-c", script, + ], + capture_output=True, text=True, timeout=120, + ) + if out.returncode != 0: + raise RuntimeError(f"kuma events read failed: {out.stderr.strip()}") + data = json.loads(out.stdout.strip() or "{}") + return data.get("watermark"), (data.get("events") or []) + + +def detect_outages(events, open_outages): + """Pair DOWN→UP transitions into self-recovered outage windows. + + `open_outages` (monitor_id str -> {name, down_at, msg}) carries outages + still open from earlier cycles and is mutated in place, so an outage that + spans a cycle boundary is still paired correctly. Returns the windows that + have now recovered (a service that went DOWN and is UP again). + """ + recovered = [] + for ev in events: + mid = str(ev["monitor_id"]) + if ev["status"] == KUMA_DOWN: + open_outages[mid] = {"name": ev["name"], "down_at": ev["time"], + "msg": ev.get("msg")} + elif ev["status"] == KUMA_UP: + o = open_outages.pop(mid, None) + recovered.append({ + "name": ev["name"], + "down_at": (o or {}).get("down_at"), # None => began before us + "up_at": ev["time"], + "msg": (o or {}).get("msg") or ev.get("msg"), + }) + return recovered + + +def _parse_kuma_time(t): + for fmt in ("%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S"): + try: + return datetime.strptime(t, fmt) + except (ValueError, TypeError): + continue + return None + + +def _fmt_outage_duration(down_at, up_at): + d, u = _parse_kuma_time(down_at), _parse_kuma_time(up_at) + if not d or not u: + return "?" + secs = max(0, int((u - d).total_seconds())) + h, rem = divmod(secs, 3600) + m, s = divmod(rem, 60) + if h: + return f"{h}h{m:02d}m" + return f"{m}m{s:02d}s" if m else f"{s}s" + + def docker_ps(): out = subprocess.run( ["docker", "ps", "-a", "--format", @@ -387,6 +475,42 @@ def status_label(code): KUMA_MAINT: "MAINTENANCE"}.get(code, f"?{code}") +def report_recovered_outages(cfg, state, recovered): + """A tracked service went DOWN then recovered on its own. Surface the + window and, unless notify-only, have the agent dig into logs for that + period so recurring faults get caught even though status is now UP.""" + def fmt(o): + when = o["down_at"] or "(began before last check)" + dur = _fmt_outage_duration(o["down_at"], o["up_at"]) if o["down_at"] else "?" + return (f" • **{o['name']}**: DOWN {when} → recovered {o['up_at']} " + f"({dur}) — {o.get('msg') or 'n/a'}") + lines = "\n".join(fmt(o) for o in recovered) + + if cfg["autonomy"] == "notify": + matrix_send(cfg, "🟡 **Self-recovered outage(s)** (notify-only — not " + f"investigated):\n{lines}") + return + + task = ( + "POST-INCIDENT REVIEW — one or more tracked services went DOWN and then " + "recovered on their own since the last check. Current status is UP, so " + "no remediation is needed; investigate the likely cause so recurring " + "faults are caught early.\n\n" + f"Self-recovered outage windows:\n{lines}\n\n" + "Current containers:\n" + f"{docker_ps()}\n\n" + "For each: map the monitor target to its container, inspect logs and " + "events around the outage window (e.g. `docker logs --since " + "--until `, container restart counts, journald), and identify the " + "probable cause (OOM, restart/redeploy, dependency flap, network). Do " + "NOT change anything unless something is actively still wrong. Report " + "concise per-service findings; set status=escalate only if a fix needs " + "a human decision." + ) + result = parse_agent_result(run_agent(cfg, task)) + relay_result(cfg, state, "Outage post-mortem", result) + + def health_cycle(cfg, state): log("Health cycle: reading Uptime Kuma…") monitors = read_kuma_monitors(cfg) @@ -407,6 +531,19 @@ def health_cycle(cfg, state): log(f" {len(monitors)} tracked, {len(down_now)} down, " f"{len(confirmed)} confirmed down") + # Surface services that blipped DOWN→UP since we last looked. Current status + # may be fine, but a self-recovered outage is a pointer worth investigating. + try: + open_outages = state.setdefault("open_outages", {}) + watermark, events = read_kuma_events(cfg, state.get("last_event_check")) + recovered = detect_outages(events, open_outages) + state["last_event_check"] = watermark + if recovered: + log(f" {len(recovered)} self-recovered outage(s) since last check") + report_recovered_outages(cfg, state, recovered) + except Exception as e: + log(f"WARN: outage-history check failed: {e}") + if not confirmed: # quiet heartbeat if (time.time() - state.get("last_heartbeat", 0) diff --git a/register-matrix-bot.sh b/register-matrix-bot.sh index 6952794..5f1131e 100755 --- a/register-matrix-bot.sh +++ b/register-matrix-bot.sh @@ -43,7 +43,7 @@ BOT_USER_ID="$(cfg_get "['matrix']['user_id']")" ADMIN_USER_ID="$(cfg_get "['matrix']['admin_user_id']")" ROOM_ID="$(cfg_get "['matrix'].get('room_id','')" 2>/dev/null || echo "")" -# localpart from @claude-code:semprini.me -> claude-code +# localpart from @maintainer-bot:semprini.me -> maintainer-bot BOT_LOCAL="${BOT_USER_ID#@}"; BOT_LOCAL="${BOT_LOCAL%%:*}" echo "→ Homeserver : $HOMESERVER"