Update bot user ID and model name in configuration files; enhance outage recovery reporting

maintainer.py

@@ -57,7 +57,7 @@ DEFAULTS = {
     # default, which may be a 1M-context ([1m]) model; once a cycle's context
     # crosses 200K that escalates to the 1M tier and fails on subscription
     # plans with "Usage credits are required for long context requests" (429).
-    "claude_model": "claude-opus-4-8",
+    "claude_model": "claude-sonnet-4-6",
     "claude_extra_args": [],
     "agent_timeout_seconds": 1800,
     "kuma_volume": "uptime-kuma-data",
@@ -112,6 +112,8 @@ def load_state(cfg):
         "down_streak": {},      # monitor name -> consecutive-down count
         "matrix_since": None,   # /sync pagination token
         "pending": None,        # outstanding question awaiting admin reply
+        "last_event_check": None,  # max kuma heartbeat id seen (outage history)
+        "open_outages": {},     # monitor_id -> open DOWN window, cross-cycle
     }
 
 
@@ -168,6 +170,92 @@ def read_kuma_monitors(cfg):
     return json.loads(data)
 
 
+# Kuma flags every up/down transition with important=1. This reads those
+# transitions since a given heartbeat id, plus the current max id (watermark),
+# in one shot — so we can spot outages that have already self-recovered.
+KUMA_EVENTS_QUERY = (
+    "select json_object("
+    "'watermark', (select max(id) from heartbeat), "
+    "'events', (select json_group_array(json_object("
+    "'monitor_id', h.monitor_id, 'name', m.name, 'status', h.status, "
+    "'time', h.time, 'msg', h.msg)) "
+    "from heartbeat h join monitor m on m.id = h.monitor_id "
+    "where h.important = 1 {clause} order by h.id));"
+)
+
+
+def read_kuma_events(cfg, since_id):
+    """Return (watermark, events) of monitor up/down transitions.
+
+    `events` are the important (transition) heartbeats with id > since_id, in
+    chronological order; `watermark` is the current max heartbeat id to pass
+    back next time. since_id=None means first call: returns the watermark with
+    NO events (a baseline, so we don't replay weeks of history). Uses the same
+    throwaway sqlite container as read_kuma_monitors.
+    """
+    clause = "and 1=0" if since_id is None else f"and h.id > {int(since_id)}"
+    query = KUMA_EVENTS_QUERY.format(clause=clause)
+    script = f"cp /src/kuma.db /tmp/k.db && sqlite3 /tmp/k.db \"{query}\""
+    out = subprocess.run(
+        [
+            "docker", "run", "--rm",
+            "-v", f"{cfg['kuma_volume']}:/src:ro",
+            cfg["sqlite_image"], "sh", "-c", script,
+        ],
+        capture_output=True, text=True, timeout=120,
+    )
+    if out.returncode != 0:
+        raise RuntimeError(f"kuma events read failed: {out.stderr.strip()}")
+    data = json.loads(out.stdout.strip() or "{}")
+    return data.get("watermark"), (data.get("events") or [])
+
+
+def detect_outages(events, open_outages):
+    """Pair DOWN→UP transitions into self-recovered outage windows.
+
+    `open_outages` (monitor_id str -> {name, down_at, msg}) carries outages
+    still open from earlier cycles and is mutated in place, so an outage that
+    spans a cycle boundary is still paired correctly. Returns the windows that
+    have now recovered (a service that went DOWN and is UP again).
+    """
+    recovered = []
+    for ev in events:
+        mid = str(ev["monitor_id"])
+        if ev["status"] == KUMA_DOWN:
+            open_outages[mid] = {"name": ev["name"], "down_at": ev["time"],
+                                 "msg": ev.get("msg")}
+        elif ev["status"] == KUMA_UP:
+            o = open_outages.pop(mid, None)
+            recovered.append({
+                "name": ev["name"],
+                "down_at": (o or {}).get("down_at"),   # None => began before us
+                "up_at": ev["time"],
+                "msg": (o or {}).get("msg") or ev.get("msg"),
+            })
+    return recovered
+
+
+def _parse_kuma_time(t):
+    for fmt in ("%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S"):
+        try:
+            return datetime.strptime(t, fmt)
+        except (ValueError, TypeError):
+            continue
+    return None
+
+
+def _fmt_outage_duration(down_at, up_at):
+    d, u = _parse_kuma_time(down_at), _parse_kuma_time(up_at)
+    if not d or not u:
+        return "?"
+    secs = max(0, int((u - d).total_seconds()))
+    h, rem = divmod(secs, 3600)
+    m, s = divmod(rem, 60)
+    if h:
+        return f"{h}h{m:02d}m"
+    return f"{m}m{s:02d}s" if m else f"{s}s"
+
+
 def docker_ps():
     out = subprocess.run(
         ["docker", "ps", "-a", "--format",
@@ -387,6 +475,42 @@ def status_label(code):
             KUMA_MAINT: "MAINTENANCE"}.get(code, f"?{code}")
 
 
+def report_recovered_outages(cfg, state, recovered):
+    """A tracked service went DOWN then recovered on its own. Surface the
+    window and, unless notify-only, have the agent dig into logs for that
+    period so recurring faults get caught even though status is now UP."""
+    def fmt(o):
+        when = o["down_at"] or "(began before last check)"
+        dur = _fmt_outage_duration(o["down_at"], o["up_at"]) if o["down_at"] else "?"
+        return (f"  • **{o['name']}**: DOWN {when} → recovered {o['up_at']} "
+                f"({dur}) — {o.get('msg') or 'n/a'}")
+    lines = "\n".join(fmt(o) for o in recovered)
+
+    if cfg["autonomy"] == "notify":
+        matrix_send(cfg, "🟡 **Self-recovered outage(s)** (notify-only — not "
+                         f"investigated):\n{lines}")
+        return
+
+    task = (
+        "POST-INCIDENT REVIEW — one or more tracked services went DOWN and then "
+        "recovered on their own since the last check. Current status is UP, so "
+        "no remediation is needed; investigate the likely cause so recurring "
+        "faults are caught early.\n\n"
+        f"Self-recovered outage windows:\n{lines}\n\n"
+        "Current containers:\n"
+        f"{docker_ps()}\n\n"
+        "For each: map the monitor target to its container, inspect logs and "
+        "events around the outage window (e.g. `docker logs --since <down> "
+        "--until <up>`, container restart counts, journald), and identify the "
+        "probable cause (OOM, restart/redeploy, dependency flap, network). Do "
+        "NOT change anything unless something is actively still wrong. Report "
+        "concise per-service findings; set status=escalate only if a fix needs "
+        "a human decision."
+    )
+    result = parse_agent_result(run_agent(cfg, task))
+    relay_result(cfg, state, "Outage post-mortem", result)
+
+
 def health_cycle(cfg, state):
     log("Health cycle: reading Uptime Kuma…")
     monitors = read_kuma_monitors(cfg)
@@ -407,6 +531,19 @@ def health_cycle(cfg, state):
     log(f"  {len(monitors)} tracked, {len(down_now)} down, "
         f"{len(confirmed)} confirmed down")
 
+    # Surface services that blipped DOWN→UP since we last looked. Current status
+    # may be fine, but a self-recovered outage is a pointer worth investigating.
+    try:
+        open_outages = state.setdefault("open_outages", {})
+        watermark, events = read_kuma_events(cfg, state.get("last_event_check"))
+        recovered = detect_outages(events, open_outages)
+        state["last_event_check"] = watermark
+        if recovered:
+            log(f"  {len(recovered)} self-recovered outage(s) since last check")
+            report_recovered_outages(cfg, state, recovered)
+    except Exception as e:
+        log(f"WARN: outage-history check failed: {e}")
+
     if not confirmed:
         # quiet heartbeat
         if (time.time() - state.get("last_heartbeat", 0)